From 7a10d17ae29e4a68f39774e39d2d53f0845a2618 Mon Sep 17 00:00:00 2001 From: droctothorpe Date: Fri, 10 Apr 2026 12:56:58 -0400 Subject: [PATCH] [WIP/PoC] refactor(driver): centralize driver as standalone ml-pipeline-driver service Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: test Signed-off-by: droctothorpe --- .github/actions/deploy/action.yml | 10 - .../manifests/base/apiserver-env.yaml | 2 - .../default/kustomization.yaml | 3 + .../artifact-proxy/kustomization.yaml | 4 +- .../cache-disabled/kustomization.yaml | 3 + .../multiuser/default/kustomization.yaml | 3 + .../cache-disabled/kustomization.yaml | 3 + .../standalone/default/kustomization.yaml | 3 + .../standalone/proxy/kustomization.yaml | 3 + .../standalone/tls-enabled/kustomization.yaml | 3 + .github/resources/scripts/collect-logs.sh | 18 +- .../scripts/kfp-readiness/wait_for_pods.py | 3 +- .github/workflows/api-server-tests.yml | 8 +- .github/workflows/e2e-test.yml | 13 +- .github/workflows/image-builds-master.yml | 8 +- .github/workflows/image-builds-release.yml | 7 +- .github/workflows/image-builds.yml | 6 +- .github/workflows/kfp-sdk-tests.yml | 4 +- .../legacy-v2-api-integration-tests.yml | 2 +- .github/workflows/sdk-component-yaml.yml | 7 +- Makefile | 10 - README.md | 8 +- .../go/pipelinespec/pipeline_spec.pb.go | 63 +- api/v2alpha1/pipeline_spec.proto | 24 +- backend/Dockerfile | 4 + backend/Dockerfile.driver | 31 +- backend/Makefile | 33 +- backend/src/apiserver/config/config.go | 35 - backend/src/apiserver/config/config_test.go | 216 ----- backend/src/apiserver/filter/filter.go | 15 - backend/src/apiserver/list/list.go | 121 +-- backend/src/apiserver/list/list_test.go | 48 +- backend/src/apiserver/main.go | 125 +-- backend/src/apiserver/main_test.go | 188 ---- backend/src/apiserver/model/experiment.go | 6 +- backend/src/apiserver/model/job.go | 6 +- backend/src/apiserver/model/pipeline.go | 6 +- .../src/apiserver/model/pipeline_version.go | 6 +- backend/src/apiserver/model/run.go | 13 +- backend/src/apiserver/model/task.go | 6 +- .../server/list_request_util_test.go | 7 +- backend/src/apiserver/storage/run_store.go | 12 +- .../src/apiserver/storage/run_store_test.go | 65 -- backend/src/common/util/context_logger.go | 73 ++ backend/src/driver/api/request.go | 55 ++ backend/src/driver/api/response.go | 27 + backend/src/driver/cmd/main.go | 55 ++ backend/src/driver/handler.go | 534 ++++++++++++ backend/src/driver/handler_test.go | 64 ++ backend/src/v2/cmd/driver/execution_paths.go | 9 - backend/src/v2/cmd/driver/main.go | 384 --------- backend/src/v2/cmd/driver/main_test.go | 144 ---- backend/src/v2/compiler/argocompiler/argo.go | 114 +-- .../v2/compiler/argocompiler/argo_ttl_test.go | 170 ---- .../src/v2/compiler/argocompiler/common.go | 12 - .../src/v2/compiler/argocompiler/container.go | 155 ++-- .../compiler/argocompiler/container_test.go | 8 +- backend/src/v2/compiler/argocompiler/dag.go | 132 ++- .../src/v2/compiler/argocompiler/plugin.go | 49 ++ backend/src/v2/component/constants.go | 5 +- backend/src/v2/component/launcher_v2.go | 75 -- backend/src/v2/component/launcher_v2_test.go | 183 +--- backend/src/v2/driver/container.go | 27 +- backend/src/v2/driver/dag.go | 34 +- backend/src/v2/driver/driver.go | 5 +- backend/src/v2/driver/driver_test.go | 6 +- backend/src/v2/driver/k8s.go | 52 +- backend/src/v2/driver/k8s_test.go | 48 +- backend/src/v2/driver/root_dag.go | 37 +- backend/src/v2/metadata/client.go | 33 +- .../test/compiler/compiler_visitor_test.go | 2 +- backend/test/compiler/utils/workflow_utils.go | 49 +- backend/test/constants/test_features.go | 2 - backend/test/end2end/e2e_suite_test.go | 3 +- backend/test/end2end/pipeline_e2e_test.go | 12 +- backend/test/end2end/utils/e2e_utils.go | 23 +- backend/test/proto_tests/objects.go | 2 +- .../testdata/generated-1791485/pipeline.json | 24 +- .../generated-1791485/pipeline_spec.pb | Bin 262 -> 262 bytes .../generated-1791485/pipeline_version.json | 78 +- .../generated-1791485/pipeline_version.pb | Bin 1678 -> 1678 bytes .../generated-1791485/platform_spec.json | 7 +- .../run_completed_with_spec.pb | Bin 1537 -> 1537 bytes backend/test/testutil/kubernetes_utils.go | 10 +- .../docs/react-18-19-upgrade-checklist.md | 13 +- frontend/package-lock.json | 6 +- frontend/package.json | 8 +- .../generate_openapi_typescript_fetch.js | 200 +---- .../generate_openapi_typescript_fetch.test.js | 284 ------ .../integration-tests/artifact-proxy.test.ts | 15 - .../apis/auth/models/GooglerpcStatus.ts | 98 ++- .../generated/apis/auth/models/ProtobufAny.ts | 179 +++- .../server/src/generated/apis/auth/runtime.ts | 490 ++++++++++- .../auth/models/GooglerpcStatus.ts | 107 ++- .../apisv2beta1/auth/models/ProtobufAny.ts | 179 +++- .../src/generated/apisv2beta1/auth/runtime.ts | 490 ++++++++++- .../openapi/models/GooglerpcStatus.ts | 64 -- .../generated/openapi/models/ProtobufAny.ts | 44 - .../server/src/generated/openapi/runtime.ts | 462 ---------- .../apis/experiment/models/GooglerpcStatus.ts | 98 ++- .../src/apis/experiment/models/ProtobufAny.ts | 179 +++- frontend/src/apis/experiment/runtime.ts | 490 ++++++++++- .../src/apis/filter/models/GooglerpcStatus.ts | 98 ++- .../src/apis/filter/models/ProtobufAny.ts | 179 +++- frontend/src/apis/filter/runtime.ts | 490 ++++++++++- .../src/apis/job/models/GooglerpcStatus.ts | 98 ++- frontend/src/apis/job/models/ProtobufAny.ts | 179 +++- frontend/src/apis/job/runtime.ts | 490 ++++++++++- .../apis/pipeline/models/GooglerpcStatus.ts | 98 ++- .../src/apis/pipeline/models/ProtobufAny.ts | 179 +++- frontend/src/apis/pipeline/runtime.ts | 490 ++++++++++- .../src/apis/run/models/GooglerpcStatus.ts | 98 ++- frontend/src/apis/run/models/ProtobufAny.ts | 179 +++- frontend/src/apis/run/runtime.ts | 490 ++++++++++- .../visualization/models/GooglerpcStatus.ts | 98 ++- .../apis/visualization/models/ProtobufAny.ts | 179 +++- frontend/src/apis/visualization/runtime.ts | 490 ++++++++++- .../experiment/models/GooglerpcStatus.ts | 107 ++- .../experiment/models/ProtobufAny.ts | 179 +++- .../src/apisv2beta1/experiment/runtime.ts | 490 ++++++++++- .../filter/models/GooglerpcStatus.ts | 107 ++- .../apisv2beta1/filter/models/ProtobufAny.ts | 179 +++- frontend/src/apisv2beta1/filter/runtime.ts | 490 ++++++++++- .../pipeline/models/GooglerpcStatus.ts | 107 ++- .../pipeline/models/ProtobufAny.ts | 179 +++- frontend/src/apisv2beta1/pipeline/runtime.ts | 490 ++++++++++- .../recurringrun/models/GooglerpcStatus.ts | 107 ++- .../recurringrun/models/ProtobufAny.ts | 179 +++- .../src/apisv2beta1/recurringrun/runtime.ts | 490 ++++++++++- .../apisv2beta1/run/models/GooglerpcStatus.ts | 107 ++- .../src/apisv2beta1/run/models/ProtobufAny.ts | 179 +++- frontend/src/apisv2beta1/run/runtime.ts | 490 ++++++++++- .../visualization/models/GooglerpcStatus.ts | 107 ++- .../visualization/models/ProtobufAny.ts | 179 +++- .../src/apisv2beta1/visualization/runtime.ts | 490 ++++++++++- frontend/src/components/CustomTable.test.tsx | 39 - frontend/src/components/CustomTable.tsx | 27 +- .../components/NewRunParametersV2.test.tsx | 29 - .../components/tabs/RuntimeNodeDetailsV2.tsx | 82 +- .../components/viewers/MetricsDropdown.tsx | 53 +- .../openapi/models/GooglerpcStatus.ts | 64 -- .../generated/openapi/models/ProtobufAny.ts | 44 - frontend/src/generated/openapi/runtime.ts | 462 ---------- frontend/src/hooks/queryKeys.ts | 3 + frontend/src/lib/LocalStorage.test.ts | 33 - frontend/src/lib/LocalStorage.ts | 18 - frontend/src/pages/CompareV2.test.tsx | 153 ---- frontend/src/pages/CompareV2.tsx | 473 ++++------ .../python/kfp/kubernetes/common.py | 25 +- .../python/kfp/kubernetes/volume.py | 6 +- .../python/test/unit/test_secret.py | 75 -- manifests/kustomize/README.md | 9 + .../decorator-controller.yaml | 14 +- .../deployment.yaml | 2 +- .../pipelines-profile-controller-admin.yaml | 42 + .../requirements-dev.txt | 2 +- .../pipelines-profile-controller/sync.py | 663 +++++++++----- .../pipelines-profile-controller/test_sync.py | 521 ++++++----- .../base/pipeline/kustomization.yaml | 7 + .../ml-pipeline-apiserver-deployment.yaml | 2 - .../ml-pipeline-driver-deployment.yaml | 62 ++ .../pipeline/ml-pipeline-driver-role.yaml | 19 + .../ml-pipeline-driver-rolebinding.yaml | 13 + .../base/pipeline/ml-pipeline-driver-sa.yaml | 4 + .../pipeline/ml-pipeline-driver-service.yaml | 14 + .../base/pipeline/pipeline-runner-role.yaml | 103 ++- .../base/pipeline/pipeline-runner-sa.yaml | 9 + .../base-tls-certs/kfp-api-cert.yaml | 4 +- .../patches/metadata-envoy-deployment.yaml | 2 +- .../patches/metadata-grpc-deployment.yaml | 2 +- .../patches/metadata-writer-deployment.yaml | 2 +- .../ml-pipeline-apiserver-deployment.yaml | 4 +- ...-pipeline-persistenceagent-deployment.yaml | 2 +- ...pipeline-scheduledworkflow-deployment.yaml | 2 +- .../patches/ml-pipeline-ui-deployment.yaml | 2 +- .../kustomize/env/dev/api-server-patch.yaml | 2 - .../kustomization.yaml | 4 + .../kustomization.yaml | 4 + .../kustomization.yaml | 25 + .../kustomize/gcp-workload-identity-setup.sh | 202 +++++ manifests/kustomize/hack/release.sh | 4 - manifests/kustomize/hack/test.sh | 2 + manifests/kustomize/sample/README.md | 9 +- .../application-controller-deployment.yaml | 2 +- .../third-party/argo/base/kustomization.yaml | 1 + ...w-controller-argo-taskset-clusterrole.yaml | 27 + .../workflow-controller-deployment-patch.yaml | 4 + .../installs/namespace/kustomization.yaml | 2 +- .../workflow-controller-argo-role-patch.json | 11 + .../metacontroller/base/cluster-role.yaml | 8 +- manifests/kustomize/wi-utils.sh | 85 ++ proposals/12862-mlflow-integration/README.md | 67 +- sdk/python/kfp/compiler/compiler_test.py | 224 ----- .../kfp/compiler/pipeline_spec_builder.py | 99 +-- .../compiler/pipeline_spec_builder_test.py | 424 --------- sdk/python/kfp/dsl/pipeline_config.py | 56 +- sdk/python/kfp/dsl/pipeline_task.py | 34 +- sdk/python/kfp/dsl/pipeline_task_test.py | 16 - .../compilation/pipeline_compilation_test.py | 51 +- .../local_execution/local_execution_test.py | 30 +- test/deploy-pipeline-lite.sh | 35 +- .../helloworld.spec.js | 40 +- .../literal-input.spec.js | 289 ------- .../literal-input.yaml | 105 --- .../tensorboard-example.spec.js | 31 +- .../frontend-integration-test/test-helpers.js | 56 +- test/frontend-integration-test/wdio.conf.js | 2 +- test/install-argo.sh | 19 + test/presubmit-tests-sdk.sh | 4 +- test_data/compiled-workflows/add_numbers.yaml | 200 +---- .../arguments_parameters.yaml | 200 +---- .../compiled-workflows/artifact_cache.yaml | 212 +---- .../compiled-workflows/artifact_crust.yaml | 212 +---- .../compiled-workflows/artifacts_complex.yaml | 245 ++---- .../compiled-workflows/artifacts_simple.yaml | 216 +---- .../collected_artifacts.yaml | 264 ++---- .../collected_parameters.yaml | 232 +---- .../component_with_metadata_fields.yaml | 200 +---- .../component_with_optional_inputs.yaml | 200 +---- .../component_with_pip_index_urls.yaml | 200 +---- .../component_with_pip_install.yaml | 202 +---- .../component_with_pip_install_in_venv.yaml | 202 +---- .../components_with_optional_artifacts.yaml | 230 +---- .../compiled-workflows/concat_message.yaml | 200 +---- .../conditional_producer_and_consumers.yaml | 226 +---- .../container_component_with_no_inputs.yaml | 202 +---- .../compiled-workflows/container_io.yaml | 200 +---- .../container_no_input.yaml | 202 +---- .../container_with_artifact_output.yaml | 202 +---- .../container_with_concat_placeholder.yaml | 202 +---- .../container_with_if_placeholder.yaml | 202 +---- ...container_with_placeholder_in_fstring.yaml | 202 +---- .../containerized_python_component.yaml | 200 +---- .../create_pod_metadata_complex.yaml | 214 +---- .../cross_loop_after_topology.yaml | 284 ++---- test_data/compiled-workflows/dict_input.yaml | 200 +---- .../compiled-workflows/embedded_artifact.yaml | 208 +---- test_data/compiled-workflows/env-var.yaml | 200 +---- test_data/compiled-workflows/fail_v2.yaml | 200 +---- test_data/compiled-workflows/flip_coin.yaml | 266 ++---- test_data/compiled-workflows/hello_world.yaml | 200 +---- test_data/compiled-workflows/identity.yaml | 200 +---- .../if_elif_else_complex.yaml | 372 +++----- .../if_elif_else_with_oneof_parameters.yaml | 259 ++---- .../if_else_with_oneof_artifacts.yaml | 246 ++---- .../if_else_with_oneof_parameters.yaml | 238 ++--- .../compiled-workflows/input_artifact.yaml | 200 +---- .../iris_pipeline_compiled.yaml | 210 +---- ...lightweight_python_functions_pipeline.yaml | 204 +---- ...tweight_python_functions_with_outputs.yaml | 216 +---- .../log_streaming_compiled.yaml | 200 +---- .../compiled-workflows/long-running.yaml | 204 +---- .../loop_consume_upstream.yaml | 220 +---- .../metrics_visualization_v2.yaml | 226 +---- .../missing_kubernetes_optional_inputs.yaml | 200 +---- .../compiled-workflows/mixed_parameters.yaml | 212 +---- test_data/compiled-workflows/modelcar.yaml | 208 +---- .../mounted_cabundle_configmap.yaml | 224 +---- .../mounted_cabundle_secret.yaml | 224 +---- .../multiple_artifacts_namedtuple.yaml | 212 +---- .../multiple_parameters_namedtuple.yaml | 212 +---- .../nested_parallel_for_secret.yaml | 533 ------------ ...peline_opt_input_child_level_compiled.yaml | 238 ++--- ...sted_pipeline_opt_inputs_nil_compiled.yaml | 214 +---- ...line_opt_inputs_parent_level_compiled.yaml | 244 ++---- .../compiled-workflows/nested_return.yaml | 200 +---- .../nested_with_parameters.yaml | 228 +---- .../notebook_component_mixed.yaml | 208 +---- .../notebook_component_simple.yaml | 202 +---- .../compiled-workflows/output_metrics.yaml | 200 +---- .../parallel_for_after_dependency.yaml | 216 +---- .../parallel_for_mount_pvc.yaml | 526 ----------- .../parallel_for_secret.yaml | 494 ----------- .../compiled-workflows/parameter_cache.yaml | 212 +---- .../compiled-workflows/parameter_oneof.yaml | 242 ++---- .../parameters_complex.yaml | 250 ++---- .../compiled-workflows/parameters_simple.yaml | 216 +---- .../pipeline_as_exit_task.yaml | 229 +---- .../pipeline_in_pipeline.yaml | 212 +---- .../pipeline_in_pipeline_complex.yaml | 238 +---- ...pipeline_in_pipeline_loaded_from_yaml.yaml | 222 +---- .../pipeline_producer_consumer.yaml | 248 ++---- .../pipeline_with_after.yaml | 208 +---- .../pipeline_with_artifact_custom_path.yaml | 208 +---- ...ipeline_with_artifact_upload_download.yaml | 208 +---- .../pipeline_with_concat_placeholder.yaml | 202 +---- .../pipeline_with_condition.yaml | 223 +---- ...namic_task_output_custom_training_job.yaml | 231 +---- ...peline_with_dynamic_importer_metadata.yaml | 200 +---- ...namic_task_output_custom_training_job.yaml | 218 +---- .../compiled-workflows/pipeline_with_env.yaml | 204 +---- .../pipeline_with_exit_handler.yaml | 212 +---- .../pipeline_with_google_artifact_type.yaml | 204 +---- .../pipeline_with_importer.yaml | 211 +---- ...pipeline_with_importer_and_gcpc_types.yaml | 200 +---- .../pipeline_with_importer_workspace.yaml | 228 +---- .../pipeline_with_input_status_state.yaml | 208 +---- .../pipeline_with_loops.yaml | 308 ++----- .../pipeline_with_loops_and_conditions.yaml | 358 +++----- .../pipeline_with_metadata_fields.yaml | 204 +---- .../pipeline_with_metrics_outputs.yaml | 214 +---- .../pipeline_with_multiple_exit_handlers.yaml | 236 +---- .../pipeline_with_nested_conditions.yaml | 242 +----- .../pipeline_with_nested_conditions_yaml.yaml | 270 ++---- .../pipeline_with_nested_loops.yaml | 240 +----- .../pipeline_with_only_display_name.yaml | 200 +---- .../pipeline_with_outputs.yaml | 212 +---- ...pipeline_with_parallelfor_parallelism.yaml | 354 +++----- ...eline_with_parallelfor_pipeline_param.yaml | 813 ------------------ ...ipeline_with_params_containing_format.yaml | 216 +---- .../pipeline_with_placeholders.yaml | 202 +---- .../pipeline_with_pod_metadata.yaml | 226 +---- .../pipeline_with_retry.yaml | 202 +---- .../pipeline_with_reused_component.yaml | 208 +---- .../pipeline_with_secret_as_env.yaml | 206 +---- .../pipeline_with_secret_as_volume.yaml | 200 +---- ..._string_machine_fields_pipeline_input.yaml | 200 +---- ...ith_string_machine_fields_task_output.yaml | 228 +---- .../pipeline_with_submit_request.yaml | 206 +---- .../pipeline_with_task_final_status.yaml | 212 +---- .../pipeline_with_task_final_status_yaml.yaml | 208 +---- ...th_task_using_ignore_upstream_failure.yaml | 204 +---- .../pipeline_with_utils.yaml | 200 +---- .../pipeline_with_various_io_types.yaml | 204 +---- .../pipeline_with_volume.yaml | 204 +---- .../pipeline_with_volume_long_name.yaml | 200 +---- .../pipeline_with_volume_no_cache.yaml | 204 +---- .../pipeline_with_workspace.yaml | 208 +---- ..._with_if_placeholder_none_input_value.yaml | 202 +---- test_data/compiled-workflows/preprocess.yaml | 200 +---- .../producer_consumer_param_pipeline.yaml | 204 +---- test_data/compiled-workflows/pvc_mount.yaml | 204 +---- .../pythonic_artifact_with_single_return.yaml | 202 +---- .../pythonic_artifacts_test_pipeline.yaml | 204 +---- ...onic_artifacts_with_list_of_artifacts.yaml | 212 +---- ...honic_artifacts_with_multiple_returns.yaml | 212 +---- .../ray_integration_compiled.yaml | 200 +---- .../run_as_user_cache_disabled.yaml | 204 +---- .../run_as_user_cache_enabled.yaml | 204 +---- .../compiled-workflows/sequential_v1.yaml | 204 +---- .../compiled-workflows/sequential_v2.yaml | 204 +---- .../compiled-workflows/take_nap_compiled.yaml | 204 +---- .../take_nap_pipeline_root_compiled.yaml | 204 +---- .../compiled-workflows/two_step_pipeline.yaml | 204 +---- .../two_step_pipeline_containerized.yaml | 204 +---- .../upload_download_compiled.yaml | 210 +---- .../xgboost_sample_pipeline.yaml | 242 ++---- .../seaweedfs/kubeflow-edit-clusterrole.yaml | 3 + .../loop_consume_upstream.py | 0 .../loop_consume_upstream.yaml | 0 .../nested_pipeline_opt_input_child_level.py | 0 ...peline_opt_input_child_level_compiled.yaml | 0 .../nested_pipeline_opt_inputs_nil.py | 0 ...sted_pipeline_opt_inputs_nil_compiled.yaml | 0 ...nested_pipeline_opt_inputs_parent_level.py | 0 ...line_opt_inputs_parent_level_compiled.yaml | 0 .../valid/{essential => }/dict_input.py | 0 .../valid/dict_input.yaml | 63 ++ .../pipeline_in_pipeline.py | 0 .../pipeline_in_pipeline.yaml | 0 .../pipeline_in_pipeline_complex.py | 0 .../pipeline_in_pipeline_complex.yaml | 0 .../pipeline_in_pipeline_loaded_from_yaml.py | 0 ...pipeline_in_pipeline_loaded_from_yaml.yaml | 0 .../pipeline_with_loops.py | 2 +- .../pipeline_with_loops.yaml | 24 +- .../valid/parallel_and_nested/__init__.py | 0 .../nested_parallel_for_secret.py | 45 - .../nested_parallel_for_secret.yaml | 148 ---- .../parallel_for_mount_pvc.py | 41 - .../parallel_for_mount_pvc.yaml | 195 ----- .../parallel_for_secret.py | 39 - .../parallel_for_secret.yaml | 110 --- ...ipeline_with_parallelfor_pipeline_param.py | 63 -- ...eline_with_parallelfor_pipeline_param.yaml | 338 -------- 375 files changed, 18681 insertions(+), 34778 deletions(-) create mode 100644 backend/src/common/util/context_logger.go create mode 100644 backend/src/driver/api/request.go create mode 100644 backend/src/driver/api/response.go create mode 100644 backend/src/driver/cmd/main.go create mode 100644 backend/src/driver/handler.go create mode 100644 backend/src/driver/handler_test.go delete mode 100644 backend/src/v2/cmd/driver/execution_paths.go delete mode 100644 backend/src/v2/cmd/driver/main.go delete mode 100644 backend/src/v2/cmd/driver/main_test.go delete mode 100644 backend/src/v2/compiler/argocompiler/argo_ttl_test.go create mode 100644 backend/src/v2/compiler/argocompiler/plugin.go delete mode 100644 frontend/server/src/generated/openapi/models/GooglerpcStatus.ts delete mode 100644 frontend/server/src/generated/openapi/models/ProtobufAny.ts delete mode 100644 frontend/server/src/generated/openapi/runtime.ts delete mode 100644 frontend/src/generated/openapi/models/GooglerpcStatus.ts delete mode 100644 frontend/src/generated/openapi/models/ProtobufAny.ts delete mode 100644 frontend/src/generated/openapi/runtime.ts create mode 100644 manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml create mode 100644 manifests/kustomize/base/pipeline/ml-pipeline-driver-deployment.yaml create mode 100644 manifests/kustomize/base/pipeline/ml-pipeline-driver-role.yaml create mode 100644 manifests/kustomize/base/pipeline/ml-pipeline-driver-rolebinding.yaml create mode 100644 manifests/kustomize/base/pipeline/ml-pipeline-driver-sa.yaml create mode 100644 manifests/kustomize/base/pipeline/ml-pipeline-driver-service.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-emissary/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-multi-user-emissary/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-multi-user-legacy/kustomization.yaml create mode 100755 manifests/kustomize/gcp-workload-identity-setup.sh create mode 100644 manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole.yaml create mode 100644 manifests/kustomize/third-party/argo/installs/namespace/workflow-controller-argo-role-patch.json create mode 100644 manifests/kustomize/wi-utils.sh delete mode 100644 test/frontend-integration-test/literal-input.spec.js delete mode 100644 test/frontend-integration-test/literal-input.yaml delete mode 100644 test_data/compiled-workflows/nested_parallel_for_secret.yaml delete mode 100644 test_data/compiled-workflows/parallel_for_mount_pvc.yaml delete mode 100644 test_data/compiled-workflows/parallel_for_secret.yaml delete mode 100644 test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/loop_consume_upstream.py (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/loop_consume_upstream.yaml (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/nested_pipeline_opt_input_child_level.py (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/nested_pipeline_opt_input_child_level_compiled.yaml (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/nested_pipeline_opt_inputs_nil.py (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/nested_pipeline_opt_inputs_nil_compiled.yaml (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/nested_pipeline_opt_inputs_parent_level.py (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => critical}/nested_pipeline_opt_inputs_parent_level_compiled.yaml (100%) rename test_data/sdk_compiled_pipelines/valid/{essential => }/dict_input.py (100%) create mode 100644 test_data/sdk_compiled_pipelines/valid/dict_input.yaml rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_in_pipeline.py (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_in_pipeline.yaml (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_in_pipeline_complex.py (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_in_pipeline_complex.yaml (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_in_pipeline_loaded_from_yaml.py (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_in_pipeline_loaded_from_yaml.yaml (100%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_with_loops.py (96%) rename test_data/sdk_compiled_pipelines/valid/{parallel_and_nested => essential}/pipeline_with_loops.yaml (98%) delete mode 100644 test_data/sdk_compiled_pipelines/valid/parallel_and_nested/__init__.py delete mode 100644 test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.py delete mode 100644 test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.yaml delete mode 100644 test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.py delete mode 100644 test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.yaml delete mode 100644 test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.py delete mode 100644 test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.yaml delete mode 100644 test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.py delete mode 100644 test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.yaml diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 954064cb1d0..5bf87671f9c 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -2,10 +2,6 @@ name: "Set up KFP on KinD" description: "Step to start and configure KFP on Kind" inputs: - cluster_name: - description: "Kind cluster name" - default: 'kfp' - required: false pipeline_store: description: "Flag to deploy KFP with K8s Native API" default: 'database' @@ -74,12 +70,6 @@ runs: docker image rm ${{ inputs.image_registry }}/$app:${{ inputs.image_tag }} docker image rm localhost:5000/$app:${{ inputs.image_tag }} done - docker pull python:3.11 - docker pull registry.access.redhat.com/ubi9/python-311:latest - kind load docker-image python:3.11 --name ${{ inputs.cluster_name }} - kind load docker-image registry.access.redhat.com/ubi9/python-311:latest --name ${{ inputs.cluster_name }} - docker image rm python:3.11 - docker image rm registry.access.redhat.com/ubi9/python-311:latest - name: Configure Args shell: bash diff --git a/.github/resources/manifests/base/apiserver-env.yaml b/.github/resources/manifests/base/apiserver-env.yaml index 510e3814118..ca919567e6c 100644 --- a/.github/resources/manifests/base/apiserver-env.yaml +++ b/.github/resources/manifests/base/apiserver-env.yaml @@ -21,8 +21,6 @@ spec: containers: - name: ml-pipeline-api-server env: - - name: V2_DRIVER_IMAGE - value: kind-registry:5000/driver - name: V2_LAUNCHER_IMAGE value: kind-registry:5000/launcher - name: LOG_LEVEL diff --git a/.github/resources/manifests/kubernetes-native/default/kustomization.yaml b/.github/resources/manifests/kubernetes-native/default/kustomization.yaml index 20a754898c8..b3a928924e8 100644 --- a/.github/resources/manifests/kubernetes-native/default/kustomization.yaml +++ b/.github/resources/manifests/kubernetes-native/default/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../../base/apiserver-env.yaml diff --git a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml b/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml index fbc76e27771..7ee74476fb1 100644 --- a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml +++ b/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../pipeline-crd-rbac.yaml @@ -96,4 +99,3 @@ replacements: name: cache-server fieldPaths: - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local] - diff --git a/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml b/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml index ce8d85bcc36..7983bc4eaf0 100644 --- a/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml +++ b/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../pipeline-crd-rbac.yaml diff --git a/.github/resources/manifests/multiuser/default/kustomization.yaml b/.github/resources/manifests/multiuser/default/kustomization.yaml index 7151c53b506..6cef39f8f98 100644 --- a/.github/resources/manifests/multiuser/default/kustomization.yaml +++ b/.github/resources/manifests/multiuser/default/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../pipeline-crd-rbac.yaml diff --git a/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml b/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml index 9f6c63ef1e4..25b0f4035d7 100644 --- a/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml +++ b/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../../base/apiserver-env.yaml diff --git a/.github/resources/manifests/standalone/default/kustomization.yaml b/.github/resources/manifests/standalone/default/kustomization.yaml index 8572e632121..1eb22822b25 100644 --- a/.github/resources/manifests/standalone/default/kustomization.yaml +++ b/.github/resources/manifests/standalone/default/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../../base/apiserver-env.yaml diff --git a/.github/resources/manifests/standalone/proxy/kustomization.yaml b/.github/resources/manifests/standalone/proxy/kustomization.yaml index 4d432795d1e..43d073da30f 100644 --- a/.github/resources/manifests/standalone/proxy/kustomization.yaml +++ b/.github/resources/manifests/standalone/proxy/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../../base/apiserver-env.yaml diff --git a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml index f7dba7acc6e..3b184531254 100644 --- a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml @@ -36,6 +36,9 @@ images: - name: ghcr.io/kubeflow/kfp-metadata-envoy newName: kind-registry:5000/metadata-envoy newTag: latest + - name: ghcr.io/kubeflow/kfp-driver + newName: kind-registry:5000/driver + newTag: latest patches: - path: ../../base/apiserver-env.yaml diff --git a/.github/resources/scripts/collect-logs.sh b/.github/resources/scripts/collect-logs.sh index 89ea75324b1..95f9e0ea9f2 100755 --- a/.github/resources/scripts/collect-logs.sh +++ b/.github/resources/scripts/collect-logs.sh @@ -29,6 +29,15 @@ function check_namespace { return 0 } +function describe_argo_workflows { + local NAMESPACE=$1 + echo "===== Argo Workflows list =====" + kubectl describe wf -n "${NAMESPACE}" + echo "===== Argo Workflows data =====" + kubectl get events -n "${NAMESPACE}" --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' + echo "===============================" +} + function display_pod_info { local NAMESPACE=$1 @@ -52,7 +61,13 @@ function display_pod_info { kubectl describe pod "${POD_NAME}" -n "${NAMESPACE}" | grep -A 100 Events || echo "No events found for pod ${POD_NAME}." echo "----- LOGS -----" - kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || echo "No logs found for pod ${POD_NAME}." + if [[ "${POD_NAME}" == *-agent* ]]; then + kubectl logs "${POD_NAME}" -n "${NAMESPACE}" -c driver-plugin || \ + echo "No logs found for pod ${POD_NAME}." + else + kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || \ + echo "No logs found for pod ${POD_NAME}." + fi echo "===========================" echo "" @@ -64,6 +79,7 @@ function display_pod_info { if check_namespace "$NS"; then display_pod_info "$NS" + describe_argo_workflows "$NS" else exit 0 fi diff --git a/.github/resources/scripts/kfp-readiness/wait_for_pods.py b/.github/resources/scripts/kfp-readiness/wait_for_pods.py index fc67d5fda0e..fc604586585 100644 --- a/.github/resources/scripts/kfp-readiness/wait_for_pods.py +++ b/.github/resources/scripts/kfp-readiness/wait_for_pods.py @@ -30,7 +30,8 @@ def get_pod_statuses(): statuses = {} for pod in pods.items: pod_name = pod.metadata.name - if "system" not in pod_name: + # This filter is safe: 'ml-pipeline-persistenceagent-' will not be excluded and will be processed. + if not ("system" in pod_name or pod_name.endswith("-agent")): pod_status = pod.status.phase container_statuses = pod.status.container_statuses or [] ready = 0 diff --git a/.github/workflows/api-server-tests.yml b/.github/workflows/api-server-tests.yml index b7916fe547d..db0659400fa 100644 --- a/.github/workflows/api-server-tests.yml +++ b/.github/workflows/api-server-tests.yml @@ -66,13 +66,13 @@ jobs: k8s_version: [ "v1.34.0" ] cache_enabled: [ "true", "false" ] proxy: [ "true", "false" ] - argo_version: [ "v3.7.3", "v3.5.14" ] + argo_version: [ "v3.7.3", "v3.6.7" ] pipeline_store: [ "database" ] pod_to_pod_tls_enabled: [ "false" ] include: - k8s_version: "v1.29.2" cache_enabled: "true" - argo_version: "v3.7.3" + argo_version: "v3.6.7" - k8s_version: "v1.29.2" cache_enabled: "true" argo_version: "v3.5.14" @@ -111,7 +111,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" @@ -167,7 +167,7 @@ jobs: k8s_version: [ "v1.34.0", "v1.29.2" ] cache_enabled: [ "true" ] uploadPipelinesWithKubernetesClient: [ "true", "false" ] - argo_version: [ "v3.7.3", "v3.5.14" ] + argo_version: [ "v3.7.3", "v3.6.7" ] pipeline_store: [ "kubernetes" ] pod_to_pod_tls_enabled: [ "false" ] include: diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index d7b5bcf5082..e216cba41de 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -63,9 +63,9 @@ jobs: matrix: k8s_version: ["v1.34.0"] cache_enabled: ["true", "false"] - argo_version: [ "v3.7.3", "v3.5.14"] + argo_version: [ "v3.7.3", "v3.6.7", "v3.5.14"] proxy: [ "false" ] - test_label: [ "E2ECritical", "E2EEssential", "E2EParallelNested" ] + test_label: [ "E2ECritical" ] pod_to_pod_tls_enabled: [ "false" ] include: - k8s_version: "v1.29.2" @@ -76,6 +76,9 @@ jobs: cache_enabled: "false" proxy: "true" test_label: "E2EProxy" + - k8s_version: "v1.34.0" + cache_enabled: "false" + test_label: "E2EEssential" - k8s_version: "v1.34.0" cache_enabled: "false" test_label: "E2EFailure" @@ -112,7 +115,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" - name: Configure Input Variables shell: bash @@ -128,10 +131,6 @@ jobs: NAMESPACE=${{ inputs.namespace }} fi - if [ "$TEST_LABEL" == "E2EParallelNested" ]; then - NUMBER_OF_NODES=3 - fi - { echo "NUMBER_OF_NODES=$NUMBER_OF_NODES" echo "TEST_LABEL=$TEST_LABEL" diff --git a/.github/workflows/image-builds-master.yml b/.github/workflows/image-builds-master.yml index 7b4fd0e9029..d66c9819a98 100644 --- a/.github/workflows/image-builds-master.yml +++ b/.github/workflows/image-builds-master.yml @@ -30,9 +30,6 @@ jobs: - name: kfp-launcher dockerfile: backend/Dockerfile.launcher context: . - - name: kfp-driver - dockerfile: backend/Dockerfile.driver - context: . - name: kfp-cache-deployer dockerfile: backend/src/cache/deployer/Dockerfile context: . @@ -48,6 +45,9 @@ jobs: - name: kfp-inverse-proxy-agent dockerfile: proxy/Dockerfile context: ./proxy + - name: kfp-driver + dockerfile: backend/Dockerfile.driver + context: . arch: - arch_name: amd64 @@ -93,12 +93,12 @@ jobs: - image: kfp-viewer-crd-controller - image: kfp-visualization-server - image: kfp-launcher - - image: kfp-driver - image: kfp-cache-deployer - image: kfp-cache-server - image: kfp-metadata-writer - image: kfp-metadata-envoy - image: kfp-inverse-proxy-agent + - image: kfp-driver uses: ./.github/workflows/create-manifest.yml secrets: inherit with: diff --git a/.github/workflows/image-builds-release.yml b/.github/workflows/image-builds-release.yml index 843063c67c1..21bb65d8763 100644 --- a/.github/workflows/image-builds-release.yml +++ b/.github/workflows/image-builds-release.yml @@ -59,9 +59,6 @@ jobs: - name: kfp-launcher dockerfile: backend/Dockerfile.launcher context: . - - name: kfp-driver - dockerfile: backend/Dockerfile.driver - context: . - name: kfp-cache-deployer dockerfile: backend/src/cache/deployer/Dockerfile context: . @@ -77,6 +74,9 @@ jobs: - name: kfp-inverse-proxy-agent dockerfile: proxy/Dockerfile context: ./proxy + - name: kfp-driver + dockerfile: backend/Dockerfile.driver + context: . arch: - arch_name: amd64 @@ -123,7 +123,6 @@ jobs: - image: kfp-viewer-crd-controller - image: kfp-visualization-server - image: kfp-launcher - - image: kfp-driver - image: kfp-cache-deployer - image: kfp-cache-server - image: kfp-metadata-writer diff --git a/.github/workflows/image-builds.yml b/.github/workflows/image-builds.yml index 17fe40c099c..899d74cf277 100644 --- a/.github/workflows/image-builds.yml +++ b/.github/workflows/image-builds.yml @@ -41,9 +41,6 @@ jobs: - image: launcher dockerfile: backend/Dockerfile.launcher context: . - - image: driver - dockerfile: backend/Dockerfile.driver - context: . - image: frontend dockerfile: frontend/Dockerfile context: . @@ -62,6 +59,9 @@ jobs: - image: cache-server dockerfile: backend/Dockerfile.cacheserver context: . + - image: driver + dockerfile: backend/Dockerfile.driver + context: . - image: metadata-envoy dockerfile: third_party/metadata_envoy/Dockerfile context: . diff --git a/.github/workflows/kfp-sdk-tests.yml b/.github/workflows/kfp-sdk-tests.yml index 3183e9b15c4..2e9c4ce7b1c 100644 --- a/.github/workflows/kfp-sdk-tests.yml +++ b/.github/workflows/kfp-sdk-tests.yml @@ -20,7 +20,7 @@ concurrency: jobs: sdk-tests: runs-on: ubuntu-latest - timeout-minutes: 50 + timeout-minutes: 45 strategy: matrix: python-version: ['3.9', '3.13'] @@ -49,7 +49,6 @@ jobs: pip install pytest pip install docker pip install pytest-cov - pip install pytest-xdist pip install google_cloud_pipeline_components pip install requests==2.28.1 @@ -59,6 +58,5 @@ jobs: SETUP_ENV: false REPO_NAME: ${{ github.repository }} PULL_NUMBER: ${{ github.event.pull_request.number }} - PYTEST_PARALLEL_WORKERS: 2 run: | ./test/presubmit-tests-sdk.sh diff --git a/.github/workflows/legacy-v2-api-integration-tests.yml b/.github/workflows/legacy-v2-api-integration-tests.yml index 1551dd0e27f..e3aea45d02c 100644 --- a/.github/workflows/legacy-v2-api-integration-tests.yml +++ b/.github/workflows/legacy-v2-api-integration-tests.yml @@ -79,7 +79,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true' }} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" - name: Forward MLMD port diff --git a/.github/workflows/sdk-component-yaml.yml b/.github/workflows/sdk-component-yaml.yml index dc43d981859..e321edf81cb 100644 --- a/.github/workflows/sdk-component-yaml.yml +++ b/.github/workflows/sdk-component-yaml.yml @@ -39,10 +39,13 @@ jobs: uses: ./.github/actions/kfp-k8s - name: Install Test dependencies - run: make test-component-yaml-install-deps + run: | + python3 -m pip install pytest + python3 -m pip install pytest-asyncio-cooperative==0.37.0 + python3 -m pip install --upgrade pip - name: Run component YAML tests env: # We setup the env in the CI SETUP_ENV: false - run: make test-component-yaml-run + run: ./test/presubmit-component-yaml.sh diff --git a/Makefile b/Makefile index 5e881ae9754..ccf1e5051f6 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,3 @@ ginkgo: mkdir -p $(BIN_DIR) GOBIN=$(BIN_DIR) go install github.com/onsi/ginkgo/v2/ginkgo@latest @echo "Ginkgo installed to $(BIN_DIR)/ginkgo" - -# Component YAML Tests -.PHONY: test-component-yaml-install-deps -test-component-yaml-install-deps: - python3 -m pip install pytest - python3 -m pip install pytest-asyncio-cooperative==0.37.0 - -.PHONY: test-component-yaml-run -test-component-yaml-run: - ./test/presubmit-component-yaml.sh diff --git a/README.md b/README.md index b94fd7a2503..e72b9d6fae6 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,10 @@ The Kubeflow pipelines service has the following goals: ### Dependencies Compatibility Matrix -| Dependency | Versions | -| -------------- |--------------| -| Argo Workflows | v3.5, v3.7 | -| MySQL | v8 | +| Dependency | Versions | +| -------------- |------------------| +| Argo Workflows | v3.5, v3.6, v3.7 | +| MySQL | v8 | ## Documentation diff --git a/api/v2alpha1/go/pipelinespec/pipeline_spec.pb.go b/api/v2alpha1/go/pipelinespec/pipeline_spec.pb.go index 9745db6da11..48f559e70c6 100644 --- a/api/v2alpha1/go/pipelinespec/pipeline_spec.pb.go +++ b/api/v2alpha1/go/pipelinespec/pipeline_spec.pb.go @@ -2755,29 +2755,14 @@ func (x *KubernetesWorkspaceConfig) GetPvcSpecPatch() *structpb.Struct { // Spec for pipeline-level config options. See PipelineConfig DSL class. type PipelineConfig struct { state protoimpl.MessageState `protogen:"open.v1"` - // Seconds to retain the Argo Workflow object after the run completes, - // regardless of success or failure. Maps to Argo's - // TTLStrategy.secondsAfterCompletion. A value of 0 or unset means no TTL - // is applied by this field. - ResourceTtlOnCompletion int32 `protobuf:"varint,1,opt,name=resource_ttl_on_completion,json=resourceTtlOnCompletion,proto3" json:"resource_ttl_on_completion,omitempty"` + // Time to live configuration after the pipeline run is completed for + // ephemeral resources created by the pipeline run. + ResourceTtl int32 `protobuf:"varint,1,opt,name=resource_ttl,json=resourceTtl,proto3" json:"resource_ttl,omitempty"` // Configuration for a shared storage workspace that persists for the duration of the pipeline run. // The workspace can be configured with size and Kubernetes-specific settings to override default PVC configurations. - Workspace *WorkspaceConfig `protobuf:"bytes,2,opt,name=workspace,proto3,oneof" json:"workspace,omitempty"` - // Seconds to retain the Argo Workflow object after a *successful* run. - // Maps to Argo's TTLStrategy.secondsAfterSuccess. Takes precedence over - // resource_ttl_on_completion for successful runs when both are set. - ResourceTtlOnSuccess int32 `protobuf:"varint,3,opt,name=resource_ttl_on_success,json=resourceTtlOnSuccess,proto3" json:"resource_ttl_on_success,omitempty"` - // Seconds to retain the Argo Workflow object after a *failed* run. - // Maps to Argo's TTLStrategy.secondsAfterFailure. Takes precedence over - // resource_ttl_on_completion for failed runs when both are set. - ResourceTtlOnFailure int32 `protobuf:"varint,4,opt,name=resource_ttl_on_failure,json=resourceTtlOnFailure,proto3" json:"resource_ttl_on_failure,omitempty"` - // Maximum number of seconds a workflow is allowed to run before it is - // forcibly terminated. Maps to Argo's activeDeadlineSeconds. A value - // of 0 (proto default) means no deadline is applied. Set a positive - // value to enforce a maximum execution time. - ActiveDeadlineSeconds int32 `protobuf:"varint,5,opt,name=active_deadline_seconds,json=activeDeadlineSeconds,proto3" json:"active_deadline_seconds,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Workspace *WorkspaceConfig `protobuf:"bytes,2,opt,name=workspace,proto3,oneof" json:"workspace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PipelineConfig) Reset() { @@ -2810,9 +2795,9 @@ func (*PipelineConfig) Descriptor() ([]byte, []int) { return file_pipeline_spec_proto_rawDescGZIP(), []int{34} } -func (x *PipelineConfig) GetResourceTtlOnCompletion() int32 { +func (x *PipelineConfig) GetResourceTtl() int32 { if x != nil { - return x.ResourceTtlOnCompletion + return x.ResourceTtl } return 0 } @@ -2824,27 +2809,6 @@ func (x *PipelineConfig) GetWorkspace() *WorkspaceConfig { return nil } -func (x *PipelineConfig) GetResourceTtlOnSuccess() int32 { - if x != nil { - return x.ResourceTtlOnSuccess - } - return 0 -} - -func (x *PipelineConfig) GetResourceTtlOnFailure() int32 { - if x != nil { - return x.ResourceTtlOnFailure - } - return 0 -} - -func (x *PipelineConfig) GetActiveDeadlineSeconds() int32 { - if x != nil { - return x.ActiveDeadlineSeconds - } - return 0 -} - // The runtime config of a PipelineJob. type PipelineJob_RuntimeConfig struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -6157,13 +6121,10 @@ const file_pipeline_spec_proto_rawDesc = "" + "\v_kubernetes\"r\n" + "\x19KubernetesWorkspaceConfig\x12B\n" + "\x0epvc_spec_patch\x18\x01 \x01(\v2\x17.google.protobuf.StructH\x00R\fpvcSpecPatch\x88\x01\x01B\x11\n" + - "\x0f_pvc_spec_patch\"\xc3\x02\n" + - "\x0ePipelineConfig\x12;\n" + - "\x1aresource_ttl_on_completion\x18\x01 \x01(\x05R\x17resourceTtlOnCompletion\x12@\n" + - "\tworkspace\x18\x02 \x01(\v2\x1d.ml_pipelines.WorkspaceConfigH\x00R\tworkspace\x88\x01\x01\x125\n" + - "\x17resource_ttl_on_success\x18\x03 \x01(\x05R\x14resourceTtlOnSuccess\x125\n" + - "\x17resource_ttl_on_failure\x18\x04 \x01(\x05R\x14resourceTtlOnFailure\x126\n" + - "\x17active_deadline_seconds\x18\x05 \x01(\x05R\x15activeDeadlineSecondsB\f\n" + + "\x0f_pvc_spec_patch\"\x83\x01\n" + + "\x0ePipelineConfig\x12!\n" + + "\fresource_ttl\x18\x01 \x01(\x05R\vresourceTtl\x12@\n" + + "\tworkspace\x18\x02 \x01(\v2\x1d.ml_pipelines.WorkspaceConfigH\x00R\tworkspace\x88\x01\x01B\f\n" + "\n" + "_workspaceB backend/Dockerfile.driver-debug - cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build --platform linux/amd64 --build-arg GCFLAGS="all=-N -l" -t ${IMG_REGISTRY}${IMG_TAG_DRIVER}:debug -f backend/Dockerfile.driver-debug . .PHONY: image_launcher image_launcher: cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build --platform linux/amd64 -t ${IMG_REGISTRY}${IMG_TAG_LAUNCHER} -f backend/Dockerfile.launcher . +.PHONY: image_driver +image_driver: + cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build --platform linux/amd64 -t ${IMG_REGISTRY}${IMG_TAG_DRIVER} -f backend/Dockerfile.driver . .PHONY: install-cert-manager install-cert-manager: @@ -129,17 +119,6 @@ dev-kind-cluster: kubectl -n kubeflow wait --for condition=Available --timeout=10m deployment/mysql kubectl -n kubeflow wait --for condition=Available --timeout=3m deployment/metadata-grpc-deployment -.PHONY: kind-load-driver-debug -kind-load-driver-debug: - @if [ "${CONTAINER_ENGINE}" = "docker" ]; then \ - kind --name ${KIND_NAME} load docker-image ${IMG_TAG_DRIVER}:debug; \ - else \ - bash -c "kind load --name ${KIND_NAME} image-archive <( ${CONTAINER_ENGINE} save ${IMG_TAG_DRIVER})"; \ - fi - -.PHONY: kind-build-and-load-driver-debug -kind-build-and-load-driver-debug: image_driver_debug kind-load-driver-debug - .PHONY: lint-and-format lint-and-format: lint format @@ -149,4 +128,4 @@ lint: .PHONY: format format: - golangci-lint fmt \ No newline at end of file + golangci-lint fmt diff --git a/backend/src/apiserver/config/config.go b/backend/src/apiserver/config/config.go index 68cfa483277..5bba54e80fb 100644 --- a/backend/src/apiserver/config/config.go +++ b/backend/src/apiserver/config/config.go @@ -18,7 +18,6 @@ import ( "encoding/json" "fmt" "os" - "strings" "time" "github.com/golang/glog" @@ -31,31 +30,6 @@ import ( "google.golang.org/grpc/codes" ) -const managedPipelinesUploadTagsEnv = "MANAGED_PIPELINES_UPLOAD_TAGS" - -// parseManagedPipelinesTags reads MANAGED_PIPELINES_UPLOAD_TAGS from the -// environment and returns the parsed key=value pairs. Returns nil when the -// variable is unset or empty (backward-compatible no-op). -func parseManagedPipelinesTags() (map[string]string, error) { - raw := os.Getenv(managedPipelinesUploadTagsEnv) - if raw == "" { - return nil, nil - } - tags := make(map[string]string) - for _, entry := range strings.Split(raw, ",") { - idx := strings.Index(entry, "=") - if idx < 0 { - return nil, fmt.Errorf("malformed %s entry %q: missing '='", managedPipelinesUploadTagsEnv, entry) - } - key := entry[:idx] - if key == "" { - return nil, fmt.Errorf("malformed %s entry %q: empty key", managedPipelinesUploadTagsEnv, entry) - } - tags[key] = entry[idx+1:] - } - return tags, nil -} - // deprecated type deprecatedConfig struct { Name string @@ -142,14 +116,6 @@ func LoadSamples(resourceManager *resource.ResourceManager, sampleConfigPath str return nil } - tags, err := parseManagedPipelinesTags() - if err != nil { - return err - } - if len(tags) > 0 { - glog.Infof("Parsed %d managed pipeline upload tag(s) from %s", len(tags), managedPipelinesUploadTagsEnv) - } - processedPipelines := map[string]bool{} for _, cfg := range pipelineConfig.Pipelines { @@ -176,7 +142,6 @@ func LoadSamples(resourceManager *resource.ResourceManager, sampleConfigPath str Name: cfg.Name, DisplayName: pipelineDisplayName, Description: model.LargeText(cfg.Description), - Tags: tags, }) if configErr != nil { // Log the error but not fail. The API Server pod can restart and it could potentially cause diff --git a/backend/src/apiserver/config/config_test.go b/backend/src/apiserver/config/config_test.go index 4e3abafc1e8..1b86a7a873d 100644 --- a/backend/src/apiserver/config/config_test.go +++ b/backend/src/apiserver/config/config_test.go @@ -292,222 +292,6 @@ func TestLoadSamplesMultiplePipelineVersionsInConfig(t *testing.T) { require.Equal(t, totalSize, 2) } -func TestParseManagedPipelinesTags(t *testing.T) { - tests := []struct { - name string - envVal string - want map[string]string - wantErr bool - }{ - { - name: "multiple tags", - envVal: "managed=true,platform-version=v2.18.0", - want: map[string]string{"managed": "true", "platform-version": "v2.18.0"}, - }, - { - name: "single tag", - envVal: "managed=true", - want: map[string]string{"managed": "true"}, - }, - { - name: "empty string returns nil", - envVal: "", - want: nil, - }, - { - name: "value containing equals splits on first only", - envVal: "key=val=ue", - want: map[string]string{"key": "val=ue"}, - }, - { - name: "missing equals is malformed", - envVal: "badentry", - wantErr: true, - }, - { - name: "empty key is malformed", - envVal: "=value", - wantErr: true, - }, - { - name: "whitespace in key and value is preserved", - envVal: " key = value ", - want: map[string]string{" key ": " value "}, - }, - { - name: "trailing comma produces empty entry", - envVal: "managed=true,", - wantErr: true, - }, - { - name: "empty value is valid", - envVal: "key=", - want: map[string]string{"key": ""}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Setenv(managedPipelinesUploadTagsEnv, tt.envVal) - got, err := parseManagedPipelinesTags() - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestLoadSamples_TagsApplied(t *testing.T) { - viper.Set("POD_NAMESPACE", "") - t.Setenv(managedPipelinesUploadTagsEnv, "managed=true,platform-version=v2.18.0") - wantTags := map[string]string{"managed": "true", "platform-version": "v2.18.0"} - - rm := fakeResourceManager() - pc := config{ - LoadSamplesOnRestart: true, - Pipelines: []configPipelines{ - { - Name: "Tagged Pipeline", - Description: "test", - File: "testdata/sample_pipeline.yaml", - VersionName: "Tagged Pipeline - Ver 1", - }, - }, - } - - path, err := writeSampleConfig(t, pc, "sample.json") - require.NoError(t, err) - require.NoError(t, LoadSamples(rm, path)) - - pipeline, err := rm.GetPipelineByNameAndNamespace("Tagged Pipeline", "") - require.NoError(t, err) - assert.Equal(t, wantTags, pipeline.Tags) - - version, err := rm.GetPipelineVersionByName("Tagged Pipeline - Ver 1") - require.NoError(t, err) - assert.Empty(t, version.Tags) -} - -func TestLoadSamples_NoTagsWhenEnvUnset(t *testing.T) { - viper.Set("POD_NAMESPACE", "") - t.Setenv(managedPipelinesUploadTagsEnv, "") - - rm := fakeResourceManager() - pc := config{ - LoadSamplesOnRestart: true, - Pipelines: []configPipelines{ - { - Name: "Untagged Pipeline", - Description: "test", - File: "testdata/sample_pipeline.yaml", - VersionName: "Untagged Pipeline - Ver 1", - }, - }, - } - - path, err := writeSampleConfig(t, pc, "sample.json") - require.NoError(t, err) - require.NoError(t, LoadSamples(rm, path)) - - pipeline, err := rm.GetPipelineByNameAndNamespace("Untagged Pipeline", "") - require.NoError(t, err) - assert.Empty(t, pipeline.Tags) - - version, err := rm.GetPipelineVersionByName("Untagged Pipeline - Ver 1") - require.NoError(t, err) - assert.Empty(t, version.Tags) -} - -func TestLoadSamples_MalformedTagsReturnsError(t *testing.T) { - viper.Set("POD_NAMESPACE", "") - t.Setenv(managedPipelinesUploadTagsEnv, "badentry") - - rm := fakeResourceManager() - pc := config{ - LoadSamplesOnRestart: true, - Pipelines: []configPipelines{ - { - Name: "Should Not Load", - Description: "test", - File: "testdata/sample_pipeline.yaml", - VersionName: "Should Not Load - Ver 1", - }, - }, - } - - path, err := writeSampleConfig(t, pc, "sample.json") - require.NoError(t, err) - err = LoadSamples(rm, path) - require.Error(t, err) - assert.Contains(t, err.Error(), "badentry") -} - -func TestLoadSamples_MalformedTagsIgnoredWhenLoadingSkipped(t *testing.T) { - viper.Set("POD_NAMESPACE", "") - rm := fakeResourceManager() - - pc := config{ - LoadSamplesOnRestart: false, - Pipelines: []configPipelines{ - { - Name: "Already Loaded Pipeline", - Description: "test", - File: "testdata/sample_pipeline.yaml", - VersionName: "Already Loaded Pipeline - Ver 1", - }, - }, - } - - // First load succeeds (no malformed tags, samples not yet loaded). - t.Setenv(managedPipelinesUploadTagsEnv, "") - path, err := writeSampleConfig(t, pc, "sample.json") - require.NoError(t, err) - require.NoError(t, LoadSamples(rm, path)) - - // Second load: samples already loaded + LoadSamplesOnRestart=false → skip. - // A malformed env var must NOT cause an error because loading is skipped. - t.Setenv(managedPipelinesUploadTagsEnv, "badentry") - path, err = writeSampleConfig(t, pc, "sample.json") - require.NoError(t, err) - require.NoError(t, LoadSamples(rm, path)) -} - -func TestLoadSamples_ExistingPipelineNotRetagged(t *testing.T) { - viper.Set("POD_NAMESPACE", "") - rm := fakeResourceManager() - - // First load: create pipeline + version without tags - t.Setenv(managedPipelinesUploadTagsEnv, "") - pc := config{ - LoadSamplesOnRestart: true, - Pipelines: []configPipelines{ - { - Name: "Existing Pipeline", - Description: "test", - File: "testdata/sample_pipeline.yaml", - VersionName: "Existing Pipeline - Ver 1", - }, - }, - } - path, err := writeSampleConfig(t, pc, "sample.json") - require.NoError(t, err) - require.NoError(t, LoadSamples(rm, path)) - - // Second load: add a new version with tags enabled - t.Setenv(managedPipelinesUploadTagsEnv, "managed=true") - pc.Pipelines[0].VersionName = "Existing Pipeline - Ver 2" - path, err = writeSampleConfig(t, pc, "sample.json") - require.NoError(t, err) - require.NoError(t, LoadSamples(rm, path)) - - // Existing pipeline should NOT have been re-tagged - pipeline, err := rm.GetPipelineByNameAndNamespace("Existing Pipeline", "") - require.NoError(t, err) - assert.Empty(t, pipeline.Tags) -} - func writeSampleConfig(t *testing.T, config config, path string) (string, error) { return writeContents(t, config, path) } diff --git a/backend/src/apiserver/filter/filter.go b/backend/src/apiserver/filter/filter.go index 542cbb4ff73..4bce3995e1a 100644 --- a/backend/src/apiserver/filter/filter.go +++ b/backend/src/apiserver/filter/filter.go @@ -161,21 +161,6 @@ func NewFromPredicate(predicates []*Predicate) (*Filter, error) { return f, nil } -// ValidateKeys checks all filter keys using the provided validator function. -// Keys may be qualified identifiers like "table.Column"; each segment is validated separately. -func (f *Filter) ValidateKeys(validator func(segment string) error) error { - for _, m := range []map[string][]interface{}{f.eq, f.neq, f.gt, f.gte, f.lt, f.lte, f.in, f.substring} { - for k := range m { - for _, segment := range strings.Split(k, ".") { - if err := validator(segment); err != nil { - return fmt.Errorf("invalid filter key %q: %w", k, err) - } - } - } - } - return nil -} - // Replaces and adds a prefix to the keys for an existing filter. // This is useful when someone wants to extend the filter with a table name. func (f *Filter) ReplaceKeys(keyMap map[string]string, prefix string) error { diff --git a/backend/src/apiserver/list/list.go b/backend/src/apiserver/list/list.go index ad7cae70fa6..bce7bdb59a5 100644 --- a/backend/src/apiserver/list/list.go +++ b/backend/src/apiserver/list/list.go @@ -24,7 +24,6 @@ import ( "fmt" "math" "reflect" - "regexp" "strings" sq "github.com/Masterminds/squirrel" @@ -41,14 +40,8 @@ import ( // page of results), with the two values pointing to the first record in the // next set of results. type token struct { - // SortByFieldName is the user-facing field name used for pagination state - // and GetFieldValue lookups. For metric sorts this is the raw metric name - // (e.g. "accuracy"). Never use this field directly in SQL identifiers. + // SortByFieldName is the field name to use when sorting. SortByFieldName string - // SortBySQLColumn is the safe SQL column name used in ORDER BY and WHERE - // clauses. For regular fields it equals SortByFieldName. For metric sorts - // it is always the fixed alias "sort_metric_value", never user input. - SortBySQLColumn string // SortByFieldValue is the value of the sorted field of the next row to be // returned. SortByFieldValue interface{} @@ -71,49 +64,9 @@ type token struct { Filter *filter.Filter } -// identifierPattern matches valid SQL identifier names: start with a letter, -// followed by letters, digits, or underscores, max 128 characters. -// Used to validate pageToken fields before they are used in SQL queries. -var identifierPattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_]{0,127}$`) - -// metricNamePattern matches valid metric names. Metric names follow the same -// rules as SQL identifiers but additionally allow hyphens ("-"), since ML -// frameworks commonly use names like "log-loss" or "val-accuracy". -// Metric names are never used as SQL identifiers — they are passed as bind -// parameters — so allowing "-" here is safe. -var metricNamePattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_\-]{0,127}$`) - -// validateMetricName validates that a metric name only contains safe characters. -// Unlike validateIdentifierName, hyphens are permitted. -func validateMetricName(name string) error { - if name == "" { - return nil - } - if !metricNamePattern.MatchString(name) { - return util.NewInvalidInputError( - "Invalid metric name: %q. Metric names must start with a letter and contain only letters, numbers, underscores, and hyphens (max 128 characters)", - name) - } - return nil -} - -// validateIdentifierName validates that a field name or table name only contains -// safe characters to prevent SQL injection through pageToken parameters. -func validateIdentifierName(name, fieldType string) error { - if name == "" { - return nil // empty names are handled elsewhere - } - if !identifierPattern.MatchString(name) { - return util.NewInvalidInputError( - "Invalid %s: %q. Field names must start with a letter and contain only letters, numbers, and underscores (max 128 characters)", - fieldType, name) - } - return nil -} - func (t *token) unmarshal(pageToken string) error { errorF := func(err error) error { - return util.NewInvalidInputErrorWithDetails(err, "Invalid page token") + return util.NewInvalidInputErrorWithDetails(err, "Invalid package token") } b, err := base64.StdEncoding.DecodeString(pageToken) if err != nil { @@ -124,52 +77,6 @@ func (t *token) unmarshal(pageToken string) error { return errorF(err) } - // Validate all identifier fields to prevent SQL injection attacks. - // pageToken fields are used to construct SQL queries; unvalidated field names - // allow injection of arbitrary SQL through the pageToken parameter. - if err := validateIdentifierName(t.KeyFieldName, "key field name"); err != nil { - return err - } - // SortByFieldName is the user-facing metric name when sorting by a run - // metric (e.g. "log-loss"). Metric names allow hyphens, so they must not - // be validated with the SQL identifier regex. SortBySQLColumn carries the - // fixed safe alias ("sort_metric_value") and is always a valid identifier. - if t.SortBySQLColumn == model.MetricSortSQLAlias { - if err := validateMetricName(t.SortByFieldName); err != nil { - return err - } - } else { - if err := validateIdentifierName(t.SortByFieldName, "sort field name"); err != nil { - return err - } - } - if err := validateIdentifierName(t.SortBySQLColumn, "sort SQL column"); err != nil { - return err - } - if err := validateIdentifierName(t.ModelName, "model name"); err != nil { - return err - } - if t.KeyFieldPrefix != "" { - prefix := strings.TrimSuffix(t.KeyFieldPrefix, ".") - if err := validateIdentifierName(prefix, "key field prefix"); err != nil { - return err - } - } - if t.SortByFieldPrefix != "" { - prefix := strings.TrimSuffix(t.SortByFieldPrefix, ".") - if err := validateIdentifierName(prefix, "sort field prefix"); err != nil { - return err - } - } - - if t.Filter != nil { - if err := t.Filter.ValidateKeys(func(segment string) error { - return validateIdentifierName(segment, "filter key") - }); err != nil { - return err - } - } - return nil } @@ -248,12 +155,10 @@ func NewOptions(listable Listable, pageSize int, sortBy string, filter *filter.F } token.SortByFieldName = listable.DefaultSortField() - token.SortBySQLColumn = token.SortByFieldName if len(queryList) > 0 { - n, sqlCol, ok := listable.GetField(queryList[0]) + n, ok := listable.GetField(queryList[0]) if ok { token.SortByFieldName = n - token.SortBySQLColumn = sqlCol } else { return nil, util.NewInvalidInputError("Invalid sorting field: %q on listable type %s", queryList[0], reflect.ValueOf(listable).Elem().Type().Name()) } @@ -294,18 +199,18 @@ func (o *Options) AddSortingToSelect(sqlBuilder sq.SelectBuilder) sq.SelectBuild if o.IsDesc { sqlBuilder = sqlBuilder. Where(sq.Or{ - sq.Lt{o.SortByFieldPrefix + o.SortBySQLColumn: o.SortByFieldValue}, + sq.Lt{o.SortByFieldPrefix + o.SortByFieldName: o.SortByFieldValue}, sq.And{ - sq.Eq{o.SortByFieldPrefix + o.SortBySQLColumn: o.SortByFieldValue}, + sq.Eq{o.SortByFieldPrefix + o.SortByFieldName: o.SortByFieldValue}, sq.LtOrEq{o.KeyFieldPrefix + o.KeyFieldName: o.KeyFieldValue}, }, }) } else { sqlBuilder = sqlBuilder. Where(sq.Or{ - sq.Gt{o.SortByFieldPrefix + o.SortBySQLColumn: o.SortByFieldValue}, + sq.Gt{o.SortByFieldPrefix + o.SortByFieldName: o.SortByFieldValue}, sq.And{ - sq.Eq{o.SortByFieldPrefix + o.SortBySQLColumn: o.SortByFieldValue}, + sq.Eq{o.SortByFieldPrefix + o.SortByFieldName: o.SortByFieldValue}, sq.GtOrEq{o.KeyFieldPrefix + o.KeyFieldName: o.KeyFieldValue}, }, }) @@ -317,8 +222,8 @@ func (o *Options) AddSortingToSelect(sqlBuilder sq.SelectBuilder) sq.SelectBuild order = "DESC" } - if o.SortBySQLColumn != "" { - sqlBuilder = sqlBuilder.OrderBy(fmt.Sprintf("%v %v", o.SortByFieldPrefix+o.SortBySQLColumn, order)) + if o.SortByFieldName != "" { + sqlBuilder = sqlBuilder.OrderBy(fmt.Sprintf("%v %v", o.SortByFieldPrefix+o.SortByFieldName, order)) } if o.KeyFieldName != "" { @@ -431,11 +336,8 @@ type Listable interface { GetSortByFieldPrefix(string) string // Get the prefix of key field. GetKeyFieldPrefix() string - // GetField returns the model field name and safe SQL column name for the - // given API field name. For regular fields fieldName and sqlColumn are - // identical. For metric fields (e.g. "metric:accuracy") sqlColumn is the - // fixed alias "sort_metric_value" so user input never reaches SQL structure. - GetField(name string) (fieldName string, sqlColumn string, ok bool) + // Get a valid field for sorting/filtering in a listable model from the given string. + GetField(name string) (string, bool) // Find the value of a given field in a listable object. GetFieldValue(name string) interface{} } @@ -467,7 +369,6 @@ func (o *Options) nextPageToken(listable Listable) (*token, error) { return &token{ SortByFieldName: o.SortByFieldName, - SortBySQLColumn: o.SortBySQLColumn, SortByFieldValue: sortByField, SortByFieldPrefix: listable.GetSortByFieldPrefix(o.SortByFieldName), KeyFieldName: listable.PrimaryKeyColumnName(), diff --git a/backend/src/apiserver/list/list_test.go b/backend/src/apiserver/list/list_test.go index 979f2ed4602..e207cd900ab 100644 --- a/backend/src/apiserver/list/list_test.go +++ b/backend/src/apiserver/list/list_test.go @@ -15,7 +15,6 @@ package list import ( - "encoding/base64" "fmt" "math" "reflect" @@ -67,14 +66,14 @@ func (f *fakeListable) GetModelName() string { return "" } -func (f *fakeListable) GetField(name string) (string, string, bool) { +func (f *fakeListable) GetField(name string) (string, bool) { if field, ok := fakeAPIToModelMap[name]; ok { - return field, field, true + return field, true } if strings.HasPrefix(name, "metric:") { - return name[7:], model.MetricSortSQLAlias, true + return name[7:], true } - return "", "", false + return "", false } func (f *fakeListable) GetFieldValue(name string) interface{} { @@ -299,31 +298,6 @@ func TestNewOptionsFromToken_FromInValidSerializedToken(t *testing.T) { } } -func TestNewOptionsFromToken_MaliciousFilterKey(t *testing.T) { - // Simulate a forged pageToken with a malicious filter key containing SQL injection. - // The filter key bypasses NewWithKeyMap's allowlist and reaches SQL construction directly. - tests := []struct { - name string - filterKey string - }{ - {"sql injection in EQ key", `pipelines.Name) OR 1=1 --`}, - {"semicolon injection", `Name; DROP TABLE pipelines--`}, - {"unqualified injection", `Name) OR 1=1`}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - raw := fmt.Sprintf(`{"KeyFieldName":"ID","SortByFieldName":"Name","Filter":{"EQ":{%q:[]}}}`, test.filterKey) - token := base64.StdEncoding.EncodeToString([]byte(raw)) - got, err := NewOptionsFromToken(token, 10) - if err == nil { - t.Errorf("NewOptionsFromToken with malicious filter key %q =\nGot: %+v, \nWant: _, error", - test.filterKey, got) - } - }) - } -} - func TestNewOptionsFromToken_FromInValidPageSize(t *testing.T) { tok := &token{ SortByFieldName: "SortField", @@ -361,7 +335,6 @@ func TestNewOptions_ValidSortOptions(t *testing.T) { KeyFieldName: "PrimaryKey", KeyFieldPrefix: "", SortByFieldName: "CreatedTimestamp", - SortBySQLColumn: "CreatedTimestamp", SortByFieldPrefix: "", IsDesc: false, }, @@ -375,7 +348,6 @@ func TestNewOptions_ValidSortOptions(t *testing.T) { KeyFieldName: "PrimaryKey", KeyFieldPrefix: "", SortByFieldName: "CreatedTimestamp", - SortBySQLColumn: "CreatedTimestamp", SortByFieldPrefix: "", IsDesc: false, }, @@ -389,7 +361,6 @@ func TestNewOptions_ValidSortOptions(t *testing.T) { KeyFieldName: "PrimaryKey", KeyFieldPrefix: "", SortByFieldName: "FakeName", - SortBySQLColumn: "FakeName", SortByFieldPrefix: "", IsDesc: false, }, @@ -403,7 +374,6 @@ func TestNewOptions_ValidSortOptions(t *testing.T) { KeyFieldName: "PrimaryKey", KeyFieldPrefix: "", SortByFieldName: "FakeName", - SortBySQLColumn: "FakeName", SortByFieldPrefix: "", IsDesc: false, }, @@ -417,7 +387,6 @@ func TestNewOptions_ValidSortOptions(t *testing.T) { KeyFieldName: "PrimaryKey", KeyFieldPrefix: "", SortByFieldName: "FakeName", - SortBySQLColumn: "FakeName", SortByFieldPrefix: "", IsDesc: true, }, @@ -431,7 +400,6 @@ func TestNewOptions_ValidSortOptions(t *testing.T) { KeyFieldName: "PrimaryKey", KeyFieldPrefix: "", SortByFieldName: "PrimaryKey", - SortBySQLColumn: "PrimaryKey", SortByFieldPrefix: "", IsDesc: true, }, @@ -509,7 +477,6 @@ func TestNewOptions_ValidFilter(t *testing.T) { KeyFieldName: "PrimaryKey", KeyFieldPrefix: "", SortByFieldName: "CreatedTimestamp", - SortBySQLColumn: "CreatedTimestamp", SortByFieldPrefix: "", IsDesc: false, Filter: f, @@ -579,7 +546,6 @@ func TestNewOptions_ModelFilter(t *testing.T) { token: &token{ KeyFieldName: "UUID", SortByFieldName: "DisplayName", - SortBySQLColumn: "DisplayName", IsDesc: false, Filter: f, }, @@ -622,7 +588,6 @@ func TestAddPaginationAndFilterToSelect(t *testing.T) { PageSize: 123, token: &token{ SortByFieldName: "SortField", - SortBySQLColumn: "SortField", SortByFieldValue: "value", SortByFieldPrefix: "", KeyFieldName: "KeyField", @@ -639,7 +604,6 @@ func TestAddPaginationAndFilterToSelect(t *testing.T) { PageSize: 123, token: &token{ SortByFieldName: "SortField", - SortBySQLColumn: "SortField", SortByFieldValue: "value", SortByFieldPrefix: "", KeyFieldName: "KeyField", @@ -656,7 +620,6 @@ func TestAddPaginationAndFilterToSelect(t *testing.T) { PageSize: 123, token: &token{ SortByFieldName: "SortField", - SortBySQLColumn: "SortField", SortByFieldValue: "value", SortByFieldPrefix: "", KeyFieldName: "KeyField", @@ -674,7 +637,6 @@ func TestAddPaginationAndFilterToSelect(t *testing.T) { PageSize: 123, token: &token{ SortByFieldName: "SortField", - SortBySQLColumn: "SortField", SortByFieldPrefix: "", KeyFieldName: "KeyField", KeyFieldPrefix: "", @@ -695,7 +657,6 @@ func TestAddPaginationAndFilterToSelect(t *testing.T) { PageSize: 123, token: &token{ SortByFieldName: "SortField", - SortBySQLColumn: "SortField", SortByFieldValue: "value", SortByFieldPrefix: "", KeyFieldName: "KeyField", @@ -711,7 +672,6 @@ func TestAddPaginationAndFilterToSelect(t *testing.T) { PageSize: 123, token: &token{ SortByFieldName: "SortField", - SortBySQLColumn: "SortField", SortByFieldValue: "value", SortByFieldPrefix: "", KeyFieldName: "KeyField", diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index fc1331ee9c4..3b5a2685f23 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -87,20 +87,6 @@ var ( type RegisterHttpHandlerFromEndpoint func(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) error -// HTTPRouterDeps holds the HTTP handler functions needed to build the -// top-level HTTP router. Using a struct of http.HandlerFunc values -// allows tests to supply lightweight stubs without constructing real -// server instances. -type HTTPRouterDeps struct { - UploadPipelineV1 http.HandlerFunc - UploadPipelineVersionV1 http.HandlerFunc - UploadPipeline http.HandlerFunc - UploadPipelineVersion http.HandlerFunc - ReadRunLogV1 http.HandlerFunc - ReadArtifactV1 http.HandlerFunc - ReadArtifact http.HandlerFunc -} - func initCerts() (*tls.Config, error) { switch { case *tlsCertPath == "" && *tlsCertKeyPath == "": @@ -425,21 +411,45 @@ func startHTTPProxy(resourceManager *resource.ResourceManager, usePipelinesKuber register(apiv2beta1.RegisterRunServiceHandlerFromEndpoint, "RunService") register(apiv2beta1.RegisterReportServiceHandlerFromEndpoint, "ReportService") + // Create a top level mux to include both pipeline upload server and gRPC servers. + topMux := mux.NewRouter() + + // multipart upload is only supported in HTTP. In long term, we should have gRPC endpoints that + // accept pipeline url for importing. + // https://github.com/grpc-ecosystem/grpc-gateway/issues/410 sharedPipelineUploadServer := server.NewPipelineUploadServer(resourceManager, &server.PipelineUploadServerOptions{CollectMetrics: *collectMetricsFlag}) + // API v1beta1 + topMux.HandleFunc("/apis/v1beta1/pipelines/upload", sharedPipelineUploadServer.UploadPipelineV1) + topMux.HandleFunc("/apis/v1beta1/pipelines/upload_version", sharedPipelineUploadServer.UploadPipelineVersionV1) + topMux.HandleFunc("/apis/v1beta1/healthz", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"commit_sha":"`+common.GetStringConfigWithDefault("COMMIT_SHA", "unknown")+`", "tag_name":"`+common.GetStringConfigWithDefault("TAG_NAME", "unknown")+`", "multi_user":`+strconv.FormatBool(common.IsMultiUserMode())+`}`) + }) + // API v2beta1 + topMux.HandleFunc("/apis/v2beta1/pipelines/upload", sharedPipelineUploadServer.UploadPipeline) + topMux.HandleFunc("/apis/v2beta1/pipelines/upload_version", sharedPipelineUploadServer.UploadPipelineVersion) + topMux.HandleFunc("/apis/v2beta1/healthz", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"commit_sha":"`+common.GetStringConfigWithDefault("COMMIT_SHA", "unknown")+`", "tag_name":"`+common.GetStringConfigWithDefault("TAG_NAME", "unknown")+`", "multi_user":`+strconv.FormatBool(common.IsMultiUserMode())+`, "pipeline_store": "`+pipelineStore+`"}`) + }) + + // log streaming is provided via HTTP. runLogServer := server.NewRunLogServer(resourceManager) + topMux.HandleFunc("/apis/v1alpha1/runs/{run_id}/nodes/{node_id}/log", runLogServer.ReadRunLogV1) + + // Artifact reading endpoints (implemented with streaming for memory efficiency) runArtifactServer := server.NewRunArtifactServer(resourceManager) + topMux.HandleFunc("/apis/v1beta1/runs/{run_id}/nodes/{node_id}/artifacts/{artifact_name}:read", runArtifactServer.ReadArtifactV1).Methods(http.MethodGet) + topMux.HandleFunc("/apis/v2beta1/runs/{run_id}/nodes/{node_id}/artifacts/{artifact_name}:read", runArtifactServer.ReadArtifact).Methods(http.MethodGet) - handlerDeps := HTTPRouterDeps{ - UploadPipelineV1: sharedPipelineUploadServer.UploadPipelineV1, - UploadPipelineVersionV1: sharedPipelineUploadServer.UploadPipelineVersionV1, - UploadPipeline: sharedPipelineUploadServer.UploadPipeline, - UploadPipelineVersion: sharedPipelineUploadServer.UploadPipelineVersion, - ReadRunLogV1: runLogServer.ReadRunLogV1, - ReadArtifactV1: runArtifactServer.ReadArtifactV1, - ReadArtifact: runArtifactServer.ReadArtifact, - } + // Driver endpoint moved to the standalone kfp-driver service + // (manifests/kustomize/base/pipeline/kfp-driver-*.yaml). Argo HTTP templates + // now POST to http://kfp-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute. - topMux := buildHTTPRouter(handlerDeps, runtimeMux, pipelineStore) + topMux.PathPrefix("/apis/").Handler(clearTagsMiddleware(runtimeMux)) + + // Register a handler for Prometheus to poll. + topMux.Handle("/metrics", promhttp.Handler()) if tlsCfg != nil { glog.Info("Starting Https Proxy") @@ -461,73 +471,6 @@ func startHTTPProxy(resourceManager *resource.ResourceManager, usePipelinesKuber glog.Info("Http Proxy started") } -type healthzResponse struct { - CommitSHA string `json:"commit_sha"` - TagName string `json:"tag_name"` - MultiUser bool `json:"multi_user"` - PipelineStore string `json:"pipeline_store,omitempty"` -} - -func writeJSONResponse(w http.ResponseWriter, payload any) { - responseBody, err := json.Marshal(payload) - if err != nil { - http.Error(w, "failed to encode response", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(responseBody); err != nil { - glog.Errorf("Failed to write JSON response: %v", err) - } -} - -func newHealthzResponse(pipelineStore string) healthzResponse { - return healthzResponse{ - CommitSHA: common.GetStringConfigWithDefault("COMMIT_SHA", "unknown"), - TagName: common.GetStringConfigWithDefault("TAG_NAME", "unknown"), - MultiUser: common.IsMultiUserMode(), - PipelineStore: pipelineStore, - } -} - -// buildHTTPRouter constructs the top-level HTTP router with all API routes -// registered. It does not start a listener, making it testable in isolation. -func buildHTTPRouter(handlerDeps HTTPRouterDeps, grpcGatewayHandler http.Handler, pipelineStore string) *mux.Router { - topMux := mux.NewRouter() - - // multipart upload is only supported in HTTP. In long term, we should have gRPC endpoints that - // accept pipeline url for importing. - // https://github.com/grpc-ecosystem/grpc-gateway/issues/410 - // API v1beta1 - topMux.HandleFunc("/apis/v1beta1/pipelines/upload", handlerDeps.UploadPipelineV1) - topMux.HandleFunc("/apis/v1beta1/pipelines/upload_version", handlerDeps.UploadPipelineVersionV1) - topMux.HandleFunc("/apis/v1beta1/healthz", func(w http.ResponseWriter, r *http.Request) { - writeJSONResponse(w, newHealthzResponse("")) - }) - // API v2beta1 - topMux.HandleFunc("/apis/v2beta1/pipelines/upload", handlerDeps.UploadPipeline) - topMux.HandleFunc("/apis/v2beta1/pipelines/upload_version", handlerDeps.UploadPipelineVersion) - topMux.HandleFunc("/apis/v2beta1/healthz", func(w http.ResponseWriter, r *http.Request) { - writeJSONResponse(w, newHealthzResponse(pipelineStore)) - }) - - // log streaming is provided via HTTP. - topMux.HandleFunc("/apis/v1alpha1/runs/{run_id}/nodes/{node_id}/log", handlerDeps.ReadRunLogV1) - - // Artifact reading endpoints (implemented with streaming for memory efficiency) - topMux.HandleFunc("/apis/v1beta1/runs/{run_id}/nodes/{node_id}/artifacts/{artifact_name}:read", handlerDeps.ReadArtifactV1).Methods(http.MethodGet) - topMux.HandleFunc("/apis/v2beta1/runs/{run_id}/nodes/{node_id}/artifacts/{artifact_name}:read", handlerDeps.ReadArtifact).Methods(http.MethodGet) - - topMux.PathPrefix("/apis/").Handler(clearTagsMiddleware(grpcGatewayHandler)) - - // Register a handler for Prometheus to poll. - // This must be unconditional because grpc_prometheus interceptors and Go - // runtime metrics are always produced regardless of collectMetricsFlag. - // The flag only controls the apiserver's per-handler counters. - topMux.Handle("/metrics", promhttp.Handler()) - - return topMux -} - func startWebhook(client ctrlclient.Client, clientNoCahe ctrlclient.Client, wg *sync.WaitGroup) (*http.Server, error) { glog.Info("Starting the Kubernetes webhooks...") diff --git a/backend/src/apiserver/main_test.go b/backend/src/apiserver/main_test.go index 14a960a3403..34e7989f14e 100644 --- a/backend/src/apiserver/main_test.go +++ b/backend/src/apiserver/main_test.go @@ -23,7 +23,6 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" - "encoding/json" "encoding/pem" "fmt" "io" @@ -36,7 +35,6 @@ import ( "testing" "time" - "github.com/gorilla/mux" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/kubeflow/pipelines/backend/src/apiserver/common" "github.com/spf13/viper" @@ -657,189 +655,3 @@ func TestClearTagsMiddleware_BodyPreserved(t *testing.T) { assert.Equal(t, original, string(capturedBody)) } - -func noOpHandler(w http.ResponseWriter, r *http.Request) {} - -func newNoOpHTTPRouterDeps() HTTPRouterDeps { - return HTTPRouterDeps{ - UploadPipelineV1: noOpHandler, - UploadPipelineVersionV1: noOpHandler, - UploadPipeline: noOpHandler, - UploadPipelineVersion: noOpHandler, - ReadRunLogV1: noOpHandler, - ReadArtifactV1: noOpHandler, - ReadArtifact: noOpHandler, - } -} - -// collectRegisteredRoutes walks a mux.Router and returns a map of -// path template to HTTP methods (nil means all methods accepted). -func collectRegisteredRoutes(t *testing.T, router *mux.Router) map[string][]string { - t.Helper() - registeredRoutes := make(map[string][]string) - err := router.Walk(func(route *mux.Route, _ *mux.Router, _ []*mux.Route) error { - pathTemplate, err := route.GetPathTemplate() - if err != nil { - return nil // skip routes without path templates - } - methods, _ := route.GetMethods() - registeredRoutes[pathTemplate] = methods - return nil - }) - require.NoError(t, err) - return registeredRoutes -} - -func TestBuildHTTPRouter_AllRoutesRegistered(t *testing.T) { - router := buildHTTPRouter(newNoOpHTTPRouterDeps(), http.HandlerFunc(noOpHandler), "database") - registeredRoutes := collectRegisteredRoutes(t, router) - - expectedRoutes := []struct { - path string - expectedMethods []string - }{ - {"/apis/v1beta1/pipelines/upload", nil}, - {"/apis/v1beta1/pipelines/upload_version", nil}, - {"/apis/v1beta1/healthz", nil}, - {"/apis/v2beta1/pipelines/upload", nil}, - {"/apis/v2beta1/pipelines/upload_version", nil}, - {"/apis/v2beta1/healthz", nil}, - {"/apis/v1alpha1/runs/{run_id}/nodes/{node_id}/log", nil}, - {"/apis/v1beta1/runs/{run_id}/nodes/{node_id}/artifacts/{artifact_name}:read", []string{"GET"}}, - {"/apis/v2beta1/runs/{run_id}/nodes/{node_id}/artifacts/{artifact_name}:read", []string{"GET"}}, - {"/metrics", nil}, - } - - for _, expectedRoute := range expectedRoutes { - t.Run(expectedRoute.path, func(t *testing.T) { - methods, exists := registeredRoutes[expectedRoute.path] - assert.True(t, exists, "route %s should be registered", expectedRoute.path) - if expectedRoute.expectedMethods != nil { - assert.Equal(t, expectedRoute.expectedMethods, methods) - } - }) - } -} - -func TestBuildHTTPRouter_HealthzResponses(t *testing.T) { - tests := []struct { - name string - path string - pipelineStore string - commitSHA string - tagName string - multiUser bool - wantV2Store bool - }{ - { - name: "v1beta1 healthz", - path: "/apis/v1beta1/healthz", - commitSHA: `sha-"v1"`, - tagName: "tag-v1\nline", - multiUser: true, - }, - { - name: "v2beta1 healthz with database store", - path: "/apis/v2beta1/healthz", - pipelineStore: "database", - commitSHA: `sha-"db"`, - tagName: `tag-"db"`, - multiUser: false, - wantV2Store: true, - }, - { - name: "v2beta1 healthz with kubernetes store", - path: "/apis/v2beta1/healthz", - pipelineStore: "kubernetes", - commitSHA: `sha-"k8s"`, - tagName: `tag-"k8s"`, - multiUser: true, - wantV2Store: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - viper.Reset() - t.Cleanup(viper.Reset) - viper.Set("COMMIT_SHA", tt.commitSHA) - viper.Set("TAG_NAME", tt.tagName) - viper.Set(common.MultiUserMode, fmt.Sprintf("%t", tt.multiUser)) - - pipelineStore := tt.pipelineStore - if pipelineStore == "" { - pipelineStore = "database" - } - router := buildHTTPRouter(newNoOpHTTPRouterDeps(), http.HandlerFunc(noOpHandler), pipelineStore) - - request := httptest.NewRequest(http.MethodGet, tt.path, nil) - recorder := httptest.NewRecorder() - router.ServeHTTP(recorder, request) - - assert.Equal(t, http.StatusOK, recorder.Code) - var responseBody map[string]any - require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &responseBody)) - assert.Equal(t, tt.commitSHA, responseBody["commit_sha"]) - assert.Equal(t, tt.tagName, responseBody["tag_name"]) - assert.Equal(t, tt.multiUser, responseBody["multi_user"]) - pipelineStoreValue, hasPipelineStore := responseBody["pipeline_store"] - assert.Equal(t, tt.wantV2Store, hasPipelineStore) - if tt.wantV2Store { - assert.Equal(t, tt.pipelineStore, pipelineStoreValue) - } - }) - } -} - -func TestBuildHTTPRouter_HandlersAreCalled(t *testing.T) { - tests := []struct { - name string - method string - path string - setHandler func(deps *HTTPRouterDeps, handler http.HandlerFunc) - }{ - {"v1beta1 upload pipeline", http.MethodPost, "/apis/v1beta1/pipelines/upload", - func(deps *HTTPRouterDeps, handler http.HandlerFunc) { deps.UploadPipelineV1 = handler }}, - {"v2beta1 upload pipeline version", http.MethodPost, "/apis/v2beta1/pipelines/upload_version", - func(deps *HTTPRouterDeps, handler http.HandlerFunc) { deps.UploadPipelineVersion = handler }}, - {"v1alpha1 run log", http.MethodGet, "/apis/v1alpha1/runs/run-123/nodes/node-456/log", - func(deps *HTTPRouterDeps, handler http.HandlerFunc) { deps.ReadRunLogV1 = handler }}, - {"v2beta1 artifact read", http.MethodGet, "/apis/v2beta1/runs/run-123/nodes/node-456/artifacts/my-artifact:read", - func(deps *HTTPRouterDeps, handler http.HandlerFunc) { deps.ReadArtifact = handler }}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - handlerCalled := false - instrumentedHandler := func(w http.ResponseWriter, r *http.Request) { - handlerCalled = true - w.WriteHeader(http.StatusOK) - } - - handlerDeps := newNoOpHTTPRouterDeps() - tt.setHandler(&handlerDeps, instrumentedHandler) - - router := buildHTTPRouter(handlerDeps, http.HandlerFunc(noOpHandler), "database") - - request := httptest.NewRequest(tt.method, tt.path, nil) - recorder := httptest.NewRecorder() - router.ServeHTTP(recorder, request) - - assert.True(t, handlerCalled, "handler for %s should have been called", tt.path) - }) - } -} - -func TestBuildHTTPRouter_UnmatchedAPIsGoToGateway(t *testing.T) { - gatewayHandlerCalled := false - gatewayHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gatewayHandlerCalled = true - w.WriteHeader(http.StatusOK) - }) - - router := buildHTTPRouter(newNoOpHTTPRouterDeps(), gatewayHandler, "database") - - request := httptest.NewRequest(http.MethodGet, "/apis/v2beta1/experiments", nil) - recorder := httptest.NewRecorder() - router.ServeHTTP(recorder, request) - - assert.True(t, gatewayHandlerCalled, "requests to /apis/ paths not matching explicit routes should reach the gRPC gateway handler") -} diff --git a/backend/src/apiserver/model/experiment.go b/backend/src/apiserver/model/experiment.go index e363ff00344..17d2ba8b9f9 100644 --- a/backend/src/apiserver/model/experiment.go +++ b/backend/src/apiserver/model/experiment.go @@ -74,11 +74,11 @@ func (e *Experiment) GetModelName() string { return "experiments" } -func (e *Experiment) GetField(name string) (string, string, bool) { +func (e *Experiment) GetField(name string) (string, bool) { if field, ok := experimentAPIToModelFieldMap[name]; ok { - return field, field, true + return field, true } - return "", "", false + return "", false } func (e *Experiment) GetFieldValue(name string) interface{} { diff --git a/backend/src/apiserver/model/job.go b/backend/src/apiserver/model/job.go index 334cac8a7a4..59126e3bf00 100644 --- a/backend/src/apiserver/model/job.go +++ b/backend/src/apiserver/model/job.go @@ -233,11 +233,11 @@ func (j *Job) GetModelName() string { return "jobs" } -func (j *Job) GetField(name string) (string, string, bool) { +func (j *Job) GetField(name string) (string, bool) { if field, ok := jobAPIToModelFieldMap[name]; ok { - return field, field, true + return field, true } - return "", "", false + return "", false } func (j *Job) GetFieldValue(name string) interface{} { diff --git a/backend/src/apiserver/model/pipeline.go b/backend/src/apiserver/model/pipeline.go index 849229ca901..f3c528c61f8 100644 --- a/backend/src/apiserver/model/pipeline.go +++ b/backend/src/apiserver/model/pipeline.go @@ -101,11 +101,11 @@ func (Pipeline) TableName() string { return "pipelines" } -func (p *Pipeline) GetField(name string) (string, string, bool) { +func (p *Pipeline) GetField(name string) (string, bool) { if field, ok := pipelineAPIToModelFieldMap[name]; ok { - return field, field, true + return field, true } - return "", "", false + return "", false } func (p *Pipeline) GetFieldValue(name string) interface{} { diff --git a/backend/src/apiserver/model/pipeline_version.go b/backend/src/apiserver/model/pipeline_version.go index 715dcb2faae..4891ef282c3 100644 --- a/backend/src/apiserver/model/pipeline_version.go +++ b/backend/src/apiserver/model/pipeline_version.go @@ -97,11 +97,11 @@ func (p *PipelineVersion) GetModelName() string { func (PipelineVersion) TableName() string { return "pipeline_versions" } -func (p *PipelineVersion) GetField(name string) (string, string, bool) { +func (p *PipelineVersion) GetField(name string) (string, bool) { if field, ok := p.APIToModelFieldMap()[name]; ok { - return field, field, true + return field, true } - return "", "", false + return "", false } func (p *PipelineVersion) GetFieldValue(name string) interface{} { diff --git a/backend/src/apiserver/model/run.go b/backend/src/apiserver/model/run.go index a1e2162c155..7823f4af9f1 100644 --- a/backend/src/apiserver/model/run.go +++ b/backend/src/apiserver/model/run.go @@ -396,19 +396,14 @@ func (r *Run) GetModelName() string { return "" } -// MetricSortSQLAlias is the fixed SQL column alias used when sorting runs by a -// metric value. Using a constant prevents user-supplied metric names from ever -// appearing as SQL identifiers. -const MetricSortSQLAlias = "sort_metric_value" - -func (r *Run) GetField(name string) (string, string, bool) { +func (r *Run) GetField(name string) (string, bool) { if field, ok := runAPIToModelFieldMap[name]; ok { - return field, field, true + return field, true } if strings.HasPrefix(name, "metric:") { - return name[7:], MetricSortSQLAlias, true + return name[7:], true } - return "", "", false + return "", false } func (r *Run) GetFieldValue(name string) interface{} { diff --git a/backend/src/apiserver/model/task.go b/backend/src/apiserver/model/task.go index 2ad0966d278..93ff8c24ad0 100644 --- a/backend/src/apiserver/model/task.go +++ b/backend/src/apiserver/model/task.go @@ -101,11 +101,11 @@ var taskAPIToModelFieldMap = map[string]string{ "finished_at": "FinishedTimestamp", // v1beta1 API } -func (t Task) GetField(name string) (string, string, bool) { +func (t Task) GetField(name string) (string, bool) { if field, ok := taskAPIToModelFieldMap[name]; ok { - return field, field, true + return field, true } - return "", "", false + return "", false } func (t Task) GetFieldValue(name string) interface{} { diff --git a/backend/src/apiserver/server/list_request_util_test.go b/backend/src/apiserver/server/list_request_util_test.go index 64aa37414d4..e13d8315957 100644 --- a/backend/src/apiserver/server/list_request_util_test.go +++ b/backend/src/apiserver/server/list_request_util_test.go @@ -333,11 +333,12 @@ func (f *fakeListable) GetModelName() string { return "" } -func (f *fakeListable) GetField(name string) (string, string, bool) { +func (f *fakeListable) GetField(name string) (string, bool) { if field, ok := fakeAPIToModelMap[name]; ok { - return field, field, true + return field, true + } else { + return "", false } - return "", "", false } func (f *fakeListable) GetFieldValue(name string) interface{} { diff --git a/backend/src/apiserver/storage/run_store.go b/backend/src/apiserver/storage/run_store.go index d1aa167c8a1..781c29888c3 100644 --- a/backend/src/apiserver/storage/run_store.go +++ b/backend/src/apiserver/storage/run_store.go @@ -261,7 +261,7 @@ func (s *RunStore) addMetricsResourceReferencesAndTasks(filteredSelectBuilder sq apply(func(column string) string { return "rd." + column }, runColumns), // Add prefix "rd." to runColumns resourceRefConcatQuery+" AS refs") if opts != nil && !r.IsRegularField(opts.SortByFieldName) { - columnsAfterJoiningResourceReferences = append(columnsAfterJoiningResourceReferences, "rd."+model.MetricSortSQLAlias) + columnsAfterJoiningResourceReferences = append(columnsAfterJoiningResourceReferences, "rd."+opts.SortByFieldName) } subQ := sq. Select(columnsAfterJoiningResourceReferences...). @@ -275,7 +275,7 @@ func (s *RunStore) addMetricsResourceReferencesAndTasks(filteredSelectBuilder sq "rdref.refs", tasksConcatQuery+" AS taskDetails") if opts != nil && !r.IsRegularField(opts.SortByFieldName) { - columnsAfterJoiningTasks = append(columnsAfterJoiningTasks, "rdref."+model.MetricSortSQLAlias) + columnsAfterJoiningTasks = append(columnsAfterJoiningTasks, "rdref."+opts.SortByFieldName) } subQ = sq. Select(columnsAfterJoiningTasks...). @@ -754,13 +754,11 @@ func (s *RunStore) addSortByRunMetricToSelect(sqlBuilder sq.SelectBuilder, opts if r.IsRegularField(opts.SortByFieldName) { return sqlBuilder } - // Use a fixed alias for the metric column so user input never reaches SQL - // structure. The metric name is passed as a bind parameter to the JOIN - // condition, preventing SQL injection. + // TODO(jingzhang36): address the case where runs doesn't have the specified metric. return sq. - Select("selected_runs.*, run_metrics.numbervalue as "+model.MetricSortSQLAlias). + Select("selected_runs.*, run_metrics.numbervalue as "+opts.SortByFieldName). FromSelect(sqlBuilder, "selected_runs"). - LeftJoin("run_metrics ON selected_runs.uuid=run_metrics.runuuid AND run_metrics.name=?", opts.SortByFieldName) + LeftJoin("run_metrics ON selected_runs.uuid=run_metrics.runuuid AND run_metrics.name='" + opts.SortByFieldName + "'") } func (s *RunStore) scanRowsToRunMetrics(rows *sql.Rows) ([]*model.RunMetric, error) { diff --git a/backend/src/apiserver/storage/run_store_test.go b/backend/src/apiserver/storage/run_store_test.go index 1077cf9729e..fcd17f6cc90 100644 --- a/backend/src/apiserver/storage/run_store_test.go +++ b/backend/src/apiserver/storage/run_store_test.go @@ -373,71 +373,6 @@ func TestListRuns_Pagination_WithSortingOnMetrics(t *testing.T) { assert.Empty(t, nextPageToken) } -func TestListRuns_MetricSortInjectionSafe(t *testing.T) { - db, runStore := initializeRunStore() - defer db.Close() - - // A malicious sortBy value attempting SQL injection through the metric name. - // The payload has no spaces so it passes format validation and reaches SQL - // generation. Prior to the fix, opts.SortByFieldName was concatenated directly - // into SQL structure (alias and JOIN predicate). After the fix, the metric name - // is passed as a bind parameter and the SQL alias is the fixed string - // "sort_metric_value", so this must execute without error. - maliciousSort := "metric:';DROP/**/TABLE/**/run_metrics;--" - opts, err := list.NewOptions(&model.Run{}, 10, maliciousSort, nil) - assert.Nil(t, err) - - runs, _, _, err := runStore.ListRuns( - &model.FilterContext{ReferenceKey: &model.ReferenceKey{Type: model.ExperimentResourceType, ID: defaultFakeExpId}}, opts) - assert.Nil(t, err, "SQL injection payload must not cause a query error") - // Runs are returned but without a matching metric value (NULL join), so - // ordering is stable but no metric value is injected into SQL structure. - assert.NotNil(t, runs) - - // Verify the run_metrics table was NOT dropped by confirming a normal - // metric-sort query still works. - opts2, err := list.NewOptions(&model.Run{}, 10, "metric:dummymetric", nil) - assert.Nil(t, err) - _, _, _, err = runStore.ListRuns( - &model.FilterContext{ReferenceKey: &model.ReferenceKey{Type: model.ExperimentResourceType, ID: defaultFakeExpId}}, - opts2) - assert.Nil(t, err, "run_metrics table must still exist after injection attempt") -} - -// TestListRuns_HyphenatedMetricSort verifies that metric names containing -// hyphens (e.g. "log-loss") can be used for pagination across multiple pages. -// This is a regression test for a bug where token.unmarshal() validated -// SortByFieldName with a SQL identifier regex that rejects "-", causing page-2 -// requests with hyphenated metric names to fail with "Invalid sort field name". -func TestListRuns_HyphenatedMetricSort(t *testing.T) { - db, runStore := initializeRunStore() - defer db.Close() - - // Seed runs 1 and 2 with the hyphenated metric so pagination produces a page token. - runStore.CreateMetric(&model.RunMetric{RunUUID: "1", NodeID: "node1", Name: "log-loss", NumberValue: 0.5, Format: "RAW"}) - runStore.CreateMetric(&model.RunMetric{RunUUID: "2", NodeID: "node2", Name: "log-loss", NumberValue: 0.3, Format: "RAW"}) - - // Page 1: metric:log-loss — must not error even though "log-loss" contains "-". - opts, err := list.NewOptions(&model.Run{}, 1, "metric:log-loss", nil) - assert.Nil(t, err, "NewOptions must accept hyphenated metric name") - - _, total, nextPageToken, err := runStore.ListRuns( - &model.FilterContext{ReferenceKey: &model.ReferenceKey{Type: model.ExperimentResourceType, ID: defaultFakeExpId}}, - opts) - assert.Nil(t, err, "page-1 ListRuns must succeed with hyphenated metric name") - assert.Equal(t, 2, total) - assert.NotEmpty(t, nextPageToken, "must produce a page token when there are 2 runs") - - // Page 2: the page token carries SortByFieldName="log-loss". Unmarshalling - // it must succeed (i.e. the identifier regex must NOT be applied to metric names). - opts2, err := list.NewOptionsFromToken(nextPageToken, 1) - assert.Nil(t, err, "NewOptionsFromToken must not reject hyphenated metric name in pageToken") - _, _, _, err = runStore.ListRuns( - &model.FilterContext{ReferenceKey: &model.ReferenceKey{Type: model.ExperimentResourceType, ID: defaultFakeExpId}}, - opts2) - assert.Nil(t, err, "page-2 ListRuns must succeed with hyphenated metric name in pageToken") -} - func TestListRuns_TotalSizeWithNoFilter(t *testing.T) { db, runStore := initializeRunStore() defer db.Close() diff --git a/backend/src/common/util/context_logger.go b/backend/src/common/util/context_logger.go new file mode 100644 index 00000000000..47a0b6a87be --- /dev/null +++ b/backend/src/common/util/context_logger.go @@ -0,0 +1,73 @@ +package util + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/sirupsen/logrus" +) + +type CtxKey string + +const ( + contextLoggerKey CtxKey = "driver_log_key" +) + +func newFileLogger(logFile string) (*logrus.Logger, io.Closer, error) { + f, err := os.Create(logFile) + if err != nil { + return nil, nil, err + } + + logger := logrus.New() + logger.Out = io.MultiWriter(os.Stdout, f) + logger.Formatter = &logrus.TextFormatter{} + return logger, f, nil +} + +// WithExistingLogger For testing only +func WithExistingLogger(ctx context.Context, logger *logrus.Logger) context.Context { + return context.WithValue(ctx, contextLoggerKey, logger) +} + +func WithLogger(ctx context.Context, logFile string) (context.Context, io.Closer, error) { + if ctx == nil { + return nil, nil, fmt.Errorf( + "error during creation of the logger for logId: %v. ctx can not be nil", + logFile, + ) + } + + if GetLoggerFrom(ctx) != nil { + return ctx, nil, fmt.Errorf("logger already exists in context") + } + + logger, f, err := newFileLogger(logFile) + if err != nil { + return ctx, nil, fmt.Errorf( + "error during creation of the logger for logId: %v details: %w", + logFile, + err, + ) + } + + ctx = context.WithValue(ctx, contextLoggerKey, logger) + + return ctx, f, nil +} + +func GetLoggerFrom(ctx context.Context) *logrus.Logger { + v := ctx.Value(contextLoggerKey) + if v == nil { + return nil + } + + logger, ok := v.(*logrus.Logger) + if !ok { + return nil + } + + return logger +} diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go new file mode 100644 index 00000000000..1e310cfdb58 --- /dev/null +++ b/backend/src/driver/api/request.go @@ -0,0 +1,55 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package api provides HTTP DTOs used by the driver server. +package api + +// DriverPluginArgs is the JSON body sent by Argo Workflows to the KFP driver +// HTTP endpoint (POST /api/v1/template.execute) when using Argo HTTP templates. +// All fields are populated by the KFP Argo compiler and substituted by Argo at +// workflow execution time. +type DriverPluginArgs struct { + // Namespace is the Kubernetes namespace of the executing workflow. + // Populated from {{workflow.namespace}} by the Argo HTTP template. + Namespace string `json:"namespace"` + Component string `json:"component,omitempty"` + Container string `json:"container,omitempty"` + DagExecutionID string `json:"dag_execution_id"` + IterationIndex string `json:"iteration_index"` + HTTPProxy string `json:"http_proxy"` + HTTPSProxy string `json:"https_proxy"` + NoProxy string `json:"no_proxy"` + KubernetesConfig string `json:"kubernetes_config,omitempty"` + RuntimeConfig string `json:"runtime_config,omitempty"` + PipelineName string `json:"pipeline_name"` + PublishLogs string `json:"publish_logs,omitempty"` + RunID string `json:"run_id"` + RunName string `json:"run_name"` + RunDisplayName string `json:"run_display_name"` + TaskName string `json:"task_name"` + Task string `json:"task"` + Type string `json:"type"` + CacheDisabledFlag bool `json:"cache_disabled"` + MLMDServerAddress string `json:"mlmd_server_address"` + MLMDServerPort string `json:"mlmd_server_port"` + MlPipelineServerAddress string `json:"ml_pipeline_server_address"` + MlPipelineServerPort string `json:"ml_pipeline_server_port"` + MlPipelineTLSEnabled bool `json:"ml_pipeline_tls_enabled"` + MetadataTLSEnabled bool `json:"metadata_tls_enabled"` + CACertPath string `json:"ca_cert_path"` + LogLevel string `json:"log_level"` + DefaultRunAsUser *int64 `json:"default_run_as_user,omitempty"` + DefaultRunAsGroup *int64 `json:"default_run_as_group,omitempty"` + DefaultRunAsNonRoot string `json:"default_run_as_non_root,omitempty"` +} diff --git a/backend/src/driver/api/response.go b/backend/src/driver/api/response.go new file mode 100644 index 00000000000..8750eacb701 --- /dev/null +++ b/backend/src/driver/api/response.go @@ -0,0 +1,27 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +// DriverHTTPResponse is the JSON body returned by the KFP driver HTTP endpoint +// (POST /api/v1/template.execute) when invoked via an Argo HTTP template. +// All fields correspond to Argo workflow output parameters extracted by the +// compiler using jsonpath(response.body, "$.FIELD") expressions. +type DriverHTTPResponse struct { + ExecutionID string `json:"execution-id,omitempty"` + IterationCount string `json:"iteration-count,omitempty"` + CachedDecision string `json:"cached-decision,omitempty"` + Condition string `json:"condition,omitempty"` + PodSpecPatch string `json:"pod-spec-patch,omitempty"` +} diff --git a/backend/src/driver/cmd/main.go b/backend/src/driver/cmd/main.go new file mode 100644 index 00000000000..c9f886bc7ba --- /dev/null +++ b/backend/src/driver/cmd/main.go @@ -0,0 +1,55 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Binary driver-service is a standalone HTTP service that exposes the KFP +// driver as a centralized endpoint in the kubeflow namespace. Argo Workflows +// calls POST /api/v1/template.execute for each workflow driver node via an +// Argo HTTP template, eliminating per-workflow driver sidecar pods. +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + + "github.com/golang/glog" + "github.com/gorilla/mux" + driver "github.com/kubeflow/pipelines/backend/src/driver" +) + +func main() { + // glog writes to files by default; redirect to stderr so logs appear in + // kubectl logs / container stdout. + flag.Set("logtostderr", "true") + flag.Parse() + + port := os.Getenv("KFP_DRIVER_PORT") + if port == "" { + port = "8080" + } + + router := mux.NewRouter() + router.HandleFunc("/api/v1/template.execute", driver.ExecutePlugin).Methods(http.MethodPost) + router.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "ok") + }).Methods(http.MethodGet) + + addr := fmt.Sprintf(":%s", port) + glog.Infof("KFP driver service listening on %s", addr) + if err := http.ListenAndServe(addr, router); err != nil { + glog.Fatalf("driver service failed: %v", err) + } +} diff --git a/backend/src/driver/handler.go b/backend/src/driver/handler.go new file mode 100644 index 00000000000..26e7ea3e76a --- /dev/null +++ b/backend/src/driver/handler.go @@ -0,0 +1,534 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package driver implements an HTTP handler compatible with the Argo Workflows +// executor plugin protocol. It exposes the KFP driver logic as a centralized +// service endpoint within the KFP API server, eliminating the need for a +// per-workflow driver sidecar container. +package driver + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/golang/glog" + "github.com/google/uuid" + "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/kubeflow/pipelines/backend/src/driver/api" + "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" + "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/driver" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" + "github.com/kubeflow/pipelines/backend/src/v2/objectstore" + "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" + "google.golang.org/protobuf/encoding/protojson" + "k8s.io/client-go/kubernetes" +) + +const ( + unsetProxyArgValue = "unset" + // RootDag denotes the root DAG driver type. + RootDag = "ROOT_DAG" + // DAG denotes the DAG driver type. + DAG = "DAG" + // CONTAINER denotes the container driver type. + CONTAINER = "CONTAINER" +) + +type driverLogArtifactContext struct { + Execution *driver.Execution + Task string + LocalPath string + OutputPathPrefix string + Namespace string + PipelineRoot string + StoreSessionInfo string + LogID string +} + +// ExecutePlugin handles POST /api/v1/template.execute requests from Argo Workflows. +// It is invoked by the Argo workflow controller via an Argo HTTP template. The +// request body is a flat JSON object of DriverPluginArgs; the response is a flat +// JSON object (DriverHTTPResponse) whose fields are extracted by the compiler as +// Argo output parameters using jsonpath(response.body, "$.FIELD") expressions. +func ExecutePlugin(w http.ResponseWriter, r *http.Request) { + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + glog.Errorf("Error closing response body: %v", err) + } + }(r.Body) + + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + glog.Infof("Received driver execute request: %v", r) + args, err := parseDriverRequestArgs(r) + if err != nil { + glog.Errorf("Failed to parse driver request args: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + execution, err := drive(*args, args.Namespace) + if err != nil { + glog.Errorf("unable to drive execution: %v", err) + http.Error(w, fmt.Sprintf("driver execution failed: %v", err), http.StatusInternalServerError) + return + } + if execution != nil && execution.ExecutorInput != nil { + executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) + if err != nil { + glog.Errorf("failed to marshal ExecutorInput: %v", err) + http.Error(w, fmt.Sprintf("failed to marshal ExecutorInput: %v", err), http.StatusInternalServerError) + return + } + glog.Infof("output ExecutorInput:%s\n", prettyPrint(string(executorInputBytes))) + } + WriteJSONResponse(w, buildHTTPResponse(execution, args.Type)) +} + +// parseDriverRequestArgs parses the Argo HTTP template request body, which is a +// flat JSON object of DriverPluginArgs. The namespace is read from the Namespace +// field, which the compiler populates with {{workflow.namespace}}. +// +// The compiler base64-encodes JSON-valued fields (component, task, container, +// kubernetes_config, runtime_config) to avoid Argo's HTTP body substitution +// breaking the enclosing JSON when those values contain quotes. This function +// decodes those fields transparently. +func parseDriverRequestArgs(r *http.Request) (*api.DriverPluginArgs, error) { + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("failed to read request body: %v", err) + } + var args api.DriverPluginArgs + if err := json.Unmarshal(bodyBytes, &args); err != nil { + // Log up to 2000 bytes of the body to aid debugging. + snippet := bodyBytes + if len(snippet) > 2000 { + snippet = snippet[:2000] + } + glog.Errorf("Request body (first 2000 bytes): %s", snippet) + return nil, fmt.Errorf("failed to parse driver request body: %v", err) + } + if err := decodeBase64Fields(&args); err != nil { + return nil, fmt.Errorf("failed to decode base64 fields: %v", err) + } + if err := validate(args); err != nil { + return nil, err + } + return &args, nil +} + +// decodeBase64Fields decodes the base64-encoded JSON fields that the compiler +// wraps in {{=base64encode(...)}} expressions to survive Argo HTTP body +// template substitution without JSON-escaping issues. +func decodeBase64Fields(args *api.DriverPluginArgs) error { + var err error + if args.Component, err = decodeBase64Field(args.Component); err != nil { + return fmt.Errorf("component: %w", err) + } + if args.Task, err = decodeBase64Field(args.Task); err != nil { + return fmt.Errorf("task: %w", err) + } + if args.Container, err = decodeBase64Field(args.Container); err != nil { + return fmt.Errorf("container: %w", err) + } + if args.KubernetesConfig, err = decodeBase64Field(args.KubernetesConfig); err != nil { + return fmt.Errorf("kubernetes_config: %w", err) + } + if args.RuntimeConfig, err = decodeBase64Field(args.RuntimeConfig); err != nil { + return fmt.Errorf("runtime_config: %w", err) + } + return nil +} + +// decodeBase64Field base64-decodes a field value. Returns empty string for +// empty input without error (matching the behavior of absent optional fields). +func decodeBase64Field(s string) (string, error) { + if s == "" { + return "", nil + } + decoded, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return "", err + } + return string(decoded), nil +} + +// drive executes the KFP driver logic for the given args and workflow namespace. +// +// namespace is the Kubernetes namespace in which the workflow runs. When running +// centrally in the API server this is supplied from the Argo request body. When +// empty (e.g., in tests or legacy sidecar mode) the pod's own namespace is used +// as a fallback. +func drive(args api.DriverPluginArgs, namespace string) (execution *driver.Execution, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("KFP driver: %w", err) + } + }() + var ( + pipelineRoot string + storeSessionInfo string + outputPathPrefix string + ) + var pipeline *metadata.Pipeline + logID := fmt.Sprintf("%d-%v-%v-%v", time.Now().UnixMilli(), args.IterationIndex, args.Type, args.TaskName) + // Use a temp file for driver logs; the file is uploaded to object storage then + // deleted. In centralized mode there is no shared Argo volume to write to. + logFile := filepath.Join(os.TempDir(), fmt.Sprintf("kfp-driver-%s.log", logID)) + ctx, f, err := util.WithLogger(context.Background(), logFile) + if err != nil { + return nil, fmt.Errorf("failed to create driver logger: %v", err) + } + defer func() { + removeErr := os.Remove(logFile) + if removeErr != nil { + glog.Errorf("Failed to remove processed log file: %v", removeErr) + } + }() + defer func() { + if pipelineRoot != "" { + logContext := &driverLogArtifactContext{ + Execution: execution, + Task: args.TaskName, + LocalPath: logFile, + LogID: logID, + Namespace: namespace, + PipelineRoot: pipelineRoot, + StoreSessionInfo: storeSessionInfo, + OutputPathPrefix: outputPathPrefix, + } + uploadErr := uploadDriverLogArtifact(ctx, logContext) + if uploadErr != nil { + glog.Errorf("Failed to upload driver-logs artifact: %v", uploadErr) + } + } + }() + defer func() { + if f != nil { + closeErr := f.Close() + if closeErr != nil { + glog.Errorf("Failed to close file: %v", closeErr) + } + } + }() + + log := util.GetLoggerFrom(ctx) + + log.Infof("driver plugin arguments: %v", args) + // Support reading component spec from a file if value starts with @. + // This bypasses exec() argument size limits for large workflows. + if strings.HasPrefix(args.Component, "@") { + filePath := (args.Component)[1:] // Remove the "@" prefix + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read component spec from file %s: %w", filePath, err) + } + args.Component = string(data) + log.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) + } + + proxy.InitializeConfig(args.HTTPProxy, args.HTTPSProxy, args.NoProxy) + + log.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) + componentSpec := &pipelinespec.ComponentSpec{} + if err := util.UnmarshalString(args.Component, componentSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(args.Component)) + } + var taskSpec *pipelinespec.PipelineTaskSpec + if args.Task != "" { + log.Infof("input TaskSpec:%s\n", prettyPrint(args.Task)) + taskSpec = &pipelinespec.PipelineTaskSpec{} + if err := util.UnmarshalString(args.Task, taskSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, args.Task) + } + } + + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} + if args.Container != "" { + log.Infof("input ContainerSpec:%s\n", prettyPrint(args.Container)) + if err := util.UnmarshalString(args.Container, containerSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, args.Container) + } + } + var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig + if args.RuntimeConfig != "" { + log.Infof("input RuntimeConfig:%s\n", prettyPrint(args.RuntimeConfig)) + runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} + if err := util.UnmarshalString(args.RuntimeConfig, runtimeConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, args.RuntimeConfig) + } + } + if args.KubernetesConfig != "" { + log.Infof("input kubernetesConfig:%s\n", prettyPrint(args.KubernetesConfig)) + } + k8sExecCfg, err := parseExecConfigJSON(&args.KubernetesConfig) + if err != nil { + return nil, err + } + + // Resolve namespace: prefer the workflow namespace from the Argo request body + // (populated in executor plugin address mode). Fall back to the pod's own namespace + // for backward compatibility with sidecar mode or local testing. + if namespace == "" { + namespace, err = config.InPodNamespace() + if err != nil { + return nil, err + } + } + + var tlsCfg *tls.Config + if args.MetadataTLSEnabled { + tlsCfg, err = util.GetTLSConfig(args.CACertPath) + if err != nil { + return nil, fmt.Errorf("unable to drive driver: failed to load TLS configuration: %v", err) + } + } + client, err := newMlmdClient(args.MLMDServerAddress, args.MLMDServerPort, tlsCfg) + if err != nil { + return nil, err + } + cacheClient, err := cacheutils.NewClient(args.MlPipelineServerAddress, args.MlPipelineServerPort, args.CacheDisabledFlag, tlsCfg) + if err != nil { + return nil, err + } + + dagExecutionID, err := strconv.ParseInt(args.DagExecutionID, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse dag execution id, error: %w", err) + } + iterationIndex, err := strconv.Atoi(args.IterationIndex) + if err != nil { + return nil, fmt.Errorf("failed to parse iteration index, error: %w", err) + } + options := driver.Options{ + PipelineName: args.PipelineName, + RunID: args.RunID, + RunName: args.RunName, + RunDisplayName: args.RunDisplayName, + Namespace: namespace, + Component: componentSpec, + Task: taskSpec, + DAGExecutionID: dagExecutionID, + IterationIndex: iterationIndex, + PipelineLogLevel: args.LogLevel, + PublishLogs: args.PublishLogs, + CacheDisabled: args.CacheDisabledFlag, + DriverType: args.Type, + TaskName: args.TaskName, + MLPipelineServerAddress: args.MlPipelineServerAddress, + MLPipelineServerPort: args.MlPipelineServerPort, + MLPipelineTLSEnabled: args.MlPipelineTLSEnabled, + MLMDServerAddress: args.MLMDServerAddress, + MLMDServerPort: args.MLMDServerPort, + MLMDTLSEnabled: args.MetadataTLSEnabled, + CaCertPath: args.CACertPath, + } + + var driverErr error + switch args.Type { + case RootDag: + options.RuntimeConfig = runtimeConfig + execution, pipeline, driverErr = driver.RootDAG(ctx, options, client) + if driverErr != nil { + return nil, err + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + case DAG: + pipeline, driverErr = client.GetPipeline(ctx, options.PipelineName, options.RunID, "", "", "", "") + if driverErr != nil { + return nil, driverErr + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + execution, driverErr = driver.DAG(ctx, pipeline, options, client) + case CONTAINER: + options.Container = containerSpec + options.KubernetesExecutorConfig = k8sExecCfg + if args.DefaultRunAsUser != nil && *args.DefaultRunAsUser >= 0 { + options.DefaultRunAsUser = args.DefaultRunAsUser + } + if args.DefaultRunAsGroup != nil && *args.DefaultRunAsGroup >= 0 { + options.DefaultRunAsGroup = args.DefaultRunAsGroup + } + if args.DefaultRunAsNonRoot != "" { + v, err := strconv.ParseBool(args.DefaultRunAsNonRoot) + if err == nil { + options.DefaultRunAsNonRoot = &v + } + } + pipeline, driverErr = client.GetPipeline(ctx, options.PipelineName, options.RunID, "", "", "", "") + if driverErr != nil { + return nil, driverErr + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + outputPathPrefix = uuid.NewString() + execution, driverErr = driver.Container(ctx, pipeline, options, client, cacheClient, outputPathPrefix) + default: + err = fmt.Errorf("unknown driverType %s", args.Type) + } + if driverErr != nil { + log.Errorf("driver execution failed with error: %v", driverErr) + if execution == nil { + return nil, driverErr + } + defer func() { + // Override error with driver error, because driver error is more important. + // However, we continue running, because the following code prints debug info that + // may be helpful for figuring out why this failed. + err = driverErr + }() + } + + return execution, nil +} + +func uploadDriverLogArtifact(ctx context.Context, logContext *driverLogArtifactContext) error { + if logContext == nil { + return fmt.Errorf("logContext is nil") + } + if logContext.PipelineRoot != "" { + restConfig, err := util.GetKubernetesConfig() + if err != nil { + return fmt.Errorf("failed to get kubernetes config: %v", err) + } + k8sClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to initialize kubernetes client set: %w", err) + } + session, err := objectstore.GetSessionInfoFromString(logContext.StoreSessionInfo) + if err != nil { + return fmt.Errorf("failed to get session info from store: %v", err) + } + bucketConfig, err := objectstore.ParseBucketConfig(logContext.PipelineRoot, session) + if err != nil { + return fmt.Errorf("failed to parse bucket config: %v", err) + } + bucket, err := objectstore.OpenBucket(ctx, k8sClient, logContext.Namespace, bucketConfig) + if err != nil { + return fmt.Errorf("failed to open bucket: %v", err) + } + key := fmt.Sprintf("driver/%s-logs", logContext.LogID) + if logContext.Execution != nil && logContext.OutputPathPrefix != "" { + key = fmt.Sprintf("%s/%s/driver-logs", logContext.Task, logContext.OutputPathPrefix) + } + glog.Infof("Uploading log key: %s ...", key) + err = objectstore.UploadBlob(ctx, bucket, logContext.LocalPath, key) + if err != nil { + return fmt.Errorf("failed to upload log: %v", err) + } + } + return nil +} + +func validate(args api.DriverPluginArgs) error { + switch { + case args.Type == "": + return fmt.Errorf("argument type must be specified") + case args.HTTPProxy == unsetProxyArgValue: + return fmt.Errorf("argument http_proxy is required but can be an empty value") + case args.HTTPSProxy == unsetProxyArgValue: + return fmt.Errorf("argument https_proxy is required but can be an empty value") + case args.NoProxy == unsetProxyArgValue: + return fmt.Errorf("argument no_proxy is required but can be an empty value") + } + return nil +} + +// buildHTTPResponse converts a driver Execution into the flat DriverHTTPResponse +// JSON that Argo extracts output parameters from via jsonpath expressions. +func buildHTTPResponse(execution *driver.Execution, driverType string) api.DriverHTTPResponse { + resp := api.DriverHTTPResponse{} + if execution == nil { + return resp + } + if execution.ID != 0 { + resp.ExecutionID = fmt.Sprint(execution.ID) + } + switch { + case execution.IterationCount != nil: + resp.IterationCount = fmt.Sprint(*execution.IterationCount) + case driverType == RootDag: + resp.IterationCount = "0" + } + if execution.Cached != nil { + resp.CachedDecision = strconv.FormatBool(*execution.Cached) + } + if execution.Condition != nil { + resp.Condition = strconv.FormatBool(*execution.Condition) + } + resp.PodSpecPatch = execution.PodSpecPatch + return resp +} + +// WriteJSONResponse writes a DriverHTTPResponse as JSON to the HTTP response writer. +// json.Marshal is used instead of json.Encoder.Encode to avoid a trailing newline in +// the response body, which can cause Argo's jsonpath function to fail when parsing +// tasks[...].outputs.result. +func WriteJSONResponse(w http.ResponseWriter, payload api.DriverHTTPResponse) { + w.Header().Set("Content-Type", "application/json") + b, err := json.Marshal(payload) + if err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + if _, err := w.Write(b); err != nil { + glog.Errorf("Failed to write response: %v", err) + } +} + +// parseExecConfigJSON parses the Kubernetes executor configuration from a JSON string. +func parseExecConfigJSON(k8sExecConfigJSON *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { + var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig + if *k8sExecConfigJSON != "" { + k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} + if err := util.UnmarshalString(*k8sExecConfigJSON, k8sExecCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJSON) + } + } + return k8sExecCfg, nil +} + +// prettyPrint formats a JSON string with indentation for readable logging output. +func prettyPrint(jsonStr string) string { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") + if err != nil { + return jsonStr + } + return prettyJSON.String() +} + +// newMlmdClient creates a new MLMD metadata client with optional TLS. +func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { + return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) +} diff --git a/backend/src/driver/handler_test.go b/backend/src/driver/handler_test.go new file mode 100644 index 00000000000..c5d1f2ca105 --- /dev/null +++ b/backend/src/driver/handler_test.go @@ -0,0 +1,64 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package driver + +import ( + "testing" + + "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" +) + +func strPtr(s string) *string { + return &s +} + +func TestSpecParsing(t *testing.T) { + tt := []struct { + name string + input *string + expected *kubernetesplatform.KubernetesExecutorConfig + wantErr bool + }{ + { + "Valid - test kubecfg value parse.", + strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}]}"), + &kubernetesplatform.KubernetesExecutorConfig{ + ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ + {SecretName: "value1"}, + }, + }, + false, + }, + { + "Valid - test kubecfg value ignores unknown field.", + strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}], \"unknown_field\": \"something\"}"), + &kubernetesplatform.KubernetesExecutorConfig{ + ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ + {SecretName: "value1"}, + }, + }, + false, + }, + } + + for _, tc := range tt { + t.Logf("Running test case: %s", tc.name) + cfg, err := parseExecConfigJSON(tc.input) + assert.Equal(t, tc.wantErr, err != nil) + assert.True(t, proto.Equal(tc.expected, cfg)) + } +} diff --git a/backend/src/v2/cmd/driver/execution_paths.go b/backend/src/v2/cmd/driver/execution_paths.go deleted file mode 100644 index 584d29065d5..00000000000 --- a/backend/src/v2/cmd/driver/execution_paths.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -type ExecutionPaths struct { - ExecutionID string - IterationCount string - CachedDecision string - Condition string - PodSpecPatch string -} diff --git a/backend/src/v2/cmd/driver/main.go b/backend/src/v2/cmd/driver/main.go deleted file mode 100644 index 428832e6bea..00000000000 --- a/backend/src/v2/cmd/driver/main.go +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright 2021-2023 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package main - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "flag" - "fmt" - - "google.golang.org/protobuf/encoding/protojson" - - "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" - "github.com/kubeflow/pipelines/backend/src/common/util" - - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/golang/glog" - "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" - "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" - "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/driver" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" - "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" -) - -const ( - driverTypeArg = "type" - httpProxyArg = "http_proxy" - httpsProxyArg = "https_proxy" - noProxyArg = "no_proxy" - unsetProxyArgValue = "unset" - ROOT_DAG = "ROOT_DAG" - DAG = "DAG" - CONTAINER = "CONTAINER" -) - -var ( - // inputs - driverType = flag.String(driverTypeArg, "", "task driver type, one of ROOT_DAG, DAG, CONTAINER") - pipelineName = flag.String("pipeline_name", "", "pipeline context name") - runID = flag.String("run_id", "", "pipeline run uid") - runName = flag.String("run_name", "", "pipeline run name (Kubernetes object name)") - runDisplayName = flag.String("run_display_name", "", "pipeline run display name") - componentSpecJson = flag.String("component", "{}", "component spec") - taskSpecJson = flag.String("task", "", "task spec") - runtimeConfigJson = flag.String("runtime_config", "", "jobruntime config") - iterationIndex = flag.Int("iteration_index", -1, "iteration index, -1 means not an interation") - taskName = flag.String("task_name", "", "original task name, used for proper input resolution in the container/dag driver") - - // container inputs - dagExecutionID = flag.Int64("dag_execution_id", 0, "DAG execution ID") - containerSpecJson = flag.String("container", "{}", "container spec") - k8sExecConfigJson = flag.String("kubernetes_config", "{}", "kubernetes executor config") - - // config - mlPipelineServerAddress = flag.String("ml_pipeline_server_address", "ml-pipeline", "The name of the ML pipeline API server address.") - mlPipelineServerPort = flag.String("ml_pipeline_server_port", "8887", "The port of the ML pipeline API server.") - mlmdServerAddress = flag.String("mlmd_server_address", "", "MLMD server address") - mlmdServerPort = flag.String("mlmd_server_port", "", "MLMD server port") - - // output paths - executionIDPath = flag.String("execution_id_path", "", "Exeucution ID output path") - iterationCountPath = flag.String("iteration_count_path", "", "Iteration Count output path") - podSpecPatchPath = flag.String("pod_spec_patch_path", "", "Pod Spec Patch output path") - // the value stored in the paths will be either 'true' or 'false' - cachedDecisionPath = flag.String("cached_decision_path", "", "Cached Decision output path") - conditionPath = flag.String("condition_path", "", "Condition output path") - logLevel = flag.String("log_level", "1", "The verbosity level to log.") - - // proxy - httpProxy = flag.String(httpProxyArg, unsetProxyArgValue, "The proxy for HTTP connections.") - httpsProxy = flag.String(httpsProxyArg, unsetProxyArgValue, "The proxy for HTTPS connections.") - noProxy = flag.String(noProxyArg, unsetProxyArgValue, "Addresses that should ignore the proxy.") - publishLogs = flag.String("publish_logs", "true", "Whether to publish component logs to the object store") - cacheDisabledFlag = flag.Bool("cache_disabled", false, "Disable cache globally.") - mlPipelineTLSEnabled = flag.Bool("ml_pipeline_tls_enabled", false, "Set to true if mlpipeline API server serves over TLS.") - metadataTLSEnabled = flag.Bool("metadata_tls_enabled", false, "Set to true if MLMD serves over TLS.") - caCertPath = flag.String("ca_cert_path", "", "The path to the CA certificate to trust on connections to the ML pipeline API server and metadata server.") - defaultRunAsUser = flag.Int64("default_run_as_user", -1, "Admin-configured default runAsUser for user containers. -1 means not set.") - defaultRunAsGroup = flag.Int64("default_run_as_group", -1, "Admin-configured default runAsGroup for user containers. -1 means not set.") - defaultRunAsNonRoot = flag.String("default_run_as_non_root", "", "Admin-configured default runAsNonRoot for user containers. Empty means not set.") -) - -// func RootDAG(pipelineName string, runID string, component *pipelinespec.ComponentSpec, task *pipelinespec.PipelineTaskSpec, mlmd *metadata.Client) (*Execution, error) { - -func main() { - flag.Parse() - - glog.Infof("Setting log level to: '%s'", *logLevel) - err := flag.Set("v", *logLevel) - if err != nil { - glog.Warningf("Failed to set log level: %s", err.Error()) - } - - err = drive() - if err != nil { - glog.Exitf("%v", err) - } -} - -// Use WARNING default logging level to facilitate troubleshooting. -func init() { - flag.Set("logtostderr", "true") - // Change the WARNING to INFO level for debugging. - flag.Set("stderrthreshold", "WARNING") -} - -func validate() error { - if *driverType == "" { - return fmt.Errorf("argument --%s must be specified", driverTypeArg) - } - if *httpProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpProxyArg) - } - if *httpsProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpsProxyArg) - } - if *noProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", noProxyArg) - } - // validation responsibility lives in driver itself, so we do not validate all other args - return nil -} - -func drive() (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("KFP driver: %w", err) - } - }() - ctx := context.Background() - if err = validate(); err != nil { - return err - } - - // Support reading component spec from a file if value starts with @ - // This bypasses exec() argument size limits for large workflows - if strings.HasPrefix(*componentSpecJson, "@") { - filePath := (*componentSpecJson)[1:] // Remove the "@" prefix - data, err := os.ReadFile(filePath) - if err != nil { - return fmt.Errorf("failed to read component spec from file %s: %w", filePath, err) - } - *componentSpecJson = string(data) - glog.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) - } - - proxy.InitializeConfig(*httpProxy, *httpsProxy, *noProxy) - glog.Infof("input ComponentSpec:%s\n", prettyPrint(*componentSpecJson)) - componentSpec := &pipelinespec.ComponentSpec{} - if err := util.UnmarshalString(*componentSpecJson, componentSpec); err != nil { - return fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(*componentSpecJson)) - } - var taskSpec *pipelinespec.PipelineTaskSpec - if *taskSpecJson != "" { - glog.Infof("input TaskSpec:%s\n", prettyPrint(*taskSpecJson)) - taskSpec = &pipelinespec.PipelineTaskSpec{} - if err := util.UnmarshalString(*taskSpecJson, taskSpec); err != nil { - return fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, taskSpecJson) - } - } - glog.Infof("input ContainerSpec:%s\n", prettyPrint(*containerSpecJson)) - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} - if err := util.UnmarshalString(*containerSpecJson, containerSpec); err != nil { - return fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, containerSpecJson) - } - var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig - if *runtimeConfigJson != "" { - glog.Infof("input RuntimeConfig:%s\n", prettyPrint(*runtimeConfigJson)) - runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} - if err := util.UnmarshalString(*runtimeConfigJson, runtimeConfig); err != nil { - return fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, runtimeConfigJson) - } - } - k8sExecCfg, err := parseExecConfigJson(k8sExecConfigJson) - if err != nil { - return err - } - namespace, err := config.InPodNamespace() - if err != nil { - return err - } - var tlsCfg *tls.Config - if *metadataTLSEnabled { - tlsCfg, err = util.GetTLSConfig(*caCertPath) - if err != nil { - return err - } - } - client, err := newMlmdClient(*mlmdServerAddress, *mlmdServerPort, tlsCfg) - if err != nil { - return err - } - cacheClient, err := cacheutils.NewClient(*mlPipelineServerAddress, *mlPipelineServerPort, *cacheDisabledFlag, tlsCfg) - if err != nil { - return err - } - options := driver.Options{ - PipelineName: *pipelineName, - RunID: *runID, - RunName: *runName, - RunDisplayName: *runDisplayName, - Namespace: namespace, - Component: componentSpec, - Task: taskSpec, - DAGExecutionID: *dagExecutionID, - IterationIndex: *iterationIndex, - PipelineLogLevel: *logLevel, - PublishLogs: *publishLogs, - CacheDisabled: *cacheDisabledFlag, - DriverType: *driverType, - TaskName: *taskName, - MLPipelineServerAddress: *mlPipelineServerAddress, - MLPipelineServerPort: *mlPipelineServerPort, - MLMDServerAddress: *mlmdServerAddress, - MLMDServerPort: *mlmdServerPort, - MLPipelineTLSEnabled: *mlPipelineTLSEnabled, - MLMDTLSEnabled: *metadataTLSEnabled, - CaCertPath: *caCertPath, - } - var execution *driver.Execution - var driverErr error - switch *driverType { - case ROOT_DAG: - options.RuntimeConfig = runtimeConfig - execution, driverErr = driver.RootDAG(ctx, options, client) - case DAG: - execution, driverErr = driver.DAG(ctx, options, client) - case CONTAINER: - options.Container = containerSpec - options.KubernetesExecutorConfig = k8sExecCfg - // Set admin defaults only when explicitly configured (non-negative). - if *defaultRunAsUser >= 0 { - options.DefaultRunAsUser = defaultRunAsUser - } - if *defaultRunAsGroup >= 0 { - options.DefaultRunAsGroup = defaultRunAsGroup - } - if *defaultRunAsNonRoot != "" { - v, err := strconv.ParseBool(*defaultRunAsNonRoot) - if err == nil { - options.DefaultRunAsNonRoot = &v - } - } - execution, driverErr = driver.Container(ctx, options, client, cacheClient) - default: - err = fmt.Errorf("unknown driverType %s", *driverType) - } - if driverErr != nil { - if execution == nil { - return driverErr - } - defer func() { - // Override error with driver error, because driver error is more important. - // However, we continue running, because the following code prints debug info that - // may be helpful for figuring out why this failed. - err = driverErr - }() - } - - executionPaths := &ExecutionPaths{ - ExecutionID: *executionIDPath, - IterationCount: *iterationCountPath, - CachedDecision: *cachedDecisionPath, - Condition: *conditionPath, - PodSpecPatch: *podSpecPatchPath, - } - - return handleExecution(execution, *driverType, executionPaths) -} - -func parseExecConfigJson(k8sExecConfigJson *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { - var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig - if *k8sExecConfigJson != "" { - glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJson)) - k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} - if err := util.UnmarshalString(*k8sExecConfigJson, k8sExecCfg); err != nil { - return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJson) - } - } - return k8sExecCfg, nil -} - -func handleExecution(execution *driver.Execution, driverType string, executionPaths *ExecutionPaths) error { - if execution.ID != 0 { - glog.Infof("output execution.ID=%v", execution.ID) - if executionPaths.ExecutionID != "" { - if err := writeFile(executionPaths.ExecutionID, []byte(fmt.Sprint(execution.ID))); err != nil { - return fmt.Errorf("failed to write execution ID to file: %w", err) - } - } - } - if execution.IterationCount != nil { - if err := writeFile(executionPaths.IterationCount, []byte(fmt.Sprintf("%v", *execution.IterationCount))); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } else { - if driverType == ROOT_DAG || driverType == DAG { - if err := writeFile(executionPaths.IterationCount, []byte("0")); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } - } - if execution.Cached != nil { - if err := writeFile(executionPaths.CachedDecision, []byte(strconv.FormatBool(*execution.Cached))); err != nil { - return fmt.Errorf("failed to write cached decision to file: %w", err) - } - } - if execution.Condition != nil { - if err := writeFile(executionPaths.Condition, []byte(strconv.FormatBool(*execution.Condition))); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } else { - // nil is a valid value for Condition - if driverType == ROOT_DAG || driverType == DAG || driverType == CONTAINER { - if err := writeFile(executionPaths.Condition, []byte("nil")); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } - } - if execution.PodSpecPatch != "" { - glog.Infof("output podSpecPatch=\n%s\n", execution.PodSpecPatch) - if executionPaths.PodSpecPatch == "" { - return fmt.Errorf("--pod_spec_patch_path is required for container executor drivers") - } - if err := writeFile(executionPaths.PodSpecPatch, []byte(execution.PodSpecPatch)); err != nil { - return fmt.Errorf("failed to write pod spec patch to file: %w", err) - } - } - if execution.ExecutorInput != nil { - executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) - if err != nil { - return fmt.Errorf("failed to marshal ExecutorInput to JSON: %w", err) - } - executorInputJSON := string(executorInputBytes) - glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) - } - return nil -} - -func prettyPrint(jsonStr string) string { - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") - if err != nil { - return jsonStr - } - return prettyJSON.String() -} - -func writeFile(path string, data []byte) (err error) { - if path == "" { - return fmt.Errorf("path is not specified") - } - defer func() { - if err != nil { - err = fmt.Errorf("failed to write to %s: %w", path, err) - } - }() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - return os.WriteFile(path, data, 0o644) -} - -func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { - return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) -} diff --git a/backend/src/v2/cmd/driver/main_test.go b/backend/src/v2/cmd/driver/main_test.go deleted file mode 100644 index 302600d2fee..00000000000 --- a/backend/src/v2/cmd/driver/main_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "os" - "testing" - - "github.com/kubeflow/pipelines/backend/src/v2/driver" - "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/proto" -) - -func strPtr(s string) *string { - return &s -} - -func TestSpecParsing(t *testing.T) { - tt := []struct { - name string - input *string - expected *kubernetesplatform.KubernetesExecutorConfig - wantErr bool - }{ - { - "Valid - test kubecfg value parse.", - strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}]}"), - &kubernetesplatform.KubernetesExecutorConfig{ - ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ - {SecretName: "value1"}, - }, - }, - false, - }, - { - "Valid - test kubecfg value ignores unknown field.", - strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}], \"unknown_field\": \"something\"}"), - &kubernetesplatform.KubernetesExecutorConfig{ - ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ - {SecretName: "value1"}, - }, - }, - false, - }, - } - - for _, tc := range tt { - t.Logf("Running test case: %s", tc.name) - cfg, err := parseExecConfigJson(tc.input) - assert.Equal(t, tc.wantErr, err != nil) - assert.True(t, proto.Equal(tc.expected, cfg)) - } -} - -func Test_handleExecutionContainer(t *testing.T) { - execution := &driver.Execution{} - - executionPaths := &ExecutionPaths{ - Condition: "condition.txt", - } - - err := handleExecution(execution, CONTAINER, executionPaths) - - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - verifyFileContent(t, executionPaths.Condition, "nil") - - cleanup(t, executionPaths) -} - -func Test_handleExecutionRootDAG(t *testing.T) { - execution := &driver.Execution{} - - executionPaths := &ExecutionPaths{ - IterationCount: "iteration_count.txt", - Condition: "condition.txt", - } - - err := handleExecution(execution, ROOT_DAG, executionPaths) - - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - verifyFileContent(t, executionPaths.IterationCount, "0") - verifyFileContent(t, executionPaths.Condition, "nil") - - cleanup(t, executionPaths) -} - -func Test_handleExecutionDAG(t *testing.T) { - execution := &driver.Execution{} - - executionPaths := &ExecutionPaths{ - IterationCount: "iteration_count.txt", - Condition: "condition.txt", - } - - err := handleExecution(execution, DAG, executionPaths) - - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - verifyFileContent(t, executionPaths.IterationCount, "0") - verifyFileContent(t, executionPaths.Condition, "nil") - - cleanup(t, executionPaths) -} - -func cleanup(t *testing.T, executionPaths *ExecutionPaths) { - removeIfExists(t, executionPaths.IterationCount) - removeIfExists(t, executionPaths.ExecutionID) - removeIfExists(t, executionPaths.Condition) - removeIfExists(t, executionPaths.PodSpecPatch) - removeIfExists(t, executionPaths.CachedDecision) -} - -func removeIfExists(t *testing.T, filePath string) { - _, err := os.Stat(filePath) - if err == nil { - err = os.Remove(filePath) - if err != nil { - t.Errorf("Unexpected error while removing the created file: %v", err) - } - } -} - -func verifyFileContent(t *testing.T, filePath string, expectedContent string) { - _, err := os.Stat(filePath) - if os.IsNotExist(err) { - t.Errorf("Expected file %s to be created, but it doesn't exist", filePath) - } - - fileContent, err := os.ReadFile(filePath) - if err != nil { - t.Errorf("Failed to read file contents: %v", err) - } - - if string(fileContent) != expectedContent { - t.Errorf("Expected file fileContent to be %q, got %q", expectedContent, string(fileContent)) - } -} diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 78a2cd1f464..4f22a16ad97 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -124,23 +124,6 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S } } - // Resolve the workflow-level TTL strategy from the pipeline config. - // resource_ttl_on_completion is interpreted as seconds-after-completion so that Argo's - // TTL controller removes the Workflow object after the run finishes, - // regardless of success or failure. - var ttlStrategy *wfapi.TTLStrategy - if hasPipelineConfig { - ttlStrategy = buildTTLStrategy(kubernetesSpec.GetPipelineConfig()) - } - - // Resolve the workflow-level active deadline from the pipeline config. - // This sets a hard timeout after which Argo forcibly terminates the - // workflow, preventing stuck/zombie runs that never complete. - var activeDeadlineSeconds *int64 - if kubernetesSpec != nil { - activeDeadlineSeconds = buildActiveDeadlineSeconds(kubernetesSpec.GetPipelineConfig()) - } - // initialization wf := &wfapi.Workflow{ TypeMeta: k8smeta.TypeMeta{ @@ -162,6 +145,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S PodMetadata: &wfapi.Metadata{ Annotations: map[string]string{ "pipelines.kubeflow.org/v2_component": "true", + util.AnnotationKeyIstioSidecarInject: util.AnnotationValueIstioSidecarInjectDisabled, }, Labels: map[string]string{ "pipelines.kubeflow.org/v2_component": "true", @@ -170,11 +154,9 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S Arguments: wfapi.Arguments{ Parameters: []wfapi.Parameter{}, }, - ServiceAccountName: common.GetStringConfigWithDefault(common.DefaultPipelineRunnerServiceAccountFlag, common.DefaultPipelineRunnerServiceAccount), - Entrypoint: tmplEntrypoint, - VolumeClaimTemplates: volumeClaimTemplates, - TTLStrategy: ttlStrategy, - ActiveDeadlineSeconds: activeDeadlineSeconds, + ServiceAccountName: common.GetStringConfigWithDefault(common.DefaultPipelineRunnerServiceAccountFlag, common.DefaultPipelineRunnerServiceAccount), + Entrypoint: tmplEntrypoint, + VolumeClaimTemplates: volumeClaimTemplates, }, } @@ -200,8 +182,6 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S // TODO(chensun): release process and update the images. launcherImage: GetLauncherImage(), launcherCommand: GetLauncherCommand(), - driverImage: GetDriverImage(), - driverCommand: GetDriverCommand(), job: job, spec: spec, executors: deploy.GetExecutors(), @@ -244,62 +224,6 @@ func retrieveLastValidString(s string) string { return sections[len(sections)-1] } -// buildTTLStrategy constructs an Argo TTLStrategy from a pipeline config. -// -// The three proto fields map to Argo's three TTL knobs: -// - resource_ttl_on_completion → SecondsAfterCompletion (regardless of outcome) -// - resource_ttl_on_success → SecondsAfterSuccess (successful runs only) -// - resource_ttl_on_failure → SecondsAfterFailure (failed runs only) -// -// Only positive values are applied; a value of 0 (proto default) is treated as -// "not set". Returns nil when none of the fields is positive. -func buildTTLStrategy(pipelineConfig *pipelinespec.PipelineConfig) *wfapi.TTLStrategy { - if pipelineConfig == nil { - return nil - } - - afterCompletion := pipelineConfig.GetResourceTtlOnCompletion() - afterSuccess := pipelineConfig.GetResourceTtlOnSuccess() - afterFailure := pipelineConfig.GetResourceTtlOnFailure() - - if afterCompletion <= 0 && afterSuccess <= 0 && afterFailure <= 0 { - return nil - } - - strategy := &wfapi.TTLStrategy{} - if afterCompletion > 0 { - v := int32(afterCompletion) - strategy.SecondsAfterCompletion = &v - } - if afterSuccess > 0 { - v := int32(afterSuccess) - strategy.SecondsAfterSuccess = &v - } - if afterFailure > 0 { - v := int32(afterFailure) - strategy.SecondsAfterFailure = &v - } - return strategy -} - -// buildActiveDeadlineSeconds returns a pointer to the active-deadline value -// that should be set on the Argo Workflow spec. -// -// Semantics (opt-in, backward compatible): -// - pipelineConfig == nil or field <= 0 → no deadline (nil) -// - field > 0 → use the user-supplied value -func buildActiveDeadlineSeconds(pipelineConfig *pipelinespec.PipelineConfig) *int64 { - if pipelineConfig == nil { - return nil - } - userValue := pipelineConfig.GetActiveDeadlineSeconds() - if userValue <= 0 { - return nil - } - v := int64(userValue) - return &v -} - type workflowCompiler struct { // inputs job *pipelinespec.PipelineJob @@ -519,19 +443,41 @@ func inputValue(parameter string) string { return fmt.Sprintf("{{inputs.parameters.%s}}", parameter) } +// b64InputValue returns an Argo expression that base64-encodes an input +// parameter value. This is required for parameters whose values are JSON +// objects (e.g., component/task/container specs): Argo's HTTP template body +// substitution does NOT JSON-escape substituted values, so raw JSON strings +// would break the enclosing JSON body. Base64 encoding avoids this because the +// encoded output contains no characters that require JSON escaping. +// +// Uses sprig.b64enc because Argo's expression environment exposes the full +// Sprig function map; there is no built-in standalone base64encode function. +func b64InputValue(parameter string) string { + return fmt.Sprintf("{{=sprig.b64enc(inputs.parameters['%s'])}}", parameter) +} + // In a DAG/steps template, refer to inputs to the parent template. func inputParameter(parameter string) string { return fmt.Sprintf("{{inputs.parameters.%s}}", parameter) } -func outputPath(parameter string) string { - return fmt.Sprintf("{{outputs.parameters.%s.path}}", parameter) -} - func taskOutputParameter(task string, param string) string { return fmt.Sprintf("{{tasks.%s.outputs.parameters.%s}}", task, param) } +// taskResultExtract returns an Argo expression that extracts a named JSON field +// from the raw result produced by an HTTP-template task. Argo HTTP templates +// only populate outputs.result (the response body); named output parameters +// with ValueFrom.Expression are never evaluated for HTTP templates. +// Use this wherever a downstream task argument or when-condition needs a +// specific value from a driver's JSON response. +func taskResultExtract(task, jsonKey string) string { + // Use bracket notation $['key'] instead of dot notation $.key so that + // hyphenated keys (e.g. "execution-id") are handled correctly. Dot + // notation parses hyphens as subtraction operators in JSONPath. + return fmt.Sprintf("{{=jsonpath(tasks['%s'].outputs.result, \"$['%s']\")}}", task, jsonKey) +} + func loopItem() string { // https://github.com/argoproj/argo-workflows/blob/13bf15309567ff10ec23b6e5cfed846312d6a1ab/examples/loops-sequence.yaml#L20 return "{{item}}" diff --git a/backend/src/v2/compiler/argocompiler/argo_ttl_test.go b/backend/src/v2/compiler/argocompiler/argo_ttl_test.go deleted file mode 100644 index 6ae896ea89e..00000000000 --- a/backend/src/v2/compiler/argocompiler/argo_ttl_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2025 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package argocompiler - -import ( - "testing" - - "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBuildTTLStrategy(t *testing.T) { - tests := []struct { - name string - pipelineConfig *pipelinespec.PipelineConfig - // nil means the whole strategy should be nil - expectedNil bool - wantAfterCompletion *int32 - wantAfterSuccess *int32 - wantAfterFailure *int32 - }{ - { - name: "nil config returns nil", - expectedNil: true, - }, - { - name: "all-zero config returns nil", - pipelineConfig: &pipelinespec.PipelineConfig{ - ResourceTtlOnCompletion: 0, - ResourceTtlOnSuccess: 0, - ResourceTtlOnFailure: 0, - }, - expectedNil: true, - }, - { - name: "resource_ttl_on_completion only → SecondsAfterCompletion", - pipelineConfig: &pipelinespec.PipelineConfig{ResourceTtlOnCompletion: 300}, - wantAfterCompletion: int32Ptr(300), - }, - { - name: "resource_ttl_on_success only → SecondsAfterSuccess", - pipelineConfig: &pipelinespec.PipelineConfig{ResourceTtlOnSuccess: 600}, - wantAfterSuccess: int32Ptr(600), - }, - { - name: "resource_ttl_on_failure only → SecondsAfterFailure", - pipelineConfig: &pipelinespec.PipelineConfig{ResourceTtlOnFailure: 120}, - wantAfterFailure: int32Ptr(120), - }, - { - name: "all three fields set independently", - pipelineConfig: &pipelinespec.PipelineConfig{ - ResourceTtlOnCompletion: 300, - ResourceTtlOnSuccess: 3600, - ResourceTtlOnFailure: 60, - }, - wantAfterCompletion: int32Ptr(300), - wantAfterSuccess: int32Ptr(3600), - wantAfterFailure: int32Ptr(60), - }, - { - name: "success and failure without completion", - pipelineConfig: &pipelinespec.PipelineConfig{ - ResourceTtlOnSuccess: 7200, - ResourceTtlOnFailure: 1800, - }, - wantAfterSuccess: int32Ptr(7200), - wantAfterFailure: int32Ptr(1800), - }, - { - name: "negative resource_ttl_on_completion is treated as zero (not set)", - pipelineConfig: &pipelinespec.PipelineConfig{ResourceTtlOnCompletion: -1}, - expectedNil: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - strategy := buildTTLStrategy(tt.pipelineConfig) - - if tt.expectedNil { - assert.Nil(t, strategy) - return - } - - require.NotNil(t, strategy) - - if tt.wantAfterCompletion != nil { - require.NotNil(t, strategy.SecondsAfterCompletion) - assert.Equal(t, *tt.wantAfterCompletion, *strategy.SecondsAfterCompletion) - } else { - assert.Nil(t, strategy.SecondsAfterCompletion) - } - - if tt.wantAfterSuccess != nil { - require.NotNil(t, strategy.SecondsAfterSuccess) - assert.Equal(t, *tt.wantAfterSuccess, *strategy.SecondsAfterSuccess) - } else { - assert.Nil(t, strategy.SecondsAfterSuccess) - } - - if tt.wantAfterFailure != nil { - require.NotNil(t, strategy.SecondsAfterFailure) - assert.Equal(t, *tt.wantAfterFailure, *strategy.SecondsAfterFailure) - } else { - assert.Nil(t, strategy.SecondsAfterFailure) - } - }) - } -} - -func TestBuildActiveDeadlineSeconds(t *testing.T) { - tests := []struct { - name string - pipelineConfig *pipelinespec.PipelineConfig - expected *int64 - }{ - { - name: "nil config returns nil (no deadline)", - expected: nil, - }, - { - name: "zero value returns nil (no deadline)", - pipelineConfig: &pipelinespec.PipelineConfig{ActiveDeadlineSeconds: 0}, - expected: nil, - }, - { - name: "negative value returns nil (no deadline)", - pipelineConfig: &pipelinespec.PipelineConfig{ActiveDeadlineSeconds: -1}, - expected: nil, - }, - { - name: "positive value sets deadline", - pipelineConfig: &pipelinespec.PipelineConfig{ActiveDeadlineSeconds: 3600}, - expected: int64Ptr(3600), - }, - { - name: "large positive value is honored", - pipelineConfig: &pipelinespec.PipelineConfig{ActiveDeadlineSeconds: 2147483647}, - expected: int64Ptr(2147483647), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := buildActiveDeadlineSeconds(tt.pipelineConfig) - if tt.expected == nil { - assert.Nil(t, result) - } else { - require.NotNil(t, result) - assert.Equal(t, *tt.expected, *result) - } - }) - } -} - -func int32Ptr(v int32) *int32 { return &v } diff --git a/backend/src/v2/compiler/argocompiler/common.go b/backend/src/v2/compiler/argocompiler/common.go index ce167539ea8..893724fc98f 100644 --- a/backend/src/v2/compiler/argocompiler/common.go +++ b/backend/src/v2/compiler/argocompiler/common.go @@ -18,7 +18,6 @@ import ( wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/golang/glog" "github.com/kubeflow/pipelines/backend/src/apiserver/common" - "github.com/kubeflow/pipelines/backend/src/v2/component" k8score "k8s.io/api/core/v1" ) @@ -54,17 +53,6 @@ var commonEnvs = []k8score.EnvVar{{ }, }} -// retryIndexEnv injects the Argo retry attempt index into the executor -// container. Argo substitutes {{retries}} with the 0-based attempt index at -// pod creation time, so each retry attempt writes executor-logs to a distinct, -// sequentially-numbered path (executor-logs-0, executor-logs-1, …). -// This variable is only valid inside a template that has a retryStrategy, so -// it must NOT be added to commonEnvs (which is applied to all templates). -var retryIndexEnv = k8score.EnvVar{ - Name: component.EnvRetryIndex, - Value: "{{retries}}", -} - // ConfigureCustomCABundle adds CABundle environment variables and volume mounts if CABUNDLE_SECRET_NAME is set. func ConfigureCustomCABundle(tmpl *wfapi.Template) { caBundleSecretName := common.GetCaBundleSecretName() diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index ce4b89e4fac..9c6dcebb009 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -39,9 +39,7 @@ import ( const ( volumeNameKFPLauncher = "kfp-launcher" - volumeNameCABundle = "ca-bundle" LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" - DriverImageEnvVar = "V2_DRIVER_IMAGE" // DefaultLauncherImage & DefaultDriverImage are set as latest here // but are overridden by environment variables set via k8s manifests. // For releases, the manifest will have the correct release version set. @@ -49,9 +47,6 @@ const ( DefaultLauncherImage = "ghcr.io/kubeflow/kfp-launcher:latest" LauncherCommandEnvVar = "V2_LAUNCHER_COMMAND" DefaultLauncherCommand = "launcher-v2" - DefaultDriverImage = "ghcr.io/kubeflow/kfp-driver:latest" - DefaultDriverCommand = "driver" - DriverCommandEnvVar = "V2_DRIVER_COMMAND" PipelineRunAsUserEnvVar = "PIPELINE_RUN_AS_USER" PipelineLogLevelEnvVar = "PIPELINE_LOG_LEVEL" PublishLogsEnvVar = "PUBLISH_LOGS" @@ -105,22 +100,6 @@ func GetLauncherImage() string { return launcherImage } -func GetDriverImage() string { - driverImage := os.Getenv(DriverImageEnvVar) - if driverImage == "" { - driverImage = DefaultDriverImage - } - return driverImage -} - -func GetDriverCommand() []string { - driverCommand := os.Getenv(DriverCommandEnvVar) - if driverCommand == "" { - driverCommand = DefaultDriverCommand - } - return strings.Split(driverCommand, " ") -} - func GetLauncherCommand() []string { launcherCommand := os.Getenv(LauncherCommandEnvVar) if launcherCommand == "" { @@ -148,10 +127,14 @@ func GetPipelineRunAsUser() *int64 { return &runAsUser } -func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs) { +func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs, error) { + template, err := c.addContainerDriverTemplate() + if err != nil { + return nil, nil, err + } dagTask := &wfapi.DAGTask{ Name: name, - Template: c.addContainerDriverTemplate(), + Template: template, Arguments: wfapi.Arguments{ Parameters: []wfapi.Parameter{ {Name: paramComponent, Value: wfapi.AnyStringPtr(inputs.component)}, @@ -175,75 +158,75 @@ func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriv ) } outputs := &containerDriverOutputs{ - podSpecPatch: taskOutputParameter(name, paramPodSpecPatch), - cached: taskOutputParameter(name, paramCachedDecision), - condition: taskOutputParameter(name, paramCondition), + podSpecPatch: taskResultExtract(name, "pod-spec-patch"), + cached: taskResultExtract(name, "cached-decision"), + condition: taskResultExtract(name, "condition"), } - return dagTask, outputs + return dagTask, outputs, nil } -func (c *workflowCompiler) addContainerDriverTemplate() string { +// addContainerDriverTemplate adds (once) the Argo HTTP template that invokes +// the centralized KFP container-driver endpoint. The workflow controller POSTs +// the driver args to the KFP API server. The response body is the raw result +// (outputs.result); downstream tasks extract individual values from it using +// taskResultExtract expressions in their arguments. +func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { name := "system-container-driver" _, ok := c.templates[name] if ok { - return name + return name, nil + } + + args := map[string]interface{}{ + "type": "CONTAINER", + "namespace": "{{workflow.namespace}}", + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": b64InputValue(paramComponent), + "task": b64InputValue(paramTask), + "task_name": inputValue(paramTaskName), + "container": b64InputValue(paramContainer), + "iteration_index": inputValue(paramIterationIndex), + "kubernetes_config": b64InputValue(paramKubernetesConfig), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.GetMetadataConfig().Address, + "mlmd_server_port": metadata.GetMetadataConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } - args := []string{ - "--type", "CONTAINER", - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--container", inputValue(paramContainer), - "--iteration_index", inputValue(paramIterationIndex), - "--cached_decision_path", outputPath(paramCachedDecision), - "--pod_spec_patch_path", outputPath(paramPodSpecPatch), - "--condition_path", outputPath(paramCondition), - "--kubernetes_config", inputValue(paramKubernetesConfig), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - "--ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "--ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "--mlmd_server_address", metadata.GetMetadataConfig().Address, - "--mlmd_server_port", metadata.GetMetadataConfig().Port, - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if c.mlPipelineTLSEnabled { - args = append(args, "--ml_pipeline_tls_enabled") - } - if common.GetMetadataTLSEnabled() { - args = append(args, "--metadata_tls_enabled") - } - - setCABundle := false // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to container driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args = append(args, "--ca_cert_path", common.CustomCaCertPath) - setCABundle = true + args["ca_cert_path"] = common.CustomCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) + args["log_level"] = value } if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + args["publish_logs"] = value } if c.defaultRunAsUser != nil { - args = append(args, "--default_run_as_user", strconv.FormatInt(*c.defaultRunAsUser, 10)) + args["default_run_as_user"] = strconv.FormatInt(*c.defaultRunAsUser, 10) } if c.defaultRunAsGroup != nil { - args = append(args, "--default_run_as_group", strconv.FormatInt(*c.defaultRunAsGroup, 10)) + args["default_run_as_group"] = strconv.FormatInt(*c.defaultRunAsGroup, 10) } if c.defaultRunAsNonRoot != nil { - args = append(args, "--default_run_as_non_root", strconv.FormatBool(*c.defaultRunAsNonRoot)) + args["default_run_as_non_root"] = strconv.FormatBool(*c.defaultRunAsNonRoot) + } + + httpTemplate, err := driverHTTPTemplate(args) + if err != nil { + return name, fmt.Errorf("failed to build container driver HTTP template: %v", err) } template := &wfapi.Template{ @@ -259,29 +242,12 @@ func (c *workflowCompiler) addContainerDriverTemplate() string { {Name: paramKubernetesConfig, Default: wfapi.AnyStringPtr("")}, }, }, - Outputs: wfapi.Outputs{ - Parameters: []wfapi.Parameter{ - {Name: paramPodSpecPatch, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/pod-spec-patch", Default: wfapi.AnyStringPtr("")}}, - {Name: paramCachedDecision, Default: wfapi.AnyStringPtr("false"), ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/cached-decision", Default: wfapi.AnyStringPtr("false")}}, - {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, - }, - }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: append(proxy.GetConfig().GetEnvVars(), commonEnvs...), - }, - } - applySecurityContextToTemplate(template) - // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the container driver template. - if setCABundle { - ConfigureCustomCABundle(template) + Outputs: wfapi.Outputs{}, + HTTP: httpTemplate, } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) - return name + return name, err } type containerExecutorInputs struct { @@ -567,17 +533,12 @@ func (c *workflowCompiler) addContainerExecutorTemplate(task *pipelinespec.Pipel } applySecurityContextToExecutorTemplate(executor, c.defaultRunAsUser, c.defaultRunAsGroup, c.defaultRunAsNonRoot) - // If retry policy is set, add retryStrategy to executor and inject - // KFP_RETRY_INDEX so the launcher can resolve the per-attempt log path - // without a Kubernetes API call. {{retries}} is only valid inside a - // template that has a retryStrategy; adding it elsewhere resolves to an - // empty string, which forces an unnecessary pod-annotation lookup. + // If retry policy is set, add retryStrategy to executor if taskRetrySpec != nil { executor.RetryStrategy = c.getTaskRetryStrategyFromInput(inputParameter(paramRetryMaxCount), inputParameter(paramRetryBackOffDuration), inputParameter(paramRetryBackOffFactor), inputParameter(paramRetryBackOffMaxDuration)) - executor.Container.Env = append(executor.Container.Env, retryIndexEnv) } // Update pod metadata if it defined in the Kubernetes Spec if k8sExecCfg.GetPodMetadata() != nil { diff --git a/backend/src/v2/compiler/argocompiler/container_test.go b/backend/src/v2/compiler/argocompiler/container_test.go index 1682874d790..0388015dff5 100644 --- a/backend/src/v2/compiler/argocompiler/container_test.go +++ b/backend/src/v2/compiler/argocompiler/container_test.go @@ -70,7 +70,12 @@ func TestAddContainerExecutorTemplate(t *testing.T) { } +// With the transition from Container to Plugin, environment variable configuration +// is now defined in manifests rather than in code, so we can no longer verify +// their behavior in a unit test. However, this is still fully tested within the +// workflow-compiler integration tests. func TestContainerDriverTemplate_IncludesKFPPodNameEnv(t *testing.T) { + t.Skip("Skipping test: env vars are now defined in manifests and cannot be verified in unit tests") proxy.InitializeConfigWithEmptyForTests() c := &workflowCompiler{ templates: make(map[string]*wfapi.Template), @@ -85,7 +90,8 @@ func TestContainerDriverTemplate_IncludesKFPPodNameEnv(t *testing.T) { job: &pipelinespec.PipelineJob{}, } - name := c.addContainerDriverTemplate() + name, err := c.addContainerDriverTemplate() + require.NoError(t, err) require.Equal(t, "system-container-driver", name) tmpl, exists := c.templates[name] diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index 658ea931fb2..c4089dd2019 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -19,15 +19,13 @@ import ( "sort" "strings" - "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" - "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" - wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/apiserver/common" + "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" "github.com/kubeflow/pipelines/backend/src/v2/compiler" - k8score "k8s.io/api/core/v1" + "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -297,7 +295,7 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec // The following call will return an empty string for tasks without kubernetes-specific annotation. kubernetesConfigPlaceholder, _ := c.useKubernetesImpl(componentName) - driver, driverOutputs := c.containerDriverTask(driverTaskName, containerDriverInputs{ + driver, driverOutputs, err := c.containerDriverTask(driverTaskName, containerDriverInputs{ component: componentSpecPlaceholder, task: taskSpecJson, container: containerPlaceholder, @@ -306,6 +304,9 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec kubernetesConfig: kubernetesConfigPlaceholder, taskName: effectiveTaskName, }) + if err != nil { + return nil, err + } if task.GetTriggerPolicy().GetCondition() == "" { driverOutputs.condition = "" } @@ -547,72 +548,74 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* Value: wfapi.AnyStringPtr(inputs.taskName), }) } + dagTemplate, err := c.addDAGDriverTemplate() + if err != nil { + return nil, nil, err + } t := &wfapi.DAGTask{ Name: name, - Template: c.addDAGDriverTemplate(), + Template: dagTemplate, Arguments: wfapi.Arguments{ Parameters: params, }, } return t, &dagDriverOutputs{ - executionID: taskOutputParameter(name, paramExecutionID), - iterationCount: taskOutputParameter(name, paramIterationCount), - condition: taskOutputParameter(name, paramCondition), + executionID: taskResultExtract(name, "execution-id"), + iterationCount: taskResultExtract(name, "iteration-count"), + condition: taskResultExtract(name, "condition"), }, nil } -func (c *workflowCompiler) addDAGDriverTemplate() string { +// addDAGDriverTemplate adds (once) the Argo HTTP template that invokes the +// centralized KFP DAG-driver endpoint. The response body is the raw result +// (outputs.result); downstream tasks extract individual values from it using +// taskResultExtract expressions in their arguments. +func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { name := "system-dag-driver" _, ok := c.templates[name] if ok { - return name - } - - args := []string{ - "--type", inputValue(paramDriverType), - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--runtime_config", inputValue(paramRuntimeConfig), - "--iteration_index", inputValue(paramIterationIndex), - "--execution_id_path", outputPath(paramExecutionID), - "--iteration_count_path", outputPath(paramIterationCount), - "--condition_path", outputPath(paramCondition), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - "--ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "--ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "--mlmd_server_address", metadata.GetMetadataConfig().Address, - "--mlmd_server_port", metadata.GetMetadataConfig().Port, - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if c.mlPipelineTLSEnabled { - args = append(args, "--ml_pipeline_tls_enabled") - } - if common.GetMetadataTLSEnabled() { - args = append(args, "--metadata_tls_enabled") - } - - setCABundle := false - // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to DAG driver. + return name, nil + } + + args := map[string]interface{}{ + "type": inputValue(paramDriverType), + "namespace": "{{workflow.namespace}}", + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": b64InputValue(paramComponent), + "task": b64InputValue(paramTask), + "task_name": inputValue(paramTaskName), + "runtime_config": b64InputValue(paramRuntimeConfig), + "iteration_index": inputValue(paramIterationIndex), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.GetMetadataConfig().Address, + "mlmd_server_port": metadata.GetMetadataConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), + } + // If CABUNDLE_SECRET_NAME add ca_cert_path arg to DAG driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args = append(args, "--ca_cert_path", common.CustomCaCertPath) - setCABundle = true + args["ca_cert_path"] = common.CustomCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) + args["log_level"] = value } if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + args["publish_logs"] = value + } + + httpTemplate, err := driverHTTPTemplate(args) + if err != nil { + return "", fmt.Errorf("failed to build DAG driver HTTP template: %v", err) } template := &wfapi.Template{ @@ -628,29 +631,12 @@ func (c *workflowCompiler) addDAGDriverTemplate() string { {Name: paramDriverType, Default: wfapi.AnyStringPtr("DAG")}, }, }, - Outputs: wfapi.Outputs{ - Parameters: []wfapi.Parameter{ - {Name: paramExecutionID, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/execution-id"}}, - {Name: paramIterationCount, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/iteration-count", Default: wfapi.AnyStringPtr("0")}}, - {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, - }, - }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: proxy.GetConfig().GetEnvVars(), - }, - } - applySecurityContextToTemplate(template) - // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the DAG driver template. - if setCABundle { - ConfigureCustomCABundle(template) + Outputs: wfapi.Outputs{}, + HTTP: httpTemplate, } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) - return name + return name, nil } func addImplicitDependencies(dagSpec *pipelinespec.DagSpec) error { diff --git a/backend/src/v2/compiler/argocompiler/plugin.go b/backend/src/v2/compiler/argocompiler/plugin.go new file mode 100644 index 00000000000..542b1be87e3 --- /dev/null +++ b/backend/src/v2/compiler/argocompiler/plugin.go @@ -0,0 +1,49 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package argocompiler + +import ( + "encoding/json" + "fmt" + + wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" +) + +// driverEndpointURL is the in-cluster URL for the standalone KFP driver +// service. Argo HTTP templates POST to this URL for every driver node, +// eliminating per-workflow sidecar or agent pods. +const driverEndpointURL = "http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute" + +// driverHTTPTemplate builds an Argo HTTP template that invokes the centralized +// KFP driver endpoint. The params map is marshaled as the JSON request body; +// Argo performs template-variable substitution on the body at runtime. +// +// The successCondition ensures Argo marks the node as failed when the driver +// returns a non-200 status (e.g. on driver logic errors). +func driverHTTPTemplate(params map[string]interface{}) (*wfapi.HTTP, error) { + bodyJSON, err := json.Marshal(params) + if err != nil { + return nil, fmt.Errorf("driver HTTP template: marshaling args to JSON failed: %w", err) + } + timeout := int64(600) + return &wfapi.HTTP{ + Method: "POST", + URL: driverEndpointURL, + Headers: wfapi.HTTPHeaders{{Name: "Content-Type", Value: "application/json"}}, + Body: string(bodyJSON), + SuccessCondition: "response.statusCode == 200", + TimeoutSeconds: &timeout, + }, nil +} diff --git a/backend/src/v2/component/constants.go b/backend/src/v2/component/constants.go index 8d069e5d92e..f4020263553 100644 --- a/backend/src/v2/component/constants.go +++ b/backend/src/v2/component/constants.go @@ -19,9 +19,8 @@ const ( KFPLauncherPath = VolumePathKFPLauncher + "/launch" // Env var names - EnvPodName = "KFP_POD_NAME" - EnvPodUID = "KFP_POD_UID" - EnvRetryIndex = "KFP_RETRY_INDEX" + EnvPodName = "KFP_POD_NAME" + EnvPodUID = "KFP_POD_UID" // Env vars in metadata-grpc-configmap EnvMetadataHost = "METADATA_GRPC_SERVICE_HOST" diff --git a/backend/src/v2/component/launcher_v2.go b/backend/src/v2/component/launcher_v2.go index 2e8da0b1935..f6851a1fee6 100644 --- a/backend/src/v2/component/launcher_v2.go +++ b/backend/src/v2/component/launcher_v2.go @@ -41,7 +41,6 @@ import ( "gocloud.dev/blob" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/structpb" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -521,56 +520,6 @@ const OutputMetadataFilepath = "/tmp/kfp_outputs/output_metadata.json" // We overwrite this as a DI mechanism for testing getLogWriter. var osCreateFunc = os.Create -// qualifyExecutorLogsURI appends the retry index to the executor-logs artifact -// URI so that each Argo retry attempt writes to and registers a distinct, -// human-readable path in the object store and MLMD (e.g. executor-logs-0, -// executor-logs-1, …). Without this, all retry pods share the single URI that -// the driver provisioned before any attempt ran, causing every retry to -// overwrite the same S3 key and register duplicate MLMD artifacts pointing at -// stale data. -func qualifyExecutorLogsURI(artifacts map[string]*pipelinespec.ArtifactList, retryIndex string) { - if retryIndex == "" { - return - } - logsArtifactList, ok := artifacts["executor-logs"] - if !ok || logsArtifactList == nil || len(logsArtifactList.Artifacts) != 1 { - return - } - art := logsArtifactList.Artifacts[0] - if art == nil { - return - } - art.Uri = art.Uri + "-" + retryIndex -} - -// retryIndexFromPodAnnotation reads the Argo node-name annotation on the current -// pod and parses the 0-based retry index from it. Argo encodes the index as a -// parenthesised suffix on the node name, e.g. "…executor(3)". This provides a -// fallback when the KFP_RETRY_INDEX env var (set via {{retries}} in the Argo -// template) is unavailable, which is the case when an older API server that -// does not yet inject KFP_RETRY_INDEX is in use. -func retryIndexFromPodAnnotation(ctx context.Context, k8sClient kubernetes.Interface, namespace, podName string) (string, error) { - pod, err := k8sClient.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) - if err != nil { - return "", fmt.Errorf("failed to get pod %s/%s: %w", namespace, podName, err) - } - nodeName, ok := pod.Annotations["workflows.argoproj.io/node-name"] - if !ok { - return "", fmt.Errorf("pod %s/%s has no argo node-name annotation", namespace, podName) - } - // Extract the trailing "(N)" from node names like "…executor(3)". - open := strings.LastIndex(nodeName, "(") - close := strings.LastIndex(nodeName, ")") - if open < 0 || close <= open { - return "", fmt.Errorf("argo node-name %q has no retry index suffix", nodeName) - } - index := nodeName[open+1 : close] - if _, err := strconv.Atoi(index); err != nil { - return "", fmt.Errorf("argo node-name %q retry index %q is not an integer: %w", nodeName, index, err) - } - return index, nil -} - // getLogWriter returns an io.Writer that can either be single-channel to stdout // or dual-channel to stdout AND a log file based on the URI of a log artifact // in the supplied ArtifactList. Downstream, the resulting log file gets @@ -638,30 +587,6 @@ func execute( return nil, err } - // Qualify the executor-logs URI with the retry index before preparing output - // folders so the per-attempt directory is created at the correct path. - // Prefer KFP_RETRY_INDEX, which is injected by the Argo compiler as - // "{{retries}}" and resolved by Argo at pod creation time. Fall back to - // reading the Argo node-name pod annotation, which encodes the index as - // executor(N), for compatibility with older API server versions. - if publishLogs == "true" { - retryIndex := os.Getenv(EnvRetryIndex) - if retryIndex == "" { - podName := os.Getenv(EnvPodName) - if podName != "" && k8sClient != nil && namespace != "" { - if idx, err := retryIndexFromPodAnnotation(ctx, k8sClient, namespace, podName); err == nil { - retryIndex = idx - } else { - glog.Warningf("Could not determine retry index from pod annotation, defaulting to 0: %v", err) - } - } - } - if retryIndex == "" { - retryIndex = "0" - } - qualifyExecutorLogsURI(executorInput.Outputs.GetArtifacts(), retryIndex) - } - if err := prepareOutputFolders(executorInput); err != nil { return nil, err } diff --git a/backend/src/v2/component/launcher_v2_test.go b/backend/src/v2/component/launcher_v2_test.go index ea2e7f19580..495016f5dee 100644 --- a/backend/src/v2/component/launcher_v2_test.go +++ b/backend/src/v2/component/launcher_v2_test.go @@ -34,8 +34,6 @@ import ( "gocloud.dev/blob" _ "gocloud.dev/blob/memblob" "google.golang.org/protobuf/types/known/structpb" - k8score "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) @@ -123,7 +121,6 @@ func Test_executeV2_publishLogs(t *testing.T) { name string executorInput *pipelinespec.ExecutorInput executorArgs []string - retryIndex string wantErr bool }{ { @@ -134,7 +131,6 @@ func Test_executeV2_publishLogs(t *testing.T) { }, }, []string{"-c", "echo testoutput && test {{$.inputs.parameters['a']}} -eq 1 || exit 1\ntest {{$.inputs.parameters['b']}} -eq 2 || exit 1"}, - "", false, }, { @@ -145,7 +141,6 @@ func Test_executeV2_publishLogs(t *testing.T) { }, }, []string{"-c", "echo testoutput && test {{$.inputs.parameters['a']}} -eq 5 || exit 1\ntest {{$.inputs.parameters['b']}} -eq 2 || exit 1"}, - "", false, }, { @@ -156,24 +151,8 @@ func Test_executeV2_publishLogs(t *testing.T) { }, }, []string{"-c", "echo testoutput && exit 1"}, - "", true, }, - { - // KFP_RETRY_INDEX is injected by the Argo compiler via "{{retries}}". - // The executor-logs URI must be qualified with the retry index so each - // attempt writes to a distinct, human-readable path (executor-logs-0, - // executor-logs-1, …). - "retry index qualifies executor-logs URI", - &pipelinespec.ExecutorInput{ - Inputs: &pipelinespec.ExecutorInput_Inputs{ - ParameterValues: map[string]*structpb.Value{"a": structpb.NewNumberValue(1), "b": structpb.NewNumberValue(2)}, - }, - }, - []string{"-c", "echo testoutput && test {{$.inputs.parameters['a']}} -eq 1 || exit 1"}, - "3", - false, - }, } for _, test := range tests { @@ -204,11 +183,6 @@ func Test_executeV2_publishLogs(t *testing.T) { }, } - // Simulate Argo injecting KFP_RETRY_INDEX into the pod env. - if test.retryIndex != "" { - t.Setenv(EnvRetryIndex, test.retryIndex) - } - _, outputArtifacts, err := executeV2( context.Background(), test.executorInput, @@ -231,20 +205,8 @@ func Test_executeV2_publishLogs(t *testing.T) { assert.Nil(t, err) } - // When a retry index is set, the executor-logs URI (and therefore the - // object-store key) must be suffixed with the index so retries don't - // overwrite each other (e.g. executor-logs-3). - effectiveIndex := test.retryIndex - if effectiveIndex == "" { - effectiveIndex = "0" - } - logKey := "executor-logs-" + effectiveIndex - logArt := test.executorInput.Outputs.Artifacts["executor-logs"].Artifacts[0] - assert.Contains(t, logArt.Uri, effectiveIndex, - "executor-logs URI should contain the retry index for attempt isolation") - - outputLog, err := bucket.ReadAll(context.TODO(), logKey) - assert.Nil(t, err, "Expected executor-logs to be readable at key %q", logKey) + outputLog, err := bucket.ReadAll(context.TODO(), "executor-logs") + assert.Nil(t, err, "Expected executor-logs to be readable") assert.Equal(t, "testoutput\n", string(outputLog)) assert.Len(t, outputArtifacts, 1, "Expected 1 output artifact (executor-logs)") @@ -415,147 +377,6 @@ func Test_get_log_Writer(t *testing.T) { } } -func Test_qualifyExecutorLogsURI(t *testing.T) { - baseURI := "minio://mlpipeline/v2/artifacts/my-pipeline/run-id/always-fail/salt123/executor-logs" - - tests := []struct { - name string - artifacts map[string]*pipelinespec.ArtifactList - retryIndex string - wantURI string - }{ - { - name: "appends retry index to executor-logs URI", - artifacts: map[string]*pipelinespec.ArtifactList{ - "executor-logs": {Artifacts: []*pipelinespec.RuntimeArtifact{{Uri: baseURI}}}, - }, - retryIndex: "2", - wantURI: baseURI + "-2", - }, - { - name: "no-op when retry index is empty", - artifacts: map[string]*pipelinespec.ArtifactList{ - "executor-logs": {Artifacts: []*pipelinespec.RuntimeArtifact{{Uri: baseURI}}}, - }, - retryIndex: "", - wantURI: baseURI, - }, - { - name: "no-op when executor-logs key is absent", - artifacts: map[string]*pipelinespec.ArtifactList{}, - retryIndex: "1", - wantURI: "", // no artifact to check - }, - { - name: "no-op when executor-logs list is empty", - artifacts: map[string]*pipelinespec.ArtifactList{ - "executor-logs": {Artifacts: []*pipelinespec.RuntimeArtifact{}}, - }, - retryIndex: "1", - wantURI: "", // no artifact to check - }, - { - name: "no-op when executor-logs list has multiple artifacts", - artifacts: map[string]*pipelinespec.ArtifactList{ - "executor-logs": {Artifacts: []*pipelinespec.RuntimeArtifact{ - {Uri: baseURI}, - {Uri: baseURI + "-2"}, - }}, - }, - retryIndex: "1", - // list len != 1: guard should skip, original URIs unchanged - wantURI: baseURI, - }, - { - name: "no-op when ArtifactList value is nil", - artifacts: map[string]*pipelinespec.ArtifactList{"executor-logs": nil}, - retryIndex: "1", - wantURI: "", // nil list: no artifact to check - }, - { - name: "no-op when first artifact is nil", - artifacts: map[string]*pipelinespec.ArtifactList{ - "executor-logs": {Artifacts: []*pipelinespec.RuntimeArtifact{nil}}, - }, - retryIndex: "1", - wantURI: "", // nil artifact: no URI to check - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.NotPanics(t, func() { - qualifyExecutorLogsURI(tc.artifacts, tc.retryIndex) - }) - list, ok := tc.artifacts["executor-logs"] - if !ok || list == nil || len(list.Artifacts) == 0 || list.Artifacts[0] == nil { - // Cases where there is nothing to assert on - return - } - assert.Equal(t, tc.wantURI, list.Artifacts[0].Uri) - }) - } -} - -func Test_retryIndexFromPodAnnotation(t *testing.T) { - tests := []struct { - name string - annotation string - wantIndex string - wantErr bool - }{ - { - name: "parses first attempt (0)", - annotation: "my-pipeline-abc.root.always-fail.executor(0)", - wantIndex: "0", - }, - { - name: "parses fourth retry (4)", - annotation: "retry-e2e-pzhkb.root.always-fail.executor(4)", - wantIndex: "4", - }, - { - name: "no annotation", - annotation: "", - wantErr: true, - }, - { - name: "annotation without parenthesised suffix", - annotation: "my-pipeline-abc.root.always-fail.executor", - wantErr: true, - }, - { - name: "annotation with non-integer index", - annotation: "my-pipeline-abc.root.always-fail.executor(abc)", - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - clientset := fake.NewClientset() - if tc.annotation != "" { - pod := &k8score.Pod{} - pod.Name = "test-pod" - pod.Namespace = "test-ns" - pod.Annotations = map[string]string{ - "workflows.argoproj.io/node-name": tc.annotation, - } - _, err := clientset.CoreV1().Pods("test-ns").Create(context.Background(), pod, metav1.CreateOptions{}) - assert.NoError(t, err) - } - - idx, err := retryIndexFromPodAnnotation(context.Background(), clientset, "test-ns", "test-pod") - if tc.wantErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.wantIndex, idx) - } - }) - } -} - // Tests happy and unhappy paths for constructing a new LauncherV2 func Test_NewLauncherV2(t *testing.T) { var testCmdArgs = []string{"sh", "-c", "echo \"hello world\""} diff --git a/backend/src/v2/driver/container.go b/backend/src/v2/driver/container.go index 7a39a38cef4..bef4fee3340 100644 --- a/backend/src/v2/driver/container.go +++ b/backend/src/v2/driver/container.go @@ -20,9 +20,8 @@ import ( "fmt" "strconv" - "github.com/golang/glog" - "github.com/google/uuid" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" "github.com/kubeflow/pipelines/backend/src/v2/expression" "github.com/kubeflow/pipelines/backend/src/v2/metadata" @@ -43,14 +42,18 @@ func validateContainer(opts Options) (err error) { return validateNonRoot(opts) } -func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheClient cacheutils.Client) (execution *Execution, err error) { +func Container(ctx context.Context, pipeline *metadata.Pipeline, opts Options, mlmd *metadata.Client, cacheClient cacheutils.Client, outputPathPrefix string) (execution *Execution, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.Container(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("Container opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, fmt.Errorf("failed to get logger for container: %s", string(b)) + } + log.Trace("Container opts: ", string(b)) err = validateContainer(opts) if err != nil { return nil, err @@ -62,15 +65,11 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl } // TODO(Bobgy): there's no need to pass any parameters, because pipeline // and pipeline run context have been created by root DAG driver. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, "", "", "", "") - if err != nil { - return nil, err - } dag, err := mlmd.GetDAG(ctx, opts.DAGExecutionID) if err != nil { return nil, err } - glog.Infof("parent DAG: %+v", dag.Execution) + log.Infof("parent DAG: %+v", dag.Execution) expr, err := expression.New() if err != nil { return nil, err @@ -116,7 +115,7 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl pipeline.GetPipelineRoot(), opts.TaskName, opts.Component.GetOutputDefinitions(), - uuid.NewString(), + outputPathPrefix, opts.PublishLogs, ) } @@ -126,11 +125,13 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl return execution, err } ecfg.TaskName = opts.TaskName + ecfg.Namespace = opts.Namespace ecfg.DisplayName = opts.Task.GetTaskInfo().GetName() ecfg.ExecutionType = metadata.ContainerExecutionTypeName ecfg.ParentDagID = dag.Execution.GetID() ecfg.IterationIndex = iterationIndex ecfg.NotTriggered = !execution.WillTrigger() + ecfg.DriverLogURI = metadata.GenerateOutputURI(pipeline.GetPipelineRoot(), []string{opts.TaskName, outputPathPrefix, "driver-logs"}, false) if isKubernetesPlatformOp { return execution, kubernetesPlatformOps(ctx, mlmd, cacheClient, execution, ecfg, &opts) @@ -182,7 +183,7 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl if err != nil { return execution, err } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return execution, nil @@ -206,12 +207,12 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl if err := mlmd.PublishExecution(ctx, createdExecution, executorOutput.GetParameterValues(), outputArtifacts, pb.Execution_CACHED); err != nil { return execution, fmt.Errorf("failed to publish cached execution: %w", err) } - glog.Infof("Use cache for task %s", opts.Task.GetTaskInfo().GetName()) + log.Infof("Use cache for task %s", opts.Task.GetTaskInfo().GetName()) *execution.Cached = true return execution, nil } } else { - glog.Info("Cache disabled globally at the server level.") + log.Info("Cache disabled globally at the server level.") } taskConfig := &TaskConfig{} diff --git a/backend/src/v2/driver/dag.go b/backend/src/v2/driver/dag.go index 362fac66f7f..39464abe2e5 100644 --- a/backend/src/v2/driver/dag.go +++ b/backend/src/v2/driver/dag.go @@ -19,8 +19,8 @@ import ( "encoding/json" "fmt" - "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/expression" "github.com/kubeflow/pipelines/backend/src/v2/metadata" "google.golang.org/protobuf/types/known/structpb" @@ -38,14 +38,18 @@ func validateDAG(opts Options) (err error) { return validateNonRoot(opts) } -func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { +func DAG(ctx context.Context, pipeline *metadata.Pipeline, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.DAG(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("DAG opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, fmt.Errorf("failed to get logger for DAG driver options: %s", string(b)) + } + log.Trace("DAG opts: ", string(b)) err = validateDAG(opts) if err != nil { return nil, err @@ -55,17 +59,11 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E index := opts.IterationIndex iterationIndex = &index } - // TODO(Bobgy): there's no need to pass any parameters, because pipeline - // and pipeline run context have been created by root DAG driver. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, "", "", "", "") - if err != nil { - return nil, err - } dag, err := mlmd.GetDAG(ctx, opts.DAGExecutionID) if err != nil { return nil, err } - glog.Infof("parent DAG: %+v", dag.Execution) + log.Infof("parent DAG: %+v", dag.Execution) expr, err := expression.New() if err != nil { return nil, err @@ -77,7 +75,7 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E executorInput := &pipelinespec.ExecutorInput{ Inputs: inputs, } - glog.Infof("executorInput value: %+v", executorInput) + log.Infof("executorInput value: %+v", executorInput) execution = &Execution{ExecutorInput: executorInput} condition := opts.Task.GetTriggerPolicy().GetCondition() if condition != "" { @@ -109,15 +107,15 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E // Handle writing output parameters to MLMD. ecfg.OutputParameters = opts.Component.GetDag().GetOutputs().GetParameters() - glog.V(4).Info("outputParameters: ", ecfg.OutputParameters) + log.Trace("outputParameters: ", ecfg.OutputParameters) // Handle writing output artifacts to MLMD. ecfg.OutputArtifacts = opts.Component.GetDag().GetOutputs().GetArtifacts() - glog.V(4).Info("outputArtifacts: ", ecfg.OutputArtifacts) + log.Trace("outputArtifacts: ", ecfg.OutputArtifacts) totalDagTasks := len(opts.Component.GetDag().GetTasks()) ecfg.TotalDagTasks = &totalDagTasks - glog.V(4).Info("totalDagTasks: ", *ecfg.TotalDagTasks) + log.Trace("totalDagTasks: ", *ecfg.TotalDagTasks) if opts.Task.GetArtifactIterator() != nil { return execution, fmt.Errorf("ArtifactIterator is not implemented") @@ -164,17 +162,17 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E execution.IterationCount = &count } - glog.V(4).Info("pipeline: ", pipeline) + log.Trace("pipeline: ", pipeline) b, _ = json.Marshal(*ecfg) - glog.V(4).Info("ecfg: ", string(b)) - glog.V(4).Infof("dag: %v", dag) + log.Trace("ecfg: ", string(b)) + log.Tracef("dag: %v", dag) // TODO(Bobgy): change execution state to pending, because this is driver, execution hasn't started. createdExecution, err := mlmd.CreateExecution(ctx, pipeline, ecfg) if err != nil { return execution, err } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() return execution, nil } diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index 6d6188c2e61..8dfc1caeb81 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -706,7 +706,7 @@ func provisionOutputs( pipelineRoot, taskName string, outputsSpec *pipelinespec.ComponentOutputsSpec, - outputURISalt string, + prefix string, publishOutput string, ) *pipelinespec.ExecutorInput_Outputs { outputs := &pipelinespec.ExecutorInput_Outputs{ @@ -736,7 +736,7 @@ func provisionOutputs( // artifacts (dsl.get_uri) by allowing the SDK to infer the task root from // the executor output file's directory (set below) and convert it back to // a remote URI at runtime. - taskRootRemote := metadata.GenerateOutputURI(pipelineRoot, []string{taskName, outputURISalt}, false) + taskRootRemote := metadata.GenerateOutputURI(pipelineRoot, []string{taskName, prefix}, false) // Set per-artifact output URIs under the task root. for name, artifact := range artifacts { @@ -769,7 +769,6 @@ func provisionOutputs( // Fallback to legacy path if the pipeline root scheme is not recognized. outputs.OutputFile = component.OutputMetadataFilepath } - return outputs } diff --git a/backend/src/v2/driver/driver_test.go b/backend/src/v2/driver/driver_test.go index e0a1f9d0858..a7b39fe529f 100644 --- a/backend/src/v2/driver/driver_test.go +++ b/backend/src/v2/driver/driver_test.go @@ -19,6 +19,8 @@ import ( "fmt" "testing" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" @@ -1469,7 +1471,7 @@ func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_Passthro assert.Nil(t, err) err = extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, opts, nil, @@ -1572,7 +1574,7 @@ func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_ApplyAnd assert.Nil(t, err) err = extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, opts, nil, diff --git a/backend/src/v2/driver/k8s.go b/backend/src/v2/driver/k8s.go index 617e9163ceb..e7a241ced44 100644 --- a/backend/src/v2/driver/k8s.go +++ b/backend/src/v2/driver/k8s.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/google/uuid" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/common/util" @@ -127,6 +126,10 @@ func extendPodSpecPatch( kubernetesExecutorConfig := opts.KubernetesExecutorConfig setOnTaskConfig, setOnPod := getTaskConfigOptions(opts.Component) + log := util.GetLoggerFrom(ctx) + if log == nil { + return fmt.Errorf("cannot get log from context for extendPodSpecPatch") + } // Always set setOnTaskConfig to an empty map if taskConfig is nil to avoid nil pointer dereference. if taskConfig == nil { @@ -210,7 +213,7 @@ func extendPodSpecPatch( if tolerations := kubernetesExecutorConfig.GetTolerations(); tolerations != nil { var k8sTolerations []k8score.Toleration - glog.Infof("Tolerations passed: %+v", tolerations) + log.Infof("Tolerations passed: %+v", tolerations) for _, toleration := range tolerations { if toleration != nil { @@ -244,7 +247,7 @@ func extendPodSpecPatch( } k8sTolerations = append(k8sTolerations, singleToleration) } else { - glog.V(4).Info("encountered empty tolerations struct, ignoring.") + log.Trace("encountered empty tolerations struct, ignoring.") } } else if isListToleration { listVal := resolvedParam.GetListValue() @@ -259,7 +262,7 @@ func extendPodSpecPatch( } k8sTolerations = append(k8sTolerations, k8sTolerationsList...) } else { - glog.V(4).Info("encountered empty tolerations list, ignoring.") + log.Trace("encountered empty tolerations list, ignoring.") } } else { return fmt.Errorf("encountered unexpected toleration proto value, must be either struct or list type") @@ -608,7 +611,7 @@ func extendPodSpecPatch( if nodeAffinityTerm.GetNodeAffinityJson() == nil && len(nodeAffinityTerm.GetMatchExpressions()) == 0 && len(nodeAffinityTerm.GetMatchFields()) == 0 { - glog.Warningf("NodeAffinityTerm %d is empty, skipping", i) + log.Warningf("NodeAffinityTerm %d is empty, skipping", i) continue } if nodeAffinityTerm.GetNodeAffinityJson() != nil { @@ -657,10 +660,10 @@ func extendPodSpecPatch( Weight: *nodeAffinityTerm.Weight, Preference: nodeSelectorTerm, }) - glog.V(4).Infof("Added preferred node affinity: %+v", nodeSelectorTerm) + log.Tracef("Added preferred node affinity: %+v", nodeSelectorTerm) } else { requiredTerms = append(requiredTerms, nodeSelectorTerm) - glog.V(4).Infof("Added required node affinity: %+v", nodeSelectorTerm) + log.Tracef("Added required node affinity: %+v", nodeSelectorTerm) } } @@ -729,18 +732,18 @@ func extendPodSpecPatch( isCompilerHardened := existingSecurityContext.AllowPrivilegeEscalation != nil && !*existingSecurityContext.AllowPrivilegeEscalation if userSecurityContext.RunAsUser != nil { if existingSecurityContext.RunAsUser != nil { - glog.Warningf("Ignoring user-specified runAsUser (%d): security context already set by admin (runAsUser=%d)", + log.Warningf("Ignoring user-specified runAsUser (%d): security context already set by admin (runAsUser=%d)", *userSecurityContext.RunAsUser, *existingSecurityContext.RunAsUser) } else { if isCompilerHardened && *userSecurityContext.RunAsUser == 0 { - glog.Warningf("Setting runAsUser=0 (root) on a container with hardened security context; consider using a non-root UID") + log.Warningf("Setting runAsUser=0 (root) on a container with hardened security context; consider using a non-root UID") } podSpec.Containers[0].SecurityContext.RunAsUser = userSecurityContext.RunAsUser } } if userSecurityContext.RunAsGroup != nil { if existingSecurityContext.RunAsGroup != nil { - glog.Warningf("Ignoring user-specified runAsGroup (%d): security context already set by admin (runAsGroup=%d)", + log.Warningf("Ignoring user-specified runAsGroup (%d): security context already set by admin (runAsGroup=%d)", *userSecurityContext.RunAsGroup, *existingSecurityContext.RunAsGroup) } else { podSpec.Containers[0].SecurityContext.RunAsGroup = userSecurityContext.RunAsGroup @@ -748,7 +751,7 @@ func extendPodSpecPatch( } if userSecurityContext.RunAsNonRoot != nil { if existingSecurityContext.RunAsNonRoot != nil { - glog.Warningf("Ignoring user-specified runAsNonRoot (%v): security context already set by admin (runAsNonRoot=%v)", + log.Warningf("Ignoring user-specified runAsNonRoot (%v): security context already set by admin (runAsNonRoot=%v)", *userSecurityContext.RunAsNonRoot, *existingSecurityContext.RunAsNonRoot) } else { podSpec.Containers[0].SecurityContext.RunAsNonRoot = userSecurityContext.RunAsNonRoot @@ -786,8 +789,13 @@ func createPVC( taskStartedTime := time.Now().Unix() + log := util.GetLoggerFrom(ctx) + if log == nil { + return "", nil, 0, fmt.Errorf("failed to get log from context") + } + inputs := execution.ExecutorInput.Inputs - glog.Infof("Input parameter values: %+v", inputs.ParameterValues) + log.Infof("Input parameter values: %+v", inputs.ParameterValues) // Required input: access_modes accessModeInput, ok := inputs.ParameterValues["access_modes"] @@ -869,7 +877,7 @@ func createPVC( if err != nil { return "", createdExecution, pb.Execution_FAILED, fmt.Errorf("error creating MLMD execution for createpvc: %w", err) } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return "", createdExecution, pb.Execution_COMPLETE, nil @@ -919,7 +927,7 @@ func createPVC( if err != nil { return "", createdExecution, pb.Execution_FAILED, fmt.Errorf("failed to create pvc: %w", err) } - glog.Infof("Created PVC %s\n", createdPVC.ObjectMeta.Name) + log.Infof("Created PVC %s\n", createdPVC.Name) // Create a cache entry if !opts.CacheDisabled && opts.Task.GetCachingOptions().GetEnableCache() { @@ -953,9 +961,13 @@ func deletePVC( }() taskStartedTime := time.Now().Unix() + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, pb.Execution_FAILED, fmt.Errorf("no logs available for execution") + } inputs := execution.ExecutorInput.Inputs - glog.Infof("Input parameter values: %+v", inputs.ParameterValues) + log.Infof("Input parameter values: %+v", inputs.ParameterValues) // Required input: pvc_name pvcNameInput, ok := inputs.ParameterValues["pvc_name"] @@ -985,7 +997,7 @@ func deletePVC( if err != nil { return createdExecution, pb.Execution_FAILED, fmt.Errorf("error creating MLMD execution for createpvc: %w", err) } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return createdExecution, pb.Execution_COMPLETE, nil @@ -1024,7 +1036,7 @@ func deletePVC( return createdExecution, pb.Execution_FAILED, fmt.Errorf("failed to delete pvc %s: %v", pvcName, err) } - glog.Infof("Deleted PVC %s\n", pvcName) + log.Infof("Deleted PVC %s\n", pvcName) // Create a cache entry if !opts.CacheDisabled && opts.Task.GetCachingOptions().GetEnableCache() && ecfg.CachedMLMDExecutionID != "" { @@ -1140,7 +1152,11 @@ func publishDriverExecution( if err = mlmd.PublishExecution(ctx, execution, outputParameters, outputArtifacts, status); err != nil { return fmt.Errorf("failed to publish: %w", err) } - glog.Infof("Published execution of Kubernetes platform task %s.", execution.TaskName()) + log := util.GetLoggerFrom(ctx) + if log == nil { + return fmt.Errorf("can not get log from the context ") + } + log.Infof("Published execution of Kubernetes platform task %s.", execution.TaskName()) return nil } diff --git a/backend/src/v2/driver/k8s_test.go b/backend/src/v2/driver/k8s_test.go index 884f448a656..2225063600f 100644 --- a/backend/src/v2/driver/k8s_test.go +++ b/backend/src/v2/driver/k8s_test.go @@ -5,8 +5,10 @@ import ( "testing" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/metadata" "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/structpb" @@ -102,7 +104,7 @@ func Test_makeVolumeMountPatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { volumeMounts, volumes, err := makeVolumeMountPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), Options{}, tt.args.pvcMount, tt.args.dag, @@ -238,7 +240,7 @@ func Test_makePodSpecPatch_nodeSelector(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -717,7 +719,7 @@ func Test_extendPodSpecPatch_Secret(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1214,7 +1216,7 @@ func Test_extendPodSpecPatch_ConfigMap(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1388,7 +1390,7 @@ func Test_extendPodSpecPatch_EmptyVolumeMount(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1514,7 +1516,7 @@ func Test_extendPodSpecPatch_ImagePullSecrets(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1953,7 +1955,7 @@ func Test_extendPodSpecPatch_Tolerations(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2060,7 +2062,7 @@ func Test_extendPodSpecPatch_FieldPathAsEnv(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2133,7 +2135,7 @@ func Test_extendPodSpecPatch_ActiveDeadlineSeconds(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2246,7 +2248,7 @@ func Test_extendPodSpecPatch_SecurityContext(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2274,7 +2276,7 @@ func Test_extendPodSpecPatch_SecurityContext_CombinedWithOtherFeatures(t *testin }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ SecurityContext: &kubernetesplatform.SecurityContext{ @@ -2321,7 +2323,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminSetPreserved(t *testing.T) { {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsUser: &adminUID, @@ -2355,7 +2357,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminDefaultsNoUserOverride(t *test {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsUser: &adminUID, @@ -2383,7 +2385,7 @@ func Test_extendPodSpecPatch_SecurityContext_RootOnHardenedContainer(t *testing. }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ SecurityContext: &kubernetesplatform.SecurityContext{ @@ -2424,7 +2426,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminRunAsNonRoot(t *testing.T) { {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsNonRoot: &adminRunAsNonRoot, @@ -2453,7 +2455,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminRunAsNonRootNoUserOverride(t * {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsNonRoot: &adminRunAsNonRoot, @@ -2475,7 +2477,7 @@ func Test_extendPodSpecPatch_SecurityContext_UserRunAsNonRootNoAdmin(t *testing. {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ @@ -2567,7 +2569,7 @@ func Test_extendPodSpecPatch_ImagePullPolicy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2765,7 +2767,7 @@ func Test_extendPodSpecPatch_GenericEphemeralVolume(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -3069,7 +3071,7 @@ func Test_extendPodSpecPatch_NodeAffinity(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -3154,7 +3156,7 @@ func Test_extendPodSpecPatch_TaskConfig_CapturesAndApplies(t *testing.T) { taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, @@ -3305,7 +3307,7 @@ func Test_extendPodSpecPatch_PvcMounts_Passthrough_NotAppliedToPod(t *testing.T) } taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, @@ -3339,7 +3341,7 @@ func Test_extendPodSpecPatch_PvcMounts_Passthrough_AppliedToPod(t *testing.T) { } taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, diff --git a/backend/src/v2/driver/root_dag.go b/backend/src/v2/driver/root_dag.go index c017fcd7bbf..2e02b922d40 100644 --- a/backend/src/v2/driver/root_dag.go +++ b/backend/src/v2/driver/root_dag.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" - "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/config" @@ -64,54 +63,58 @@ func validateRootDAG(opts Options) (err error) { return nil } -func RootDAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { +func RootDAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *Execution, pipeline *metadata.Pipeline, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.RootDAG(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("RootDAG opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, nil, fmt.Errorf("driver.RootDAG(%s) failed: invalid log configuration", opts.info()) + } + log.Trace("RootDAG opts: ", string(b)) err = validateRootDAG(opts) if err != nil { - return nil, err + return nil, nil, err } // TODO(v2): in pipeline spec, rename GCS output directory to pipeline root. pipelineRoot := opts.RuntimeConfig.GetGcsOutputDirectory() restConfig, err := util.GetKubernetesConfig() if err != nil { - return nil, fmt.Errorf("failed to initialize kubernetes client: %w", err) + return nil, nil, fmt.Errorf("failed to initialize kubernetes client: %w", err) } k8sClient, err := kubernetes.NewForConfig(restConfig) if err != nil { - return nil, fmt.Errorf("failed to initialize kubernetes client set: %w", err) + return nil, nil, fmt.Errorf("failed to initialize kubernetes client set: %w", err) } cfg, err := config.FromConfigMap(ctx, k8sClient, opts.Namespace) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfo := objectstore.SessionInfo{} if pipelineRoot != "" { - glog.Infof("PipelineRoot=%q", pipelineRoot) + log.Infof("PipelineRoot=%q", pipelineRoot) } else { pipelineRoot = cfg.DefaultPipelineRoot() - glog.Infof("PipelineRoot=%q from default config", pipelineRoot) + log.Infof("PipelineRoot=%q from default config", pipelineRoot) } storeSessionInfo, err = cfg.GetStoreSessionInfo(pipelineRoot) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfoJSON, err := json.Marshal(storeSessionInfo) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfoStr := string(storeSessionInfoJSON) // TODO(Bobgy): fill in run resource. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, opts.Namespace, "run-resource", pipelineRoot, storeSessionInfoStr) + pipeline, err = mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, opts.Namespace, "run-resource", pipelineRoot, storeSessionInfoStr) if err != nil { - return nil, err + return nil, nil, err } executorInput := &pipelinespec.ExecutorInput{ @@ -122,16 +125,16 @@ func RootDAG(ctx context.Context, opts Options, mlmd *metadata.Client) (executio // TODO(Bobgy): validate executorInput matches component spec types ecfg, err := metadata.GenerateExecutionConfig(executorInput) if err != nil { - return nil, err + return nil, pipeline, err } ecfg.ExecutionType = metadata.DagExecutionTypeName ecfg.Name = fmt.Sprintf("run/%s", opts.RunID) exec, err := mlmd.CreateExecution(ctx, pipeline, ecfg) if err != nil { - return nil, err + return nil, pipeline, err } - glog.Infof("Created execution: %s", exec) + log.Infof("Created execution: %s", exec) // No need to return ExecutorInput, because tasks in the DAG will resolve // needed info from MLMD. - return &Execution{ID: exec.GetID()}, nil + return &Execution{ID: exec.GetID()}, pipeline, nil } diff --git a/backend/src/v2/metadata/client.go b/backend/src/v2/metadata/client.go index ef54b9acbb3..0e8e2fc08ad 100644 --- a/backend/src/v2/metadata/client.go +++ b/backend/src/v2/metadata/client.go @@ -55,19 +55,9 @@ const ( pipelineRunContextTypeName = "system.PipelineRun" ImporterExecutionTypeName = "system.ImporterExecution" ImporterWorkspaceExecutionTypeName = "system.ImporterWorkspaceExecution" - // mlmdClientSideMaxRetries is the number of times the MLMD client will retry - // a failed gRPC call before returning an error. This is intentionally higher - // than a typical RPC retry budget because MySQL deadlocks (codes.Aborted) on - // the MLMD server are transient and require time to resolve under sustained - // parallel execution. - mlmdClientSideMaxRetries = 10 - mlmdClientSideBackoffBase = 1 * time.Second - mlmdClientSideBackoffJitter = 0.25 - // mlmdClientSideBackoffCap limits the maximum per-attempt wait so a single - // deadlock retry never stalls a pod for more than 30 s. - mlmdClientSideBackoffCap = 30 * time.Second - defaultMaxGRPCMessageSize = 100 * 1024 * 1024 // 100MB - maxGRPCMessageSizeEnv = "METADATA_GRPC_MESSAGE_SIZE" + mlmdClientSideMaxRetries = 3 + defaultMaxGRPCMessageSize = 100 * 1024 * 1024 // 100MB + maxGRPCMessageSizeEnv = "METADATA_GRPC_MESSAGE_SIZE" ) // MaxGRPCMessageSize is the max gRPC message size for the metadata client. @@ -148,17 +138,10 @@ type Client struct { // NewClient creates a Client given the MLMD server address and port. func NewClient(serverAddress, serverPort string, tlsCfg *tls.Config) (*Client, error) { - // Retry on Aborted (MySQL deadlock) and Unavailable (transient connectivity). - // Use bounded exponential backoff so that high-concurrency deadlock storms - // are given enough time to resolve without stalling a pod indefinitely. opts := []grpc_retry.CallOption{ grpc_retry.WithMax(mlmdClientSideMaxRetries), - grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitterBounded( - mlmdClientSideBackoffBase, - mlmdClientSideBackoffJitter, - mlmdClientSideBackoffCap, - )), - grpc_retry.WithCodes(codes.Aborted, codes.Unavailable), + grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(300*time.Millisecond, 0.20)), + grpc_retry.WithCodes(codes.Aborted), } creds := insecure.NewCredentials() @@ -198,6 +181,7 @@ type ExecutionConfig struct { OutputArtifacts map[string]*pipelinespec.DagOutputsSpec_DagOutputArtifactSpec InputArtifactIDs map[string][]int64 IterationIndex *int // Index of the iteration. + DriverLogURI string // ContainerExecution custom properties Image, CachedMLMDExecutionID, FingerPrint string @@ -594,6 +578,7 @@ const ( keyOutputs = "outputs" keyParameterProducerTask = "parameter_producer_task" keyOutputArtifacts = "output_artifacts" + keyDriverLogURI = "driver_logs_uri" keyArtifactProducerTask = "artifact_producer_task" keyParentDagID = "parent_dag_id" // Parent DAG Execution ID. keyIterationIndex = "iteration_index" @@ -651,6 +636,10 @@ func (c *Client) CreateExecution(ctx context.Context, pipeline *Pipeline, config e.CustomProperties[keyCacheFingerPrint] = StringValue(config.FingerPrint) } } + if config.DriverLogURI != "" { + e.CustomProperties[keyDriverLogURI] = StringValue(config.DriverLogURI) + e.CustomProperties[keyStoreSessionInfo] = StringValue(pipeline.GetStoreSessionInfo()) + } if config.InputParameters != nil { e.CustomProperties[keyInputs] = &pb.Value{Value: &pb.Value_StructValue{ StructValue: &structpb.Struct{ diff --git a/backend/test/compiler/compiler_visitor_test.go b/backend/test/compiler/compiler_visitor_test.go index 611287fc6c3..280dc3608f5 100644 --- a/backend/test/compiler/compiler_visitor_test.go +++ b/backend/test/compiler/compiler_visitor_test.go @@ -92,7 +92,7 @@ var _ = Describe("Verify iteration over the pipeline components >", Label(POSITI }, }, { - pipelineSpecPath: "parallel_and_nested/nested_pipeline_opt_input_child_level_compiled.yaml", + pipelineSpecPath: "critical/nested_pipeline_opt_input_child_level_compiled.yaml", expectedVisited: []string{ "container(name=\"comp-component-a-bool\")", "container(name=\"comp-component-a-int\")", diff --git a/backend/test/compiler/utils/workflow_utils.go b/backend/test/compiler/utils/workflow_utils.go index 1cb7a8ac5f6..5a3a707fb51 100644 --- a/backend/test/compiler/utils/workflow_utils.go +++ b/backend/test/compiler/utils/workflow_utils.go @@ -36,6 +36,8 @@ import ( v1 "k8s.io/api/core/v1" ) +type driverPlugin map[string]map[string]map[string]interface{} + // LoadPipelineSpecsFromIR - Unmarshall Pipeline Spec IR into a tuple of (pipelinespec.PipelineJob, pipelinespec.SinglePlatformSpec) func LoadPipelineSpecsFromIR(pipelineIRFilePath string, cacheDisabled bool, defaultWorkspace *v1.PersistentVolumeClaimSpec) (*pipelinespec.PipelineJob, *pipelinespec.SinglePlatformSpec) { pipelineSpecsFromFile := testutil.ParseFileToSpecs(pipelineIRFilePath, cacheDisabled, defaultWorkspace) @@ -89,6 +91,51 @@ func CreateCompiledWorkflowFile(compiledWorflow *v1alpha1.Workflow, compiledWork return testutil.CreateFile(compiledWorkflowFilePath, [][]byte{fileContents}) } +// ConfigurePluginSettings - Add/Remove cache_disabled args in the driver-plugin or HTTP template body. +// When remove=true, sets cache_disabled=false (normalizes to baseline for golden-file generation). +// When remove=false, sets cache_disabled=true (restores for cacheDisabled=true validation). +func ConfigurePluginSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1.Workflow { + configuredWorkflow := workflow.DeepCopy() + for i, template := range configuredWorkflow.Spec.Templates { + if template.Plugin != nil { + var pluginMap driverPlugin + if err := json.Unmarshal(template.Plugin.Value, &pluginMap); err == nil { + if driverPlugin, ok := pluginMap["driver-plugin"]; ok { + if args, ok := driverPlugin["args"]; ok { + if remove { + args["cache_disabled"] = false + } else { + args["cache_disabled"] = true + } + } + } + jsonPlugin, err := json.Marshal(pluginMap) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to marshal plugin map") + configuredWorkflow.Spec.Templates[i].Plugin.Value = jsonPlugin + } + } + // Handle centralized HTTP driver templates: normalize cache_disabled in the JSON body + // so that the same golden file validates under both CacheDisabled=true and CacheDisabled=false. + if template.HTTP != nil && template.HTTP.Body != "" { + var bodyMap map[string]interface{} + if err := json.Unmarshal([]byte(template.HTTP.Body), &bodyMap); err == nil { + if _, ok := bodyMap["cache_disabled"]; ok { + if remove { + bodyMap["cache_disabled"] = false + } else { + bodyMap["cache_disabled"] = true + } + jsonBody, err := json.Marshal(bodyMap) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to marshal HTTP template body") + configuredWorkflow.Spec.Templates[i].HTTP.Body = string(jsonBody) + } + } + } + } + + return configuredWorkflow +} + // ConfigureCacheSettings - Add/Remove cache_disabled args in the workflow func ConfigureCacheSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1.Workflow { cacheDisabledArg := "--cache_disabled" @@ -138,5 +185,5 @@ func ConfigureCacheSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1. } } } - return configuredWorkflow + return ConfigurePluginSettings(configuredWorkflow, remove) } diff --git a/backend/test/constants/test_features.go b/backend/test/constants/test_features.go index b65d44e5e59..2e7f9800254 100644 --- a/backend/test/constants/test_features.go +++ b/backend/test/constants/test_features.go @@ -28,8 +28,6 @@ const ( E2eFailed string = "E2EFailure" // E2eCritical - For pipelines that verify the critical functionality of the system E2eCritical string = "E2ECritical" - // E2eParallelNested - For parallel or nested pipelines for regression testing - E2eParallelNested string = "E2EParallelNested" // E2eProxy - For pipeline that runs with a proxy E2eProxy string = "E2EProxy" diff --git a/backend/test/end2end/e2e_suite_test.go b/backend/test/end2end/e2e_suite_test.go index a765a3dab9d..621fb6de0e7 100644 --- a/backend/test/end2end/e2e_suite_test.go +++ b/backend/test/end2end/e2e_suite_test.go @@ -38,7 +38,7 @@ var randomName string var experimentID *string = nil var userToken string -const maxPipelineWaitTime = 720 // In Seconds +const maxPipelineWaitTime = 540 // In Seconds var ( pipelineUploadClient apiserver.PipelineUploadInterface @@ -135,7 +135,6 @@ var _ = BeforeSuite(func() { }) var _ = BeforeEach(func() { - // Create Experiment so that we can use it to associate pipeline runs with experimentName := fmt.Sprintf("E2EExperiment-%s", strconv.FormatInt(time.Now().UnixNano(), 10)) experiment := testutil.CreateExperiment(experimentClient, experimentName, testutil.GetNamespace()) diff --git a/backend/test/end2end/pipeline_e2e_test.go b/backend/test/end2end/pipeline_e2e_test.go index fad03f50d2c..41ced61cdc2 100644 --- a/backend/test/end2end/pipeline_e2e_test.go +++ b/backend/test/end2end/pipeline_e2e_test.go @@ -140,16 +140,6 @@ var _ = Describe("Upload and Verify Pipeline Run >", Label(FullRegression), func } }) - Context("Upload a nested or a parallel pipeline file, run it and verify that pipeline run succeeds >", FlakeAttempts(2), Label(E2eParallelNested), func() { - var pipelineDir = "valid/parallel_and_nested" - pipelineFiles := testutil.GetListOfFilesInADir(filepath.Join(testutil.GetPipelineFilesDir(), pipelineDir)) - for _, pipelineFile := range pipelineFiles { - It(fmt.Sprintf("Upload %s pipeline", pipelineFile), func() { - validatePipelineRunSuccess(pipelineFile, pipelineDir, testContext) - }) - } - }) - // Few of the following pipelines randomly fail in Multi User Mode during CI run - which is why a FlakeAttempt is added, but we need to investigate, create ticket and fix it in the future Context("Upload a pipeline file, run it and verify that pipeline run succeeds >", FlakeAttempts(2), Label("Sample", E2eCritical), func() { var pipelineDir = "valid/critical" @@ -182,7 +172,7 @@ var _ = Describe("Upload and Verify Pipeline Run >", Label(FullRegression), func pipelineFiles := []string{ "essential/component_with_pip_index_urls.yaml", "essential/lightweight_python_functions_pipeline.yaml", - "parallel_and_nested/pipeline_in_pipeline.yaml", + "essential/pipeline_in_pipeline.yaml", "critical/pipeline_with_secret_as_env.yaml", "critical/pipeline_with_input_status_state.yaml", "critical/notebook_component_simple.yaml", diff --git a/backend/test/end2end/utils/e2e_utils.go b/backend/test/end2end/utils/e2e_utils.go index 96d5cf78c73..1ef0546e606 100644 --- a/backend/test/end2end/utils/e2e_utils.go +++ b/backend/test/end2end/utils/e2e_utils.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "sort" + "strings" "time" runparams "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service" @@ -74,7 +75,8 @@ func ValidateComponentStatuses(runClient *apiserver.RunClient, k8Client *kuberne logger.Log("Updated pipeline run details") expectedTaskDetails := GetTasksFromWorkflow(compiledWorkflow) if *updatedRun.State == run_model.V2beta1RuntimeStateRUNNING { - logger.Log("Pipeline run did not finish, checking workflow controller logs") + logger.Log("Pipeline run did not finish") + logger.Log("Checking workflow controller logs") podLog := testutil.ReadContainerLogs(k8Client, *config.Namespace, "workflow-controller", nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Attaching Workflow Controller logs to the report") ginkgo.AddReportEntry("Workflow Controller Logs", podLog) @@ -83,6 +85,10 @@ func ValidateComponentStatuses(runClient *apiserver.RunClient, k8Client *kuberne } else { if *updatedRun.State != run_model.V2beta1RuntimeStateSUCCEEDED { logger.Log("Looks like the run %s FAILED, so capture pod logs for the failed task", runID) + logger.Log("Checking workflow controller logs") + podLog := testutil.ReadContainerLogs(k8Client, *config.Namespace, "workflow-controller", nil, &testContext.TestStartTimeUTC, config.PodLogLimit) + logger.Log("Attaching Workflow Controller logs to the report") + ginkgo.AddReportEntry("Workflow Controller Logs", podLog) CapturePodLogsForUnsuccessfulTasks(k8Client, testContext, actualTaskDetails) ginkgo.Fail("Failing test because the pipeline run was not SUCCESSFUL") } else { @@ -121,11 +127,24 @@ func CapturePodLogsForUnsuccessfulTasks(k8Client *kubernetes.Clientset, testCont } case run_model.V2beta1RuntimeStateFAILED: { + agentLogsNotYetAcquired := true logger.Log("%s - Task %s for Run %s did not complete successfully", *task.State, task.DisplayName, task.RunID) for _, childTask := range task.ChildTasks { podName := childTask.PodName + isDriver := strings.Contains(task.DisplayName, "-driver") if podName != "" { - logger.Log("Capturing pod logs for task %s, with pod name %s", task.DisplayName, podName) + if isDriver { + if agentLogsNotYetAcquired { + logger.Log("Capturing pod logs for task executor agent, with pod name %s", podName) + podName += "-agent" + agentLogsNotYetAcquired = false + } else { + logger.Log("Logs from agent pod was already captured") + continue + } + } else { + logger.Log("Capturing pod logs for task %s, with pod name %s", task.DisplayName, podName) + } podLog := testutil.ReadPodLogs(k8Client, *config.Namespace, podName, nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Pod logs captured for task %s in pod %s", task.DisplayName, podName) logger.Log("Attaching pod logs to the report") diff --git a/backend/test/proto_tests/objects.go b/backend/test/proto_tests/objects.go index dc13e7249d9..1cb2dec0966 100644 --- a/backend/test/proto_tests/objects.go +++ b/backend/test/proto_tests/objects.go @@ -306,7 +306,7 @@ var platformSpec = &specPB.PlatformSpec{ }, }, PipelineConfig: &specPB.PipelineConfig{ - ResourceTtlOnCompletion: 24, + ResourceTtl: 24, }, }, }, diff --git a/backend/test/proto_tests/testdata/generated-1791485/pipeline.json b/backend/test/proto_tests/testdata/generated-1791485/pipeline.json index e3ca87700b8..187c192df32 100644 --- a/backend/test/proto_tests/testdata/generated-1791485/pipeline.json +++ b/backend/test/proto_tests/testdata/generated-1791485/pipeline.json @@ -1,14 +1,14 @@ { - "pipeline_id": "9b187b86-7c0a-42ae-a0bc-2a746b6eb7a3", - "display_name": "Production Data Processing Pipeline", - "name": "pipeline1", - "description": "Pipeline for processing and analyzing production data", - "created_at": "2024-01-01T12:00:00Z", - "namespace": "namespace1", - "error": { - "code": 0, - "message": "This a successful pipeline.", - "details": [] + "pipeline_id": "9b187b86-7c0a-42ae-a0bc-2a746b6eb7a3", + "display_name": "Production Data Processing Pipeline", + "name": "pipeline1", + "description": "Pipeline for processing and analyzing production data", + "created_at": "2024-01-01T12:00:00Z", + "namespace": "namespace1", + "error": { + "code": 0, + "message": "This a successful pipeline.", + "details": [] }, - "tags": {} -} + "tags": {} +} \ No newline at end of file diff --git a/backend/test/proto_tests/testdata/generated-1791485/pipeline_spec.pb b/backend/test/proto_tests/testdata/generated-1791485/pipeline_spec.pb index 39a55ae76aa0f0ba276c5460cf3c429be72171a2..4f4f8023f0559da8109a6c8b7f3c8ab07d3c26cb 100644 GIT binary patch delta 88 zcmZo;YGazPHBgv~Ei=3.7.4,<5; python_version<\"3.9\"' && \"$0\" \"$@\"\n", @@ -34,36 +34,36 @@ "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef hello_world():\n print(\"hello world\")\n\n" ], - "image": "python:3.11" + "image": "python:3.11" } } } }, - "pipelineInfo": { - "name": "pipeline-hello-world" + "pipelineInfo": { + "name": "pipeline-hello-world" }, - "root": { - "dag": { - "tasks": { - "hello-world": { - "cachingOptions": {}, - "componentRef": { - "name": "comp-hello-world" + "root": { + "dag": { + "tasks": { + "hello-world": { + "cachingOptions": {}, + "componentRef": { + "name": "comp-hello-world" }, - "taskInfo": { - "name": "hello-world" + "taskInfo": { + "name": "hello-world" } } } } }, - "schemaVersion": "2.1.0", - "sdkVersion": "kfp-2.14.0" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-2.14.0" }, - "error": { - "code": 0, - "message": "This is a successful pipeline version.", - "details": [] + "error": { + "code": 0, + "message": "This is a successful pipeline version.", + "details": [] }, - "tags": {} -} + "tags": {} +} \ No newline at end of file diff --git a/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.pb b/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.pb index 1550e8778d86e1246d0c96a4841fb4bb8d8ad9da..2347dadb954e3e35b5842b6a4079f1c9faf4fcd3 100644 GIT binary patch delta 21 dcmeC delta 14 WcmZqVY2=x3bMqcXSC)x49s>X?`UZ&r diff --git a/backend/test/testutil/kubernetes_utils.go b/backend/test/testutil/kubernetes_utils.go index 6cdc218ed70..9e1acbebf0c 100644 --- a/backend/test/testutil/kubernetes_utils.go +++ b/backend/test/testutil/kubernetes_utils.go @@ -25,7 +25,6 @@ import ( "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/test/logger" - "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" v1 "k8s.io/api/core/v1" @@ -77,12 +76,7 @@ func ReadPodLogs(client *kubernetes.Clientset, namespace string, podName string, podLogsRequest := client.CoreV1().Pods(namespace).GetLogs(podFromPodName.Name, podLogOptions) podLogs, err := podLogsRequest.Stream(context.Background()) // Pass a context for cancellation if err != nil { - logger.Log("Failed to stream pod logs for container '%s' due to %v", container.Name, err) - continue - } - if podLogs == nil { - logger.Log("Pod log stream is nil for container '%s'", container.Name) - continue + logger.Log("Failed to stream pod logs due to %v", err) } defer func(podLogs io.ReadCloser) { err = podLogs.Close() @@ -92,7 +86,7 @@ func ReadPodLogs(client *kubernetes.Clientset, namespace string, podName string, }(podLogs) _, err = io.Copy(buf, podLogs) if err != nil { - logger.Log("Failed to add pod logs to buffer for container '%s' due to: %v", container.Name, err) + logger.Log("Failed to add pod logs to buffer due to: %v", err) } } } else { diff --git a/frontend/docs/react-18-19-upgrade-checklist.md b/frontend/docs/react-18-19-upgrade-checklist.md index 0b7e72bf534..640ad8ca910 100644 --- a/frontend/docs/react-18-19-upgrade-checklist.md +++ b/frontend/docs/react-18-19-upgrade-checklist.md @@ -58,10 +58,9 @@ Completed via [#12881](https://github.com/kubeflow/pipelines/pull/12881). **Description**: Add `check-react-peers.mjs` and wire it into `npm run test:ci`. The repo now exposes: -- `npm run check:react-peers` (targets React 19 by default) +- `npm run check:react-peers` - `npm run check:react-peers:18` - -> **Note**: `check:react-peers:19` was originally a separate alias but was removed after the React 19 upgrade landed; `check:react-peers` now targets React 19 directly. +- `npm run check:react-peers:19` This CI guardrail prevents new dependency additions from silently breaking the targeted React major. @@ -290,10 +289,10 @@ Make the React 18.3 state explicit in package metadata, run the full verificatio **Depends on**: #11 **Status**: -Completed via [#13082](https://github.com/kubeflow/pipelines/pull/13082), merged on March 20, 2026 UTC. `npm run check:react-peers` (targeting React 19) now passes with a single allowlisted React core blocker that remains until `#13`. Issue [#12900](https://github.com/kubeflow/pipelines/issues/12900) was closed on March 20, 2026. +Completed via [#13082](https://github.com/kubeflow/pipelines/pull/13082), merged on March 20, 2026 UTC. `npm run check:react-peers:19` now passes with a single allowlisted React core blocker that remains until `#13`. Issue [#12900](https://github.com/kubeflow/pipelines/issues/12900) was closed on March 20, 2026. **Description**: -Run `npm run check:react-peers` (targeting React 19), upgrade any remaining React 19-incompatible dependencies, and allowlist the expected React core blocker that is resolved in `#13`. +Run `npm run check:react-peers:19`, upgrade any remaining React 19-incompatible dependencies, and allowlist the expected React core blocker that is resolved in `#13`. **Result**: - Cleared `react-ace` by upgrading to `14.0.1` @@ -303,7 +302,7 @@ Run `npm run check:react-peers` (targeting React 19), upgrade any remaining Reac **Current state after `#13`**: That allowlist entry is gone; `frontend/docs/react-peer-allowlist.json` is `[]` for targets `17`, `18`, and `19`. **Acceptance Criteria**: -- [x] `npm run check:react-peers` (targeting React 19) passed with only the expected allowlisted `react-dom` core blocker until `#13` landed +- [x] `npm run check:react-peers:19` passed with only the expected allowlisted `react-dom` core blocker until `#13` landed - [x] No non-core React 19 blockers remain - [x] `npm run test:ci` passes @@ -329,7 +328,7 @@ Bump `react`, `react-dom`, `@types/react`, and `@types/react-dom` to v19, addres - [x] `npm run test:ci && npm run build` pass - [x] `npm install` completes without peer-dependency warnings (and the `#12` core allowlist is removed) - [x] Manual smoke testing completed as part of the React 19 rollout -- [x] `npm run check:react-peers` (targeting React 19) passes with an empty allowlist +- [x] `npm run check:react-peers:19` passes with an empty allowlist --- diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c9fb55c58fb..60943fdbe35 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -27,7 +27,7 @@ "http-proxy-middleware": "^2.0.7", "immer": "^9.0.6", "js-yaml": "^3.14.1", - "lodash": "^4.18.1", + "lodash": "^4.17.23", "lodash.debounce": "^4.0.8", "lodash.flatten": "^4.4.0", "lodash.groupby": "^4.6.0", @@ -8227,9 +8227,7 @@ "license": "MIT" }, "node_modules/lodash": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", - "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "version": "4.17.23", "license": "MIT" }, "node_modules/lodash.debounce": { diff --git a/frontend/package.json b/frontend/package.json index c65ab42c76d..46b0880c5f5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -24,7 +24,7 @@ "http-proxy-middleware": "^2.0.7", "immer": "^9.0.6", "js-yaml": "^3.14.1", - "lodash": "^4.18.1", + "lodash": "^4.17.23", "lodash.debounce": "^4.0.8", "lodash.flatten": "^4.4.0", "lodash.groupby": "^4.6.0", @@ -80,6 +80,8 @@ "format:check": "prettier --check './**/*.{ts,tsx}' || node ./scripts/check-format-error-info.js", "check:react-peers": "node scripts/check-react-peers.mjs --target 19", "check:react-peers:18": "node scripts/check-react-peers.mjs --target 18", + "check:react-peers:19": "node scripts/check-react-peers.mjs --target 19", + "java": "java -version", "lint": "npm run lint:ui && npm run lint:server", "lint:ui": "eslint --ext js,ts,tsx src --max-warnings=0", "lint:server": "eslint --ext ts server --max-warnings=0", @@ -88,7 +90,7 @@ "mock:api": "ts-node-dev -O '{\"module\": \"commonjs\"}' --project mock-backend/tsconfig.json -r tsconfig-paths/register mock-backend/mock-api-server.ts 3001", "mock:server": "node server/dist/server.js build", "mock:server:inspect": "node inspect server/dist/server.js build", - "postinstall": "npm ci --prefix ./server && npm ci --prefix ./mock-backend", + "postinstall": "cd ./server && npm ci && cd ../mock-backend && npm ci", "start:proxy": "./scripts/start-proxy.sh", "start:proxy-and-server": "./scripts/start-proxy-and-server.sh", "start:proxy-and-server-inspect": "./scripts/start-proxy-and-server.sh --inspect", @@ -107,7 +109,7 @@ "visual:baseline": "node scripts/visual-compare.mjs capture --out-dir .visual/baseline", "visual:current": "node scripts/visual-compare.mjs capture --out-dir .visual/current", "visual:diff": "node scripts/visual-compare.mjs diff --baseline-dir .visual/baseline --current-dir .visual/current --diff-dir .visual/diff --report .visual/report.html", - "test:server:coverage": "npm --prefix ./server test -- --coverage", + "test:server:coverage": "cd ./server && npm test -- --coverage && cd ..", "test:coverage": "npm run test:ui:coverage && npm run test:server:coverage", "test:ci": "export CI=true && npm run format:check && npm run lint && npm run typecheck && npm run typecheck:mock-backend && npm run check:react-peers && npm run test:coverage", "storybook": "storybook dev -p 6006", diff --git a/frontend/scripts/generate_openapi_typescript_fetch.js b/frontend/scripts/generate_openapi_typescript_fetch.js index 6c1d84d4619..89d3a1478e1 100644 --- a/frontend/scripts/generate_openapi_typescript_fetch.js +++ b/frontend/scripts/generate_openapi_typescript_fetch.js @@ -88,46 +88,12 @@ const GROUPS = { v2beta1: Object.keys(SPEC_TARGETS).filter((key) => key.startsWith('v2beta1:')), all: Object.keys(SPEC_TARGETS), }; -const SHARED_OPENAPI_SUPPORT_GROUPS = [ - { - outputPrefixes: ['frontend/src/apis/', 'frontend/src/apisv2beta1/'], - sharedRoot: 'frontend/src/generated/openapi', - importExtension: '', - }, - { - outputPrefixes: ['frontend/server/src/generated/apis/', 'frontend/server/src/generated/apisv2beta1/'], - sharedRoot: 'frontend/server/src/generated/openapi', - importExtension: '.js', - }, -]; -const SHARED_OPENAPI_SUPPORT_FILES = [ - { relativePath: 'runtime.ts' }, - { relativePath: path.join('models', 'ProtobufAny.ts') }, - { relativePath: path.join('models', 'GooglerpcStatus.ts') }, -]; function fatal(message) { console.error(message); process.exit(1); } -function ensureTrailingNewline(source) { - return source.endsWith('\n') ? source : `${source}\n`; -} - -function writeFileIfChanged(filePath, source) { - const nextSource = ensureTrailingNewline(source); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - if (fs.existsSync(filePath) && fs.readFileSync(filePath, 'utf8') === nextSource) { - return; - } - fs.writeFileSync(filePath, nextSource); -} - -function toImportSpecifier(filePath) { - return filePath.split(path.sep).join('/').replace(/\.ts$/, ''); -} - function resolveTargets(args) { if (args.length === 0) { return GROUPS.all; @@ -164,13 +130,6 @@ function resolveTargets(args) { return [...new Set(resolved)]; } -function getSharedOpenApiSupportGroup(target) { - const normalizedOutput = target.output.split(path.sep).join('/'); - return SHARED_OPENAPI_SUPPORT_GROUPS.find((group) => - group.outputPrefixes.some((prefix) => normalizedOutput.startsWith(prefix)), - ); -} - function runCommand(command, args) { const result = spawnSync(command, args, { stdio: 'inherit' }); if (result.error) { @@ -293,147 +252,6 @@ function normalizeGeneratedTypeScript(outputDir, options = {}) { } } -function stripGeneratedJSDocComments(source) { - return source.replace(/\/\*\*[\s\S]*?\*\/\n*/g, ''); -} - -function createSharedOpenApiSupportSource(source) { - const lines = source.split('\n'); - const preservedPrefix = []; - - while ( - lines.length > 0 && - (lines[0] === '/* tslint:disable */' || lines[0] === '/* eslint-disable */') - ) { - preservedPrefix.push(lines.shift()); - } - - const strippedBody = stripGeneratedJSDocComments(lines.join('\n')) - .replace(/^\s+/, '') - .replace(/\n{3,}/g, '\n\n') - .trimStart(); - - return [ - ...preservedPrefix, - '// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually.', - strippedBody, - ] - .filter(Boolean) - .join('\n'); -} - -function createOpenApiReExportShim(sharedFilePath, sourceFilePath, importExtension = '') { - const relativeImportPath = toImportSpecifier(path.relative(path.dirname(sourceFilePath), sharedFilePath)); - const importSpecifier = `${relativeImportPath}${importExtension}`; - return [ - '/* tslint:disable */', - '/* eslint-disable */', - '// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually.', - `export * from '${importSpecifier}';`, - ].join('\n'); -} - -function assertSharedOpenApiSupportMatches(sharedFilePath, sharedSource, sourceFilePath) { - if (!fs.existsSync(sharedFilePath)) { - return; - } - - const expectedSharedSource = ensureTrailingNewline(sharedSource); - const existingSharedSource = fs.readFileSync(sharedFilePath, 'utf8'); - if (existingSharedSource === expectedSharedSource) { - return; - } - - throw new Error( - [ - `Shared OpenAPI support mismatch for ${path.basename(sharedFilePath)}.`, - `Existing shared file: ${sharedFilePath}`, - `Current generated source: ${sourceFilePath}`, - 'All targets in a shared dedupe group must normalize to identical support content.', - ].join('\n'), - ); -} - -function getTargetKeysForSharedOpenApiSupportGroup(sharedGroup) { - return Object.keys(SPEC_TARGETS).filter((targetKey) => { - const targetGroup = getSharedOpenApiSupportGroup(SPEC_TARGETS[targetKey]); - return targetGroup && targetGroup.sharedRoot === sharedGroup.sharedRoot; - }); -} - -function dedupeGeneratedOpenApiSupportFiles(repoRoot, targetKey) { - const target = SPEC_TARGETS[targetKey]; - const sharedGroup = getSharedOpenApiSupportGroup(target); - if (!sharedGroup) { - return []; - } - - const outputPath = path.join(repoRoot, target.output); - const sharedRootPath = path.join(repoRoot, sharedGroup.sharedRoot); - const sharedDirs = new Set(); - - for (const supportFile of SHARED_OPENAPI_SUPPORT_FILES) { - const sourceFilePath = path.join(outputPath, supportFile.relativePath); - if (!fs.existsSync(sourceFilePath)) { - continue; - } - - const sharedFilePath = path.join(sharedRootPath, supportFile.relativePath); - const sharedSource = createSharedOpenApiSupportSource(fs.readFileSync(sourceFilePath, 'utf8')); - - assertSharedOpenApiSupportMatches(sharedFilePath, sharedSource, sourceFilePath); - writeFileIfChanged(sharedFilePath, sharedSource); - writeFileIfChanged( - sourceFilePath, - createOpenApiReExportShim(sharedFilePath, sourceFilePath, sharedGroup.importExtension), - ); - sharedDirs.add(sharedRootPath); - } - - return [...sharedDirs]; -} - -function resetFullySelectedSharedOpenApiSupportDirs(repoRoot, targets) { - const selectedTargets = new Set(targets); - for (const sharedGroup of SHARED_OPENAPI_SUPPORT_GROUPS) { - const groupTargetKeys = getTargetKeysForSharedOpenApiSupportGroup(sharedGroup); - const selectedGroupTargetCount = groupTargetKeys.filter((targetKey) => selectedTargets.has(targetKey)) - .length; - - if (selectedGroupTargetCount === 0 || selectedGroupTargetCount !== groupTargetKeys.length) { - continue; - } - - fs.rmSync(path.join(repoRoot, sharedGroup.sharedRoot), { - recursive: true, - force: true, - }); - } -} - -function getSharedOpenApiSupportDirs(repoRoot, targets) { - return [ - ...new Set( - targets - .map((targetKey) => { - const sharedGroup = getSharedOpenApiSupportGroup(SPEC_TARGETS[targetKey]); - return sharedGroup ? path.join(repoRoot, sharedGroup.sharedRoot) : null; - }) - .filter(Boolean), - ), - ]; -} - -function dedupeTargetsOpenApiSupportFiles(repoRoot, targets) { - const sharedDirs = new Set(); - for (const targetKey of targets) { - for (const sharedDir of dedupeGeneratedOpenApiSupportFiles(repoRoot, targetKey)) { - sharedDirs.add(sharedDir); - } - } - return [...sharedDirs]; -} - function listTypeScriptFiles(rootDir) { if (!fs.existsSync(rootDir)) { return []; @@ -647,14 +465,10 @@ async function main() { await generateTargetsParallel(repoRoot, targets, concurrency); } - const outputDirs = [...new Set(targets.map((targetKey) => path.join(repoRoot, SPEC_TARGETS[targetKey].output)))]; + const outputDirs = [ + ...new Set(targets.map((targetKey) => path.join(repoRoot, SPEC_TARGETS[targetKey].output))), + ]; await formatGeneratedTypeScript(repoRoot, outputDirs); - - resetFullySelectedSharedOpenApiSupportDirs(repoRoot, targets); - const sharedDirs = dedupeTargetsOpenApiSupportFiles(repoRoot, targets); - - const allFormattedDirs = [...new Set([...outputDirs, ...sharedDirs, ...getSharedOpenApiSupportDirs(repoRoot, targets)])]; - await formatGeneratedTypeScript(repoRoot, allFormattedDirs); } if (require.main === module) { @@ -663,21 +477,13 @@ if (require.main === module) { module.exports = { formatGeneratedTypeScript, - assertSharedOpenApiSupportMatches, - createOpenApiReExportShim, - createSharedOpenApiSupportSource, - dedupeGeneratedOpenApiSupportFiles, - dedupeTargetsOpenApiSupportFiles, generateTargetsParallel, - getSharedOpenApiSupportDirs, - getTargetKeysForSharedOpenApiSupportGroup, listTypeScriptFiles, main, normalizeGeneratedTypeScript, normalizeGeneratedTypeScriptSource, normalizeNodeCompatibleFetchTypes, normalizeV1ApiSymbols, - resetFullySelectedSharedOpenApiSupportDirs, resolvePrettierModule, resolveTargets, runPool, diff --git a/frontend/scripts/generate_openapi_typescript_fetch.test.js b/frontend/scripts/generate_openapi_typescript_fetch.test.js index 3b5749f6e10..f77612a3f00 100644 --- a/frontend/scripts/generate_openapi_typescript_fetch.test.js +++ b/frontend/scripts/generate_openapi_typescript_fetch.test.js @@ -1,10 +1,6 @@ -const fs = require('fs'); -const os = require('os'); const path = require('path'); const { - createSharedOpenApiSupportSource, - dedupeGeneratedOpenApiSupportFiles, generateTargetsParallel, normalizeGeneratedTypeScriptSource, resolvePrettierModule, @@ -243,284 +239,4 @@ export class ResponseError extends Error { expect(updated).toContain("import type { Foo } from './Foo';"); expect(updated).toContain('override name: "ResponseError" = "ResponseError";'); }); - - describe('shared OpenAPI support dedupe', () => { - it('normalizes version-specific generated comments out of shared support files', () => { - const v1Status = ` -/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v1beta1/run.proto - * No description provided. - */ - -import { mapValues } from '../runtime'; -import type { ProtobufAny } from './ProtobufAny'; - -/** - * - */ -export interface GooglerpcStatus { - /** - * - */ - code?: number; -} -`; - const v2Status = ` -/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v2beta1/run.proto - * No description provided. - */ - -import { mapValues } from '../runtime'; -import type { ProtobufAny } from './ProtobufAny'; - -/** - * Richer status docs in v2beta1. - */ -export interface GooglerpcStatus { - /** - * The status code. - */ - code?: number; -} -`; - - const normalizedV1 = createSharedOpenApiSupportSource(v1Status); - const normalizedV2 = createSharedOpenApiSupportSource(v2Status); - - expect(normalizedV1).toEqual(normalizedV2); - expect(normalizedV1).not.toContain('backend/api/'); - expect(normalizedV1).not.toContain('/**'); - expect(normalizedV1).toContain('export interface GooglerpcStatus'); - }); - - it('rewrites generated client and server support files to shared re-export shims', () => { - const tempRepoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'openapi-dedupe-')); - const write = (relativePath, source) => { - const filePath = path.join(tempRepoRoot, relativePath); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync(filePath, source); - }; - - try { - write( - 'frontend/src/apis/run/runtime.ts', - `/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v1beta1/run.proto - */ - -export const BASE_PATH = 'http://localhost'; -`, - ); - write( - 'frontend/src/apis/run/models/ProtobufAny.ts', - `/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v1beta1/run.proto - */ - -import { mapValues } from '../runtime'; - -/** - * Shared protobuf any docs. - */ -export interface ProtobufAny { - type?: string; -} -`, - ); - write( - 'frontend/src/apis/run/models/GooglerpcStatus.ts', - `/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v1beta1/run.proto - */ - -import { mapValues } from '../runtime'; -import type { ProtobufAny } from './ProtobufAny'; - -/** - * Status docs. - */ -export interface GooglerpcStatus { - code?: number; - details?: Array; -} -`, - ); - write( - 'frontend/server/src/generated/apis/auth/runtime.ts', - `/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v1beta1/auth.proto - */ - -export const BASE_PATH = 'http://localhost'; -`, - ); - write( - 'frontend/server/src/generated/apis/auth/models/ProtobufAny.ts', - `/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v1beta1/auth.proto - */ - -import { mapValues } from '../runtime.js'; - -export interface ProtobufAny { - type?: string; -} -`, - ); - write( - 'frontend/server/src/generated/apis/auth/models/GooglerpcStatus.ts', - `/* tslint:disable */ -/* eslint-disable */ -/** - * backend/api/v1beta1/auth.proto - */ - -import { mapValues } from '../runtime.js'; -import type { ProtobufAny } from './ProtobufAny.js'; - -export interface GooglerpcStatus { - code?: number; - details?: Array; -} -`, - ); - - dedupeGeneratedOpenApiSupportFiles(tempRepoRoot, 'v1:run'); - dedupeGeneratedOpenApiSupportFiles(tempRepoRoot, 'v1:auth'); - - expect( - fs.readFileSync(path.join(tempRepoRoot, 'frontend/src/apis/run/runtime.ts'), 'utf8'), - ).toContain("export * from '../../generated/openapi/runtime';"); - expect( - fs.readFileSync( - path.join(tempRepoRoot, 'frontend/src/apis/run/models/ProtobufAny.ts'), - 'utf8', - ), - ).toContain("export * from '../../../generated/openapi/models/ProtobufAny';"); - expect( - fs.readFileSync( - path.join(tempRepoRoot, 'frontend/server/src/generated/apis/auth/runtime.ts'), - 'utf8', - ), - ).toContain("export * from '../../openapi/runtime.js';"); - expect( - fs.readFileSync( - path.join( - tempRepoRoot, - 'frontend/server/src/generated/apis/auth/models/GooglerpcStatus.ts', - ), - 'utf8', - ), - ).toContain("export * from '../../../openapi/models/GooglerpcStatus.js';"); - - const clientSharedRuntime = fs.readFileSync( - path.join(tempRepoRoot, 'frontend/src/generated/openapi/runtime.ts'), - 'utf8', - ); - expect(clientSharedRuntime).toContain("export const BASE_PATH = 'http://localhost';"); - expect(clientSharedRuntime).not.toContain('backend/api/v1beta1/run.proto'); - expect(clientSharedRuntime).not.toContain('/**'); - - const serverSharedStatus = fs.readFileSync( - path.join(tempRepoRoot, 'frontend/server/src/generated/openapi/models/GooglerpcStatus.ts'), - 'utf8', - ); - expect(serverSharedStatus).toContain("import type { ProtobufAny } from './ProtobufAny.js';"); - expect(serverSharedStatus).not.toContain('backend/api/v1beta1/auth.proto'); - } finally { - fs.rmSync(tempRepoRoot, { recursive: true, force: true }); - } - }); - - it('fails when a later target normalizes to different shared support content', () => { - const tempRepoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'openapi-dedupe-mismatch-')); - const write = (relativePath, source) => { - const filePath = path.join(tempRepoRoot, relativePath); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync(filePath, source); - }; - - try { - write( - 'frontend/src/apis/run/runtime.ts', - `/* tslint:disable */ -/* eslint-disable */ -export const BASE_PATH = 'run'; -`, - ); - write( - 'frontend/src/apis/run/models/ProtobufAny.ts', - `/* tslint:disable */ -/* eslint-disable */ -export interface ProtobufAny { - type?: string; -} -`, - ); - write( - 'frontend/src/apis/run/models/GooglerpcStatus.ts', - `/* tslint:disable */ -/* eslint-disable */ -export interface GooglerpcStatus { - code?: number; -} -`, - ); - write( - 'frontend/src/apis/pipeline/runtime.ts', - `/* tslint:disable */ -/* eslint-disable */ -export const BASE_PATH = 'pipeline'; -`, - ); - write( - 'frontend/src/apis/pipeline/models/ProtobufAny.ts', - `/* tslint:disable */ -/* eslint-disable */ -export interface ProtobufAny { - type?: string; -} -`, - ); - write( - 'frontend/src/apis/pipeline/models/GooglerpcStatus.ts', - `/* tslint:disable */ -/* eslint-disable */ -export interface GooglerpcStatus { - code?: number; -} -`, - ); - - dedupeGeneratedOpenApiSupportFiles(tempRepoRoot, 'v1:run'); - - expect(() => dedupeGeneratedOpenApiSupportFiles(tempRepoRoot, 'v1:pipeline')).toThrow( - 'Shared OpenAPI support mismatch for runtime.ts.', - ); - expect( - fs.readFileSync(path.join(tempRepoRoot, 'frontend/src/generated/openapi/runtime.ts'), 'utf8'), - ).toContain("export const BASE_PATH = 'run';"); - expect( - fs.readFileSync(path.join(tempRepoRoot, 'frontend/src/apis/pipeline/runtime.ts'), 'utf8'), - ).toContain("export const BASE_PATH = 'pipeline';"); - } finally { - fs.rmSync(tempRepoRoot, { recursive: true, force: true }); - } - }); - }); }); diff --git a/frontend/server/integration-tests/artifact-proxy.test.ts b/frontend/server/integration-tests/artifact-proxy.test.ts index a7e39f7ff09..a2e502fb808 100644 --- a/frontend/server/integration-tests/artifact-proxy.test.ts +++ b/frontend/server/integration-tests/artifact-proxy.test.ts @@ -152,21 +152,6 @@ describe('/artifacts/get namespaced proxy', () => { .expect(200, 'text-data2'); }); - it('returns 400 for invalid namespace without leaking namespace value', async () => { - const configs = loadConfigs(argv, { ARTIFACTS_SERVICE_PROXY_ENABLED: 'true' }); - app = new UIServer(configs); - const res = await requests(app.app) - .get( - `/artifacts/get${buildQuery({ - ...commonParams, - namespace: '../../etc', - })}`, - ) - .expect(400); - expect(res.text).not.toContain('../../etc'); - expect(res.text).not.toContain('stack'); - }); - it('proxies a request with basePath too', async () => { const { receivedUrls, response } = await setUpNamespacedArtifactService({}); const configs = loadConfigs(argv, { diff --git a/frontend/server/src/generated/apis/auth/models/GooglerpcStatus.ts b/frontend/server/src/generated/apis/auth/models/GooglerpcStatus.ts index 6ab9d3fc0d6..9022fe74493 100644 --- a/frontend/server/src/generated/apis/auth/models/GooglerpcStatus.ts +++ b/frontend/server/src/generated/apis/auth/models/GooglerpcStatus.ts @@ -1,4 +1,98 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../openapi/models/GooglerpcStatus.js'; +/** + * backend/api/v1beta1/auth.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime.js'; +import type { ProtobufAny } from './ProtobufAny.js'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny.js'; + +/** + * + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/server/src/generated/apis/auth/models/ProtobufAny.ts b/frontend/server/src/generated/apis/auth/models/ProtobufAny.ts index 8eec4e0ebdc..21b13c9c4f6 100644 --- a/frontend/server/src/generated/apis/auth/models/ProtobufAny.ts +++ b/frontend/server/src/generated/apis/auth/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../openapi/models/ProtobufAny.js'; +/** + * backend/api/v1beta1/auth.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime.js'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/server/src/generated/apis/auth/runtime.ts b/frontend/server/src/generated/apis/auth/runtime.ts index 5bf590019df..301c5d878cb 100644 --- a/frontend/server/src/generated/apis/auth/runtime.ts +++ b/frontend/server/src/generated/apis/auth/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../openapi/runtime.js'; +/** + * backend/api/v1beta1/auth.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestInit['credentials']; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestInit['credentials'] | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit): Promise => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = (input: string, init: RequestInit) => Promise; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestInit['credentials']; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/server/src/generated/apisv2beta1/auth/models/GooglerpcStatus.ts b/frontend/server/src/generated/apisv2beta1/auth/models/GooglerpcStatus.ts index 6ab9d3fc0d6..3ced30a7616 100644 --- a/frontend/server/src/generated/apisv2beta1/auth/models/GooglerpcStatus.ts +++ b/frontend/server/src/generated/apisv2beta1/auth/models/GooglerpcStatus.ts @@ -1,4 +1,107 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../openapi/models/GooglerpcStatus.js'; +/** + * backend/api/v2beta1/auth.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime.js'; +import type { ProtobufAny } from './ProtobufAny.js'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny.js'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/server/src/generated/apisv2beta1/auth/models/ProtobufAny.ts b/frontend/server/src/generated/apisv2beta1/auth/models/ProtobufAny.ts index 8eec4e0ebdc..88a043b7c00 100644 --- a/frontend/server/src/generated/apisv2beta1/auth/models/ProtobufAny.ts +++ b/frontend/server/src/generated/apisv2beta1/auth/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../openapi/models/ProtobufAny.js'; +/** + * backend/api/v2beta1/auth.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime.js'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/server/src/generated/apisv2beta1/auth/runtime.ts b/frontend/server/src/generated/apisv2beta1/auth/runtime.ts index 5bf590019df..b682b88e478 100644 --- a/frontend/server/src/generated/apisv2beta1/auth/runtime.ts +++ b/frontend/server/src/generated/apisv2beta1/auth/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../openapi/runtime.js'; +/** + * backend/api/v2beta1/auth.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestInit['credentials']; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestInit['credentials'] | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit): Promise => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = (input: string, init: RequestInit) => Promise; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestInit['credentials']; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/server/src/generated/openapi/models/GooglerpcStatus.ts b/frontend/server/src/generated/openapi/models/GooglerpcStatus.ts deleted file mode 100644 index 0b0ed9a5355..00000000000 --- a/frontend/server/src/generated/openapi/models/GooglerpcStatus.ts +++ /dev/null @@ -1,64 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -import { mapValues } from '../runtime.js'; -import type { ProtobufAny } from './ProtobufAny.js'; -import { - ProtobufAnyFromJSON, - ProtobufAnyFromJSONTyped, - ProtobufAnyToJSON, - ProtobufAnyToJSONTyped, -} from './ProtobufAny.js'; - -export interface GooglerpcStatus { - code?: number; - message?: string; - details?: Array; -} - -export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { - return true; -} - -export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { - return GooglerpcStatusFromJSONTyped(json, false); -} - -export function GooglerpcStatusFromJSONTyped( - json: any, - ignoreDiscriminator: boolean, -): GooglerpcStatus { - if (json == null) { - return json; - } - return { - code: json['code'] == null ? undefined : json['code'], - message: json['message'] == null ? undefined : json['message'], - details: - json['details'] == null - ? undefined - : (json['details'] as Array).map(ProtobufAnyFromJSON), - }; -} - -export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { - return GooglerpcStatusToJSONTyped(json, false); -} - -export function GooglerpcStatusToJSONTyped( - value?: GooglerpcStatus | null, - ignoreDiscriminator: boolean = false, -): any { - if (value == null) { - return value; - } - - return { - code: value['code'], - message: value['message'], - details: - value['details'] == null - ? undefined - : (value['details'] as Array).map(ProtobufAnyToJSON), - }; -} diff --git a/frontend/server/src/generated/openapi/models/ProtobufAny.ts b/frontend/server/src/generated/openapi/models/ProtobufAny.ts deleted file mode 100644 index 900d73dab6c..00000000000 --- a/frontend/server/src/generated/openapi/models/ProtobufAny.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -import { mapValues } from '../runtime.js'; -export interface ProtobufAny { - [key: string]: object | any; - type?: string; -} - -export function instanceOfProtobufAny(value: object): value is ProtobufAny { - return true; -} - -export function ProtobufAnyFromJSON(json: any): ProtobufAny { - return ProtobufAnyFromJSONTyped(json, false); -} - -export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { - if (json == null) { - return json; - } - return { - ...json, - type: json['@type'] == null ? undefined : json['@type'], - }; -} - -export function ProtobufAnyToJSON(json: any): ProtobufAny { - return ProtobufAnyToJSONTyped(json, false); -} - -export function ProtobufAnyToJSONTyped( - value?: ProtobufAny | null, - ignoreDiscriminator: boolean = false, -): any { - if (value == null) { - return value; - } - - return { - ...value, - '@type': value['type'], - }; -} diff --git a/frontend/server/src/generated/openapi/runtime.ts b/frontend/server/src/generated/openapi/runtime.ts deleted file mode 100644 index 24efbf8b495..00000000000 --- a/frontend/server/src/generated/openapi/runtime.ts +++ /dev/null @@ -1,462 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); - -export interface ConfigurationParameters { - basePath?: string; // override base path - fetchApi?: FetchAPI; // override for fetch implementation - middleware?: Middleware[]; // middleware to apply before/after fetch requests - queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings - username?: string; // parameter for basic security - password?: string; // parameter for basic security - apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security - accessToken?: - | string - | Promise - | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security - headers?: HTTPHeaders; //header params we want to use on every request - credentials?: RequestInit['credentials']; //value for the credentials param we want to use on each request -} - -export class Configuration { - constructor(private configuration: ConfigurationParameters = {}) {} - - set config(configuration: Configuration) { - this.configuration = configuration; - } - - get basePath(): string { - return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; - } - - get fetchApi(): FetchAPI | undefined { - return this.configuration.fetchApi; - } - - get middleware(): Middleware[] { - return this.configuration.middleware || []; - } - - get queryParamsStringify(): (params: HTTPQuery) => string { - return this.configuration.queryParamsStringify || querystring; - } - - get username(): string | undefined { - return this.configuration.username; - } - - get password(): string | undefined { - return this.configuration.password; - } - - get apiKey(): ((name: string) => string | Promise) | undefined { - const apiKey = this.configuration.apiKey; - if (apiKey) { - return typeof apiKey === 'function' ? apiKey : () => apiKey; - } - return undefined; - } - - get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { - const accessToken = this.configuration.accessToken; - if (accessToken) { - return typeof accessToken === 'function' ? accessToken : async () => accessToken; - } - return undefined; - } - - get headers(): HTTPHeaders | undefined { - return this.configuration.headers; - } - - get credentials(): RequestInit['credentials'] | undefined { - return this.configuration.credentials; - } -} - -export const DefaultConfig = new Configuration(); - -export class BaseAPI { - private static readonly jsonRegex = new RegExp( - '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', - 'i', - ); - private middleware: Middleware[]; - - constructor(protected configuration = DefaultConfig) { - this.middleware = configuration.middleware; - } - - withMiddleware(this: T, ...middlewares: Middleware[]) { - const next = this.clone(); - next.middleware = next.middleware.concat(...middlewares); - return next; - } - - withPreMiddleware(this: T, ...preMiddlewares: Array) { - const middlewares = preMiddlewares.map((pre) => ({ pre })); - return this.withMiddleware(...middlewares); - } - - withPostMiddleware(this: T, ...postMiddlewares: Array) { - const middlewares = postMiddlewares.map((post) => ({ post })); - return this.withMiddleware(...middlewares); - } - - protected isJsonMime(mime: string | null | undefined): boolean { - if (!mime) { - return false; - } - return BaseAPI.jsonRegex.test(mime); - } - - protected async request( - context: RequestOpts, - initOverrides?: RequestInit | InitOverrideFunction, - ): Promise { - const { url, init } = await this.createFetchParams(context, initOverrides); - const response = await this.fetchApi(url, init); - if (response && response.status >= 200 && response.status < 300) { - return response; - } - throw new ResponseError(response, 'Response returned an error code'); - } - - private async createFetchParams( - context: RequestOpts, - initOverrides?: RequestInit | InitOverrideFunction, - ) { - let url = this.configuration.basePath + context.path; - if (context.query !== undefined && Object.keys(context.query).length !== 0) { - // only add the querystring to the URL if there are query parameters. - // this is done to avoid urls ending with a "?" character which buggy webservers - // do not handle correctly sometimes. - url += '?' + this.configuration.queryParamsStringify(context.query); - } - - const headers = Object.assign({}, this.configuration.headers, context.headers); - Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); - - const initOverrideFn = - typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; - - const initParams = { - method: context.method, - headers, - body: context.body, - credentials: this.configuration.credentials, - }; - - const overriddenInit: RequestInit = { - ...initParams, - ...(await initOverrideFn({ - init: initParams, - context, - })), - }; - - let body: any; - if ( - isFormData(overriddenInit.body) || - overriddenInit.body instanceof URLSearchParams || - isBlob(overriddenInit.body) - ) { - body = overriddenInit.body; - } else if (this.isJsonMime(headers['Content-Type'])) { - body = JSON.stringify(overriddenInit.body); - } else { - body = overriddenInit.body; - } - - const init: RequestInit = { - ...overriddenInit, - body, - }; - - return { url, init }; - } - - private fetchApi = async (url: string, init: RequestInit): Promise => { - let fetchParams = { url, init }; - for (const middleware of this.middleware) { - if (middleware.pre) { - fetchParams = - (await middleware.pre({ - fetch: this.fetchApi, - ...fetchParams, - })) || fetchParams; - } - } - let response: Response | undefined = undefined; - try { - response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); - } catch (e) { - for (const middleware of this.middleware) { - if (middleware.onError) { - response = - (await middleware.onError({ - fetch: this.fetchApi, - url: fetchParams.url, - init: fetchParams.init, - error: e, - response: response ? response.clone() : undefined, - })) || response; - } - } - if (response === undefined) { - if (e instanceof Error) { - throw new FetchError( - e, - 'The request failed and the interceptors did not return an alternative response', - ); - } else { - throw e; - } - } - } - for (const middleware of this.middleware) { - if (middleware.post) { - response = - (await middleware.post({ - fetch: this.fetchApi, - url: fetchParams.url, - init: fetchParams.init, - response: response.clone(), - })) || response; - } - } - return response; - }; - - private clone(this: T): T { - const constructor = this.constructor as any; - const next = new constructor(this.configuration); - next.middleware = this.middleware.slice(); - return next; - } -} - -function isBlob(value: any): value is Blob { - return typeof Blob !== 'undefined' && value instanceof Blob; -} - -function isFormData(value: any): value is FormData { - return typeof FormData !== 'undefined' && value instanceof FormData; -} - -export class ResponseError extends Error { - override name: 'ResponseError' = 'ResponseError'; - constructor( - public response: Response, - msg?: string, - ) { - super(msg); - } -} - -export class FetchError extends Error { - override name: 'FetchError' = 'FetchError'; - constructor( - public cause: Error, - msg?: string, - ) { - super(msg); - } -} - -export class RequiredError extends Error { - override name: 'RequiredError' = 'RequiredError'; - constructor( - public field: string, - msg?: string, - ) { - super(msg); - } -} - -export const COLLECTION_FORMATS = { - csv: ',', - ssv: ' ', - tsv: '\t', - pipes: '|', -}; - -export type FetchAPI = (input: string, init: RequestInit) => Promise; - -export type Json = any; -export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; -export type HTTPHeaders = { [key: string]: string }; -export type HTTPQuery = { - [key: string]: - | string - | number - | null - | boolean - | Array - | Set - | HTTPQuery; -}; -export type HTTPBody = Json | FormData | URLSearchParams; -export type HTTPRequestInit = { - headers?: HTTPHeaders; - method: HTTPMethod; - credentials?: RequestInit['credentials']; - body?: HTTPBody; -}; -export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; - -export type InitOverrideFunction = (requestContext: { - init: HTTPRequestInit; - context: RequestOpts; -}) => Promise; - -export interface FetchParams { - url: string; - init: RequestInit; -} - -export interface RequestOpts { - path: string; - method: HTTPMethod; - headers: HTTPHeaders; - query?: HTTPQuery; - body?: HTTPBody; -} - -export function querystring(params: HTTPQuery, prefix: string = ''): string { - return Object.keys(params) - .map((key) => querystringSingleKey(key, params[key], prefix)) - .filter((part) => part.length > 0) - .join('&'); -} - -function querystringSingleKey( - key: string, - value: - | string - | number - | null - | undefined - | boolean - | Array - | Set - | HTTPQuery, - keyPrefix: string = '', -): string { - const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); - if (value instanceof Array) { - const multiValue = value - .map((singleValue) => encodeURIComponent(String(singleValue))) - .join(`&${encodeURIComponent(fullKey)}=`); - return `${encodeURIComponent(fullKey)}=${multiValue}`; - } - if (value instanceof Set) { - const valueAsArray = Array.from(value); - return querystringSingleKey(key, valueAsArray, keyPrefix); - } - if (value instanceof Date) { - return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; - } - if (value instanceof Object) { - return querystring(value as HTTPQuery, fullKey); - } - return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; -} - -export function exists(json: any, key: string) { - const value = json[key]; - return value !== null && value !== undefined; -} - -export function mapValues(data: any, fn: (item: any) => any) { - const result: { [key: string]: any } = {}; - for (const key of Object.keys(data)) { - result[key] = fn(data[key]); - } - return result; -} - -export function canConsumeForm(consumes: Consume[]): boolean { - for (const consume of consumes) { - if ('multipart/form-data' === consume.contentType) { - return true; - } - } - return false; -} - -export interface Consume { - contentType: string; -} - -export interface RequestContext { - fetch: FetchAPI; - url: string; - init: RequestInit; -} - -export interface ResponseContext { - fetch: FetchAPI; - url: string; - init: RequestInit; - response: Response; -} - -export interface ErrorContext { - fetch: FetchAPI; - url: string; - init: RequestInit; - error: unknown; - response?: Response; -} - -export interface Middleware { - pre?(context: RequestContext): Promise; - post?(context: ResponseContext): Promise; - onError?(context: ErrorContext): Promise; -} - -export interface ApiResponse { - raw: Response; - value(): Promise; -} - -export interface ResponseTransformer { - (json: any): T; -} - -export class JSONApiResponse { - constructor( - public raw: Response, - private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, - ) {} - - async value(): Promise { - return this.transformer(await this.raw.json()); - } -} - -export class VoidApiResponse { - constructor(public raw: Response) {} - - async value(): Promise { - return undefined; - } -} - -export class BlobApiResponse { - constructor(public raw: Response) {} - - async value(): Promise { - return await this.raw.blob(); - } -} - -export class TextApiResponse { - constructor(public raw: Response) {} - - async value(): Promise { - return await this.raw.text(); - } -} diff --git a/frontend/src/apis/experiment/models/GooglerpcStatus.ts b/frontend/src/apis/experiment/models/GooglerpcStatus.ts index c0fb39cac1f..338005d98c4 100644 --- a/frontend/src/apis/experiment/models/GooglerpcStatus.ts +++ b/frontend/src/apis/experiment/models/GooglerpcStatus.ts @@ -1,4 +1,98 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v1beta1/experiment.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apis/experiment/models/ProtobufAny.ts b/frontend/src/apis/experiment/models/ProtobufAny.ts index 9a216fddd07..9935d8eb870 100644 --- a/frontend/src/apis/experiment/models/ProtobufAny.ts +++ b/frontend/src/apis/experiment/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v1beta1/experiment.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apis/experiment/runtime.ts b/frontend/src/apis/experiment/runtime.ts index 371fc44d523..74a99315f42 100644 --- a/frontend/src/apis/experiment/runtime.ts +++ b/frontend/src/apis/experiment/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v1beta1/experiment.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apis/filter/models/GooglerpcStatus.ts b/frontend/src/apis/filter/models/GooglerpcStatus.ts index c0fb39cac1f..1b0cbd38c58 100644 --- a/frontend/src/apis/filter/models/GooglerpcStatus.ts +++ b/frontend/src/apis/filter/models/GooglerpcStatus.ts @@ -1,4 +1,98 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v1beta1/filter.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apis/filter/models/ProtobufAny.ts b/frontend/src/apis/filter/models/ProtobufAny.ts index 9a216fddd07..28a34b2910c 100644 --- a/frontend/src/apis/filter/models/ProtobufAny.ts +++ b/frontend/src/apis/filter/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v1beta1/filter.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apis/filter/runtime.ts b/frontend/src/apis/filter/runtime.ts index 371fc44d523..ae63a7251a7 100644 --- a/frontend/src/apis/filter/runtime.ts +++ b/frontend/src/apis/filter/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v1beta1/filter.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apis/job/models/GooglerpcStatus.ts b/frontend/src/apis/job/models/GooglerpcStatus.ts index c0fb39cac1f..a9ebf3c8b04 100644 --- a/frontend/src/apis/job/models/GooglerpcStatus.ts +++ b/frontend/src/apis/job/models/GooglerpcStatus.ts @@ -1,4 +1,98 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v1beta1/job.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apis/job/models/ProtobufAny.ts b/frontend/src/apis/job/models/ProtobufAny.ts index 9a216fddd07..069fffed41c 100644 --- a/frontend/src/apis/job/models/ProtobufAny.ts +++ b/frontend/src/apis/job/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v1beta1/job.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apis/job/runtime.ts b/frontend/src/apis/job/runtime.ts index 371fc44d523..d6f16d6ff0c 100644 --- a/frontend/src/apis/job/runtime.ts +++ b/frontend/src/apis/job/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v1beta1/job.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apis/pipeline/models/GooglerpcStatus.ts b/frontend/src/apis/pipeline/models/GooglerpcStatus.ts index c0fb39cac1f..070a45d3f6c 100644 --- a/frontend/src/apis/pipeline/models/GooglerpcStatus.ts +++ b/frontend/src/apis/pipeline/models/GooglerpcStatus.ts @@ -1,4 +1,98 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v1beta1/pipeline.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apis/pipeline/models/ProtobufAny.ts b/frontend/src/apis/pipeline/models/ProtobufAny.ts index 9a216fddd07..6c178f2fbb0 100644 --- a/frontend/src/apis/pipeline/models/ProtobufAny.ts +++ b/frontend/src/apis/pipeline/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v1beta1/pipeline.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apis/pipeline/runtime.ts b/frontend/src/apis/pipeline/runtime.ts index 371fc44d523..573de06df17 100644 --- a/frontend/src/apis/pipeline/runtime.ts +++ b/frontend/src/apis/pipeline/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v1beta1/pipeline.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apis/run/models/GooglerpcStatus.ts b/frontend/src/apis/run/models/GooglerpcStatus.ts index c0fb39cac1f..9e08e7fb133 100644 --- a/frontend/src/apis/run/models/GooglerpcStatus.ts +++ b/frontend/src/apis/run/models/GooglerpcStatus.ts @@ -1,4 +1,98 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v1beta1/run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apis/run/models/ProtobufAny.ts b/frontend/src/apis/run/models/ProtobufAny.ts index 9a216fddd07..2653dcba55a 100644 --- a/frontend/src/apis/run/models/ProtobufAny.ts +++ b/frontend/src/apis/run/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v1beta1/run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apis/run/runtime.ts b/frontend/src/apis/run/runtime.ts index 371fc44d523..e8a32decfac 100644 --- a/frontend/src/apis/run/runtime.ts +++ b/frontend/src/apis/run/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v1beta1/run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apis/visualization/models/GooglerpcStatus.ts b/frontend/src/apis/visualization/models/GooglerpcStatus.ts index c0fb39cac1f..5ecb63d2e51 100644 --- a/frontend/src/apis/visualization/models/GooglerpcStatus.ts +++ b/frontend/src/apis/visualization/models/GooglerpcStatus.ts @@ -1,4 +1,98 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v1beta1/visualization.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apis/visualization/models/ProtobufAny.ts b/frontend/src/apis/visualization/models/ProtobufAny.ts index 9a216fddd07..a08fbc1db5d 100644 --- a/frontend/src/apis/visualization/models/ProtobufAny.ts +++ b/frontend/src/apis/visualization/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v1beta1/visualization.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apis/visualization/runtime.ts b/frontend/src/apis/visualization/runtime.ts index 371fc44d523..84e6acd6ff5 100644 --- a/frontend/src/apis/visualization/runtime.ts +++ b/frontend/src/apis/visualization/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v1beta1/visualization.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apisv2beta1/experiment/models/GooglerpcStatus.ts b/frontend/src/apisv2beta1/experiment/models/GooglerpcStatus.ts index c0fb39cac1f..7c2856ca050 100644 --- a/frontend/src/apisv2beta1/experiment/models/GooglerpcStatus.ts +++ b/frontend/src/apisv2beta1/experiment/models/GooglerpcStatus.ts @@ -1,4 +1,107 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v2beta1/experiment.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apisv2beta1/experiment/models/ProtobufAny.ts b/frontend/src/apisv2beta1/experiment/models/ProtobufAny.ts index 9a216fddd07..9d84ff46376 100644 --- a/frontend/src/apisv2beta1/experiment/models/ProtobufAny.ts +++ b/frontend/src/apisv2beta1/experiment/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v2beta1/experiment.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apisv2beta1/experiment/runtime.ts b/frontend/src/apisv2beta1/experiment/runtime.ts index 371fc44d523..3820c43ef5f 100644 --- a/frontend/src/apisv2beta1/experiment/runtime.ts +++ b/frontend/src/apisv2beta1/experiment/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v2beta1/experiment.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apisv2beta1/filter/models/GooglerpcStatus.ts b/frontend/src/apisv2beta1/filter/models/GooglerpcStatus.ts index c0fb39cac1f..6145dfe0222 100644 --- a/frontend/src/apisv2beta1/filter/models/GooglerpcStatus.ts +++ b/frontend/src/apisv2beta1/filter/models/GooglerpcStatus.ts @@ -1,4 +1,107 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v2beta1/filter.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apisv2beta1/filter/models/ProtobufAny.ts b/frontend/src/apisv2beta1/filter/models/ProtobufAny.ts index 9a216fddd07..45af930b655 100644 --- a/frontend/src/apisv2beta1/filter/models/ProtobufAny.ts +++ b/frontend/src/apisv2beta1/filter/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v2beta1/filter.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apisv2beta1/filter/runtime.ts b/frontend/src/apisv2beta1/filter/runtime.ts index 371fc44d523..49ec326db74 100644 --- a/frontend/src/apisv2beta1/filter/runtime.ts +++ b/frontend/src/apisv2beta1/filter/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v2beta1/filter.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apisv2beta1/pipeline/models/GooglerpcStatus.ts b/frontend/src/apisv2beta1/pipeline/models/GooglerpcStatus.ts index c0fb39cac1f..41e1aa9b01e 100644 --- a/frontend/src/apisv2beta1/pipeline/models/GooglerpcStatus.ts +++ b/frontend/src/apisv2beta1/pipeline/models/GooglerpcStatus.ts @@ -1,4 +1,107 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v2beta1/pipeline.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apisv2beta1/pipeline/models/ProtobufAny.ts b/frontend/src/apisv2beta1/pipeline/models/ProtobufAny.ts index 9a216fddd07..f5906742204 100644 --- a/frontend/src/apisv2beta1/pipeline/models/ProtobufAny.ts +++ b/frontend/src/apisv2beta1/pipeline/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v2beta1/pipeline.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apisv2beta1/pipeline/runtime.ts b/frontend/src/apisv2beta1/pipeline/runtime.ts index 371fc44d523..aa3eb8171db 100644 --- a/frontend/src/apisv2beta1/pipeline/runtime.ts +++ b/frontend/src/apisv2beta1/pipeline/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v2beta1/pipeline.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apisv2beta1/recurringrun/models/GooglerpcStatus.ts b/frontend/src/apisv2beta1/recurringrun/models/GooglerpcStatus.ts index c0fb39cac1f..d2efb9f28d2 100644 --- a/frontend/src/apisv2beta1/recurringrun/models/GooglerpcStatus.ts +++ b/frontend/src/apisv2beta1/recurringrun/models/GooglerpcStatus.ts @@ -1,4 +1,107 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v2beta1/recurring_run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apisv2beta1/recurringrun/models/ProtobufAny.ts b/frontend/src/apisv2beta1/recurringrun/models/ProtobufAny.ts index 9a216fddd07..2040b175031 100644 --- a/frontend/src/apisv2beta1/recurringrun/models/ProtobufAny.ts +++ b/frontend/src/apisv2beta1/recurringrun/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v2beta1/recurring_run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apisv2beta1/recurringrun/runtime.ts b/frontend/src/apisv2beta1/recurringrun/runtime.ts index 371fc44d523..e5dad0a3c8d 100644 --- a/frontend/src/apisv2beta1/recurringrun/runtime.ts +++ b/frontend/src/apisv2beta1/recurringrun/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v2beta1/recurring_run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apisv2beta1/run/models/GooglerpcStatus.ts b/frontend/src/apisv2beta1/run/models/GooglerpcStatus.ts index c0fb39cac1f..f97c650716f 100644 --- a/frontend/src/apisv2beta1/run/models/GooglerpcStatus.ts +++ b/frontend/src/apisv2beta1/run/models/GooglerpcStatus.ts @@ -1,4 +1,107 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v2beta1/run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apisv2beta1/run/models/ProtobufAny.ts b/frontend/src/apisv2beta1/run/models/ProtobufAny.ts index 9a216fddd07..83485304e66 100644 --- a/frontend/src/apisv2beta1/run/models/ProtobufAny.ts +++ b/frontend/src/apisv2beta1/run/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v2beta1/run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apisv2beta1/run/runtime.ts b/frontend/src/apisv2beta1/run/runtime.ts index 371fc44d523..7703a05aa28 100644 --- a/frontend/src/apisv2beta1/run/runtime.ts +++ b/frontend/src/apisv2beta1/run/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v2beta1/run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/apisv2beta1/visualization/models/GooglerpcStatus.ts b/frontend/src/apisv2beta1/visualization/models/GooglerpcStatus.ts index c0fb39cac1f..019a7438b18 100644 --- a/frontend/src/apisv2beta1/visualization/models/GooglerpcStatus.ts +++ b/frontend/src/apisv2beta1/visualization/models/GooglerpcStatus.ts @@ -1,4 +1,107 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/GooglerpcStatus'; +/** + * backend/api/v2beta1/visualization.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { ProtobufAny } from './ProtobufAny'; +import { + ProtobufAnyFromJSON, + ProtobufAnyFromJSONTyped, + ProtobufAnyToJSON, + ProtobufAnyToJSONTyped, +} from './ProtobufAny'; + +/** + * The `Status` type defines a logical error model that is suitable for + * different programming environments, including REST APIs and RPC APIs. It is + * used by [gRPC](https://github.com/grpc). Each `Status` message contains + * three pieces of data: error code, error message, and error details. + * + * You can find out more about this error model and how to work with it in the + * [API Design Guide](https://cloud.google.com/apis/design/errors). + * @export + * @interface GooglerpcStatus + */ +export interface GooglerpcStatus { + /** + * The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + * @type {number} + * @memberof GooglerpcStatus + */ + code?: number; + /** + * A developer-facing error message, which should be in English. Any + * user-facing error message should be localized and sent in the + * [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + * @type {string} + * @memberof GooglerpcStatus + */ + message?: string; + /** + * A list of messages that carry the error details. There is a common set of + * message types for APIs to use. + * @type {Array} + * @memberof GooglerpcStatus + */ + details?: Array; +} + +/** + * Check if a given object implements the GooglerpcStatus interface. + */ +export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { + return true; +} + +export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { + return GooglerpcStatusFromJSONTyped(json, false); +} + +export function GooglerpcStatusFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): GooglerpcStatus { + if (json == null) { + return json; + } + return { + code: json['code'] == null ? undefined : json['code'], + message: json['message'] == null ? undefined : json['message'], + details: + json['details'] == null + ? undefined + : (json['details'] as Array).map(ProtobufAnyFromJSON), + }; +} + +export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { + return GooglerpcStatusToJSONTyped(json, false); +} + +export function GooglerpcStatusToJSONTyped( + value?: GooglerpcStatus | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + code: value['code'], + message: value['message'], + details: + value['details'] == null + ? undefined + : (value['details'] as Array).map(ProtobufAnyToJSON), + }; +} diff --git a/frontend/src/apisv2beta1/visualization/models/ProtobufAny.ts b/frontend/src/apisv2beta1/visualization/models/ProtobufAny.ts index 9a216fddd07..452c1c0d17a 100644 --- a/frontend/src/apisv2beta1/visualization/models/ProtobufAny.ts +++ b/frontend/src/apisv2beta1/visualization/models/ProtobufAny.ts @@ -1,4 +1,179 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../../generated/openapi/models/ProtobufAny'; +/** + * backend/api/v2beta1/visualization.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a + * URL that describes the type of the serialized message. + * + * Protobuf library provides support to pack/unpack Any values in the form + * of utility functions or additional generated methods of the Any type. + * + * Example 1: Pack and unpack a message in C++. + * + * Foo foo = ...; + * Any any; + * any.PackFrom(foo); + * ... + * if (any.UnpackTo(&foo)) { + * ... + * } + * + * Example 2: Pack and unpack a message in Java. + * + * Foo foo = ...; + * Any any = Any.pack(foo); + * ... + * if (any.is(Foo.class)) { + * foo = any.unpack(Foo.class); + * } + * // or ... + * if (any.isSameTypeAs(Foo.getDefaultInstance())) { + * foo = any.unpack(Foo.getDefaultInstance()); + * } + * + * Example 3: Pack and unpack a message in Python. + * + * foo = Foo(...) + * any = Any() + * any.Pack(foo) + * ... + * if any.Is(Foo.DESCRIPTOR): + * any.Unpack(foo) + * ... + * + * Example 4: Pack and unpack a message in Go + * + * foo := &pb.Foo{...} + * any, err := anypb.New(foo) + * if err != nil { + * ... + * } + * ... + * foo := &pb.Foo{} + * if err := any.UnmarshalTo(foo); err != nil { + * ... + * } + * + * The pack methods provided by protobuf library will by default use + * 'type.googleapis.com/full.type.name' as the type URL and the unpack + * methods only use the fully qualified type name after the last '/' + * in the type URL, for example "foo.bar.com/x/y.z" will yield type + * name "y.z". + * + * JSON + * ==== + * The JSON representation of an `Any` value uses the regular + * representation of the deserialized, embedded message, with an + * additional field `@type` which contains the type URL. Example: + * + * package google.profile; + * message Person { + * string first_name = 1; + * string last_name = 2; + * } + * + * { + * "@type": "type.googleapis.com/google.profile.Person", + * "firstName": , + * "lastName": + * } + * + * If the embedded message type is well-known and has a custom JSON + * representation, that representation will be embedded adding a field + * `value` which holds the custom JSON in addition to the `@type` + * field. Example (for message [google.protobuf.Duration][]): + * + * { + * "@type": "type.googleapis.com/google.protobuf.Duration", + * "value": "1.212s" + * } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + [key: string]: object | any; + /** + * A URL/resource name that uniquely identifies the type of the serialized + * protocol buffer message. This string must contain at least + * one "/" character. The last segment of the URL's path must represent + * the fully qualified name of the type (as in + * `path/google.protobuf.Duration`). The name should be in a canonical form + * (e.g., leading "." is not accepted). + * + * In practice, teams usually precompile into the binary all types that they + * expect it to use in the context of Any. However, for URLs which use the + * scheme `http`, `https`, or no scheme, one can optionally set up a type + * server that maps type URLs to message definitions as follows: + * + * * If no scheme is provided, `https` is assumed. + * * An HTTP GET on the URL must yield a [google.protobuf.Type][] + * value in binary format, or produce an error. + * * Applications are allowed to cache lookup results based on the + * URL, or have them precompiled into a binary to avoid any + * lookup. Therefore, binary compatibility needs to be preserved + * on changes to types. (Use versioned type names to manage + * breaking changes.) + * + * Note: this functionality is not currently available in the official + * protobuf release, and it is not used for type URLs beginning with + * type.googleapis.com. As of May 2023, there are no widely used type server + * implementations and no plans to implement one. + * + * Schemes other than `http`, `https` (or the empty scheme) might be + * used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type?: string; +} + +/** + * Check if a given object implements the ProtobufAny interface. + */ +export function instanceOfProtobufAny(value: object): value is ProtobufAny { + return true; +} + +export function ProtobufAnyFromJSON(json: any): ProtobufAny { + return ProtobufAnyFromJSONTyped(json, false); +} + +export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { + if (json == null) { + return json; + } + return { + ...json, + type: json['@type'] == null ? undefined : json['@type'], + }; +} + +export function ProtobufAnyToJSON(json: any): ProtobufAny { + return ProtobufAnyToJSONTyped(json, false); +} + +export function ProtobufAnyToJSONTyped( + value?: ProtobufAny | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + ...value, + '@type': value['type'], + }; +} diff --git a/frontend/src/apisv2beta1/visualization/runtime.ts b/frontend/src/apisv2beta1/visualization/runtime.ts index 371fc44d523..ab877533cb8 100644 --- a/frontend/src/apisv2beta1/visualization/runtime.ts +++ b/frontend/src/apisv2beta1/visualization/runtime.ts @@ -1,4 +1,490 @@ /* tslint:disable */ /* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export * from '../../generated/openapi/runtime'; +/** + * backend/api/v2beta1/visualization.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); + +export interface ConfigurationParameters { + basePath?: string; // override base path + fetchApi?: FetchAPI; // override for fetch implementation + middleware?: Middleware[]; // middleware to apply before/after fetch requests + queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings + username?: string; // parameter for basic security + password?: string; // parameter for basic security + apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security + accessToken?: + | string + | Promise + | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security + headers?: HTTPHeaders; //header params we want to use on every request + credentials?: RequestCredentials; //value for the credentials param we want to use on each request +} + +export class Configuration { + constructor(private configuration: ConfigurationParameters = {}) {} + + set config(configuration: Configuration) { + this.configuration = configuration; + } + + get basePath(): string { + return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; + } + + get fetchApi(): FetchAPI | undefined { + return this.configuration.fetchApi; + } + + get middleware(): Middleware[] { + return this.configuration.middleware || []; + } + + get queryParamsStringify(): (params: HTTPQuery) => string { + return this.configuration.queryParamsStringify || querystring; + } + + get username(): string | undefined { + return this.configuration.username; + } + + get password(): string | undefined { + return this.configuration.password; + } + + get apiKey(): ((name: string) => string | Promise) | undefined { + const apiKey = this.configuration.apiKey; + if (apiKey) { + return typeof apiKey === 'function' ? apiKey : () => apiKey; + } + return undefined; + } + + get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { + const accessToken = this.configuration.accessToken; + if (accessToken) { + return typeof accessToken === 'function' ? accessToken : async () => accessToken; + } + return undefined; + } + + get headers(): HTTPHeaders | undefined { + return this.configuration.headers; + } + + get credentials(): RequestCredentials | undefined { + return this.configuration.credentials; + } +} + +export const DefaultConfig = new Configuration(); + +/** + * This is the base class for all generated API classes. + */ +export class BaseAPI { + private static readonly jsonRegex = new RegExp( + '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', + 'i', + ); + private middleware: Middleware[]; + + constructor(protected configuration = DefaultConfig) { + this.middleware = configuration.middleware; + } + + withMiddleware(this: T, ...middlewares: Middleware[]) { + const next = this.clone(); + next.middleware = next.middleware.concat(...middlewares); + return next; + } + + withPreMiddleware(this: T, ...preMiddlewares: Array) { + const middlewares = preMiddlewares.map((pre) => ({ pre })); + return this.withMiddleware(...middlewares); + } + + withPostMiddleware(this: T, ...postMiddlewares: Array) { + const middlewares = postMiddlewares.map((post) => ({ post })); + return this.withMiddleware(...middlewares); + } + + /** + * Check if the given MIME is a JSON MIME. + * JSON MIME examples: + * application/json + * application/json; charset=UTF8 + * APPLICATION/JSON + * application/vnd.company+json + * @param mime - MIME (Multipurpose Internet Mail Extensions) + * @return True if the given MIME is JSON, false otherwise. + */ + protected isJsonMime(mime: string | null | undefined): boolean { + if (!mime) { + return false; + } + return BaseAPI.jsonRegex.test(mime); + } + + protected async request( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ): Promise { + const { url, init } = await this.createFetchParams(context, initOverrides); + const response = await this.fetchApi(url, init); + if (response && response.status >= 200 && response.status < 300) { + return response; + } + throw new ResponseError(response, 'Response returned an error code'); + } + + private async createFetchParams( + context: RequestOpts, + initOverrides?: RequestInit | InitOverrideFunction, + ) { + let url = this.configuration.basePath + context.path; + if (context.query !== undefined && Object.keys(context.query).length !== 0) { + // only add the querystring to the URL if there are query parameters. + // this is done to avoid urls ending with a "?" character which buggy webservers + // do not handle correctly sometimes. + url += '?' + this.configuration.queryParamsStringify(context.query); + } + + const headers = Object.assign({}, this.configuration.headers, context.headers); + Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); + + const initOverrideFn = + typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; + + const initParams = { + method: context.method, + headers, + body: context.body, + credentials: this.configuration.credentials, + }; + + const overriddenInit: RequestInit = { + ...initParams, + ...(await initOverrideFn({ + init: initParams, + context, + })), + }; + + let body: any; + if ( + isFormData(overriddenInit.body) || + overriddenInit.body instanceof URLSearchParams || + isBlob(overriddenInit.body) + ) { + body = overriddenInit.body; + } else if (this.isJsonMime(headers['Content-Type'])) { + body = JSON.stringify(overriddenInit.body); + } else { + body = overriddenInit.body; + } + + const init: RequestInit = { + ...overriddenInit, + body, + }; + + return { url, init }; + } + + private fetchApi = async (url: string, init: RequestInit) => { + let fetchParams = { url, init }; + for (const middleware of this.middleware) { + if (middleware.pre) { + fetchParams = + (await middleware.pre({ + fetch: this.fetchApi, + ...fetchParams, + })) || fetchParams; + } + } + let response: Response | undefined = undefined; + try { + response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); + } catch (e) { + for (const middleware of this.middleware) { + if (middleware.onError) { + response = + (await middleware.onError({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + error: e, + response: response ? response.clone() : undefined, + })) || response; + } + } + if (response === undefined) { + if (e instanceof Error) { + throw new FetchError( + e, + 'The request failed and the interceptors did not return an alternative response', + ); + } else { + throw e; + } + } + } + for (const middleware of this.middleware) { + if (middleware.post) { + response = + (await middleware.post({ + fetch: this.fetchApi, + url: fetchParams.url, + init: fetchParams.init, + response: response.clone(), + })) || response; + } + } + return response; + }; + + /** + * Create a shallow clone of `this` by constructing a new instance + * and then shallow cloning data members. + */ + private clone(this: T): T { + const constructor = this.constructor as any; + const next = new constructor(this.configuration); + next.middleware = this.middleware.slice(); + return next; + } +} + +function isBlob(value: any): value is Blob { + return typeof Blob !== 'undefined' && value instanceof Blob; +} + +function isFormData(value: any): value is FormData { + return typeof FormData !== 'undefined' && value instanceof FormData; +} + +export class ResponseError extends Error { + override name: 'ResponseError' = 'ResponseError'; + constructor( + public response: Response, + msg?: string, + ) { + super(msg); + } +} + +export class FetchError extends Error { + override name: 'FetchError' = 'FetchError'; + constructor( + public cause: Error, + msg?: string, + ) { + super(msg); + } +} + +export class RequiredError extends Error { + override name: 'RequiredError' = 'RequiredError'; + constructor( + public field: string, + msg?: string, + ) { + super(msg); + } +} + +export const COLLECTION_FORMATS = { + csv: ',', + ssv: ' ', + tsv: '\t', + pipes: '|', +}; + +export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; + +export type Json = any; +export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; +export type HTTPHeaders = { [key: string]: string }; +export type HTTPQuery = { + [key: string]: + | string + | number + | null + | boolean + | Array + | Set + | HTTPQuery; +}; +export type HTTPBody = Json | FormData | URLSearchParams; +export type HTTPRequestInit = { + headers?: HTTPHeaders; + method: HTTPMethod; + credentials?: RequestCredentials; + body?: HTTPBody; +}; +export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; + +export type InitOverrideFunction = (requestContext: { + init: HTTPRequestInit; + context: RequestOpts; +}) => Promise; + +export interface FetchParams { + url: string; + init: RequestInit; +} + +export interface RequestOpts { + path: string; + method: HTTPMethod; + headers: HTTPHeaders; + query?: HTTPQuery; + body?: HTTPBody; +} + +export function querystring(params: HTTPQuery, prefix: string = ''): string { + return Object.keys(params) + .map((key) => querystringSingleKey(key, params[key], prefix)) + .filter((part) => part.length > 0) + .join('&'); +} + +function querystringSingleKey( + key: string, + value: + | string + | number + | null + | undefined + | boolean + | Array + | Set + | HTTPQuery, + keyPrefix: string = '', +): string { + const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); + if (value instanceof Array) { + const multiValue = value + .map((singleValue) => encodeURIComponent(String(singleValue))) + .join(`&${encodeURIComponent(fullKey)}=`); + return `${encodeURIComponent(fullKey)}=${multiValue}`; + } + if (value instanceof Set) { + const valueAsArray = Array.from(value); + return querystringSingleKey(key, valueAsArray, keyPrefix); + } + if (value instanceof Date) { + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; + } + if (value instanceof Object) { + return querystring(value as HTTPQuery, fullKey); + } + return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; +} + +export function exists(json: any, key: string) { + const value = json[key]; + return value !== null && value !== undefined; +} + +export function mapValues(data: any, fn: (item: any) => any) { + const result: { [key: string]: any } = {}; + for (const key of Object.keys(data)) { + result[key] = fn(data[key]); + } + return result; +} + +export function canConsumeForm(consumes: Consume[]): boolean { + for (const consume of consumes) { + if ('multipart/form-data' === consume.contentType) { + return true; + } + } + return false; +} + +export interface Consume { + contentType: string; +} + +export interface RequestContext { + fetch: FetchAPI; + url: string; + init: RequestInit; +} + +export interface ResponseContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + response: Response; +} + +export interface ErrorContext { + fetch: FetchAPI; + url: string; + init: RequestInit; + error: unknown; + response?: Response; +} + +export interface Middleware { + pre?(context: RequestContext): Promise; + post?(context: ResponseContext): Promise; + onError?(context: ErrorContext): Promise; +} + +export interface ApiResponse { + raw: Response; + value(): Promise; +} + +export interface ResponseTransformer { + (json: any): T; +} + +export class JSONApiResponse { + constructor( + public raw: Response, + private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, + ) {} + + async value(): Promise { + return this.transformer(await this.raw.json()); + } +} + +export class VoidApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return undefined; + } +} + +export class BlobApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.blob(); + } +} + +export class TextApiResponse { + constructor(public raw: Response) {} + + async value(): Promise { + return await this.raw.text(); + } +} diff --git a/frontend/src/components/CustomTable.test.tsx b/frontend/src/components/CustomTable.test.tsx index 7316219e632..c413c272be3 100644 --- a/frontend/src/components/CustomTable.test.tsx +++ b/frontend/src/components/CustomTable.test.tsx @@ -135,7 +135,6 @@ function getRowsPerPageCombobox(): HTMLElement { describe('CustomTable', () => { beforeEach(() => { vi.useRealTimers(); - localStorage.clear(); }); it('renders with default filter label', async () => { @@ -747,42 +746,4 @@ describe('CustomTable', () => { fireEvent.change(screen.getByLabelText('Filter'), { target: { value: 'test filter' } }); expect(setFilterString).toHaveBeenLastCalledWith('test filter'); }); - - it('reads page size from localStorage on mount', async () => { - localStorage.setItem('tablePageSize', '50'); - const reload = vi.fn(async () => ''); - const wrapper = renderTable({ rows, columns, reload }); - await waitFor(() => expect(reload).toHaveBeenCalled()); - expect(reload).toHaveBeenLastCalledWith({ - filter: '', - orderAscending: false, - pageSize: 50, - pageToken: '', - sortBy: '', - }); - wrapper.unmount(); - }); - - it('persists page size across resources on the same details page', async () => { - window.location.hash = '#/runs/details/run-abc123'; - const firstWrapper = renderTable({ rows: [], columns, reload: vi.fn(async () => '') }); - fireEvent.mouseDown(getRowsPerPageCombobox()); - fireEvent.click(await screen.findByText('20')); - await TestUtils.flushPromises(); - firstWrapper.unmount(); - const reload = vi.fn(async () => ''); - window.location.hash = '#/runs/details/run-def456'; - const secondWrapper = renderTable({ rows, columns, reload }); - await waitFor(() => - expect(reload).toHaveBeenLastCalledWith({ - filter: '', - orderAscending: false, - pageSize: 20, - pageToken: '', - sortBy: '', - }), - ); - secondWrapper.unmount(); - window.location.hash = ''; - }); }); diff --git a/frontend/src/components/CustomTable.tsx b/frontend/src/components/CustomTable.tsx index eb36b97ca5f..13494a6cfa7 100644 --- a/frontend/src/components/CustomTable.tsx +++ b/frontend/src/components/CustomTable.tsx @@ -24,7 +24,6 @@ import Separator from '../atoms/Separator'; import { ListRequest } from '../lib/Apis'; import { classes, stylesheet } from 'typestyle'; import { fonts, fontsize, dimension, commonCss, color, padding, zIndex } from '../Css'; -import { LocalStorage } from '../lib/LocalStorage'; import { logger } from '../lib/Utils'; import { debounce } from 'lodash'; import { @@ -243,7 +242,7 @@ export default class CustomTable extends React.Component, ): Promise { const pageSize = Number(event.target.value); - LocalStorage.saveTablePageSize(pageSize, this._getPageId()); - this._resetToFirstPage(await this.reload({ pageSize, pageToken: '' })); - } - private _getPageId(): string | undefined { - const pathSegments = window.location.hash - .split('?')[0] - .replace(/^#\//, '') - .split('/') - .filter(Boolean); - if (pathSegments.length === 0) { - return undefined; - } - const [root, second] = pathSegments; - if (root === 'shared' && second === 'pipelines') { - return 'shared/pipelines'; - } - if (root === 'archive' && second) { - return `${root}/${second}`; - } - if (second && ['details', 'new', 'lineage'].includes(second)) { - return `${root}/${second}`; - } - return root; + this._resetToFirstPage(await this.reload({ pageSize, pageToken: '' })); } private _resetToFirstPage(newPageToken?: string): void { diff --git a/frontend/src/components/NewRunParametersV2.test.tsx b/frontend/src/components/NewRunParametersV2.test.tsx index 0bac11934cb..96977dad729 100644 --- a/frontend/src/components/NewRunParametersV2.test.tsx +++ b/frontend/src/components/NewRunParametersV2.test.tsx @@ -837,35 +837,6 @@ describe('NewRunParametersV2', () => { screen.getByDisplayValue('gs://dummy_pipeline_root'); }); - it('shows the cloned pipeline root after remounting with a different runtime config', async () => { - const props = { - titleMessage: 'default Title', - pipelineRoot: 'default pipelineRoot', - specParameters: {}, - clonedRuntimeConfig: {}, - handlePipelineRootChange: vi.fn(), - handleParameterChange: vi.fn(), - setIsValidInput: vi.fn(), - }; - const { rerender } = render(); - - fireEvent.click(screen.getByLabelText('Set custom pipeline root.')); - fireEvent.change(screen.getByLabelText('pipeline-root'), { - target: { value: 'gs://custom_pipeline_root' }, - }); - expect(screen.getByDisplayValue('gs://custom_pipeline_root')).toBeInTheDocument(); - - rerender( - , - ); - - expect(await screen.findByDisplayValue('gs://dummy_pipeline_root')).toBeInTheDocument(); - }); - it('shows parameters from cloned RuntimeConfig', () => { const props = { titleMessage: 'default Title', diff --git a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx index 03f9ab108e1..bf08cd80fa2 100644 --- a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx +++ b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx @@ -62,6 +62,9 @@ import { getComponentSpec } from 'src/lib/v2/NodeUtils'; export const LOGS_DETAILS = 'logs_details'; export const LOGS_BANNER_MESSAGE = 'logs_banner_message'; export const LOGS_BANNER_ADDITIONAL_INFO = 'logs_banner_additional_info'; +export const SYS_LOGS_DETAILS = 'sys_log_details'; +export const SYS_LOGS_BANNER_MESSAGE = 'sys_logs_banner_message'; +export const SYS_LOGS_BANNER_ADDITIONAL_INFO = 'sys_logs_banner_additional_info'; export const K8S_PLATFORM_KEY = 'kubernetes'; const NODE_INFO_UNKNOWN = ( @@ -170,19 +173,33 @@ function TaskNodeDetail({ enabled: !!execution, }); + const { data: driverLogsInfo } = useQuery({ + queryKey: queryKeys.driverLogs(execution?.getId(), namespace), + queryFn: async (): Promise> => { + if (!execution) { + throw new Error('No execution is found.'); + } + return getDriverLogsInfo(execution, namespace); + }, + enabled: !!execution, + }); + const logsDetails = logsInfo?.get(LOGS_DETAILS); const logsBannerMessage = logsInfo?.get(LOGS_BANNER_MESSAGE) || (logsQueryFailed ? 'Failed to retrieve pod logs.' : undefined); const logsBannerAdditionalInfo = logsInfo?.get(LOGS_BANNER_ADDITIONAL_INFO) || logsQueryError?.message; + const sysLogDetails = driverLogsInfo?.get(SYS_LOGS_DETAILS); + const sysLogsBannerMessage = driverLogsInfo?.get(SYS_LOGS_BANNER_MESSAGE); + const sysLogsBannerAdditionalInfo = driverLogsInfo?.get(SYS_LOGS_BANNER_ADDITIONAL_INFO); const [selectedTab, setSelectedTab] = useState(0); return (
setSelectedTab(tab)} /> @@ -219,6 +236,24 @@ function TaskNodeDetail({ )}
)} + {/* System Logs tab */} + {selectedTab === 3 && ( +
+ {sysLogsBannerMessage && ( + + + + )} + {!sysLogsBannerMessage && ( +
+ +
+ )} +
+ )} ); @@ -306,6 +341,51 @@ function getNodeVolumeMounts( return volumeMounts; } +async function getDriverLogsInfo( + execution: Execution, + namespace?: string, +): Promise> { + const logsInfo = new Map(); + let podNameSpace = ''; + let logsDetails = ''; + let logsBannerMessage = ''; + let logsBannerAdditionalInfo = ''; + const customPropertiesMap = execution.getCustomPropertiesMap(); + + if (execution) { + podNameSpace = customPropertiesMap.get('namespace')?.getStringValue() || ''; + } + + try { + const driverLogUri = customPropertiesMap.get('driver_logs_uri')?.getStringValue(); + const storeSessionInfo = customPropertiesMap.get('store_session_info')?.getStringValue(); + + if (driverLogUri && storeSessionInfo) { + const storagePath = WorkflowParser.parseStoragePath(driverLogUri); + const providerInfo = storeSessionInfo; + const artifactNamespace = namespace || podNameSpace; + + logsDetails = await Apis.readFile({ + path: storagePath, + providerInfo: providerInfo, + namespace: artifactNamespace, + }); + logsInfo.set(SYS_LOGS_DETAILS, logsDetails); + return logsInfo; + } + } catch (artifactErr) { + let errMsg = await errorToMessage(artifactErr); + logsBannerMessage = 'Failed to retrieve system logs.'; + logsInfo.set(SYS_LOGS_BANNER_MESSAGE, logsBannerMessage); + logsBannerAdditionalInfo = 'Error response: ' + errMsg; + logsInfo.set(SYS_LOGS_BANNER_ADDITIONAL_INFO, logsBannerAdditionalInfo); + + console.error('Failed to retrieve driver-logs artifact:', artifactErr); + } + + return logsInfo; +} + async function getLogsInfo( execution: Execution, runId?: string, diff --git a/frontend/src/components/viewers/MetricsDropdown.tsx b/frontend/src/components/viewers/MetricsDropdown.tsx index e93ddb67598..5f2ffa5e11a 100644 --- a/frontend/src/components/viewers/MetricsDropdown.tsx +++ b/frontend/src/components/viewers/MetricsDropdown.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { color, commonCss, fontsize, zIndex } from 'src/Css'; import { queryKeys } from 'src/hooks/queryKeys'; import { classes, stylesheet } from 'typestyle'; @@ -98,41 +98,32 @@ export default function MetricsDropdown(props: MetricsDropdownProps) { const [secondSelectedItem, setSecondSelectedItem] = useState( selectedArtifacts[1].selectedItem, ); + const [firstLinkedArtifact, setFirstLinkedArtifact] = useState( + selectedArtifacts[0].linkedArtifact, + ); + const [secondLinkedArtifact, setSecondLinkedArtifact] = useState( + selectedArtifacts[1].linkedArtifact, + ); useEffect(() => { setFirstSelectedItem(selectedArtifacts[0].selectedItem); setSecondSelectedItem(selectedArtifacts[1].selectedItem); + setFirstLinkedArtifact(selectedArtifacts[0].linkedArtifact); + setSecondLinkedArtifact(selectedArtifacts[1].linkedArtifact); }, [selectedArtifacts]); - const selectedArtifactsForDisplay = useMemo( - () => [ - { - selectedItem: firstSelectedItem, - linkedArtifact: getLinkedArtifactFromSelectedItem(filteredRunArtifacts, firstSelectedItem), - }, - { - selectedItem: secondSelectedItem, - linkedArtifact: getLinkedArtifactFromSelectedItem(filteredRunArtifacts, secondSelectedItem), - }, - ], - [filteredRunArtifacts, firstSelectedItem, secondSelectedItem], - ); - const metricsTabText = metricsTypeToString(metricsTab); const updateSelectedItemAndArtifact = ( setSelectedItem: (selectedItem: SelectedItem) => void, + setLinkedArtifact: (linkedArtifact: LinkedArtifact | undefined) => void, panelIndex: number, selectedItem: SelectedItem, ): void => { setSelectedItem(selectedItem); const linkedArtifact = getLinkedArtifactFromSelectedItem(filteredRunArtifacts, selectedItem); - const nextSelectedArtifacts = selectedArtifactsForDisplay.map((selectedArtifact, index) => - index === panelIndex - ? { - selectedItem, - linkedArtifact, - } - : selectedArtifact, + setLinkedArtifact(linkedArtifact); + const nextSelectedArtifacts = selectedArtifacts.map((selectedArtifact, index) => + index === panelIndex ? { selectedItem, linkedArtifact } : selectedArtifact, ); updateSelectedArtifacts(nextSelectedArtifacts); }; @@ -151,12 +142,17 @@ export default function MetricsDropdown(props: MetricsDropdownProps) { title={`Choose a first ${metricsTabText} artifact`} items={dropdownItems} selectedItem={firstSelectedItem} - setSelectedItem={updateSelectedItemAndArtifact.bind(null, setFirstSelectedItem, 0)} + setSelectedItem={updateSelectedItemAndArtifact.bind( + null, + setFirstSelectedItem, + setFirstLinkedArtifact, + 0, + )} /> @@ -165,12 +161,17 @@ export default function MetricsDropdown(props: MetricsDropdownProps) { title={`Choose a second ${metricsTabText} artifact`} items={dropdownItems} selectedItem={secondSelectedItem} - setSelectedItem={updateSelectedItemAndArtifact.bind(null, setSecondSelectedItem, 1)} + setSelectedItem={updateSelectedItemAndArtifact.bind( + null, + setSecondSelectedItem, + setSecondLinkedArtifact, + 1, + )} /> diff --git a/frontend/src/generated/openapi/models/GooglerpcStatus.ts b/frontend/src/generated/openapi/models/GooglerpcStatus.ts deleted file mode 100644 index 2b88c316260..00000000000 --- a/frontend/src/generated/openapi/models/GooglerpcStatus.ts +++ /dev/null @@ -1,64 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -import { mapValues } from '../runtime'; -import type { ProtobufAny } from './ProtobufAny'; -import { - ProtobufAnyFromJSON, - ProtobufAnyFromJSONTyped, - ProtobufAnyToJSON, - ProtobufAnyToJSONTyped, -} from './ProtobufAny'; - -export interface GooglerpcStatus { - code?: number; - message?: string; - details?: Array; -} - -export function instanceOfGooglerpcStatus(value: object): value is GooglerpcStatus { - return true; -} - -export function GooglerpcStatusFromJSON(json: any): GooglerpcStatus { - return GooglerpcStatusFromJSONTyped(json, false); -} - -export function GooglerpcStatusFromJSONTyped( - json: any, - ignoreDiscriminator: boolean, -): GooglerpcStatus { - if (json == null) { - return json; - } - return { - code: json['code'] == null ? undefined : json['code'], - message: json['message'] == null ? undefined : json['message'], - details: - json['details'] == null - ? undefined - : (json['details'] as Array).map(ProtobufAnyFromJSON), - }; -} - -export function GooglerpcStatusToJSON(json: any): GooglerpcStatus { - return GooglerpcStatusToJSONTyped(json, false); -} - -export function GooglerpcStatusToJSONTyped( - value?: GooglerpcStatus | null, - ignoreDiscriminator: boolean = false, -): any { - if (value == null) { - return value; - } - - return { - code: value['code'], - message: value['message'], - details: - value['details'] == null - ? undefined - : (value['details'] as Array).map(ProtobufAnyToJSON), - }; -} diff --git a/frontend/src/generated/openapi/models/ProtobufAny.ts b/frontend/src/generated/openapi/models/ProtobufAny.ts deleted file mode 100644 index d9e94003c27..00000000000 --- a/frontend/src/generated/openapi/models/ProtobufAny.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -import { mapValues } from '../runtime'; -export interface ProtobufAny { - [key: string]: object | any; - type?: string; -} - -export function instanceOfProtobufAny(value: object): value is ProtobufAny { - return true; -} - -export function ProtobufAnyFromJSON(json: any): ProtobufAny { - return ProtobufAnyFromJSONTyped(json, false); -} - -export function ProtobufAnyFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProtobufAny { - if (json == null) { - return json; - } - return { - ...json, - type: json['@type'] == null ? undefined : json['@type'], - }; -} - -export function ProtobufAnyToJSON(json: any): ProtobufAny { - return ProtobufAnyToJSONTyped(json, false); -} - -export function ProtobufAnyToJSONTyped( - value?: ProtobufAny | null, - ignoreDiscriminator: boolean = false, -): any { - if (value == null) { - return value; - } - - return { - ...value, - '@type': value['type'], - }; -} diff --git a/frontend/src/generated/openapi/runtime.ts b/frontend/src/generated/openapi/runtime.ts deleted file mode 100644 index a4d25c02b6d..00000000000 --- a/frontend/src/generated/openapi/runtime.ts +++ /dev/null @@ -1,462 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -// Shared OpenAPI support generated by `npm run apis:all`. Do not edit manually. -export const BASE_PATH = 'http://localhost'.replace(/\/+$/, ''); - -export interface ConfigurationParameters { - basePath?: string; // override base path - fetchApi?: FetchAPI; // override for fetch implementation - middleware?: Middleware[]; // middleware to apply before/after fetch requests - queryParamsStringify?: (params: HTTPQuery) => string; // stringify function for query strings - username?: string; // parameter for basic security - password?: string; // parameter for basic security - apiKey?: string | Promise | ((name: string) => string | Promise); // parameter for apiKey security - accessToken?: - | string - | Promise - | ((name?: string, scopes?: string[]) => string | Promise); // parameter for oauth2 security - headers?: HTTPHeaders; //header params we want to use on every request - credentials?: RequestCredentials; //value for the credentials param we want to use on each request -} - -export class Configuration { - constructor(private configuration: ConfigurationParameters = {}) {} - - set config(configuration: Configuration) { - this.configuration = configuration; - } - - get basePath(): string { - return this.configuration.basePath != null ? this.configuration.basePath : BASE_PATH; - } - - get fetchApi(): FetchAPI | undefined { - return this.configuration.fetchApi; - } - - get middleware(): Middleware[] { - return this.configuration.middleware || []; - } - - get queryParamsStringify(): (params: HTTPQuery) => string { - return this.configuration.queryParamsStringify || querystring; - } - - get username(): string | undefined { - return this.configuration.username; - } - - get password(): string | undefined { - return this.configuration.password; - } - - get apiKey(): ((name: string) => string | Promise) | undefined { - const apiKey = this.configuration.apiKey; - if (apiKey) { - return typeof apiKey === 'function' ? apiKey : () => apiKey; - } - return undefined; - } - - get accessToken(): ((name?: string, scopes?: string[]) => string | Promise) | undefined { - const accessToken = this.configuration.accessToken; - if (accessToken) { - return typeof accessToken === 'function' ? accessToken : async () => accessToken; - } - return undefined; - } - - get headers(): HTTPHeaders | undefined { - return this.configuration.headers; - } - - get credentials(): RequestCredentials | undefined { - return this.configuration.credentials; - } -} - -export const DefaultConfig = new Configuration(); - -export class BaseAPI { - private static readonly jsonRegex = new RegExp( - '^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', - 'i', - ); - private middleware: Middleware[]; - - constructor(protected configuration = DefaultConfig) { - this.middleware = configuration.middleware; - } - - withMiddleware(this: T, ...middlewares: Middleware[]) { - const next = this.clone(); - next.middleware = next.middleware.concat(...middlewares); - return next; - } - - withPreMiddleware(this: T, ...preMiddlewares: Array) { - const middlewares = preMiddlewares.map((pre) => ({ pre })); - return this.withMiddleware(...middlewares); - } - - withPostMiddleware(this: T, ...postMiddlewares: Array) { - const middlewares = postMiddlewares.map((post) => ({ post })); - return this.withMiddleware(...middlewares); - } - - protected isJsonMime(mime: string | null | undefined): boolean { - if (!mime) { - return false; - } - return BaseAPI.jsonRegex.test(mime); - } - - protected async request( - context: RequestOpts, - initOverrides?: RequestInit | InitOverrideFunction, - ): Promise { - const { url, init } = await this.createFetchParams(context, initOverrides); - const response = await this.fetchApi(url, init); - if (response && response.status >= 200 && response.status < 300) { - return response; - } - throw new ResponseError(response, 'Response returned an error code'); - } - - private async createFetchParams( - context: RequestOpts, - initOverrides?: RequestInit | InitOverrideFunction, - ) { - let url = this.configuration.basePath + context.path; - if (context.query !== undefined && Object.keys(context.query).length !== 0) { - // only add the querystring to the URL if there are query parameters. - // this is done to avoid urls ending with a "?" character which buggy webservers - // do not handle correctly sometimes. - url += '?' + this.configuration.queryParamsStringify(context.query); - } - - const headers = Object.assign({}, this.configuration.headers, context.headers); - Object.keys(headers).forEach((key) => (headers[key] === undefined ? delete headers[key] : {})); - - const initOverrideFn = - typeof initOverrides === 'function' ? initOverrides : async () => initOverrides; - - const initParams = { - method: context.method, - headers, - body: context.body, - credentials: this.configuration.credentials, - }; - - const overriddenInit: RequestInit = { - ...initParams, - ...(await initOverrideFn({ - init: initParams, - context, - })), - }; - - let body: any; - if ( - isFormData(overriddenInit.body) || - overriddenInit.body instanceof URLSearchParams || - isBlob(overriddenInit.body) - ) { - body = overriddenInit.body; - } else if (this.isJsonMime(headers['Content-Type'])) { - body = JSON.stringify(overriddenInit.body); - } else { - body = overriddenInit.body; - } - - const init: RequestInit = { - ...overriddenInit, - body, - }; - - return { url, init }; - } - - private fetchApi = async (url: string, init: RequestInit) => { - let fetchParams = { url, init }; - for (const middleware of this.middleware) { - if (middleware.pre) { - fetchParams = - (await middleware.pre({ - fetch: this.fetchApi, - ...fetchParams, - })) || fetchParams; - } - } - let response: Response | undefined = undefined; - try { - response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init); - } catch (e) { - for (const middleware of this.middleware) { - if (middleware.onError) { - response = - (await middleware.onError({ - fetch: this.fetchApi, - url: fetchParams.url, - init: fetchParams.init, - error: e, - response: response ? response.clone() : undefined, - })) || response; - } - } - if (response === undefined) { - if (e instanceof Error) { - throw new FetchError( - e, - 'The request failed and the interceptors did not return an alternative response', - ); - } else { - throw e; - } - } - } - for (const middleware of this.middleware) { - if (middleware.post) { - response = - (await middleware.post({ - fetch: this.fetchApi, - url: fetchParams.url, - init: fetchParams.init, - response: response.clone(), - })) || response; - } - } - return response; - }; - - private clone(this: T): T { - const constructor = this.constructor as any; - const next = new constructor(this.configuration); - next.middleware = this.middleware.slice(); - return next; - } -} - -function isBlob(value: any): value is Blob { - return typeof Blob !== 'undefined' && value instanceof Blob; -} - -function isFormData(value: any): value is FormData { - return typeof FormData !== 'undefined' && value instanceof FormData; -} - -export class ResponseError extends Error { - override name: 'ResponseError' = 'ResponseError'; - constructor( - public response: Response, - msg?: string, - ) { - super(msg); - } -} - -export class FetchError extends Error { - override name: 'FetchError' = 'FetchError'; - constructor( - public cause: Error, - msg?: string, - ) { - super(msg); - } -} - -export class RequiredError extends Error { - override name: 'RequiredError' = 'RequiredError'; - constructor( - public field: string, - msg?: string, - ) { - super(msg); - } -} - -export const COLLECTION_FORMATS = { - csv: ',', - ssv: ' ', - tsv: '\t', - pipes: '|', -}; - -export type FetchAPI = WindowOrWorkerGlobalScope['fetch']; - -export type Json = any; -export type HTTPMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD'; -export type HTTPHeaders = { [key: string]: string }; -export type HTTPQuery = { - [key: string]: - | string - | number - | null - | boolean - | Array - | Set - | HTTPQuery; -}; -export type HTTPBody = Json | FormData | URLSearchParams; -export type HTTPRequestInit = { - headers?: HTTPHeaders; - method: HTTPMethod; - credentials?: RequestCredentials; - body?: HTTPBody; -}; -export type ModelPropertyNaming = 'camelCase' | 'snake_case' | 'PascalCase' | 'original'; - -export type InitOverrideFunction = (requestContext: { - init: HTTPRequestInit; - context: RequestOpts; -}) => Promise; - -export interface FetchParams { - url: string; - init: RequestInit; -} - -export interface RequestOpts { - path: string; - method: HTTPMethod; - headers: HTTPHeaders; - query?: HTTPQuery; - body?: HTTPBody; -} - -export function querystring(params: HTTPQuery, prefix: string = ''): string { - return Object.keys(params) - .map((key) => querystringSingleKey(key, params[key], prefix)) - .filter((part) => part.length > 0) - .join('&'); -} - -function querystringSingleKey( - key: string, - value: - | string - | number - | null - | undefined - | boolean - | Array - | Set - | HTTPQuery, - keyPrefix: string = '', -): string { - const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key); - if (value instanceof Array) { - const multiValue = value - .map((singleValue) => encodeURIComponent(String(singleValue))) - .join(`&${encodeURIComponent(fullKey)}=`); - return `${encodeURIComponent(fullKey)}=${multiValue}`; - } - if (value instanceof Set) { - const valueAsArray = Array.from(value); - return querystringSingleKey(key, valueAsArray, keyPrefix); - } - if (value instanceof Date) { - return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`; - } - if (value instanceof Object) { - return querystring(value as HTTPQuery, fullKey); - } - return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`; -} - -export function exists(json: any, key: string) { - const value = json[key]; - return value !== null && value !== undefined; -} - -export function mapValues(data: any, fn: (item: any) => any) { - const result: { [key: string]: any } = {}; - for (const key of Object.keys(data)) { - result[key] = fn(data[key]); - } - return result; -} - -export function canConsumeForm(consumes: Consume[]): boolean { - for (const consume of consumes) { - if ('multipart/form-data' === consume.contentType) { - return true; - } - } - return false; -} - -export interface Consume { - contentType: string; -} - -export interface RequestContext { - fetch: FetchAPI; - url: string; - init: RequestInit; -} - -export interface ResponseContext { - fetch: FetchAPI; - url: string; - init: RequestInit; - response: Response; -} - -export interface ErrorContext { - fetch: FetchAPI; - url: string; - init: RequestInit; - error: unknown; - response?: Response; -} - -export interface Middleware { - pre?(context: RequestContext): Promise; - post?(context: ResponseContext): Promise; - onError?(context: ErrorContext): Promise; -} - -export interface ApiResponse { - raw: Response; - value(): Promise; -} - -export interface ResponseTransformer { - (json: any): T; -} - -export class JSONApiResponse { - constructor( - public raw: Response, - private transformer: ResponseTransformer = (jsonValue: any) => jsonValue, - ) {} - - async value(): Promise { - return this.transformer(await this.raw.json()); - } -} - -export class VoidApiResponse { - constructor(public raw: Response) {} - - async value(): Promise { - return undefined; - } -} - -export class BlobApiResponse { - constructor(public raw: Response) {} - - async value(): Promise { - return await this.raw.blob(); - } -} - -export class TextApiResponse { - constructor(public raw: Response) {} - - async value(): Promise { - return await this.raw.text(); - } -} diff --git a/frontend/src/hooks/queryKeys.ts b/frontend/src/hooks/queryKeys.ts index cc70d74effd..f229524fba2 100644 --- a/frontend/src/hooks/queryKeys.ts +++ b/frontend/src/hooks/queryKeys.ts @@ -65,6 +65,9 @@ export const queryKeys = { executionLogs: (executionId: number | undefined, namespace: string | undefined) => ['execution_logs', { executionId, namespace }] as const, + driverLogs: (executionId: number | undefined, namespace: string | undefined) => + ['driver_logs', { executionId, namespace }] as const, + contextByExecution: (executionId: number, executionState: number) => ['context_by_execution', { id: executionId, state: executionState }] as const, diff --git a/frontend/src/lib/LocalStorage.test.ts b/frontend/src/lib/LocalStorage.test.ts index 170c8d08323..ec4bb3f414e 100644 --- a/frontend/src/lib/LocalStorage.test.ts +++ b/frontend/src/lib/LocalStorage.test.ts @@ -38,37 +38,4 @@ describe('LocalStorage', () => { LocalStorage.saveNavbarCollapsed(true); expect(LocalStorage.isNavbarCollapsed()).toBe(true); }); - - it('returns default page size of 10 when no value is saved', () => { - expect(LocalStorage.getTablePageSize()).toBe(10); - }); - - it('reads and writes table page size', () => { - LocalStorage.saveTablePageSize(50); - expect(LocalStorage.getTablePageSize()).toBe(50); - }); - - it('returns default page size for invalid stored values', () => { - localStorage.setItem(LocalStorageKey.tablePageSize, '25'); - expect(LocalStorage.getTablePageSize()).toBe(10); - }); - - it('accepts all valid page size options', () => { - for (const size of [10, 20, 50, 100]) { - LocalStorage.saveTablePageSize(size); - expect(LocalStorage.getTablePageSize()).toBe(size); - } - }); - - it('stores page size per page independently', () => { - LocalStorage.saveTablePageSize(50, 'runs'); - LocalStorage.saveTablePageSize(20, 'experiments'); - expect(LocalStorage.getTablePageSize('runs')).toBe(50); - expect(LocalStorage.getTablePageSize('experiments')).toBe(20); - }); - - it('returns default for a page with no saved value', () => { - LocalStorage.saveTablePageSize(50, 'runs'); - expect(LocalStorage.getTablePageSize('pipelines')).toBe(10); - }); }); diff --git a/frontend/src/lib/LocalStorage.ts b/frontend/src/lib/LocalStorage.ts index 6e795a98b5f..3995db8d080 100644 --- a/frontend/src/lib/LocalStorage.ts +++ b/frontend/src/lib/LocalStorage.ts @@ -16,7 +16,6 @@ export enum LocalStorageKey { navbarCollapsed = 'navbarCollapsed', - tablePageSize = 'tablePageSize', } export class LocalStorage { @@ -31,21 +30,4 @@ export class LocalStorage { public static saveNavbarCollapsed(value: boolean): void { localStorage.setItem(LocalStorageKey.navbarCollapsed, value.toString()); } - - public static getTablePageSize(pageId?: string): number { - const key = pageId - ? `${LocalStorageKey.tablePageSize}_${pageId}` - : LocalStorageKey.tablePageSize; - const value = localStorage.getItem(key); - const parsed = Number(value); - return [10, 20, 50, 100].includes(parsed) ? parsed : 10; - } - - public static saveTablePageSize(value: number, pageId?: string): void { - const normalizedValue = [10, 20, 50, 100].includes(value) ? value : 10; - const key = pageId - ? `${LocalStorageKey.tablePageSize}_${pageId}` - : LocalStorageKey.tablePageSize; - localStorage.setItem(key, normalizedValue.toString()); - } } diff --git a/frontend/src/pages/CompareV2.test.tsx b/frontend/src/pages/CompareV2.test.tsx index b1e7c795c6d..ae97732632d 100644 --- a/frontend/src/pages/CompareV2.test.tsx +++ b/frontend/src/pages/CompareV2.test.tsx @@ -28,7 +28,6 @@ import { PageProps } from './Page'; import { METRICS_SECTION_NAME, OVERVIEW_SECTION_NAME, PARAMS_SECTION_NAME } from './Compare'; import { Struct, Value } from 'google-protobuf/google/protobuf/struct_pb'; import { V2beta1Run, V2beta1RuntimeState } from 'src/apisv2beta1/run'; -import { MetricsType } from 'src/lib/v2/CompareUtils'; import { vi } from 'vitest'; const CompareV2 = TEST_ONLY.CompareV2; @@ -232,158 +231,6 @@ describe('CompareV2', () => { screen.getByText(OVERVIEW_SECTION_NAME); }); - it('does not mark ROC selection initialized before ROC artifacts are available', () => { - const initialSelection = TEST_ONLY.createInitialRocCurveSelectionState(); - - expect(TEST_ONLY.reconcileRocCurveSelectionState(initialSelection, [], new Set())).toBe( - initialSelection, - ); - }); - - it('reconciles invalid two-panel artifact selections against the available run artifacts', () => { - const selectedArtifactsMap = { - [MetricsType.CONFUSION_MATRIX]: [ - { - selectedItem: { - itemName: `test run ${MOCK_RUN_2_ID}`, - subItemName: 'artifactName', - }, - }, - { - selectedItem: { - itemName: 'missing run', - subItemName: 'staleArtifact', - }, - }, - ], - [MetricsType.HTML]: [ - { - selectedItem: { - itemName: `test run ${MOCK_RUN_1_ID}`, - subItemName: 'firstHtmlArtifact', - }, - }, - { - selectedItem: { - itemName: '', - subItemName: '', - }, - }, - ], - [MetricsType.MARKDOWN]: [ - { - selectedItem: { - itemName: '', - subItemName: '', - }, - }, - { - selectedItem: { - itemName: '', - subItemName: '', - }, - }, - ], - }; - - const reconciledArtifactsMap = TEST_ONLY.reconcileSelectedArtifactsMap(selectedArtifactsMap, { - scalarMetricsTableData: undefined, - confusionMatrixRunArtifacts: [ - { run: newMockRun(MOCK_RUN_2_ID), executionArtifacts: [] as any }, - ], - htmlRunArtifacts: [{ run: newMockRun(MOCK_RUN_1_ID), executionArtifacts: [] as any }], - markdownRunArtifacts: [], - rocCurveRunArtifacts: [], - }); - - expect(reconciledArtifactsMap[MetricsType.CONFUSION_MATRIX][0].selectedItem).toEqual({ - itemName: `test run ${MOCK_RUN_2_ID}`, - subItemName: 'artifactName', - }); - expect(reconciledArtifactsMap[MetricsType.CONFUSION_MATRIX][1].selectedItem).toEqual({ - itemName: '', - subItemName: '', - }); - expect(reconciledArtifactsMap[MetricsType.HTML][0].selectedItem).toEqual({ - itemName: `test run ${MOCK_RUN_1_ID}`, - subItemName: 'firstHtmlArtifact', - }); - expect(reconciledArtifactsMap[MetricsType.MARKDOWN][0].selectedItem).toEqual({ - itemName: '', - subItemName: '', - }); - expect(reconciledArtifactsMap[MetricsType.MARKDOWN][1].selectedItem).toEqual({ - itemName: '', - subItemName: '', - }); - }); - - it('keeps both two-panel selections when they reference the same run', () => { - const selectedArtifactsMap = { - [MetricsType.CONFUSION_MATRIX]: [ - { - selectedItem: { - itemName: `test run ${MOCK_RUN_2_ID}`, - subItemName: 'firstArtifact', - }, - }, - { - selectedItem: { - itemName: `test run ${MOCK_RUN_2_ID}`, - subItemName: 'secondArtifact', - }, - }, - ], - [MetricsType.HTML]: [ - { - selectedItem: { - itemName: '', - subItemName: '', - }, - }, - { - selectedItem: { - itemName: '', - subItemName: '', - }, - }, - ], - [MetricsType.MARKDOWN]: [ - { - selectedItem: { - itemName: '', - subItemName: '', - }, - }, - { - selectedItem: { - itemName: '', - subItemName: '', - }, - }, - ], - }; - - const reconciledArtifactsMap = TEST_ONLY.reconcileSelectedArtifactsMap(selectedArtifactsMap, { - scalarMetricsTableData: undefined, - confusionMatrixRunArtifacts: [ - { run: newMockRun(MOCK_RUN_2_ID), executionArtifacts: [] as any }, - ], - htmlRunArtifacts: [], - markdownRunArtifacts: [], - rocCurveRunArtifacts: [], - }); - - expect(reconciledArtifactsMap[MetricsType.CONFUSION_MATRIX][0].selectedItem).toEqual({ - itemName: `test run ${MOCK_RUN_2_ID}`, - subItemName: 'firstArtifact', - }); - expect(reconciledArtifactsMap[MetricsType.CONFUSION_MATRIX][1].selectedItem).toEqual({ - itemName: `test run ${MOCK_RUN_2_ID}`, - subItemName: 'secondArtifact', - }); - }); - it('getRun is called with query param IDs', async () => { const getRunSpy = vi.spyOn(Apis.runServiceApiV2, 'getRun'); runs = [newMockRun(MOCK_RUN_1_ID), newMockRun(MOCK_RUN_2_ID), newMockRun(MOCK_RUN_3_ID)]; diff --git a/frontend/src/pages/CompareV2.tsx b/frontend/src/pages/CompareV2.tsx index 493deac2416..8f7e6af3984 100644 --- a/frontend/src/pages/CompareV2.tsx +++ b/frontend/src/pages/CompareV2.tsx @@ -236,200 +236,6 @@ interface CompareV2Namespace { export type CompareV2Props = PageProps & CompareV2Namespace; -interface DerivedMetricsArtifacts { - scalarMetricsTableData?: CompareTableProps; - confusionMatrixRunArtifacts: RunArtifact[]; - htmlRunArtifacts: RunArtifact[]; - markdownRunArtifacts: RunArtifact[]; - rocCurveRunArtifacts: RunArtifact[]; -} - -interface RocCurveSelectionState { - hasInitialized: boolean; - selectedIds: string[]; - selectedIdColorMap: RocCurveColorMap; - lineColorsStack: string[]; -} - -const createSelectedArtifactArray = (count: number): SelectedArtifact[] => { - const array: SelectedArtifact[] = []; - for (let i = 0; i < count; i++) { - array.push({ - selectedItem: { itemName: '', subItemName: '' }, - }); - } - return array; -}; - -const createInitialSelectedArtifactsMap = () => ({ - [MetricsType.CONFUSION_MATRIX]: createSelectedArtifactArray(2), - [MetricsType.HTML]: createSelectedArtifactArray(2), - [MetricsType.MARKDOWN]: createSelectedArtifactArray(2), -}); - -const createInitialRocCurveSelectionState = (): RocCurveSelectionState => ({ - hasInitialized: false, - selectedIds: [], - selectedIdColorMap: {}, - lineColorsStack: [...lineColors].reverse(), -}); - -const areSelectedArtifactsEqual = ( - currentArtifacts: SelectedArtifact[], - nextArtifacts: SelectedArtifact[], -) => - currentArtifacts.length === nextArtifacts.length && - currentArtifacts.every((currentArtifact, index) => { - const nextArtifact = nextArtifacts[index]; - return ( - currentArtifact.selectedItem.itemName === nextArtifact.selectedItem.itemName && - currentArtifact.selectedItem.subItemName === nextArtifact.selectedItem.subItemName && - currentArtifact.linkedArtifact?.artifact.getId() === - nextArtifact.linkedArtifact?.artifact.getId() - ); - }); - -// Ensure that the two-panel selected artifacts are present in the selected valid run list. -const getVerifiedTwoPanelSelection = ( - runArtifacts: RunArtifact[], - selectedArtifacts: SelectedArtifact[], -) => { - const artifactsPresent: boolean[] = new Array(2).fill(false); - for (const runArtifact of runArtifacts) { - const runName = runArtifact.run.display_name; - if (runName === selectedArtifacts[0].selectedItem.itemName) { - artifactsPresent[0] = true; - } - if (runName === selectedArtifacts[1].selectedItem.itemName) { - artifactsPresent[1] = true; - } - } - - return selectedArtifacts.map((selectedArtifact, index) => { - if (artifactsPresent[index]) { - return selectedArtifact; - } - return { - selectedItem: { - itemName: '', - subItemName: '', - }, - }; - }); -}; - -const reconcileSelectedArtifactsMap = ( - currentSelectedArtifactsMap: { [key: string]: SelectedArtifact[] }, - metricsArtifactData?: DerivedMetricsArtifacts, -) => { - if (!metricsArtifactData) { - return currentSelectedArtifactsMap; - } - - const nextSelectedArtifactsMap = { - ...currentSelectedArtifactsMap, - [MetricsType.CONFUSION_MATRIX]: getVerifiedTwoPanelSelection( - metricsArtifactData.confusionMatrixRunArtifacts, - currentSelectedArtifactsMap[MetricsType.CONFUSION_MATRIX], - ), - [MetricsType.HTML]: getVerifiedTwoPanelSelection( - metricsArtifactData.htmlRunArtifacts, - currentSelectedArtifactsMap[MetricsType.HTML], - ), - [MetricsType.MARKDOWN]: getVerifiedTwoPanelSelection( - metricsArtifactData.markdownRunArtifacts, - currentSelectedArtifactsMap[MetricsType.MARKDOWN], - ), - }; - - if ( - areSelectedArtifactsEqual( - currentSelectedArtifactsMap[MetricsType.CONFUSION_MATRIX], - nextSelectedArtifactsMap[MetricsType.CONFUSION_MATRIX], - ) && - areSelectedArtifactsEqual( - currentSelectedArtifactsMap[MetricsType.HTML], - nextSelectedArtifactsMap[MetricsType.HTML], - ) && - areSelectedArtifactsEqual( - currentSelectedArtifactsMap[MetricsType.MARKDOWN], - nextSelectedArtifactsMap[MetricsType.MARKDOWN], - ) - ) { - return currentSelectedArtifactsMap; - } - - return nextSelectedArtifactsMap; -}; - -const reconcileRocCurveSelectionState = ( - currentSelection: RocCurveSelectionState, - linkedArtifacts: LinkedArtifact[], - validRocCurveIdSet: Set, -): RocCurveSelectionState => { - if (!currentSelection.hasInitialized) { - if (linkedArtifacts.length === 0) { - return currentSelection; - } - - const nextLineColorsStack = [...currentSelection.lineColorsStack]; - const nextSelectedIdColorMap = { ...currentSelection.selectedIdColorMap }; - const nextSelectedIds = linkedArtifacts - .map((linkedArtifact) => getRocCurveId(linkedArtifact)) - .slice(0, 3); - - nextSelectedIds.forEach((rocCurveId) => { - nextSelectedIdColorMap[rocCurveId] = nextLineColorsStack.pop()!; - }); - - return { - hasInitialized: true, - selectedIds: nextSelectedIds, - selectedIdColorMap: nextSelectedIdColorMap, - lineColorsStack: nextLineColorsStack, - }; - } - - const nextSelectedIds = currentSelection.selectedIds.filter((rocCurveId) => - validRocCurveIdSet.has(rocCurveId), - ); - const nextSelectedIdColorMap = { ...currentSelection.selectedIdColorMap }; - const nextLineColorsStack = [...currentSelection.lineColorsStack]; - - Object.keys(currentSelection.selectedIdColorMap).forEach((selectedId) => { - if (!validRocCurveIdSet.has(selectedId)) { - nextLineColorsStack.push(currentSelection.selectedIdColorMap[selectedId]); - delete nextSelectedIdColorMap[selectedId]; - } - }); - - const selectedIdsUnchanged = - currentSelection.selectedIds.length === nextSelectedIds.length && - currentSelection.selectedIds.every( - (selectedId, index) => selectedId === nextSelectedIds[index], - ); - const selectedColorsUnchanged = - Object.keys(currentSelection.selectedIdColorMap).length === - Object.keys(nextSelectedIdColorMap).length && - Object.entries(currentSelection.selectedIdColorMap).every( - ([selectedId, color]) => nextSelectedIdColorMap[selectedId] === color, - ); - const lineColorsUnchanged = - currentSelection.lineColorsStack.length === nextLineColorsStack.length && - currentSelection.lineColorsStack.every((color, index) => color === nextLineColorsStack[index]); - - if (selectedIdsUnchanged && selectedColorsUnchanged && lineColorsUnchanged) { - return currentSelection; - } - - return { - hasInitialized: true, - selectedIds: nextSelectedIds, - selectedIdColorMap: nextSelectedIdColorMap, - lineColorsStack: nextLineColorsStack, - }; -}; - function CompareV2(props: CompareV2Props) { const { updateBanner, updateToolbar, namespace } = props; @@ -439,15 +245,44 @@ function CompareV2(props: CompareV2Props) { const [isOverviewCollapsed, setIsOverviewCollapsed] = useState(false); const [isParamsCollapsed, setIsParamsCollapsed] = useState(false); const [isMetricsCollapsed, setIsMetricsCollapsed] = useState(false); + const [isLoadingArtifacts, setIsLoadingArtifacts] = useState(true); + const [isInitialArtifactsLoad, setIsInitialArtifactsLoad] = useState(true); const selectionRunIdsKeyRef = useRef(''); - const [rocCurveSelection, setRocCurveSelection] = useState( - createInitialRocCurveSelectionState, - ); + + // Scalar Metrics + const [scalarMetricsTableData, setScalarMetricsTableData] = useState< + CompareTableProps | undefined + >(undefined); + + // ROC Curve + const [rocCurveLinkedArtifacts, setRocCurveLinkedArtifacts] = useState([]); + const [selectedRocCurveIds, setSelectedRocCurveIds] = useState([]); + const [selectedIdColorMap, setSelectedIdColorMap] = useState({}); + const [lineColorsStack, setLineColorsStack] = useState([...lineColors].reverse()); + const [fullArtifactPathMap, setFullArtifactPathMap] = useState({}); + + // Two-panel display artifacts + const [confusionMatrixRunArtifacts, setConfusionMatrixRunArtifacts] = useState([]); + const [htmlRunArtifacts, setHtmlRunArtifacts] = useState([]); + const [markdownRunArtifacts, setMarkdownRunArtifacts] = useState([]); // Selected artifacts for two-panel layout. + const createSelectedArtifactArray = (count: number): SelectedArtifact[] => { + const array: SelectedArtifact[] = []; + for (let i = 0; i < count; i++) { + array.push({ + selectedItem: { itemName: '', subItemName: '' }, + }); + } + return array; + }; const [selectedArtifactsMap, setSelectedArtifactsMap] = useState<{ [key: string]: SelectedArtifact[]; - }>(createInitialSelectedArtifactsMap); + }>({ + [MetricsType.CONFUSION_MATRIX]: createSelectedArtifactArray(2), + [MetricsType.HTML]: createSelectedArtifactArray(2), + [MetricsType.MARKDOWN]: createSelectedArtifactArray(2), + }); const queryParamRunIds = new URLParser(props).get(QUERY_PARAMS.runlist); const runIds = (queryParamRunIds && queryParamRunIds.split(',')) || []; @@ -503,94 +338,141 @@ function CompareV2(props: CompareV2Props) { error: errorArtifactTypes, } = useArtifactTypes(); - const metricsArtifactData = useMemo(() => { - if (!(runs && mlmdPackages && artifactTypes)) { - return undefined; + // Ensure that the two-panel selected artifacts are present in selected valid run list. + const getVerifiedTwoPanelSelection = ( + runArtifacts: RunArtifact[], + selectedArtifacts: SelectedArtifact[], + ) => { + const artifactsPresent: boolean[] = new Array(2).fill(false); + for (const runArtifact of runArtifacts) { + const runName = runArtifact.run.display_name; + if (runName === selectedArtifacts[0].selectedItem.itemName) { + artifactsPresent[0] = true; + } else if (runName === selectedArtifacts[1].selectedItem.itemName) { + artifactsPresent[1] = true; + } } - const selectedIdsSet = new Set(selectedIds); - const runArtifacts: RunArtifact[] = getRunArtifacts(runs, mlmdPackages).filter((runArtifact) => - selectedIdsSet.has(runArtifact.run.run_id!), - ); - const scalarMetricsArtifactData = filterRunArtifactsByType( - runArtifacts, - artifactTypes, - MetricsType.SCALAR_METRICS, - ); - const confusionMatrixArtifactData = filterRunArtifactsByType( - runArtifacts, - artifactTypes, - MetricsType.CONFUSION_MATRIX, - ); - const htmlArtifactData = filterRunArtifactsByType( - runArtifacts, - artifactTypes, - MetricsType.HTML, - ); - const markdownArtifactData = filterRunArtifactsByType( - runArtifacts, - artifactTypes, - MetricsType.MARKDOWN, - ); - const rocCurveArtifactData = filterRunArtifactsByType( - runArtifacts, - artifactTypes, - MetricsType.ROC_CURVE, - ); + for (let i: number = 0; i < artifactsPresent.length; i++) { + if (!artifactsPresent[i]) { + selectedArtifacts[i] = { + selectedItem: { + itemName: '', + subItemName: '', + }, + }; + } + } - return { - scalarMetricsTableData: getScalarTableProps( - scalarMetricsArtifactData.runArtifacts, - scalarMetricsArtifactData.artifactCount, - ), - confusionMatrixRunArtifacts: confusionMatrixArtifactData.runArtifacts, - htmlRunArtifacts: htmlArtifactData.runArtifacts, - markdownRunArtifacts: markdownArtifactData.runArtifacts, - rocCurveRunArtifacts: rocCurveArtifactData.runArtifacts, - }; - }, [runs, selectedIds, mlmdPackages, artifactTypes]); + return [...selectedArtifacts]; + }; - const rocCurveData = useMemo(() => { - if (!metricsArtifactData) { - return { - fullArtifactPathMap: {} as FullArtifactPathMap, - rocCurveLinkedArtifacts: [] as LinkedArtifact[], - validRocCurveIdSet: new Set(), - }; + useEffect(() => { + if (runs && selectedIds && mlmdPackages && artifactTypes) { + const selectedIdsSet = new Set(selectedIds); + const runArtifacts: RunArtifact[] = getRunArtifacts(runs, mlmdPackages).filter( + (runArtifact) => selectedIdsSet.has(runArtifact.run.run_id!), + ); + const scalarMetricsArtifactData = filterRunArtifactsByType( + runArtifacts, + artifactTypes, + MetricsType.SCALAR_METRICS, + ); + setScalarMetricsTableData( + getScalarTableProps( + scalarMetricsArtifactData.runArtifacts, + scalarMetricsArtifactData.artifactCount, + ), + ); + + // Filter and set the two-panel layout run artifacts. + const confusionMatrixRunArtifacts: RunArtifact[] = filterRunArtifactsByType( + runArtifacts, + artifactTypes, + MetricsType.CONFUSION_MATRIX, + ).runArtifacts; + const htmlRunArtifacts: RunArtifact[] = filterRunArtifactsByType( + runArtifacts, + artifactTypes, + MetricsType.HTML, + ).runArtifacts; + const markdownRunArtifacts: RunArtifact[] = filterRunArtifactsByType( + runArtifacts, + artifactTypes, + MetricsType.MARKDOWN, + ).runArtifacts; + setConfusionMatrixRunArtifacts(confusionMatrixRunArtifacts); + setHtmlRunArtifacts(htmlRunArtifacts); + setMarkdownRunArtifacts(markdownRunArtifacts); + + // Iterate through selected runs, remove current selection if not present among runs. + setSelectedArtifactsMap({ + [MetricsType.CONFUSION_MATRIX]: getVerifiedTwoPanelSelection( + confusionMatrixRunArtifacts, + selectedArtifactsMap[MetricsType.CONFUSION_MATRIX], + ), + [MetricsType.HTML]: getVerifiedTwoPanelSelection( + htmlRunArtifacts, + selectedArtifactsMap[MetricsType.HTML], + ), + [MetricsType.MARKDOWN]: getVerifiedTwoPanelSelection( + markdownRunArtifacts, + selectedArtifactsMap[MetricsType.MARKDOWN], + ), + }); + + // Set ROC Curve run artifacts and get all valid ROC curve data for plot visualization. + const rocCurveRunArtifacts: RunArtifact[] = filterRunArtifactsByType( + runArtifacts, + artifactTypes, + MetricsType.ROC_CURVE, + ).runArtifacts; + updateRocCurveDisplay(rocCurveRunArtifacts); + setIsLoadingArtifacts(false); + setIsInitialArtifactsLoad(false); } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [runs, selectedIds, mlmdPackages, artifactTypes]); + // Update the ROC Curve colors and selection. + const updateRocCurveDisplay = (runArtifacts: RunArtifact[]) => { const { validLinkedArtifacts, fullArtifactPathMap, validRocCurveIdSet } = - getValidRocCurveArtifactData(metricsArtifactData.rocCurveRunArtifacts); - return { - fullArtifactPathMap, - rocCurveLinkedArtifacts: validLinkedArtifacts, - validRocCurveIdSet, - }; - }, [metricsArtifactData]); + getValidRocCurveArtifactData(runArtifacts); - const visibleSelectedArtifactsMap = useMemo( - () => reconcileSelectedArtifactsMap(selectedArtifactsMap, metricsArtifactData), - [selectedArtifactsMap, metricsArtifactData], - ); + setFullArtifactPathMap(fullArtifactPathMap); + setRocCurveLinkedArtifacts(validLinkedArtifacts); - useEffect(() => { - setRocCurveSelection((currentSelection) => - reconcileRocCurveSelectionState( - currentSelection, - rocCurveData.rocCurveLinkedArtifacts, - rocCurveData.validRocCurveIdSet, - ), - ); - }, [rocCurveData]); + // Remove all newly invalid ROC Curves from the selection (if run selection changes). + const removedRocCurveIds: Set = new Set(); + for (const oldSelectedId of Object.keys(selectedIdColorMap)) { + if (!validRocCurveIdSet.has(oldSelectedId)) { + removedRocCurveIds.add(oldSelectedId); + } + } - const scalarMetricsTableData = metricsArtifactData?.scalarMetricsTableData; - const confusionMatrixRunArtifacts = metricsArtifactData?.confusionMatrixRunArtifacts || []; - const htmlRunArtifacts = metricsArtifactData?.htmlRunArtifacts || []; - const markdownRunArtifacts = metricsArtifactData?.markdownRunArtifacts || []; - const rocCurveLinkedArtifacts = rocCurveData.rocCurveLinkedArtifacts; - const fullArtifactPathMap = rocCurveData.fullArtifactPathMap; - const isLoadingArtifacts = - !metricsArtifactData && !(isErrorRunDetails || isErrorMlmdPackages || isErrorArtifactTypes); + // If initial load, choose first three artifacts; ow, remove artifacts from de-selected runs. + let updatedRocCurveIds: string[] = selectedRocCurveIds; + if (isInitialArtifactsLoad) { + updatedRocCurveIds = validLinkedArtifacts + .map((linkedArtifact) => getRocCurveId(linkedArtifact)) + .slice(0, 3); + updatedRocCurveIds.forEach((rocCurveId) => { + selectedIdColorMap[rocCurveId] = lineColorsStack.pop()!; + }); + } else { + updatedRocCurveIds = updatedRocCurveIds.filter((rocCurveId) => { + if (removedRocCurveIds.has(rocCurveId)) { + lineColorsStack.push(selectedIdColorMap[rocCurveId]); + delete selectedIdColorMap[rocCurveId]; + return false; + } + return true; + }); + } + setSelectedRocCurveIds(updatedRocCurveIds); + setLineColorsStack(lineColorsStack); + setSelectedIdColorMap(selectedIdColorMap); + }; useEffect(() => { if (isLoadingRunDetails || isLoadingMlmdPackages || isLoadingArtifactTypes) { @@ -703,10 +585,8 @@ function CompareV2(props: CompareV2Props) { }; const updateSelectedArtifacts = (newArtifacts: SelectedArtifact[]) => { - setSelectedArtifactsMap((currentSelectedArtifactsMap) => ({ - ...currentSelectedArtifactsMap, - [metricsTab]: newArtifacts, - })); + selectedArtifactsMap[metricsTab] = newArtifacts; + setSelectedArtifactsMap(selectedArtifactsMap); }; const isErrorArtifacts = isErrorRunDetails || isErrorMlmdPackages || isErrorArtifactTypes; @@ -791,7 +671,7 @@ function CompareV2(props: CompareV2Props) { @@ -800,25 +680,13 @@ function CompareV2(props: CompareV2Props) { - setRocCurveSelection((currentSelection) => ({ - ...currentSelection, - selectedIds, - })), + selectedIds: selectedRocCurveIds, + setSelectedIds: setSelectedRocCurveIds, fullArtifactPathMap, - selectedIdColorMap: rocCurveSelection.selectedIdColorMap, - setSelectedIdColorMap: (selectedIdColorMap) => - setRocCurveSelection((currentSelection) => ({ - ...currentSelection, - selectedIdColorMap, - })), - lineColorsStack: rocCurveSelection.lineColorsStack, - setLineColorsStack: (lineColorsStack) => - setRocCurveSelection((currentSelection) => ({ - ...currentSelection, - lineColorsStack, - })), + selectedIdColorMap, + setSelectedIdColorMap, + lineColorsStack, + setLineColorsStack, }} /> )} @@ -826,7 +694,7 @@ function CompareV2(props: CompareV2Props) { @@ -835,7 +703,7 @@ function CompareV2(props: CompareV2Props) { @@ -867,7 +735,4 @@ export default EnhancedCompareV2; export const TEST_ONLY = { CompareV2, - createInitialRocCurveSelectionState, - reconcileSelectedArtifactsMap, - reconcileRocCurveSelectionState, }; diff --git a/kubernetes_platform/python/kfp/kubernetes/common.py b/kubernetes_platform/python/kfp/kubernetes/common.py index d60e3bc33e4..99fe1890418 100644 --- a/kubernetes_platform/python/kfp/kubernetes/common.py +++ b/kubernetes_platform/python/kfp/kubernetes/common.py @@ -14,32 +14,26 @@ from typing import Union -from google.protobuf import json_format -from kfp.compiler.pipeline_spec_builder import to_protobuf_value from kfp.dsl import pipeline_channel +from kfp.compiler.pipeline_spec_builder import to_protobuf_value from kfp.dsl import PipelineTask -from kfp.kubernetes import kubernetes_executor_config_pb2 as pb +from google.protobuf import json_format from kfp.pipeline_spec import pipeline_spec_pb2 - +from kfp.kubernetes import kubernetes_executor_config_pb2 as pb def camel_to_python_case(name: str) -> str: import re s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() - def deserialize_dict_to_k8s_model_keys(obj): if isinstance(obj, dict): - return { - camel_to_python_case(k): deserialize_dict_to_k8s_model_keys(v) - for k, v in obj.items() - } + return {camel_to_python_case(k): deserialize_dict_to_k8s_model_keys(v) for k, v in obj.items()} elif isinstance(obj, list): return [deserialize_dict_to_k8s_model_keys(i) for i in obj] else: return obj - def get_existing_kubernetes_config_as_message( task: 'PipelineTask') -> pb.KubernetesExecutorConfig: cur_k8_config_dict = task.platform_config.get('kubernetes', {}) @@ -48,16 +42,14 @@ def get_existing_kubernetes_config_as_message( def parse_k8s_parameter_input( - input_param: Union[pipeline_channel.PipelineParameterChannel, str, dict], - task: PipelineTask, + input_param: Union[pipeline_channel.PipelineParameterChannel, str, dict], + task: PipelineTask, ) -> pipeline_spec_pb2.TaskInputsSpec.InputParameterSpec: param_spec = pipeline_spec_pb2.TaskInputsSpec.InputParameterSpec() if isinstance(input_param, (str, dict)): - param_spec.runtime_value.constant.CopyFrom( - to_protobuf_value(input_param)) + param_spec.runtime_value.constant.CopyFrom(to_protobuf_value(input_param)) elif isinstance(input_param, pipeline_channel.PipelineParameterChannel): - task.register_pipeline_channels([input_param]) if input_param.task_name is None: param_spec.component_input_parameter = input_param.full_name @@ -69,6 +61,7 @@ def parse_k8s_parameter_input( else: raise ValueError( f'Argument for {"input_param"!r} must be an instance of str, dict, or PipelineChannel. ' - f'Got unknown input type: {type(input_param)!r}.') + f'Got unknown input type: {type(input_param)!r}.' + ) return param_spec diff --git a/kubernetes_platform/python/kfp/kubernetes/volume.py b/kubernetes_platform/python/kfp/kubernetes/volume.py index 4dc70f315ca..81dab6a1a14 100644 --- a/kubernetes_platform/python/kfp/kubernetes/volume.py +++ b/kubernetes_platform/python/kfp/kubernetes/volume.py @@ -109,8 +109,10 @@ def DeletePVC(pvc_name: str): return dsl.ContainerSpec(image='argostub/deletepvc') -def _assign_pvc_name_to_msg(msg: message.Message, - pvc_name: Union[str, 'PipelineChannel']) -> bool: +def _assign_pvc_name_to_msg( + msg: message.Message, + pvc_name: Union[str, 'PipelineChannel'], +) -> bool: """Assigns pvc_name to the msg's pvc_reference oneof. Returns True if pvc_name is an upstream task output; otherwise, False. diff --git a/kubernetes_platform/python/test/unit/test_secret.py b/kubernetes_platform/python/test/unit/test_secret.py index e22a4845611..896cabf30ec 100644 --- a/kubernetes_platform/python/test/unit/test_secret.py +++ b/kubernetes_platform/python/test/unit/test_secret.py @@ -971,81 +971,6 @@ def my_pipeline(): } } -class TestEnsureChannelInputPropagation: - """Tests that PipelineParameterChannel values used in Kubernetes platform - config are correctly registered in task.channel_inputs for sub-DAG - propagation.""" - - def test_pipeline_param_added_to_channel_inputs(self): - """When a PipelineParameterChannel is passed as secret_name, - it should be appended to the task's channel_inputs.""" - - @dsl.pipeline - def my_pipeline(secret_name: str): - task = comp() - initial_count = len(task.channel_inputs) - kubernetes.use_secret_as_env( - task, - secret_name=secret_name, - secret_key_to_env={"key": "VAR"}, - ) - assert len(task.channel_inputs) == initial_count + 1 - channel_patterns = {ch.pattern for ch in task.channel_inputs} - assert secret_name.pattern in channel_patterns - - def test_duplicate_pipeline_param_not_added_twice(self): - """When the same PipelineParameterChannel is used in both - use_secret_as_env and use_secret_as_volume, channel_inputs - should not contain duplicates.""" - - @dsl.pipeline - def my_pipeline(secret_name: str): - task = comp() - kubernetes.use_secret_as_env( - task, - secret_name=secret_name, - secret_key_to_env={"key1": "VAR1"}, - ) - count_after_first = len(task.channel_inputs) - kubernetes.use_secret_as_volume( - task, - secret_name=secret_name, - mount_path="/mnt/secret", - ) - assert len(task.channel_inputs) == count_after_first - - def test_string_input_does_not_add_channel_inputs(self): - """When a literal string is passed, channel_inputs should not - be modified.""" - - @dsl.pipeline - def my_pipeline(): - task = comp() - initial_count = len(task.channel_inputs) - kubernetes.use_secret_as_env( - task, - secret_name="literal-secret", - secret_key_to_env={"key": "VAR"}, - ) - assert len(task.channel_inputs) == initial_count - - def test_task_output_added_to_channel_inputs(self): - """When a task output PipelineChannel is passed as secret_name, - it should be appended to channel_inputs.""" - - @dsl.pipeline - def my_pipeline(): - name_task = comp_with_output() - task = comp() - initial_count = len(task.channel_inputs) - kubernetes.use_secret_as_env( - task, - secret_name=name_task.output, - secret_key_to_env={"key": "VAR"}, - ) - assert len(task.channel_inputs) == initial_count + 1 - - @dsl.component def comp(): pass diff --git a/manifests/kustomize/README.md b/manifests/kustomize/README.md index b45a61ee19b..8f094d22775 100644 --- a/manifests/kustomize/README.md +++ b/manifests/kustomize/README.md @@ -27,6 +27,15 @@ kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80 Now you can access Kubeflow Pipelines UI in your browser by . +Customize: + +There are two variations for platform-agnostic that uses different [argo workflow executors](https://argoproj.github.io/argo-workflows/workflow-executors/): + +* env/platform-agnostic-emissary +* env/platform-agnostic-pns + +You can install them by changing `KFP_ENV` in above instructions to the variation you want. + Data: Application data are persisted in in-cluster PersistentVolumeClaim storage. diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml index 318e5fd2fae..d2d83c4096b 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml @@ -27,6 +27,18 @@ spec: resource: services updateStrategy: method: InPlace + - apiVersion: v1 + resource: serviceaccounts + updateStrategy: + method: InPlace + - apiVersion: rbac.authorization.k8s.io/v1 + resource: roles + updateStrategy: + method: InPlace + - apiVersion: rbac.authorization.k8s.io/v1 + resource: rolebindings + updateStrategy: + method: InPlace - apiVersion: networking.istio.io/v1alpha3 resource: destinationrules updateStrategy: @@ -38,4 +50,4 @@ spec: hooks: sync: webhook: - url: http://kubeflow-pipelines-profile-controller/sync \ No newline at end of file + url: http://kubeflow-pipelines-profile-controller/sync diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml index 3652a67305f..6504ccb0b11 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml @@ -40,7 +40,7 @@ spec: - name: AWS_ENDPOINT_URL value: http://seaweedfs.kubeflow:8111 - name: S3_ENDPOINT_URL - value: http://seaweedfs.kubeflow:8333 + value: http://seaweedfs.kubeflow:8333 - name: AWS_REGION value: us-east-1 - name: AWS_ACCESS_KEY_ID diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml new file mode 100644 index 00000000000..5dfcd3bc206 --- /dev/null +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pipelines-profile-controller-admin + namespace: kubeflow +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pipelines-profile-controller-admin-clusterrole +rules: + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "create", "get", "list" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pipelines-profile-controller-admin-binding +subjects: + - kind: ServiceAccount + name: pipelines-profile-controller-admin + namespace: kubeflow +roleRef: + kind: ClusterRole + name: pipelines-profile-controller-admin-clusterrole + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/requirements-dev.txt b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/requirements-dev.txt index dc27cb5746f..19cdf1da80f 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/requirements-dev.txt +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/requirements-dev.txt @@ -1,3 +1,3 @@ pytest -pytest-lazy-fixture requests +botocore diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 12fa8bd1926..f741d02a2dd 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from http.server import BaseHTTPRequestHandler, HTTPServer -import json -import os import base64 import hashlib +from http.server import BaseHTTPRequestHandler +from http.server import HTTPServer +import json +import os # From awscli installed in alpine/k8s image import botocore.session @@ -25,8 +26,10 @@ session = botocore.session.get_session() # S3 client for lifecycle policy management -s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", "http://seaweedfs.kubeflow:8333") -s3 = session.create_client('s3', region_name='foobar', endpoint_url=s3_endpoint_url) +s3_endpoint_url = os.environ.get('S3_ENDPOINT_URL', + 'http://seaweedfs.kubeflow:8333') +s3 = session.create_client( + 's3', region_name='foobar', endpoint_url=s3_endpoint_url) def _normalize_domain(domain): @@ -35,9 +38,10 @@ def _normalize_domain(domain): def create_iam_client(): # To interact with SeaweedFS user management. Region does not matter. - endpoint_url = os.environ.get("AWS_ENDPOINT_URL") + endpoint_url = os.environ.get('AWS_ENDPOINT_URL') if endpoint_url: - return session.create_client('iam', region_name='foobar', endpoint_url=endpoint_url) + return session.create_client( + 'iam', region_name='foobar', endpoint_url=endpoint_url) return session.create_client('iam', region_name='foobar') @@ -58,8 +62,8 @@ def get_settings_from_env(controller_port=None, artifact_retention_days=None, cluster_domain=None, object_store_host=None): - """ - Returns a dict of settings from environment variables relevant to the controller + """Returns a dict of settings from environment variables relevant to the + controller. Environment settings can be overridden by passing them here as arguments. @@ -71,41 +75,41 @@ def get_settings_from_env(controller_port=None, disable_istio_sidecar: Required (no default) """ settings = dict() - settings["controller_port"] = \ + settings['controller_port'] = \ controller_port or \ - os.environ.get("CONTROLLER_PORT", "8080") + os.environ.get('CONTROLLER_PORT', '8080') - settings["frontend_image"] = \ + settings['frontend_image'] = \ frontend_image or \ - os.environ.get("FRONTEND_IMAGE", "ghcr.io/kubeflow/kfp-frontend") + os.environ.get('FRONTEND_IMAGE', 'ghcr.io/kubeflow/kfp-frontend') - settings["artifacts_proxy_enabled"] = \ + settings['artifacts_proxy_enabled'] = \ artifacts_proxy_enabled or \ - os.environ.get("ARTIFACTS_PROXY_ENABLED", "false") + os.environ.get('ARTIFACTS_PROXY_ENABLED', 'false') - settings["artifact_retention_days"] = \ + settings['artifact_retention_days'] = \ artifact_retention_days or \ - os.environ.get("ARTIFACT_RETENTION_DAYS", -1) + os.environ.get('ARTIFACT_RETENTION_DAYS', -1) - settings["cluster_domain"] = \ + settings['cluster_domain'] = \ cluster_domain or \ - os.environ.get("CLUSTER_DOMAIN", ".svc.cluster.local") + os.environ.get('CLUSTER_DOMAIN', '.svc.cluster.local') - settings["object_store_host"] = \ + settings['object_store_host'] = \ object_store_host or \ - os.environ.get("OBJECT_STORE_HOST", "seaweedfs") + os.environ.get('OBJECT_STORE_HOST', 'seaweedfs') # Look for specific tags for each image first, falling back to # previously used KFP_VERSION environment variable for backwards # compatibility - settings["frontend_tag"] = \ + settings['frontend_tag'] = \ frontend_tag or \ - os.environ.get("FRONTEND_TAG") or \ - os.environ["KFP_VERSION"] + os.environ.get('FRONTEND_TAG') or \ + os.environ['KFP_VERSION'] - settings["disable_istio_sidecar"] = \ + settings['disable_istio_sidecar'] = \ disable_istio_sidecar if disable_istio_sidecar is not None \ - else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + else os.environ.get('DISABLE_ISTIO_SIDECAR') == 'true' return settings @@ -115,269 +119,459 @@ def server_factory(frontend_image, disable_istio_sidecar, artifacts_proxy_enabled, artifact_retention_days, - cluster_domain=".svc.cluster.local", - object_store_host="seaweedfs", - url="", + cluster_domain='.svc.cluster.local', + object_store_host='seaweedfs', + url='', controller_port=8080): - """ - Returns an HTTPServer populated with Handler with customized settings - """ + """Returns an HTTPServer populated with Handler with customized + settings.""" + class Controller(BaseHTTPRequestHandler): + def upsert_lifecycle_policy(self, bucket_name, artifact_retention_days): - """Configures or deletes the lifecycle policy based on the artifact_retention_days string.""" + """Configures or deletes the lifecycle policy based on the + artifact_retention_days string.""" try: retention_days = int(artifact_retention_days) except ValueError: - print(f"ERROR: ARTIFACT_RETENTION_DAYS value '{artifact_retention_days}' is not a valid integer. Aborting policy update.") + print( + f"ERROR: ARTIFACT_RETENTION_DAYS value '{artifact_retention_days}' is not a valid integer. Aborting policy update." + ) return # To disable lifecycle policy we need to delete it if retention_days <= 0: - print(f"ARTIFACT_RETENTION_DAYS is non-positive ({retention_days} days). Attempting to delete lifecycle policy.") + print( + f'ARTIFACT_RETENTION_DAYS is non-positive ({retention_days} days). Attempting to delete lifecycle policy.' + ) try: - response = s3.get_bucket_lifecycle_configuration(Bucket=bucket_name) + response = s3.get_bucket_lifecycle_configuration( + Bucket=bucket_name) # Check if there are any enabled rules - has_enabled_rules = any(rule.get('Status') == 'Enabled' for rule in response.get('Rules', [])) + has_enabled_rules = any( + rule.get('Status') == 'Enabled' + for rule in response.get('Rules', [])) if has_enabled_rules: s3.delete_bucket_lifecycle(Bucket=bucket_name) - print("Successfully deleted lifecycle policy.") + print('Successfully deleted lifecycle policy.') else: - print("No enabled lifecycle rules found to delete.") + print('No enabled lifecycle rules found to delete.') except Exception: - print(f"Warning: No lifecycle policy exists") + print(f'Warning: No lifecycle policy exists') return # Create/update lifecycle policy life_cycle_policy = { - "Rules": [ - { - "Status": "Enabled", - "Filter": {"Prefix": "private-artifacts"}, - "Expiration": {"Days": retention_days}, - "ID": "private-artifacts", + 'Rules': [{ + 'Status': 'Enabled', + 'Filter': { + 'Prefix': 'private-artifacts' }, - ] + 'Expiration': { + 'Days': retention_days + }, + 'ID': 'private-artifacts', + },] } print('upsert_lifecycle_policy:', life_cycle_policy) try: api_response = s3.put_bucket_lifecycle_configuration( Bucket=bucket_name, - LifecycleConfiguration = life_cycle_policy - ) + LifecycleConfiguration=life_cycle_policy) print('Lifecycle policy configured successfully:', api_response) except Exception as exception: - if hasattr(exception, 'response') and 'Error' in exception.response: - print(f"ERROR: Failed to configure lifecycle policy: {exception.response['Error']['Code']} - {exception}") + if hasattr(exception, + 'response') and 'Error' in exception.response: + print( + f"ERROR: Failed to configure lifecycle policy: {exception.response['Error']['Code']} - {exception}" + ) else: - print(f"ERROR: Failed to configure lifecycle policy: {exception}") - + print( + f'ERROR: Failed to configure lifecycle policy: {exception}' + ) def sync(self, parent, attachments): # parent is a namespace - namespace = parent.get("metadata", {}).get("name") + namespace = parent.get('metadata', {}).get('name') - pipeline_enabled = parent.get("metadata", {}).get( - "labels", {}).get("pipelines.kubeflow.org/enabled") + pipeline_enabled = parent.get('metadata', {}).get( + 'labels', {}).get('pipelines.kubeflow.org/enabled') - if pipeline_enabled != "true": - return {"status": {}, "attachments": []} + if pipeline_enabled != 'true': + return {'status': {}, 'attachments': []} # Compute status based on observed state. desired_status = { - "kubeflow-pipelines-ready": - len(attachments["Secret.v1"]) == 1 and - len(attachments["ConfigMap.v1"]) == 3 and - len(attachments["Deployment.apps/v1"]) == (1 if artifacts_proxy_enabled.lower() == "true" else 0) and - len(attachments["Service.v1"]) == (1 if artifacts_proxy_enabled.lower() == "true" else 0) and - "True" or "False" + 'kubeflow-pipelines-ready': + len(attachments['Secret.v1']) == 3 and + len(attachments['ConfigMap.v1']) == 3 and + len(attachments['Deployment.apps/v1']) == + (1 if artifacts_proxy_enabled.lower() == 'true' else 0) and + len(attachments['Service.v1']) == + (1 if artifacts_proxy_enabled.lower() == 'true' else 0) and + len(attachments['ServiceAccount.v1']) == 1 and + len(attachments['Role.rbac.authorization.k8s.io/v1']) == 4 + and + len(attachments['RoleBinding.rbac.authorization.k8s.io/v1']) + == 4 and 'True' or 'False' } # Generate the desired attachment object(s). desired_resources = [ { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "kfp-launcher", - "namespace": namespace, + 'apiVersion': 'v1', + 'kind': 'ConfigMap', + 'metadata': { + 'name': 'kfp-launcher', + 'namespace': namespace, }, - "data": { - "defaultPipelineRoot": f"minio://{S3_BUCKET_NAME}/private-artifacts/{namespace}/v2/artifacts", - "clusterDomain": cluster_domain, + 'data': { + 'defaultPipelineRoot': + f'minio://{S3_BUCKET_NAME}/private-artifacts/{namespace}/v2/artifacts', + 'clusterDomain': + cluster_domain, }, }, { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "metadata-grpc-configmap", - "namespace": namespace, + 'apiVersion': 'v1', + 'kind': 'ConfigMap', + 'metadata': { + 'name': 'metadata-grpc-configmap', + 'namespace': namespace, }, - "data": { - "METADATA_GRPC_SERVICE_HOST": - "metadata-grpc-service.kubeflow", - "METADATA_GRPC_SERVICE_PORT": "8080", + 'data': { + 'METADATA_GRPC_SERVICE_HOST': + 'metadata-grpc-service.kubeflow', + 'METADATA_GRPC_SERVICE_PORT': + '8080', }, }, { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "artifact-repositories", - "namespace": namespace, - "annotations": { - "workflows.argoproj.io/default-artifact-repository": "default-namespaced" + 'apiVersion': 'v1', + 'kind': 'ConfigMap', + 'metadata': { + 'name': 'artifact-repositories', + 'namespace': namespace, + 'annotations': { + 'workflows.argoproj.io/default-artifact-repository': + 'default-namespaced' } }, - "data": { - "default-namespaced": json.dumps({ - "archiveLogs": True, - "s3": { - "endpoint": f"{object_store_host}.kubeflow{_normalize_domain(cluster_domain)}:9000", - "bucket": S3_BUCKET_NAME, - "keyFormat": f"private-artifacts/{namespace}/{{{{workflow.name}}}}/{{{{workflow.creationTimestamp.Y}}}}/{{{{workflow.creationTimestamp.m}}}}/{{{{workflow.creationTimestamp.d}}}}/{{{{pod.name}}}}", - "insecure": True, - "accessKeySecret": { - "name": "mlpipeline-minio-artifact", - "key": "accesskey", - }, - "secretKeySecret": { - "name": "mlpipeline-minio-artifact", - "key": "secretkey", + 'data': { + 'default-namespaced': + json.dumps({ + 'archiveLogs': True, + 's3': { + 'endpoint': + f'{object_store_host}.kubeflow{_normalize_domain(cluster_domain)}:9000', + 'bucket': + S3_BUCKET_NAME, + 'keyFormat': + f'private-artifacts/{namespace}/{{{{workflow.name}}}}/{{{{workflow.creationTimestamp.Y}}}}/{{{{workflow.creationTimestamp.m}}}}/{{{{workflow.creationTimestamp.d}}}}/{{{{pod.name}}}}', + 'insecure': + True, + 'accessKeySecret': { + 'name': 'mlpipeline-minio-artifact', + 'key': 'accesskey', + }, + 'secretKeySecret': { + 'name': 'mlpipeline-minio-artifact', + 'key': 'secretkey', + } } - } - }) + }) } }, ] - # Add artifact fetcher related resources if enabled - if artifacts_proxy_enabled.lower() == "true": + if artifacts_proxy_enabled.lower() == 'true': desired_resources.extend([ { - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" + 'apiVersion': 'apps/v1', + 'kind': 'Deployment', + 'metadata': { + 'labels': { + 'app': 'ml-pipeline-ui-artifact' }, - "name": "ml-pipeline-ui-artifact", - "namespace": namespace, + 'name': 'ml-pipeline-ui-artifact', + 'namespace': namespace, }, - "spec": { - "selector": { - "matchLabels": { - "app": "ml-pipeline-ui-artifact" + 'spec': { + 'selector': { + 'matchLabels': { + 'app': 'ml-pipeline-ui-artifact' } }, - "template": { - "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" + 'template': { + 'metadata': { + 'labels': { + 'app': 'ml-pipeline-ui-artifact' }, - "annotations": { - **({"sidecar.istio.io/inject": "false"} if disable_istio_sidecar else {}), - "kubeflow-pipelines/image-spec-hash": hashlib.sha256(f"{frontend_image}:{frontend_tag}".encode()).hexdigest()[:16], + 'annotations': { + **({ + 'sidecar.istio.io/inject': 'false' + } if disable_istio_sidecar else {}), + 'kubeflow-pipelines/image-spec-hash': + hashlib.sha256( + f'{frontend_image}:{frontend_tag}' + .encode()).hexdigest()[:16], }, }, - "spec": { - "containers": [{ - "name": - "ml-pipeline-ui-artifact", - "image": f"{frontend_image}:{frontend_tag}", - "imagePullPolicy": - "IfNotPresent", - "ports": [{ - "containerPort": 3000 + 'spec': { + 'containers': [{ + 'name': + 'ml-pipeline-ui-artifact', + 'image': + f'{frontend_image}:{frontend_tag}', + 'imagePullPolicy': + 'IfNotPresent', + 'ports': [{ + 'containerPort': 3000 }], - "env": [ - { - "name": "MINIO_ACCESS_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "accesskey", - "name": "mlpipeline-minio-artifact" - } + 'env': [{ + 'name': 'MINIO_ACCESS_KEY', + 'valueFrom': { + 'secretKeyRef': { + 'key': + 'accesskey', + 'name': + 'mlpipeline-minio-artifact' } - }, - { - "name": "MINIO_SECRET_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "secretkey", - "name": "mlpipeline-minio-artifact" - } + } + }, { + 'name': 'MINIO_SECRET_KEY', + 'valueFrom': { + 'secretKeyRef': { + 'key': + 'secretkey', + 'name': + 'mlpipeline-minio-artifact' } - }, - { - "name": "ML_PIPELINE_SERVICE_HOST", - "value": f"ml-pipeline.kubeflow{_normalize_domain(cluster_domain)}" - }, - { - "name": "ML_PIPELINE_SERVICE_PORT", - "value": "8888" - }, - { - "name": "FRONTEND_SERVER_NAMESPACE", - "value": namespace, - }, - { - "name": "CLUSTER_DOMAIN", - "value": cluster_domain, } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "70Mi" + }, { + 'name': + 'ML_PIPELINE_SERVICE_HOST', + 'value': + f'ml-pipeline.kubeflow{_normalize_domain(cluster_domain)}' + }, { + 'name': 'ML_PIPELINE_SERVICE_PORT', + 'value': '8888' + }, { + 'name': 'FRONTEND_SERVER_NAMESPACE', + 'value': namespace, + }, { + 'name': 'CLUSTER_DOMAIN', + 'value': cluster_domain, + }], + 'resources': { + 'requests': { + 'cpu': '10m', + 'memory': '70Mi' }, - "limits": { - "cpu": "100m", - "memory": "500Mi" + 'limits': { + 'cpu': '100m', + 'memory': '500Mi' }, } }], - "serviceAccountName": - "default-editor" + 'serviceAccountName': 'default-editor' } } } }, { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": "ml-pipeline-ui-artifact", - "namespace": namespace, - "labels": { - "app": "ml-pipeline-ui-artifact" + 'apiVersion': 'v1', + 'kind': 'Service', + 'metadata': { + 'name': 'ml-pipeline-ui-artifact', + 'namespace': namespace, + 'labels': { + 'app': 'ml-pipeline-ui-artifact' } }, - "spec": { - "ports": [{ - "name": - "http", # name is required to let istio understand request protocol - "port": 80, - "protocol": "TCP", - "targetPort": 3000 + 'spec': { + 'ports': [{ + 'name': + 'http', # name is required to let istio understand request protocol + 'port': 80, + 'protocol': 'TCP', + 'targetPort': 3000 }], - "selector": { - "app": "ml-pipeline-ui-artifact" + 'selector': { + 'app': 'ml-pipeline-ui-artifact' } } }, ]) + print('Creating executor-plugin RBAC for centralized driver') + # The KFP driver now runs as a centralized endpoint within the API server + # (ml-pipeline SA in the kubeflow namespace) rather than as a per-workflow + # sidecar. The API server needs permissions in each user namespace to + # create PVCs, read pods, and access secrets on behalf of workflows. + kfp_system_namespace = 'kubeflow' + api_server_sa = 'ml-pipeline' + desired_resources.extend([ + { + 'apiVersion': + 'rbac.authorization.k8s.io/v1', + 'kind': + 'Role', + 'metadata': { + 'name': 'configmap-reader', + 'namespace': namespace, + }, + 'rules': [{ + 'apiGroups': [''], + 'resources': ['configmaps'], + 'verbs': ['get', 'list', 'watch'], + }] + }, + { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'RoleBinding', + 'metadata': { + 'name': 'configmap-reader-binding', + 'namespace': namespace, + }, + 'subjects': [{ + 'kind': 'ServiceAccount', + 'name': api_server_sa, + 'namespace': kfp_system_namespace, + }], + 'roleRef': { + 'kind': 'Role', + 'name': 'configmap-reader', + 'apiGroup': 'rbac.authorization.k8s.io', + } + }, + { + 'apiVersion': + 'rbac.authorization.k8s.io/v1', + 'kind': + 'Role', + 'metadata': { + 'name': 'ml-pipeline-driver-pods-reader', + 'namespace': namespace, + }, + 'rules': [{ + 'apiGroups': [''], + 'resources': ['pods'], + 'verbs': ['get', 'list', 'watch'], + }] + }, + { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'RoleBinding', + 'metadata': { + 'name': 'ml-pipeline-driver-pods-reader-binding', + 'namespace': namespace, + }, + 'subjects': [{ + 'kind': 'ServiceAccount', + 'name': api_server_sa, + 'namespace': kfp_system_namespace, + }], + 'roleRef': { + 'kind': 'Role', + 'name': 'ml-pipeline-driver-pods-reader', + 'apiGroup': 'rbac.authorization.k8s.io', + } + }, + { + 'apiVersion': + 'rbac.authorization.k8s.io/v1', + 'kind': + 'Role', + 'metadata': { + 'name': 'ml-pipeline-driver-pvc-editor', + 'namespace': namespace, + }, + 'rules': [{ + 'apiGroups': [''], + 'resources': ['persistentvolumeclaims'], + 'verbs': ['create', 'get', 'list'], + }] + }, + { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'RoleBinding', + 'metadata': { + 'name': 'ml-pipeline-driver-pvc-editor-binding', + 'namespace': namespace, + }, + 'subjects': [{ + 'kind': 'ServiceAccount', + 'name': api_server_sa, + 'namespace': kfp_system_namespace, + }], + 'roleRef': { + 'kind': 'Role', + 'name': 'ml-pipeline-driver-pvc-editor', + 'apiGroup': 'rbac.authorization.k8s.io', + } + }, + { + 'apiVersion': + 'rbac.authorization.k8s.io/v1', + 'kind': + 'Role', + 'metadata': { + 'name': 'artifact-secret-reader', + 'namespace': namespace, + }, + 'rules': [{ + 'apiGroups': [''], + 'resources': ['secrets'], + 'resourceNames': ['mlpipeline-minio-artifact'], + 'verbs': ['get', 'list', 'watch'], + }] + }, + { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'RoleBinding', + 'metadata': { + 'name': 'artifact-secret-reader-binding', + 'namespace': namespace, + }, + 'subjects': [{ + 'kind': 'ServiceAccount', + 'name': api_server_sa, + 'namespace': kfp_system_namespace, + }], + 'roleRef': { + 'kind': 'Role', + 'name': 'artifact-secret-reader', + 'apiGroup': 'rbac.authorization.k8s.io', + } + }, + { + 'apiVersion': 'v1', + 'kind': 'Secret', + 'metadata': { + 'name': 'default-editor.service-account-token', + 'namespace': namespace, + 'annotations': { + 'kubernetes.io/service-account.name': + 'default-editor' + } + }, + 'type': 'kubernetes.io/service-account-token' + }, + ]) + print('Received request:\n', json.dumps(parent, sort_keys=True)) - print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True)) + print('Desired resources except secrets:\n', + json.dumps(desired_resources, sort_keys=True)) # Moved after the print argument because this is sensitive data. # Check if secret is already there when the controller made the request. If yes, then # use it. Else create a new credentials on seaweedfs for the namespace. - if s3_secret := attachments["Secret.v1"].get(f"{namespace}/mlpipeline-minio-artifact"): + if s3_secret := attachments['Secret.v1'].get( + f'{namespace}/mlpipeline-minio-artifact'): desired_resources.append(s3_secret) print('Using existing secret') else: @@ -387,59 +581,62 @@ def sync(self, parent, attachments): # This policy ensures that a user can only access artifacts from his own profile. iam.put_user_policy( UserName=namespace, - PolicyName=f"KubeflowProject{namespace}", - PolicyDocument=json.dumps( - { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "s3:Put*", - "s3:Get*", - "s3:List*" - ], - "Resource": [ - f"arn:aws:s3:::{S3_BUCKET_NAME}/artifacts/*", - f"arn:aws:s3:::{S3_BUCKET_NAME}/private-artifacts/{namespace}/*", - f"arn:aws:s3:::{S3_BUCKET_NAME}/private/{namespace}/*", - f"arn:aws:s3:::{S3_BUCKET_NAME}/shared/*", - ] - }] - }) - ) - - self.upsert_lifecycle_policy(S3_BUCKET_NAME, artifact_retention_days) + PolicyName=f'KubeflowProject{namespace}', + PolicyDocument=json.dumps({ + 'Version': + '2012-10-17', + 'Statement': [{ + 'Effect': + 'Allow', + 'Action': ['s3:Put*', 's3:Get*', 's3:List*'], + 'Resource': [ + f'arn:aws:s3:::{S3_BUCKET_NAME}/artifacts/*', + f'arn:aws:s3:::{S3_BUCKET_NAME}/private-artifacts/{namespace}/*', + f'arn:aws:s3:::{S3_BUCKET_NAME}/private/{namespace}/*', + f'arn:aws:s3:::{S3_BUCKET_NAME}/shared/*', + ] + }] + })) + + self.upsert_lifecycle_policy(S3_BUCKET_NAME, + artifact_retention_days) desired_resources.insert( - 0, - { - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "mlpipeline-minio-artifact", - "namespace": namespace, + 0, { + 'apiVersion': 'v1', + 'kind': 'Secret', + 'metadata': { + 'name': 'mlpipeline-minio-artifact', + 'namespace': namespace, }, - "data": { - "accesskey": base64.b64encode(s3_access_key["AccessKey"]["AccessKeyId"].encode('utf-8')).decode("utf-8"), - "secretkey": base64.b64encode(s3_access_key["AccessKey"]["SecretAccessKey"].encode('utf-8')).decode("utf-8"), - }, - }) + 'data': { + 'accesskey': + base64.b64encode(s3_access_key['AccessKey'] + ['AccessKeyId'].encode('utf-8') + ).decode('utf-8'), + 'secretkey': + base64.b64encode( + s3_access_key['AccessKey'] + ['SecretAccessKey'].encode('utf-8') + ).decode('utf-8'), + }, + }) - return {"status": desired_status, "attachments": desired_resources} + return {'status': desired_status, 'attachments': desired_resources} def do_POST(self): # Serve the sync() function as a JSON webhook. observed = json.loads( - self.rfile.read(int(self.headers.get("content-length")))) - desired = self.sync(observed["object"], observed["attachments"]) + self.rfile.read(int(self.headers.get('content-length')))) + desired = self.sync(observed['object'], observed['attachments']) self.send_response(200) - self.send_header("Content-type", "application/json") + self.send_header('Content-type', 'application/json') self.end_headers() self.wfile.write(bytes(json.dumps(desired), 'utf-8')) return HTTPServer((url, int(controller_port)), Controller) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/test_sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/test_sync.py index 7be8907f0fc..9a6ebd42fb5 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/test_sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/test_sync.py @@ -1,305 +1,292 @@ -import os -from unittest import mock +import base64 import threading -import sync -from sync import get_settings_from_env, server_factory -import json import pytest import requests +import sync +from sync import server_factory -# Data sets passed to server -DATA_INCORRECT_CHILDREN = { - "parent": { - "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" - }, - "name": "myName" - } - }, - "children": { - "Secret.v1": [], - "ConfigMap.v1": [], - "Deployment.apps/v1": [], - "Service.v1": [], - "DestinationRule.networking.istio.io/v1alpha3": [], - "AuthorizationPolicy.security.istio.io/v1beta1": [], +NAMESPACE = 'my-name' + + +def _base_attachments(): + return { + 'Secret.v1': {}, + 'ConfigMap.v1': {}, + 'Deployment.apps/v1': {}, + 'Service.v1': {}, + 'ServiceAccount.v1': {}, + 'Role.rbac.authorization.k8s.io/v1': {}, + 'RoleBinding.rbac.authorization.k8s.io/v1': {}, } -} -DATA_CORRECT_CHILDREN = { - "parent": { - "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" + +def _ready_attachments(artifacts_proxy_enabled=False): + attachments = _base_attachments() + attachments['Secret.v1'] = { + f'{NAMESPACE}/mlpipeline-minio-artifact': { + 'apiVersion': 'v1', + 'kind': 'Secret', + 'metadata': { + 'name': 'mlpipeline-minio-artifact', + 'namespace': NAMESPACE, + }, + 'data': { + 'accesskey': 'existing-access-key', + 'secretkey': 'existing-secret-key', + }, + }, + f'{NAMESPACE}/default-editor.service-account-token': { + 'kind': 'Secret' + }, + } + attachments['ConfigMap.v1'] = { + f'{NAMESPACE}/kfp-launcher': { + 'kind': 'ConfigMap' + }, + f'{NAMESPACE}/metadata-grpc-configmap': { + 'kind': 'ConfigMap' + }, + f'{NAMESPACE}/artifact-repositories': { + 'kind': 'ConfigMap' + }, + } + attachments['ServiceAccount.v1'] = {} + attachments['Role.rbac.authorization.k8s.io/v1'] = { + f'{NAMESPACE}/configmap-reader': { + 'kind': 'Role' + }, + f'{NAMESPACE}/ml-pipeline-driver-pods-reader': { + 'kind': 'Role' + }, + f'{NAMESPACE}/ml-pipeline-driver-pvc-editor': { + 'kind': 'Role' + }, + f'{NAMESPACE}/artifact-secret-reader': { + 'kind': 'Role' + }, + } + attachments['RoleBinding.rbac.authorization.k8s.io/v1'] = { + f'{NAMESPACE}/configmap-reader-binding': { + 'kind': 'RoleBinding' + }, + f'{NAMESPACE}/ml-pipeline-driver-pods-reader-binding': { + 'kind': 'RoleBinding' + }, + f'{NAMESPACE}/ml-pipeline-driver-pvc-editor-binding': { + 'kind': 'RoleBinding' + }, + f'{NAMESPACE}/artifact-secret-reader-binding': { + 'kind': 'RoleBinding' + }, + } + if artifacts_proxy_enabled: + attachments['Deployment.apps/v1'] = { + f'{NAMESPACE}/ml-pipeline-ui-artifact': { + 'kind': 'Deployment' }, - "name": "myName" } - }, - "children": { - "Secret.v1": [1], - "ConfigMap.v1": [1], - "Deployment.apps/v1": [1, 1], - "Service.v1": [1, 1], - "DestinationRule.networking.istio.io/v1alpha3": [1], - "AuthorizationPolicy.security.istio.io/v1beta1": [1], + attachments['Service.v1'] = { + f'{NAMESPACE}/ml-pipeline-ui-artifact': { + 'kind': 'Service' + }, + } + return attachments + + +def _observed(attachments, pipeline_enabled=True): + labels = { + 'pipelines.kubeflow.org/enabled': 'true' + } if pipeline_enabled else {} + return { + 'object': { + 'metadata': { + 'name': NAMESPACE, + 'labels': labels, + } + }, + 'attachments': attachments, } -} - -DATA_MISSING_PIPELINE_ENABLED = {"parent": {}, "children": {}} - -# Default values when environments are not explicit -DEFAULT_FRONTEND_IMAGE = "ghcr.io/kubeflow/kfp-frontend" -DEFAULT_VISUALIZATION_IMAGE = "ghcr.io/kubeflow/kfp-visualization-server" - -# Variables used for environment variable sets -VISUALIZATION_SERVER_IMAGE = "vis-image" -VISUALIZATION_SERVER_TAG = "somenumber.1.2.3" -FRONTEND_IMAGE = "frontend-image" -FRONTEND_TAG = "somehash" - -KFP_VERSION = "x.y.z" - -MINIO_ACCESS_KEY = "abcdef" -MINIO_SECRET_KEY = "uvwxyz" - -# "Environments" used in tests -ENV_VARIABLES_BASE = { - "MINIO_ACCESS_KEY": MINIO_ACCESS_KEY, - "MINIO_SECRET_KEY": MINIO_SECRET_KEY, - "CONTROLLER_PORT": "0", # HTTPServer randomly assigns the port to a free port -} - -ENV_KFP_VERSION_ONLY = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - } - ) - -ENV_IMAGES_NO_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - } - ) - -ENV_IMAGES_WITH_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - "VISUALIZATION_SERVER_TAG": VISUALIZATION_SERVER_TAG, - "FRONTEND_TAG": FRONTEND_TAG, - } - ) - -ENV_IMAGES_WITH_TAGS_AND_ISTIO = dict(ENV_IMAGES_WITH_TAGS, - **{ - "DISABLE_ISTIO_SIDECAR": "false", - } - ) - - -def generate_image_name(imagename, tag): - return f"{str(imagename)}:{str(tag)}" - - -@pytest.fixture( - scope="function", -) -def sync_server(request): - """ - Starts the sync HTTP server for a given set of environment variables on a separate thread - - Yields: - * the server (useful to interrogate for the server address) - * environment variables (useful to interrogate for correct responses) - """ - environ = request.param - with mock.patch.dict(os.environ, environ): - # Create a server at an available port and serve it on a thread as a daemon - # This will result in a collection of servers being active - not a great way - # if this fixture is run many times during a test, but ok for now - settings = get_settings_from_env() - server = server_factory(**settings) - server_thread = threading.Thread(target=server.serve_forever) - # Put on daemon so it doesn't keep pytest from ending - server_thread.daemon = True - server_thread.start() - yield server, environ - - -@pytest.fixture( - scope="function", -) -def sync_server_from_arguments(request): - """ - Starts the sync HTTP server for a given set of parameters passed as arguments, with server on a separate thread - - Yields: - * the server (useful to interrogate for the server address) - * environment variables (useful to interrogate for correct responses) - """ - environ = {k.lower(): v for k, v in request.param.items()} - settings = environ - server = server_factory(**settings) - server_thread = threading.Thread(target=server.serve_forever) - # Put on daemon so it doesn't keep pytest from ending - server_thread.daemon = True - server_thread.start() - yield server, environ - -@pytest.mark.parametrize( - "sync_server, data, expected_status, expected_visualization_server_image, expected_frontend_server_image", - [ - ( - ENV_KFP_VERSION_ONLY, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(DEFAULT_VISUALIZATION_IMAGE, KFP_VERSION), - generate_image_name(DEFAULT_FRONTEND_IMAGE, KFP_VERSION), - ), - ( - ENV_IMAGES_NO_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_NO_TAGS["VISUALIZATION_SERVER_IMAGE"], KFP_VERSION), - generate_image_name(ENV_IMAGES_NO_TAGS["FRONTEND_IMAGE"], KFP_VERSION), - ), - ( - ENV_IMAGES_WITH_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), - ), - ( - ENV_IMAGES_WITH_TAGS, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), - ), - ], - indirect=["sync_server"] -) -def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, - expected_visualization_server_image, expected_frontend_server_image): - """ - Nearly end-to-end test of how Controller serves .sync as a POST - Tests case where metadata.labels.pipelines.kubeflow.org/enabled exists, and thus - we should produce children +def _count_kind(resources, kind_name): + return sum(1 for resource in resources if resource.get('kind') == kind_name) - Only does spot checks on children to see if key properties are correct - """ - server, environ = sync_server - # server.server_address = (url, port_as_integer) - url = f"http://{server.server_address[0]}:{str(server.server_address[1])}" - print("url: ", url) - print("data") - print(json.dumps(data)) - x = requests.post(url, data=json.dumps(data)) - results = json.loads(x.text) +@pytest.fixture +def sync_server(request): + settings = dict(request.param) + settings['controller_port'] = 0 + server = server_factory(**settings) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() - # Test overall status of whether children are ok - assert results['status'] == expected_status - # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image +def _post(server, payload): + url = f'http://{server.server_address[0]}:{server.server_address[1]}' + response = requests.post(url, json=payload, timeout=5) + response.raise_for_status() + return response.json() @pytest.mark.parametrize( - "sync_server_from_arguments, data, expected_status, expected_visualization_server_image, " - "expected_frontend_server_image", - [ - ( - ENV_IMAGES_WITH_TAGS_AND_ISTIO, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), - ), - ], - indirect=["sync_server_from_arguments"] + 'sync_server', + [{ + 'frontend_image': 'frontend-image', + 'frontend_tag': 'tag123', + 'disable_istio_sidecar': False, + 'artifacts_proxy_enabled': 'false', + 'artifact_retention_days': '7', + 'cluster_domain': '.svc.cluster.local', + 'object_store_host': 'seaweedfs', + }], + indirect=True, ) -def test_sync_server_with_direct_passing_of_settings( - sync_server_from_arguments, data, expected_status, expected_visualization_server_image, - expected_frontend_server_image): - """ - Nearly end-to-end test of how Controller serves .sync as a POST, taking variables as arguments - - Only does spot checks on children to see if key properties are correct - """ - server, environ = sync_server_from_arguments - - # server.server_address = (url, port_as_integer) - url = f"http://{server.server_address[0]}:{str(server.server_address[1])}" - print("url: ", url) - print("data") - print(json.dumps(data)) - x = requests.post(url, data=json.dumps(data)) - results = json.loads(x.text) - - # Test overall status of whether children are ok - assert results['status'] == expected_status - - # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image +def test_sync_pipeline_enabled_without_proxy(sync_server): + observed = _observed(_ready_attachments(artifacts_proxy_enabled=False)) + results = _post(sync_server, observed) + + assert results['status'] == {'kubeflow-pipelines-ready': 'True'} + assert _count_kind(results['attachments'], 'ServiceAccount') == 0 + assert _count_kind(results['attachments'], 'Role') == 4 + assert _count_kind(results['attachments'], 'RoleBinding') == 4 + assert _count_kind(results['attachments'], 'Deployment') == 0 + assert _count_kind(results['attachments'], 'Service') == 0 @pytest.mark.parametrize( - "sync_server, data, expected_status, expected_children", - [ - (ENV_IMAGES_WITH_TAGS, DATA_MISSING_PIPELINE_ENABLED, {}, []), - ], - indirect=["sync_server"] + 'sync_server', + [{ + 'frontend_image': 'frontend-image', + 'frontend_tag': 'tag123', + 'disable_istio_sidecar': True, + 'artifacts_proxy_enabled': 'true', + 'artifact_retention_days': '7', + 'cluster_domain': '.svc.cluster.local', + 'object_store_host': 'seaweedfs', + }], + indirect=True, ) -def test_sync_server_without_pipeline_enabled(sync_server, data, expected_status, - expected_children): - """ - Nearly end-to-end test of how Controller serves .sync as a POST +def test_sync_pipeline_enabled_with_proxy(sync_server): + observed = _observed(_ready_attachments(artifacts_proxy_enabled=True)) + results = _post(sync_server, observed) - Tests case where metadata.labels.pipelines.kubeflow.org/enabled does not - exist and thus server returns an empty reply - """ - server, environ = sync_server + assert results['status'] == {'kubeflow-pipelines-ready': 'True'} + deployments = [ + resource for resource in results['attachments'] + if resource.get('kind') == 'Deployment' + ] + assert len(deployments) == 1 + image = deployments[0]['spec']['template']['spec']['containers'][0]['image'] + assert image == 'frontend-image:tag123' - # server.server_address = (url, port_as_integer) - url = f"http://{server.server_address[0]}:{str(server.server_address[1])}" - x = requests.post(url, data=json.dumps(data)) - results = json.loads(x.text) - # Test overall status of whether children are ok - assert results['status'] == expected_status - assert results['children'] == expected_children +@pytest.mark.parametrize( + 'sync_server', + [{ + 'frontend_image': 'frontend-image', + 'frontend_tag': 'tag123', + 'disable_istio_sidecar': True, + 'artifacts_proxy_enabled': 'false', + 'artifact_retention_days': '7', + 'cluster_domain': '.svc.cluster.local', + 'object_store_host': 'seaweedfs', + }], + indirect=True, +) +def test_sync_pipeline_disabled_returns_empty(sync_server): + results = _post(sync_server, + _observed(_base_attachments(), pipeline_enabled=False)) + assert results['status'] == {} + assert results['attachments'] == [] + + +def test_sync_creates_minio_secret_when_missing(monkeypatch): + + class DummyIAM: + + def __init__(self): + self.put_policy_called = False + + def create_access_key(self, UserName): + assert UserName == NAMESPACE + return { + 'AccessKey': { + 'AccessKeyId': 'AKIA_TEST', + 'SecretAccessKey': 'SECRET_TEST', + } + } + + def put_user_policy(self, **kwargs): + self.put_policy_called = True + assert kwargs['UserName'] == NAMESPACE + + class DummyS3: + + def put_bucket_lifecycle_configuration(self, **kwargs): + assert kwargs['Bucket'] == 'mlpipeline' + return {} + + dummy_iam = DummyIAM() + monkeypatch.setattr(sync, 'iam', dummy_iam) + monkeypatch.setattr(sync, 's3', DummyS3()) + + settings = { + 'frontend_image': 'frontend-image', + 'frontend_tag': 'tag123', + 'disable_istio_sidecar': True, + 'artifacts_proxy_enabled': 'false', + 'artifact_retention_days': '7', + 'cluster_domain': '.svc.cluster.local', + 'object_store_host': 'seaweedfs', + 'controller_port': 0, + } + server = server_factory(**settings) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + attachments = _ready_attachments(artifacts_proxy_enabled=False) + attachments['Secret.v1'].pop(f'{NAMESPACE}/mlpipeline-minio-artifact') + results = _post(server, _observed(attachments)) + finally: + server.shutdown() + server.server_close() + + minio_secrets = [ + resource for resource in results['attachments'] + if resource.get('kind') == 'Secret' and + resource.get('metadata', {}).get('name') == 'mlpipeline-minio-artifact' + ] + assert len(minio_secrets) == 1 + assert base64.b64decode( + minio_secrets[0]['data']['accesskey']).decode('utf-8') == 'AKIA_TEST' + assert base64.b64decode( + minio_secrets[0]['data']['secretkey']).decode('utf-8') == 'SECRET_TEST' + assert dummy_iam.put_policy_called def test_create_iam_client_uses_endpoint(monkeypatch): called = {} class DummySession: - def create_client(self, service_name, region_name=None, endpoint_url=None): - called["service_name"] = service_name - called["endpoint_url"] = endpoint_url + + def create_client(self, + service_name, + region_name=None, + endpoint_url=None): + called['service_name'] = service_name + called['endpoint_url'] = endpoint_url return object() - monkeypatch.setenv("AWS_ENDPOINT_URL", "http://seaweedfs.kubeflow:8111") - monkeypatch.setattr(sync, "session", DummySession()) + monkeypatch.setenv('AWS_ENDPOINT_URL', 'http://seaweedfs.kubeflow:8111') + monkeypatch.setattr(sync, 'session', DummySession()) sync.create_iam_client() - assert called["service_name"] == "iam" - assert called["endpoint_url"] == "http://seaweedfs.kubeflow:8111" + assert called['service_name'] == 'iam' + assert called['endpoint_url'] == 'http://seaweedfs.kubeflow:8111' diff --git a/manifests/kustomize/base/pipeline/kustomization.yaml b/manifests/kustomize/base/pipeline/kustomization.yaml index f5e65af608c..8ff1ef30c25 100644 --- a/manifests/kustomize/base/pipeline/kustomization.yaml +++ b/manifests/kustomize/base/pipeline/kustomization.yaml @@ -31,6 +31,11 @@ resources: - pipeline-runner-role.yaml - pipeline-runner-rolebinding.yaml - pipeline-runner-sa.yaml + - ml-pipeline-driver-deployment.yaml + - ml-pipeline-driver-role.yaml + - ml-pipeline-driver-rolebinding.yaml + - ml-pipeline-driver-sa.yaml + - ml-pipeline-driver-service.yaml - container-builder-sa.yaml - viewer-sa.yaml - kfp-launcher-configmap.yaml @@ -55,3 +60,5 @@ images: newTag: 2.16.0 - name: ghcr.io/kubeflow/kfp-visualization-server newTag: 2.16.0 + - name: ghcr.io/kubeflow/kfp-driver + newTag: 2.16.0 diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index a48ac0dc8ab..103e7394801 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -125,8 +125,6 @@ spec: secretKeyRef: name: mlpipeline-minio-artifact key: secretkey - - name: V2_DRIVER_IMAGE - value: ghcr.io/kubeflow/kfp-driver:2.16.0 - name: V2_LAUNCHER_IMAGE value: ghcr.io/kubeflow/kfp-launcher:2.16.0 # JSON patch to apply to compiled workflow specifications diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-deployment.yaml new file mode 100644 index 00000000000..ef10cda89bc --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ml-pipeline-driver + name: ml-pipeline-driver +spec: + replicas: 1 + selector: + matchLabels: + app: ml-pipeline-driver + template: + metadata: + labels: + app: ml-pipeline-driver + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: ml-pipeline-driver + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: ml-pipeline-driver + image: ghcr.io/kubeflow/kfp-driver:dummy + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http + env: + - name: KFP_DRIVER_PORT + value: "8080" + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-role.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-role.yaml new file mode 100644 index 00000000000..ab95c37bdd4 --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-role.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: ml-pipeline-driver + name: ml-pipeline-driver +rules: +# Needed to read pipeline-runner SA token for pod spec patch generation +- apiGroups: [""] + resources: [pods, pods/log] + verbs: [get, list] +# Needed to look up ConfigMaps for object store session info +- apiGroups: [""] + resources: [configmaps, secrets] + verbs: [get, list] +# Needed for cache lookups +- apiGroups: [argoproj.io] + resources: [workflows] + verbs: [get, list] diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-rolebinding.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-rolebinding.yaml new file mode 100644 index 00000000000..f83e82ce001 --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-rolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: ml-pipeline-driver + name: ml-pipeline-driver +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ml-pipeline-driver +subjects: +- kind: ServiceAccount + name: ml-pipeline-driver diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-sa.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-sa.yaml new file mode 100644 index 00000000000..e5c470fb051 --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ml-pipeline-driver diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-service.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-service.yaml new file mode 100644 index 00000000000..3bc662fd0cf --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: ml-pipeline-driver + name: ml-pipeline-driver +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: ml-pipeline-driver diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index eba0ee9f2d6..9c2c5510924 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -36,12 +36,16 @@ rules: - argoproj.io resources: - workflows + - workflowtaskresults + - workflowtasksets + - "workflowtasksets/status" verbs: - get - - list + - create + - patch - watch + - list - update - - patch - apiGroups: - "" resources: @@ -85,3 +89,98 @@ rules: verbs: - create - patch + +--- +# The following roles and bindings grant the KFP API server (ml-pipeline SA) the +# permissions it needs to run the driver logic centrally. The driver is now served +# as a built-in endpoint of the API server rather than injected as a per-workflow +# sidecar container. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ml-pipeline-driver-pods-viewer + namespace: kubeflow +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ml-pipeline-driver-pods-viewer-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline +roleRef: + kind: Role + name: ml-pipeline-driver-pods-viewer + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ml-pipeline-driver-pvc-editor + namespace: kubeflow +rules: + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ml-pipeline-driver-pvc-editor-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline +roleRef: + kind: Role + name: ml-pipeline-driver-pvc-editor + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: configmap-reader +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: configmap-reader-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline +roleRef: + kind: Role + name: configmap-reader + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: artifact-secret-reader + namespace: kubeflow +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["mlpipeline-minio-artifact"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: artifact-secret-reader-binding + namespace: kubeflow +subjects: + - kind: ServiceAccount + name: ml-pipeline + namespace: kubeflow +roleRef: + kind: Role + name: artifact-secret-reader + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml index 8cb2c669fb2..e1f42088371 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml @@ -2,3 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: name: pipeline-runner + +--- +apiVersion: v1 +kind: Secret +metadata: + name: pipeline-runner.service-account-token + annotations: + kubernetes.io/service-account.name: pipeline-runner +type: kubernetes.io/service-account-token diff --git a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml index 8dcdf1c623c..47edb66303a 100644 --- a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml +++ b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml @@ -23,4 +23,6 @@ spec: issuerRef: kind: Issuer name: kfp-api-tls-selfsigned-issuer - secretName: kfp-api-tls-cert + # Unfortunately, Argo Workflows (for security reasons) can mount certificates + # into the agent only from a Secret with a predefined name `argo-workflows-agent-ca-certificates` + secretName: argo-workflows-agent-ca-certificates \ No newline at end of file diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml index 0fa72587d37..c6a99880058 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml @@ -20,7 +20,7 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates - name: envoy-config-tls-enabled configMap: name: envoy-config diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml index 4bc193483d7..e6af2e5f545 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml @@ -10,7 +10,7 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates - name: grpc-tls-config emptyDir: { } - name: mysql-secret diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml index 1c651cc3b4b..f5f72c4b111 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml @@ -19,4 +19,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml index fef772368b5..59d268fe922 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml @@ -17,7 +17,7 @@ spec: - "--tlsCertKeyPath=/etc/pki/tls/certs/tls.key" env: - name: CABUNDLE_SECRET_NAME - value: "kfp-api-tls-cert" + value: "argo-workflows-agent-ca-certificates" - name: METADATA_TLS_ENABLED value: "true" - name: ML_PIPELINE_SERVICE_HOST @@ -53,4 +53,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml index fccb9d2fedc..73f64d3ac06 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml @@ -35,4 +35,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml index 301ca3bb816..7c08cb7d4af 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml @@ -22,4 +22,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml index cf7775542b2..aed2004e31c 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml @@ -30,4 +30,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/dev/api-server-patch.yaml b/manifests/kustomize/env/dev/api-server-patch.yaml index 1c590f556f0..785ff49022b 100644 --- a/manifests/kustomize/env/dev/api-server-patch.yaml +++ b/manifests/kustomize/env/dev/api-server-patch.yaml @@ -8,8 +8,6 @@ spec: spec: containers: - env: - - name: V2_DRIVER_IMAGE - value: ghcr.io/kubeflow/kfp-driver:master - name: V2_LAUNCHER_IMAGE value: ghcr.io/kubeflow/kfp-launcher:master name: ml-pipeline-api-server diff --git a/manifests/kustomize/env/platform-agnostic-emissary/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-emissary/kustomization.yaml new file mode 100644 index 00000000000..e4e22ecd451 --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-emissary/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../platform-agnostic diff --git a/manifests/kustomize/env/platform-agnostic-multi-user-emissary/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-multi-user-emissary/kustomization.yaml new file mode 100644 index 00000000000..6a396aca8e3 --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-multi-user-emissary/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../platform-agnostic-multi-user diff --git a/manifests/kustomize/env/platform-agnostic-multi-user-legacy/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-multi-user-legacy/kustomization.yaml new file mode 100644 index 00000000000..b8deb06d39c --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-multi-user-legacy/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../third-party/metacontroller/base +- ../../base/installs/multi-user +- ../../base/metadata/overlays/db +- ../../base/metadata/options/istio +- ../../third-party/argo/installs/cluster +- ../../third-party/mysql/base +- ../../third-party/mysql/options/istio +- ../../third-party/seaweedfs/istio + + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +labels: +- includeSelectors: true + pairs: + application-crd-id: kubeflow-pipelines diff --git a/manifests/kustomize/gcp-workload-identity-setup.sh b/manifests/kustomize/gcp-workload-identity-setup.sh new file mode 100755 index 00000000000..79f86c95c85 --- /dev/null +++ b/manifests/kustomize/gcp-workload-identity-setup.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# +# Copyright 2019 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Kubernetes Namespace +NAMESPACE=${NAMESPACE:-kubeflow} + +# Google service Account (GSA) +SYSTEM_GSA=${SYSTEM_GSA:-$RESOURCE_PREFIX-kfp-system} +USER_GSA=${USER_GSA:-$RESOURCE_PREFIX-kfp-user} + +# Kubernetes Service Account (KSA) +# Note, if deploying manifests/kustomize/env/gcp, you can add the following KSAs +# to the array of SYSTEM_KSA: +# * kubeflow-pipelines-minio-gcs-gateway needs gcs permissions +# * kubeflow-pipelines-cloudsql-proxy needs cloudsql permissions +SYSTEM_KSA=(ml-pipeline-ui ml-pipeline-visualizationserver) +USER_KSA=(pipeline-runner kubeflow-pipelines-container-builder kubeflow-pipelines-viewer) + +if [ -n $USE_GCP_MANAGED_STORAGE ]; then + SYSTEM_KSA+=(kubeflow-pipelines-minio-gcs-gateway) + SYSTEM_KSA+=(kubeflow-pipelines-cloudsql-proxy) +fi + +cat < RESOURCE_PREFIX= NAMESPACE= ./gcp-workload-identity-setup.sh +``` + +PROJECT_ID: GCP project ID your cluster belongs to. +RESOURCE_PREFIX: Your preferred resource prefix for GCP resources this script creates. +NAMESPACE: Optional. Kubernetes namespace your Kubeflow Pipelines standalone deployment belongs to. (Defaults to kubeflow) +USE_GCP_MANAGED_STORAGE: Optional. Defaults to "false", specify "true" if you intend to use GCP managed storage (Google Cloud Storage and Cloud SQL) following instructions in: +https://github.com/kubeflow/pipelines/tree/master/manifests/kustomize/sample +EOF +} +if [ -z "$PROJECT_ID" ]; then + usage + echo + echo "Error: PROJECT_ID env variable is empty!" + exit 1 +fi +if [ -z "$RESOURCE_PREFIX" ]; then + usage + echo + echo "Error: RESOURCE_PREFIX env variable is empty!" + exit 1 +fi +echo "Env variables set:" +echo "* PROJECT_ID=$PROJECT_ID" +echo "* RESOURCE_PREFIX=$RESOURCE_PREFIX" +echo "* NAMESPACE=$NAMESPACE" +echo "* USE_GCP_MANAGED_STORAGE=${USE_GCP_MANAGED_STORAGE:-false}" +echo + +SYSTEM_GSA_FULL="$SYSTEM_GSA@$PROJECT_ID.iam.gserviceaccount.com" +USER_GSA_FULL="$USER_GSA@$PROJECT_ID.iam.gserviceaccount.com" + +cat </dev/null; then + echo "KSA $name already exists" + else + kubectl create serviceaccount $name -n $NAMESPACE --save-config + echo "KSA $name created" + fi +} + +# Bind KSA to GSA through workload identity. +# Documentation: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity +function bind_gsa_and_ksa { + local gsa=${1} + local ksa=${2} + + gcloud iam service-accounts add-iam-policy-binding $gsa@$PROJECT_ID.iam.gserviceaccount.com \ + --member="serviceAccount:$PROJECT_ID.svc.id.goog[$NAMESPACE/$ksa]" \ + --role="roles/iam.workloadIdentityUser" \ + > /dev/null # hide verbose output + + create_ksa_if_not_present $ksa + kubectl annotate serviceaccount \ + --namespace $NAMESPACE \ + --overwrite \ + $ksa \ + iam.gke.io/gcp-service-account=$gsa@$PROJECT_ID.iam.gserviceaccount.com + echo "* Bound KSA $ksa to GSA $gsa" +} + +echo "Binding each kfp system KSA to $SYSTEM_GSA" +for ksa in ${SYSTEM_KSA[@]}; do + bind_gsa_and_ksa $SYSTEM_GSA $ksa +done + +echo "Binding each kfp user KSA to $USER_GSA" +for ksa in ${USER_KSA[@]}; do + bind_gsa_and_ksa $USER_GSA $ksa +done + +echo +echo "All the workload identity bindings have succeeded!" +cat < /dev/null # hide verbose output + kubectl annotate serviceaccount \ + --namespace $namespace \ + --overwrite \ + $ksa \ + iam.gke.io/gcp-service-account=$gsa_full + echo "* Bound KSA $ksa in namespace $namespace to GSA $gsa_full" +} + +# This can be used to programmatically verify workload identity binding grants corresponding GSA +# permissions successfully. +# Usage: verify_workload_identity_binding $KSA $NAMESPACE +# +# If you want to verify manually, use the following command instead: +# kubectl run test-$RANDOM --rm -it --restart=Never \ +# --image=google/cloud-sdk:slim \ +# --serviceaccount $ksa \ +# --namespace $namespace \ +# -- /bin/bash +# It connects you to a pod using specified KSA running an image with gcloud and gsutil CLI tools. +function verify_workload_identity_binding { + local ksa=${1} + local namespace=${2} + local max_attempts=10 + local workload_identity_is_ready=false + for i in $(seq 1 ${max_attempts}) + do + workload_identity_is_ready=true + kubectl run test-$RANDOM --rm -i --restart=Never \ + --image=google/cloud-sdk:slim \ + --serviceaccount $ksa \ + --namespace $namespace \ + -- gcloud auth list || workload_identity_is_ready=false + kubectl run test-$RANDOM --rm -i --restart=Never \ + --image=google/cloud-sdk:slim \ + --serviceaccount $ksa \ + --namespace $namespace \ + -- gsutil ls gs:// || workload_identity_is_ready=false + if [ "$workload_identity_is_ready" = true ]; then + break + fi + done + if [ ! "$workload_identity_is_ready" = true ]; then + echo "Workload identity bindings are not ready after $max_attempts attempts" + return 1 + fi +} diff --git a/proposals/12862-mlflow-integration/README.md b/proposals/12862-mlflow-integration/README.md index 54af36b027d..b39fe3fe9fc 100644 --- a/proposals/12862-mlflow-integration/README.md +++ b/proposals/12862-mlflow-integration/README.md @@ -274,10 +274,10 @@ run-related messages. MLflow integration requires an explicit opt-in at the API server level via a `plugins.mlflow` (JSON notation here) entry in the API server config. Existing KFP deployments that upgrade will not have MLflow enabled; an administrator must explicitly add the `plugins.mlflow` configuration to enable it. In multi-user mode, the per-namespace `kfp-launcher` -ConfigMap (`plugins.mlflow` key) provides namespace-specific overrides. For secret-based auth (`"bearer"` and -`"basic-auth"`), the namespace must also provide `credentialSecretRef` in this ConfigMap; otherwise MLflow integration -is disabled for runs in that namespace. Global API server configuration may still provide shared non-secret defaults such -as the endpoint and timeout. +ConfigMap (`plugins.mlflow` key) provides namespace-specific overrides but is only effective when the API server has +MLflow enabled. Alternatively, the administrator may choose not to configure a global default and instead require each +namespace to opt in by defining its own `plugins.mlflow` entry in the `kfp-launcher` ConfigMap. If neither the API +server nor the namespace ConfigMap provides MLflow configuration, MLflow integration is disabled for that namespace. When MLflow is enabled for a namespace (via either configuration level), it applies to **all** pipeline runs in that namespace by default. A user may opt out of MLflow tracking for a specific run by setting @@ -295,15 +295,14 @@ When a run is created and MLflow is enabled for the namespace: 1. Read `plugins_input.mlflow.experiment_id` or `plugins_input.mlflow.experiment_name` from the request if provided. `experiment_id` takes precedence if both are set. If neither is provided or `plugins_input.mlflow` is not set at all, default to the `"Default"` experiment. -1. Resolve MLflow credentials for outbound API calls. The API server uses its own SA token when `authType: "kubernetes"`. - For `"bearer"` and `"basic-auth"`, the API server reads the credential Secret from `credentialSecretRef`. In - multi-user mode, this `credentialSecretRef` must come from the namespace's `kfp-launcher` ConfigMap (see - [MLflow Configuration](#mlflow-configuration)). When secret-based auth is enabled, the Argo compiler mounts the - credential Secret on the driver, launcher, and user container templates via `env` entries with - `valueFrom.secretKeyRef`, mapping Secret keys to the standard MLflow env vars (`MLFLOW_TRACKING_TOKEN`, or - `MLFLOW_TRACKING_USERNAME` / `MLFLOW_TRACKING_PASSWORD`). This ensures the driver and launcher can authenticate their - own MLflow REST calls without reading the Secret at runtime, and no secret values appear in the Argo Workflow spec - (see [MLflow Configuration](#mlflow-configuration)). +1. Resolve MLflow credentials for outbound API calls. The API server uses its own SA token (`authType: "kubernetes"`) or + reads the credential Secret from `credentialSecretRef`. If a per-namespace ConfigMap overrides the `endpoint`, it + must also provide its own credentials; the API server's global credentials are never paired with a namespace-provided + endpoint. For `"bearer"` or `"basic-auth"` modes, the Argo compiler mounts the credential Secret on the driver, + launcher, and user container templates via `env` entries with `valueFrom.secretKeyRef`, mapping Secret keys to the + standard MLflow env vars (`MLFLOW_TRACKING_TOKEN`, or `MLFLOW_TRACKING_USERNAME` / `MLFLOW_TRACKING_PASSWORD`). This + ensures the driver and launcher can authenticate their own MLflow REST calls without reading the Secret at runtime, + and no secret values appear in the Argo Workflow spec (see [MLflow Configuration](#mlflow-configuration)). 1. Call the MLflow REST API to create the experiment if it does not already exist (`POST /api/2.0/mlflow/experiments/create`). The experiment description is set to the value of `experimentDescription` from the plugin settings. If `experimentDescription` is not set, it defaults to @@ -526,14 +525,14 @@ permissions on the corresponding `mlflow.kubeflow.org` resources (e.g., `dataset [Kubernetes workspace provider RBAC documentation](https://github.com/opendatahub-io/mlflow/tree/master/kubernetes-workspace-provider#kubernetes-rbac-requirements) for the full resource list. -**Secret-based mode.** When secret-based auth is enabled, the `ml-pipeline` API server service account will need a new -RBAC rule granting `get` on `secrets` in run namespaces to read the credential Secret at run creation time. As a best -practice, administrators should use the same Secret name (e.g., `kfp-mlflow-credentials`) across all namespaces and -scope the RBAC rule with `resourceNames` to limit the API server's access to that single Secret name. The -`pipeline-runner` service account already has `get` on `secrets` in the run's namespace. The token or credentials -stored in the Secret must have sufficient MLflow permissions to create and update experiments and runs (equivalent to -`get`, `list`, `create`, `update` on experiments in Kubernetes-native mode). If MLflow's built-in authentication is -used, this means a user account with at least editor-level access. +**Secret-based mode.** The `ml-pipeline` API server service account will need a new RBAC rule granting `get` on +`secrets` in run namespaces to read the credential Secret at run creation time. As a best practice, administrators +should use the same Secret name (e.g., `kfp-mlflow-credentials`) across all namespaces and scope the RBAC rule with +`resourceNames` to limit the API server's access to that single Secret name. The `pipeline-runner` service account +already has `get` on `secrets` in the run's namespace. The token or credentials stored in the Secret must have +sufficient MLflow permissions to create and update experiments and runs (equivalent to `get`, `list`, `create`, `update` +on experiments in Kubernetes-native mode). If MLflow's built-in authentication is used, this means a user account with +at least editor-level access. If a user specifies a custom service account for their pipeline run (via `kubernetes_platform`), that service account must also have the appropriate MLflow permissions, otherwise MLflow calls from the driver and launcher will fail. @@ -648,28 +647,24 @@ Three `authType` values are supported: This is the default authentication mechanism for self-hosted MLflow using its [built-in authentication](https://mlflow.org/docs/latest/ml/auth/). -For `"bearer"` and `"basic-auth"`, `credentialSecretRef` is required. In multi-user mode, it must come from the -namespace's `kfp-launcher` ConfigMap rather than only from the API server's global config; if it is absent there, MLflow -integration is disabled for runs in that namespace. Workspaces default to disabled but can be enabled. Administrators -should understand that these modes delegate access control to whoever can create Secrets in the namespace. +For `"bearer"` and `"basic-auth"`, `credentialSecretRef` is required. Workspaces default to disabled but can be enabled. +Administrators should understand that these modes delegate access control to whoever can create Secrets in the +namespace. Configuration is provided at two levels: - **API server config** (required for opt-in): A `plugins` object containing a `mlflow` key. This is required in both standalone and multi-user modes to enable MLflow integration. When set, this provides the global default MLflow - configuration used for all namespaces unless overridden. In standalone mode, this is the only configuration level. In - multi-user mode, these defaults may include shared non-secret fields such as `endpoint`, `workspacesEnabled`, and - timeout settings. + configuration used for all namespaces unless overridden. In standalone mode, this is the only configuration level. - **Per-namespace `kfp-launcher` ConfigMap** (multi-user mode): A `plugins.mlflow` key with a JSON value. Allows per-namespace MLflow configuration. Fields set in the per-namespace config override the corresponding global defaults; - unset fields other than `credentialSecretRef` are inherited from the global config. In multi-user mode, namespaces - using `"bearer"` or `"basic-auth"` must provide `credentialSecretRef` in this ConfigMap; - `credentialSecretRef` is never inherited from the API server's global config. If they do not provide it, MLflow - integration is disabled for runs in that namespace. If the namespace overrides the `endpoint`, it must also provide - its own credentials -- the API server's global credentials are never used with a namespace-provided endpoint. A - namespace may override only credentials while inheriting the global endpoint, for example to use a namespace-scoped - Secret instead of the API server's service account token. If neither the API server nor the namespace ConfigMap - provides MLflow configuration, MLflow integration is disabled for that namespace. + unset fields are inherited from the global config. If the namespace overrides the `endpoint`, it must also provide its + own credentials -- the API server's global credentials are never used with a namespace-provided endpoint. A namespace + may override only credentials while inheriting the global endpoint, for example to use a namespace-scoped Secret + instead of the API server's service account token. The admin may choose not to configure a default MLflow server at + the API server level and instead require each namespace to define its own MLflow configuration via the `kfp-launcher` + ConfigMap. If neither the API server nor the namespace ConfigMap provides MLflow configuration, MLflow integration is + disabled for that namespace. The Go types for the configuration use a generic `PluginConfig` wrapper with plugin-specific settings in a `json.RawMessage` field. This structure aligns with [KEP #12700](https://github.com/kubeflow/pipelines/pull/12700)'s diff --git a/sdk/python/kfp/compiler/compiler_test.py b/sdk/python/kfp/compiler/compiler_test.py index dbc17a57a71..d90c79ef027 100644 --- a/sdk/python/kfp/compiler/compiler_test.py +++ b/sdk/python/kfp/compiler/compiler_test.py @@ -4587,230 +4587,6 @@ def my_pipeline(): # Should not raise an error self.assertTrue(os.path.exists(output_yaml)) - def test_pipeline_with_ttl_seconds_after_finished(self): - """ttl_seconds_after_finished → resource_ttl_on_completion (SecondsAfterCompletion).""" - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig(ttl_seconds_after_finished=300)) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - json_format.ParseDict( - {'pipelineConfig': { - 'resourceTtlOnCompletion': 300 - }}, expected.platforms['kubernetes']) - - self.assertEqual(my_pipeline.platform_spec, expected) - - loaded_pipeline = compile_and_reload(my_pipeline) - self.assertEqual(loaded_pipeline.platform_spec, expected) - - def test_pipeline_with_ttl_seconds_after_success(self): - """ttl_seconds_after_success → resource_ttl_on_success (SecondsAfterSuccess).""" - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig(ttl_seconds_after_success=600)) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - json_format.ParseDict({'pipelineConfig': { - 'resourceTtlOnSuccess': 600 - }}, expected.platforms['kubernetes']) - - self.assertEqual(my_pipeline.platform_spec, expected) - - loaded_pipeline = compile_and_reload(my_pipeline) - self.assertEqual(loaded_pipeline.platform_spec, expected) - - def test_pipeline_with_ttl_seconds_after_failure(self): - """ttl_seconds_after_failure → resource_ttl_on_failure (SecondsAfterFailure).""" - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig(ttl_seconds_after_failure=120)) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - json_format.ParseDict({'pipelineConfig': { - 'resourceTtlOnFailure': 120 - }}, expected.platforms['kubernetes']) - - self.assertEqual(my_pipeline.platform_spec, expected) - - loaded_pipeline = compile_and_reload(my_pipeline) - self.assertEqual(loaded_pipeline.platform_spec, expected) - - def test_pipeline_with_all_three_ttl_fields(self): - """All three TTL fields can be set independently.""" - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig( - ttl_seconds_after_finished=300, - ttl_seconds_after_success=3600, - ttl_seconds_after_failure=60, - )) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - json_format.ParseDict( - { - 'pipelineConfig': { - 'resourceTtlOnCompletion': 300, - 'resourceTtlOnSuccess': 3600, - 'resourceTtlOnFailure': 60, - } - }, expected.platforms['kubernetes']) - - self.assertEqual(my_pipeline.platform_spec, expected) - - def test_pipeline_with_ttl_zero(self): - """TTL of 0 is accepted but treated as "not set" and not serialized. - - A value of 0 is indistinguishable from the proto3 int32 default, so the - serializer skips it. The resulting platform spec contains no - pipelineConfig entry at all, identical to not setting a TTL. - """ - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig(ttl_seconds_after_finished=0)) - def my_pipeline(): - comp() - - # TTL=0 is skipped by the > 0 gate, so the platform spec is empty. - expected = pipeline_spec_pb2.PlatformSpec() - - self.assertEqual(my_pipeline.platform_spec, expected) - - loaded_pipeline = compile_and_reload(my_pipeline) - self.assertEqual(loaded_pipeline.platform_spec, expected) - - def test_pipeline_with_ttl_and_workspace(self): - """TTL and workspace can coexist in PipelineConfig.""" - config = PipelineConfig( - workspace=WorkspaceConfig(size='10Gi'), - ttl_seconds_after_finished=3600, - ) - - @dsl.pipeline(pipeline_config=config) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - json_format.ParseDict( - { - 'pipelineConfig': { - 'workspace': { - 'size': '10Gi', - 'kubernetes': { - 'pvcSpecPatch': {} - } - }, - 'resourceTtlOnCompletion': 3600, - } - }, expected.platforms['kubernetes']) - - self.assertEqual(my_pipeline.platform_spec, expected) - - def test_pipeline_config_ttl_validation_rejects_negative(self): - """Any negative TTL value should raise ValueError.""" - with self.assertRaisesRegex(ValueError, 'non-negative'): - dsl.PipelineConfig(ttl_seconds_after_finished=-1) - with self.assertRaisesRegex(ValueError, 'non-negative'): - dsl.PipelineConfig(ttl_seconds_after_success=-5) - with self.assertRaisesRegex(ValueError, 'non-negative'): - dsl.PipelineConfig(ttl_seconds_after_failure=-100) - - def test_pipeline_config_ttl_validation_rejects_non_int(self): - """Float, str, and bool TTL values should raise TypeError.""" - with self.assertRaisesRegex(TypeError, 'must be an int'): - dsl.PipelineConfig(ttl_seconds_after_finished=3.5) - with self.assertRaisesRegex(TypeError, 'must be an int'): - dsl.PipelineConfig(ttl_seconds_after_success='300') - with self.assertRaisesRegex(TypeError, 'must be an int'): - dsl.PipelineConfig(ttl_seconds_after_failure=True) - - def test_pipeline_config_ttl_validation_rejects_int32_overflow(self): - """TTL values exceeding int32 max should raise ValueError.""" - with self.assertRaisesRegex(ValueError, '2147483647'): - dsl.PipelineConfig(ttl_seconds_after_finished=2147483648) - - def test_pipeline_with_active_deadline_seconds(self): - """active_deadline_seconds → activeDeadlineSeconds in platform spec.""" - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig(active_deadline_seconds=3600)) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - json_format.ParseDict( - {'pipelineConfig': { - 'activeDeadlineSeconds': 3600 - }}, expected.platforms['kubernetes']) - - self.assertEqual(my_pipeline.platform_spec, expected) - - loaded_pipeline = compile_and_reload(my_pipeline) - self.assertEqual(loaded_pipeline.platform_spec, expected) - - def test_pipeline_with_active_deadline_none_omits_field(self): - """active_deadline_seconds=None (default) produces no platform spec entry.""" - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig(active_deadline_seconds=None)) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - - self.assertEqual(my_pipeline.platform_spec, expected) - - loaded_pipeline = compile_and_reload(my_pipeline) - self.assertEqual(loaded_pipeline.platform_spec, expected) - - def test_pipeline_with_active_deadline_and_ttl(self): - """active_deadline_seconds and TTL fields coexist.""" - - @dsl.pipeline( - pipeline_config=dsl.PipelineConfig( - ttl_seconds_after_finished=300, - active_deadline_seconds=7200, - )) - def my_pipeline(): - comp() - - expected = pipeline_spec_pb2.PlatformSpec() - json_format.ParseDict( - { - 'pipelineConfig': { - 'resourceTtlOnCompletion': 300, - 'activeDeadlineSeconds': 7200, - } - }, expected.platforms['kubernetes']) - - self.assertEqual(my_pipeline.platform_spec, expected) - - loaded_pipeline = compile_and_reload(my_pipeline) - self.assertEqual(loaded_pipeline.platform_spec, expected) - - def test_pipeline_config_active_deadline_validation_rejects_negative(self): - """Negative active_deadline_seconds should raise ValueError.""" - with self.assertRaisesRegex(ValueError, 'non-negative'): - dsl.PipelineConfig(active_deadline_seconds=-1) - - def test_pipeline_config_active_deadline_validation_rejects_non_int(self): - """Float active_deadline_seconds should raise TypeError.""" - with self.assertRaisesRegex(TypeError, 'must be an int'): - dsl.PipelineConfig(active_deadline_seconds=3.5) - - def test_pipeline_config_default_has_no_active_deadline(self): - """Default PipelineConfig should not set active_deadline_seconds.""" - config = dsl.PipelineConfig() - self.assertIsNone(config.active_deadline_seconds) - class ExtractInputOutputDescription(unittest.TestCase): diff --git a/sdk/python/kfp/compiler/pipeline_spec_builder.py b/sdk/python/kfp/compiler/pipeline_spec_builder.py index 4577064eb96..3c77d4f730c 100644 --- a/sdk/python/kfp/compiler/pipeline_spec_builder.py +++ b/sdk/python/kfp/compiler/pipeline_spec_builder.py @@ -1413,9 +1413,6 @@ def build_spec_by_group( single_task_platform_spec = platform_config_to_platform_spec( subgroup.platform_config, executor_label, - parent_component_inputs=group_component_spec - .input_definitions, - tasks_in_current_dag=tasks_in_current_dag, ) merge_platform_specs( platform_spec, @@ -1621,17 +1618,9 @@ def modify_task_for_ignore_upstream_failure( def platform_config_to_platform_spec( platform_config: dict, executor_label: str, - parent_component_inputs: Optional[ - pipeline_spec_pb2.ComponentInputsSpec] = None, - tasks_in_current_dag: Optional[List[str]] = None, ) -> pipeline_spec_pb2.PlatformSpec: """Converts a single task's pipeline_task.platform_config dictionary to a PlatformSpec message using the executor_label for the task.""" - rewritten_config = _rewrite_platform_config_input_references( - platform_config=platform_config, - parent_component_inputs=parent_component_inputs, - tasks_in_current_dag=tasks_in_current_dag, - ) platform_spec_msg = pipeline_spec_pb2.PlatformSpec() json_format.ParseDict( { @@ -1642,74 +1631,12 @@ def platform_config_to_platform_spec( executor_label: config } } - } for platform_key, config in rewritten_config.items() + } for platform_key, config in platform_config.items() } }, platform_spec_msg) return platform_spec_msg -def _rewrite_platform_config_input_references( - platform_config: dict, - parent_component_inputs: Optional[pipeline_spec_pb2.ComponentInputsSpec], - tasks_in_current_dag: Optional[List[str]], -) -> Dict[str, Any]: - """Rewrites platform config input references for sub-DAG boundaries. - - When a task is inside a sub-DAG (e.g. ParallelFor), pipeline-level - inputs are propagated with a 'pipelinechannel--' prefix. This - function rewrites: - - componentInputParameter references to use the prefixed name - - taskOutputParameter references from outer tasks to the surfaced - componentInputParameter equivalent - """ - if parent_component_inputs is None: - return copy.deepcopy(platform_config) - - tasks_in_current_dag = tasks_in_current_dag or [] - - def _rewrite(data: Any) -> Any: - if isinstance(data, list): - return [_rewrite(item) for item in data] - - if not isinstance(data, dict): - return data - - rewritten = {key: _rewrite(value) for key, value in data.items()} - - # Rewrite componentInputParameter to prefixed name if needed - cip = rewritten.get('componentInputParameter') - if cip is not None: - if cip not in parent_component_inputs.parameters: - prefixed = compiler_utils.additional_input_name_for_pipeline_channel( - cip) - if prefixed in parent_component_inputs.parameters: - rewritten['componentInputParameter'] = prefixed - - # Rewrite taskOutputParameter from outer tasks to surfaced - # componentInputParameter - top = rewritten.get('taskOutputParameter') - if top is not None: - producer_task = top.get('producerTask') - output_key = top.get('outputParameterKey') - if (producer_task is not None and output_key is not None and - producer_task not in tasks_in_current_dag): - surfaced_name = compiler_utils.additional_input_name_for_pipeline_channel( - f'{producer_task}-{output_key}') - if surfaced_name in parent_component_inputs.parameters: - rewritten.pop('taskOutputParameter') - rewritten['componentInputParameter'] = surfaced_name - else: - raise compiler_utils.InvalidTopologyException( - 'Failed to rewrite cross-DAG platform config reference ' - f'for task output {producer_task}.{output_key}. ' - f'Expected surfaced input {surfaced_name!r} in parent ' - 'component inputs, but it was not found.') - - return rewritten - - return _rewrite(platform_config) - - def merge_platform_specs( main_msg: pipeline_spec_pb2.PlatformSpec, sub_msg: pipeline_spec_pb2.PlatformSpec, @@ -1779,10 +1706,6 @@ def build_exit_handler_groups_recursively( single_task_platform_spec = platform_config_to_platform_spec( exit_task.platform_config, executor_label, - parent_component_inputs=pipeline_spec.root - .input_definitions, - tasks_in_current_dag=list( - pipeline_spec.root.dag.tasks.keys()), ) merge_platform_specs( platform_spec, @@ -2387,26 +2310,6 @@ def _merge_pipeline_config(pipelineConfig: pipeline_config.PipelineConfig, if workspace is not None: config_dict['workspace'] = workspace.get_workspace() - if (pipelineConfig.ttl_seconds_after_finished is not None and - pipelineConfig.ttl_seconds_after_finished > 0): - config_dict[ - 'resourceTtlOnCompletion'] = pipelineConfig.ttl_seconds_after_finished - - if (pipelineConfig.ttl_seconds_after_success is not None and - pipelineConfig.ttl_seconds_after_success > 0): - config_dict[ - 'resourceTtlOnSuccess'] = pipelineConfig.ttl_seconds_after_success - - if (pipelineConfig.ttl_seconds_after_failure is not None and - pipelineConfig.ttl_seconds_after_failure > 0): - config_dict[ - 'resourceTtlOnFailure'] = pipelineConfig.ttl_seconds_after_failure - - if (pipelineConfig.active_deadline_seconds is not None and - pipelineConfig.active_deadline_seconds > 0): - config_dict[ - 'activeDeadlineSeconds'] = pipelineConfig.active_deadline_seconds - if config_dict: json_format.ParseDict({'pipelineConfig': config_dict}, platformSpec.platforms['kubernetes']) diff --git a/sdk/python/kfp/compiler/pipeline_spec_builder_test.py b/sdk/python/kfp/compiler/pipeline_spec_builder_test.py index dc781b47b5b..36b2e04e887 100644 --- a/sdk/python/kfp/compiler/pipeline_spec_builder_test.py +++ b/sdk/python/kfp/compiler/pipeline_spec_builder_test.py @@ -24,7 +24,6 @@ from kfp import dsl from kfp import kubernetes from kfp.compiler import compiler -from kfp.compiler import compiler_utils from kfp.compiler import pipeline_spec_builder from kfp.dsl import TaskConfigField from kfp.pipeline_spec import pipeline_spec_pb2 @@ -556,429 +555,6 @@ def pipe(): pipeline_func=pipe, package_path=package_path) -class TestPlatformConfigDAGBoundaryHandling(unittest.TestCase): - """Tests that platform config input references are correctly rewritten when - tasks are inside sub-DAGs (e.g. ParallelFor).""" - - def _compile_and_parse(self, pipeline_func): - """Compile a pipeline and return (pipeline_spec, platform_spec).""" - with tempfile.TemporaryDirectory() as tmpdir: - package_path = os.path.join(tmpdir, 'pipeline.yaml') - compiler.Compiler().compile( - pipeline_func=pipeline_func, package_path=package_path) - with open(package_path, 'r') as f: - docs = list(yaml.safe_load_all(f)) - pipeline_spec = json_format.ParseDict( - docs[0], pipeline_spec_pb2.PipelineSpec()) - platform_spec = json_format.ParseDict( - docs[1], pipeline_spec_pb2.PlatformSpec()) if len( - docs) > 1 else pipeline_spec_pb2.PlatformSpec() - return pipeline_spec, platform_spec - - def test_simple_secret_no_subdag(self): - """Baseline: secret_name from pipeline param at root level. - No rewriting needed - componentInputParameter stays unprefixed.""" - - @dsl.component - def my_comp(): - print('hello') - - @dsl.pipeline - def pipe(secret_name: str): - task = my_comp() - kubernetes.use_secret_as_env( - task, - secret_name=secret_name, - secret_key_to_env={'key': 'VAL'}, - ) - - pipeline_spec, platform_spec = self._compile_and_parse(pipe) - - # At root level the param name should NOT be prefixed - secret_param = ( - platform_spec.platforms['kubernetes'].deployment_spec - .executors['exec-my-comp'].fields['secretAsEnv'].list_value - .values[0].struct_value.fields['secretNameParameter'].struct_value - .fields['componentInputParameter'].string_value) - self.assertEqual(secret_param, 'secret_name') - - def test_parallelfor_pipeline_input_secret(self): - """Bug scenario: secret_name from pipeline param inside ParallelFor. - - The sub-DAG must surface the param, and the platform config must - reference the prefixed name. - """ - - @dsl.component - def my_comp(item: str): - print(item) - - @dsl.pipeline - def pipe(secret_name: str): - with dsl.ParallelFor(items=['a', 'b'], parallelism=1) as item: - t = my_comp(item=item) - kubernetes.use_secret_as_env( - t, - secret_name=secret_name, - secret_key_to_env={'key': 'VAL'}, - ) - - pipeline_spec, platform_spec = self._compile_and_parse(pipe) - - # Sub-DAG component must have the surfaced input - loop_component = pipeline_spec.components['comp-for-loop-2'] - self.assertIn('pipelinechannel--secret_name', - loop_component.input_definitions.parameters) - - # Root DAG task must wire the input - root_task_params = pipeline_spec.root.dag.tasks[ - 'for-loop-2'].inputs.parameters - self.assertEqual( - root_task_params['pipelinechannel--secret_name'] - .component_input_parameter, - 'secret_name', - ) - - # Platform config must reference the prefixed name - secret_param = ( - platform_spec.platforms['kubernetes'].deployment_spec - .executors['exec-my-comp'].fields['secretAsEnv'].list_value - .values[0].struct_value.fields['secretNameParameter'].struct_value - .fields['componentInputParameter'].string_value) - self.assertEqual(secret_param, 'pipelinechannel--secret_name') - - def test_parallelfor_outer_task_output_secret(self): - """Cross-DAG: secret_name from outer task output inside ParallelFor. - The taskOutputParameter must be rewritten to componentInputParameter - pointing to the surfaced input.""" - - @dsl.component - def emit_secret_name() -> str: - return 'secret' - - @dsl.component - def my_comp(): - print('hello') - - @dsl.pipeline - def pipe(): - secret_task = emit_secret_name() - with dsl.ParallelFor(items=[1, 2], parallelism=1): - t = my_comp() - kubernetes.use_secret_as_env( - t, - secret_name=secret_task.output, - secret_key_to_env={'key': 'VAL'}, - ) - - pipeline_spec, platform_spec = self._compile_and_parse(pipe) - - # Sub-DAG component must have the surfaced task output - loop_component = pipeline_spec.components['comp-for-loop-2'] - self.assertIn('pipelinechannel--emit-secret-name-Output', - loop_component.input_definitions.parameters) - - # Root DAG task must wire the task output - root_task_params = pipeline_spec.root.dag.tasks[ - 'for-loop-2'].inputs.parameters - self.assertEqual( - root_task_params['pipelinechannel--emit-secret-name-Output'] - .task_output_parameter.producer_task, - 'emit-secret-name', - ) - - # Platform config must use componentInputParameter (NOT taskOutputParameter) - secret_name_fields = ( - platform_spec.platforms['kubernetes'].deployment_spec. - executors['exec-my-comp'].fields['secretAsEnv'].list_value.values[0] - .struct_value.fields['secretNameParameter'].struct_value.fields) - self.assertEqual( - secret_name_fields['componentInputParameter'].string_value, - 'pipelinechannel--emit-secret-name-Output', - ) - self.assertNotIn('taskOutputParameter', secret_name_fields) - - def test_parallelfor_literal_secret_unchanged(self): - """Literal secret names should not be affected by the rewriting.""" - - @dsl.component - def my_comp(item: str): - print(item) - - @dsl.pipeline - def pipe(): - with dsl.ParallelFor(items=['a', 'b'], parallelism=1) as item: - t = my_comp(item=item) - kubernetes.use_secret_as_env( - t, - secret_name='my-literal-secret', - secret_key_to_env={'key': 'VAL'}, - ) - - pipeline_spec, platform_spec = self._compile_and_parse(pipe) - - # Platform config should have the constant value, not a parameter ref - secret_name_fields = ( - platform_spec.platforms['kubernetes'].deployment_spec. - executors['exec-my-comp'].fields['secretAsEnv'].list_value.values[0] - .struct_value.fields['secretNameParameter'].struct_value.fields) - self.assertNotIn('componentInputParameter', secret_name_fields) - self.assertNotIn('taskOutputParameter', secret_name_fields) - - def test_parallelfor_pipeline_input_mount_pvc(self): - """Pipeline pvc_name param inside ParallelFor is correctly surfaced and - rewritten for mount_pvc platform config.""" - - @dsl.component - def my_comp(item: str): - print(item) - - @dsl.pipeline - def pipe(pvc_name: str): - with dsl.ParallelFor(items=['a', 'b'], parallelism=1) as item: - t = my_comp(item=item) - kubernetes.mount_pvc( - t, - pvc_name=pvc_name, - mount_path='/mnt/data', - ) - - pipeline_spec, platform_spec = self._compile_and_parse(pipe) - - loop_component = pipeline_spec.components['comp-for-loop-2'] - self.assertIn('pipelinechannel--pvc_name', - loop_component.input_definitions.parameters) - - root_task_params = pipeline_spec.root.dag.tasks[ - 'for-loop-2'].inputs.parameters - self.assertEqual( - root_task_params['pipelinechannel--pvc_name'] - .component_input_parameter, - 'pvc_name', - ) - - pvc_param = ( - platform_spec.platforms['kubernetes'].deployment_spec - .executors['exec-my-comp'].fields['pvcMount'].list_value.values[0] - .struct_value.fields['pvcNameParameter'].struct_value - .fields['componentInputParameter'].string_value) - self.assertEqual(pvc_param, 'pipelinechannel--pvc_name') - - def test_exit_handler_platform_config_rewrite_path(self): - """Exit handler task platform config uses rewrite path with parent - component context.""" - - @dsl.component - def cleanup(): - print('cleanup') - - @dsl.component - def main_task(): - print('main') - - @dsl.pipeline - def pipe(secret_name: str): - exit_task = cleanup() - kubernetes.use_secret_as_env( - exit_task, - secret_name=secret_name, - secret_key_to_env={'key': 'VAL'}, - ) - with dsl.ExitHandler(exit_task=exit_task): - main_task() - - _, platform_spec = self._compile_and_parse(pipe) - - cleanup_executors = [ - executor for executor in platform_spec.platforms['kubernetes'] - .deployment_spec.executors.values() - if 'secretAsEnv' in executor.fields - ] - self.assertEqual(len(cleanup_executors), 1) - secret_param = ( - cleanup_executors[0].fields['secretAsEnv'].list_value.values[0] - .struct_value.fields['secretNameParameter'].struct_value - .fields['componentInputParameter'].string_value) - self.assertEqual(secret_param, 'secret_name') - - -class TestRewritePlatformConfigInputReferences(unittest.TestCase): - """Unit tests for the _rewrite_platform_config_input_references helper.""" - - def test_rewrites_unprefixed_component_input_param(self): - platform_config = { - 'kubernetes': { - 'secretAsEnv': [{ - 'secretNameParameter': { - 'componentInputParameter': 'secret_name' - }, - 'keyToEnv': [{ - 'secretKey': 'pw', - 'envVar': 'PASSWORD' - }], - }] - } - } - parent_inputs = pipeline_spec_pb2.ComponentInputsSpec() - parent_inputs.parameters[ - 'pipelinechannel--secret_name'].parameter_type = ( - pipeline_spec_pb2.ParameterType.STRING) - - result = pipeline_spec_builder._rewrite_platform_config_input_references( - platform_config, parent_inputs, []) - - self.assertEqual( - result['kubernetes']['secretAsEnv'][0]['secretNameParameter'] - ['componentInputParameter'], - 'pipelinechannel--secret_name', - ) - - def test_no_rewrite_when_param_exists_in_parent(self): - platform_config = { - 'kubernetes': { - 'secretAsEnv': [{ - 'secretNameParameter': { - 'componentInputParameter': 'secret_name' - }, - }] - } - } - parent_inputs = pipeline_spec_pb2.ComponentInputsSpec() - parent_inputs.parameters['secret_name'].parameter_type = ( - pipeline_spec_pb2.ParameterType.STRING) - - result = pipeline_spec_builder._rewrite_platform_config_input_references( - platform_config, parent_inputs, []) - - self.assertEqual( - result['kubernetes']['secretAsEnv'][0]['secretNameParameter'] - ['componentInputParameter'], - 'secret_name', - ) - - def test_rewrites_task_output_from_outer_task(self): - platform_config = { - 'kubernetes': { - 'secretAsEnv': [{ - 'secretNameParameter': { - 'taskOutputParameter': { - 'producerTask': 'emit-secret', - 'outputParameterKey': 'Output', - } - }, - }] - } - } - parent_inputs = pipeline_spec_pb2.ComponentInputsSpec() - parent_inputs.parameters[ - 'pipelinechannel--emit-secret-Output'].parameter_type = ( - pipeline_spec_pb2.ParameterType.STRING) - - result = pipeline_spec_builder._rewrite_platform_config_input_references( - platform_config, - parent_inputs, - tasks_in_current_dag=['worker-task']) - - # taskOutputParameter should be replaced with componentInputParameter - secret_ref = result['kubernetes']['secretAsEnv'][0][ - 'secretNameParameter'] - self.assertNotIn('taskOutputParameter', secret_ref) - self.assertEqual( - secret_ref['componentInputParameter'], - 'pipelinechannel--emit-secret-Output', - ) - - def test_no_rewrite_task_output_from_local_task(self): - platform_config = { - 'kubernetes': { - 'secretAsEnv': [{ - 'secretNameParameter': { - 'taskOutputParameter': { - 'producerTask': 'local-task', - 'outputParameterKey': 'Output', - } - }, - }] - } - } - parent_inputs = pipeline_spec_pb2.ComponentInputsSpec() - - result = pipeline_spec_builder._rewrite_platform_config_input_references( - platform_config, parent_inputs, tasks_in_current_dag=['local-task']) - - # taskOutputParameter should remain since the producer is in the current DAG - secret_ref = result['kubernetes']['secretAsEnv'][0][ - 'secretNameParameter'] - self.assertIn('taskOutputParameter', secret_ref) - self.assertNotIn('componentInputParameter', secret_ref) - - def test_returns_copy_when_no_parent_inputs(self): - platform_config = {'kubernetes': {'secretAsEnv': [{'key': 'val'}]}} - result = pipeline_spec_builder._rewrite_platform_config_input_references( - platform_config, None, None) - self.assertEqual(result, platform_config) - # Must be a copy, not the same object - self.assertIsNot(result, platform_config) - - def test_rewrites_multiple_params(self): - platform_config = { - 'kubernetes': { - 'secretAsEnv': [{ - 'secretNameParameter': { - 'componentInputParameter': 'secret1' - }, - }], - 'pvcMount': [{ - 'pvcNameParameter': { - 'componentInputParameter': 'pvc_name' - }, - }], - } - } - parent_inputs = pipeline_spec_pb2.ComponentInputsSpec() - parent_inputs.parameters['pipelinechannel--secret1'].parameter_type = ( - pipeline_spec_pb2.ParameterType.STRING) - parent_inputs.parameters['pipelinechannel--pvc_name'].parameter_type = ( - pipeline_spec_pb2.ParameterType.STRING) - - result = pipeline_spec_builder._rewrite_platform_config_input_references( - platform_config, parent_inputs, []) - - self.assertEqual( - result['kubernetes']['secretAsEnv'][0]['secretNameParameter'] - ['componentInputParameter'], - 'pipelinechannel--secret1', - ) - self.assertEqual( - result['kubernetes']['pvcMount'][0]['pvcNameParameter'] - ['componentInputParameter'], - 'pipelinechannel--pvc_name', - ) - - def test_raises_when_cross_dag_output_missing_surfaced_input(self): - platform_config = { - 'kubernetes': { - 'secretAsEnv': [{ - 'secretNameParameter': { - 'taskOutputParameter': { - 'producerTask': 'emit-secret', - 'outputParameterKey': 'Output', - } - }, - }] - } - } - parent_inputs = pipeline_spec_pb2.ComponentInputsSpec() - - with self.assertRaisesRegex(compiler_utils.InvalidTopologyException, - 'Expected surfaced input'): - pipeline_spec_builder._rewrite_platform_config_input_references( - platform_config, - parent_inputs, - tasks_in_current_dag=['worker-task'], - ) - - def pipeline_spec_from_file(filepath: str) -> str: with open(filepath, 'r') as f: dictionary = yaml.safe_load(f) diff --git a/sdk/python/kfp/dsl/pipeline_config.py b/sdk/python/kfp/dsl/pipeline_config.py index 19791405a09..42505bf0d25 100644 --- a/sdk/python/kfp/dsl/pipeline_config.py +++ b/sdk/python/kfp/dsl/pipeline_config.py @@ -94,59 +94,7 @@ def set_kubernetes_config(self, class PipelineConfig: - """PipelineConfig contains pipeline-level config options. + """PipelineConfig contains pipeline-level config options.""" - Attributes: - workspace: Optional configuration for a shared workspace PVC that - persists for the duration of the pipeline run. - ttl_seconds_after_finished: Optional number of seconds to retain the - Argo Workflow resource after the pipeline run completes, regardless - of success or failure. Maps to Argo's - ``ttlStrategy.secondsAfterCompletion``. When ``None`` (the - default) no completion TTL is set. - ttl_seconds_after_success: Optional number of seconds to retain the - Argo Workflow resource after a *successful* run. Maps to Argo's - ``ttlStrategy.secondsAfterSuccess``. Takes precedence over - ``ttl_seconds_after_finished`` for successful runs when both are - set. - ttl_seconds_after_failure: Optional number of seconds to retain the - Argo Workflow resource after a *failed* run. Maps to Argo's - ``ttlStrategy.secondsAfterFailure``. Takes precedence over - ``ttl_seconds_after_finished`` for failed runs when both are set. - active_deadline_seconds: Optional maximum number of seconds a - workflow is allowed to run before it is forcibly terminated. - Maps to Argo's ``activeDeadlineSeconds``. When ``None`` (the - default) no deadline is applied. - """ - - def __init__( - self, - workspace: Optional[WorkspaceConfig] = None, - ttl_seconds_after_finished: Optional[int] = None, - ttl_seconds_after_success: Optional[int] = None, - ttl_seconds_after_failure: Optional[int] = None, - active_deadline_seconds: Optional[int] = None, - ): + def __init__(self, workspace: Optional[WorkspaceConfig] = None): self.workspace = workspace - for name, value in [ - ('ttl_seconds_after_finished', ttl_seconds_after_finished), - ('ttl_seconds_after_success', ttl_seconds_after_success), - ('ttl_seconds_after_failure', ttl_seconds_after_failure), - ('active_deadline_seconds', active_deadline_seconds), - ]: - if value is None: - continue - if isinstance(value, bool) or not isinstance(value, int): - raise TypeError( - f'{name} must be an int. Got: {type(value).__name__}') - if value < 0: - raise ValueError( - f'{name} must be a non-negative integer. Got: {value}') - if value > 2147483647: - raise ValueError( - f'{name} must not exceed the int32 maximum (2147483647).' - f' Got: {value}') - self.ttl_seconds_after_finished = ttl_seconds_after_finished - self.ttl_seconds_after_success = ttl_seconds_after_success - self.ttl_seconds_after_failure = ttl_seconds_after_failure - self.active_deadline_seconds = active_deadline_seconds diff --git a/sdk/python/kfp/dsl/pipeline_task.py b/sdk/python/kfp/dsl/pipeline_task.py index c8cb250a97e..c77529bad6e 100644 --- a/sdk/python/kfp/dsl/pipeline_task.py +++ b/sdk/python/kfp/dsl/pipeline_task.py @@ -308,32 +308,6 @@ def _extract_container_spec_and_convert_placeholders( ] return container_spec - def register_pipeline_channels( - self, - pipeline_channels: List[pipeline_channel.PipelineChannel]) -> None: - """Registers additional pipeline channels consumed by the task. - - Args: - pipeline_channels: Pipeline channels to add to the task's tracked - inputs. - - Channels are deduplicated by their serialized ``pattern`` so the same - runtime reference is only registered once. - """ - existing_channel_patterns = { - channel.pattern for channel in self._channel_inputs - } - for channel in pipeline_channels: - if channel.pattern not in existing_channel_patterns: - self._channel_inputs.append(channel) - existing_channel_patterns.add(channel.pattern) - - def _register_pipeline_channels( - self, - pipeline_channels: List[pipeline_channel.PipelineChannel]) -> None: - """Backwards-compatible wrapper for ``register_pipeline_channels``.""" - self.register_pipeline_channels(pipeline_channels) - @block_if_final() def set_caching_options(self, enable_caching: bool, @@ -702,7 +676,13 @@ def set_container_image( self.container_spec.image = name if pipeline_channels: - self.register_pipeline_channels(pipeline_channels) + existing_channel_patterns = { + channel.pattern for channel in self._channel_inputs + } + for channel in pipeline_channels: + if channel.pattern not in existing_channel_patterns: + self._channel_inputs.append(channel) + existing_channel_patterns.add(channel.pattern) return self @block_if_final() diff --git a/sdk/python/kfp/dsl/pipeline_task_test.py b/sdk/python/kfp/dsl/pipeline_task_test.py index 998dbf3afda..1ab66cb7c67 100644 --- a/sdk/python/kfp/dsl/pipeline_task_test.py +++ b/sdk/python/kfp/dsl/pipeline_task_test.py @@ -18,7 +18,6 @@ from absl.testing import parameterized from kfp import dsl -from kfp.dsl import pipeline_channel from kfp.dsl import pipeline_task from kfp.dsl import placeholders from kfp.dsl import structures @@ -143,21 +142,6 @@ def test_set_caching_options(self): task.set_caching_options(False) self.assertEqual(False, task._task_spec.enable_caching) - def test_register_pipeline_channels_deduplicates_channels(self): - task = pipeline_task.PipelineTask( - component_spec=structures.ComponentSpec.from_yaml_documents( - V2_YAML), - args={'input1': 'value'}, - ) - channel = pipeline_channel.PipelineParameterChannel( - name='secret_name', - channel_type='String', - ) - - task.register_pipeline_channels([channel, channel]) - - self.assertEqual(task.channel_inputs, [channel]) - @parameterized.parameters( { 'cpu': '123', diff --git a/sdk/python/test/compilation/pipeline_compilation_test.py b/sdk/python/test/compilation/pipeline_compilation_test.py index 52816de59e9..abd0fc658b0 100644 --- a/sdk/python/test/compilation/pipeline_compilation_test.py +++ b/sdk/python/test/compilation/pipeline_compilation_test.py @@ -72,6 +72,8 @@ pipeline as container_no_inputs_pipeline from test_data.sdk_compiled_pipelines.valid.critical.flip_coin import \ flipcoin_pipeline as flip_coin +from test_data.sdk_compiled_pipelines.valid.critical.loop_consume_upstream import \ + loop_consume_upstream from test_data.sdk_compiled_pipelines.valid.critical.missing_kubernetes_optional_inputs import \ missing_kubernetes_optional_inputs_pipeline from test_data.sdk_compiled_pipelines.valid.critical.mixed_parameters import \ @@ -82,6 +84,12 @@ crust as multiple_artifacts_namedtuple_pipeline from test_data.sdk_compiled_pipelines.valid.critical.multiple_parameters_namedtuple import \ crust as multiple_params_namedtuple_pipeline +from test_data.sdk_compiled_pipelines.valid.critical.nested_pipeline_opt_input_child_level import \ + nested_pipeline_opt_input_child_level +from test_data.sdk_compiled_pipelines.valid.critical.nested_pipeline_opt_inputs_nil import \ + nested_pipeline_opt_inputs_nil +from test_data.sdk_compiled_pipelines.valid.critical.nested_pipeline_opt_inputs_parent_level import \ + nested_pipeline_opt_inputs_parent_level from test_data.sdk_compiled_pipelines.valid.critical.parallel_for_after_dependency import \ loop_with_after_dependency_set from test_data.sdk_compiled_pipelines.valid.critical.parameter_cache import \ @@ -114,6 +122,7 @@ my_pipeline as two_step_containerized_pipeline from test_data.sdk_compiled_pipelines.valid.cross_loop_after_topology import \ my_pipeline as cross_loop_after_topology_pipeline +from test_data.sdk_compiled_pipelines.valid.dict_input import dict_input from test_data.sdk_compiled_pipelines.valid.env_var import test_env_exists from test_data.sdk_compiled_pipelines.valid.essential.component_with_pip_index_urls import \ pipeline as pip_index_urls_pipeline @@ -125,18 +134,24 @@ concat_message from test_data.sdk_compiled_pipelines.valid.essential.container_no_input import \ container_no_input -from test_data.sdk_compiled_pipelines.valid.essential.dict_input import \ - dict_input from test_data.sdk_compiled_pipelines.valid.essential.lightweight_python_functions_pipeline import \ pipeline as lightweight_python_pipeline from test_data.sdk_compiled_pipelines.valid.essential.lightweight_python_functions_with_outputs import \ pipeline as lightweight_python_with_outputs_pipeline +from test_data.sdk_compiled_pipelines.valid.essential.pipeline_in_pipeline import \ + my_pipeline as pipeline_in_pipeline +from test_data.sdk_compiled_pipelines.valid.essential.pipeline_in_pipeline_complex import \ + my_pipeline as pipeline_in_pipeline_complex +from test_data.sdk_compiled_pipelines.valid.essential.pipeline_in_pipeline_loaded_from_yaml import \ + my_pipeline as pipeline_in_pipeline_loaded_from_yaml from test_data.sdk_compiled_pipelines.valid.essential.pipeline_with_after import \ my_pipeline as after_pipeline from test_data.sdk_compiled_pipelines.valid.essential.pipeline_with_condition import \ my_pipeline as condition_pipeline from test_data.sdk_compiled_pipelines.valid.essential.pipeline_with_if_placeholder import \ pipeline_none +from test_data.sdk_compiled_pipelines.valid.essential.pipeline_with_loops import \ + my_pipeline as loops_pipeline from test_data.sdk_compiled_pipelines.valid.essential.pipeline_with_metrics_outputs import \ my_pipeline as metrics_outputs_pipeline from test_data.sdk_compiled_pipelines.valid.essential.pipeline_with_nested_conditions import \ @@ -174,22 +189,6 @@ math_pipeline as nested_with_parameters_pipeline from test_data.sdk_compiled_pipelines.valid.output_metrics import \ output_metrics -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.loop_consume_upstream import \ - loop_consume_upstream -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.nested_pipeline_opt_input_child_level import \ - nested_pipeline_opt_input_child_level -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.nested_pipeline_opt_inputs_nil import \ - nested_pipeline_opt_inputs_nil -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.nested_pipeline_opt_inputs_parent_level import \ - nested_pipeline_opt_inputs_parent_level -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.pipeline_in_pipeline import \ - my_pipeline as pipeline_in_pipeline -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.pipeline_in_pipeline_complex import \ - my_pipeline as pipeline_in_pipeline_complex -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.pipeline_in_pipeline_loaded_from_yaml import \ - my_pipeline as pipeline_in_pipeline_loaded_from_yaml -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.pipeline_with_loops import \ - my_pipeline as loops_pipeline from test_data.sdk_compiled_pipelines.valid.parameters_complex import \ math_pipeline as parameters_complex_pipeline from test_data.sdk_compiled_pipelines.valid.pipeline_as_exit_task import \ @@ -313,7 +312,7 @@ def __repr__(self) -> str: pipeline_func=loops_pipeline, pipeline_func_args={'loop_parameter': ['item1', 'item2']}, compiled_file_name='loops_pipeline.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/pipeline_with_loops.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/essential/pipeline_with_loops.yaml' ), TestData( pipeline_name='pipeline-with-outputs', @@ -439,7 +438,7 @@ def __repr__(self) -> str: pipeline_func=loop_consume_upstream, pipeline_func_args=None, compiled_file_name='loop_consume_upstream.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/loop_consume_upstream.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/critical/loop_consume_upstream.yaml' ), TestData( pipeline_name='math-pipeline', @@ -509,7 +508,7 @@ def __repr__(self) -> str: pipeline_func=nested_pipeline_opt_input_child_level, pipeline_func_args=None, compiled_file_name='nested_pipeline_opt_input_child_level.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/nested_pipeline_opt_input_child_level_compiled.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/critical/nested_pipeline_opt_input_child_level_compiled.yaml' ), TestData( pipeline_name='split-datasets-and-return-first', @@ -701,7 +700,7 @@ def __repr__(self) -> str: pipeline_func=pipeline_in_pipeline, pipeline_func_args=None, compiled_file_name='pipeline_in_pipeline.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/pipeline_in_pipeline.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/essential/pipeline_in_pipeline.yaml' ), TestData( pipeline_name='container-with-concat-placeholder', @@ -774,7 +773,7 @@ def __repr__(self) -> str: pipeline_func=pipeline_in_pipeline_complex, pipeline_func_args=None, compiled_file_name='pipeline_in_pipeline_complex.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/pipeline_in_pipeline_complex.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/essential/pipeline_in_pipeline_complex.yaml' ), TestData( pipeline_name='lucky-number-pipeline', @@ -962,14 +961,14 @@ def __repr__(self) -> str: pipeline_func=nested_pipeline_opt_inputs_nil, pipeline_func_args=None, compiled_file_name='nested_pipeline_opt_inputs_nil.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/nested_pipeline_opt_inputs_nil_compiled.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/critical/nested_pipeline_opt_inputs_nil_compiled.yaml' ), TestData( pipeline_name='nested-pipeline-opt-inputs-parent-level', pipeline_func=nested_pipeline_opt_inputs_parent_level, pipeline_func_args=None, compiled_file_name='nested_pipeline_opt_inputs_parent_level.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/nested_pipeline_opt_inputs_parent_level_compiled.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/critical/nested_pipeline_opt_inputs_parent_level_compiled.yaml' ), TestData( pipeline_name='cross-loop-after-topology', @@ -990,7 +989,7 @@ def __repr__(self) -> str: pipeline_func=pipeline_in_pipeline_loaded_from_yaml, pipeline_func_args=None, compiled_file_name='pipeline_in_pipeline_loaded_from_yaml.yaml', - expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/parallel_and_nested/pipeline_in_pipeline_loaded_from_yaml.yaml' + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/essential/pipeline_in_pipeline_loaded_from_yaml.yaml' ), TestData( pipeline_name='dynamic-importer-metadata-pipeline', diff --git a/sdk/python/test/local_execution/local_execution_test.py b/sdk/python/test/local_execution/local_execution_test.py index e5380ce77db..554509de2cf 100644 --- a/sdk/python/test/local_execution/local_execution_test.py +++ b/sdk/python/test/local_execution/local_execution_test.py @@ -43,29 +43,26 @@ pipeline_with_importer_workspace as importer_workspace_pipeline from test_data.sdk_compiled_pipelines.valid.critical.producer_consumer_param import \ producer_consumer_param_pipeline +from test_data.sdk_compiled_pipelines.valid.dict_input import dict_input from test_data.sdk_compiled_pipelines.valid.essential.concat_message import \ concat_message from test_data.sdk_compiled_pipelines.valid.essential.container_no_input import \ container_no_input -from test_data.sdk_compiled_pipelines.valid.essential.dict_input import \ - dict_input from test_data.sdk_compiled_pipelines.valid.essential.lightweight_python_functions_with_outputs import \ pipeline as lightweight_with_outputs_pipeline +from test_data.sdk_compiled_pipelines.valid.essential.pipeline_with_loops import \ + my_pipeline as pipeline_with_loops from test_data.sdk_compiled_pipelines.valid.hello_world import echo from test_data.sdk_compiled_pipelines.valid.identity import identity from test_data.sdk_compiled_pipelines.valid.nested_return import nested_return from test_data.sdk_compiled_pipelines.valid.output_metrics import \ output_metrics -from test_data.sdk_compiled_pipelines.valid.parallel_and_nested.pipeline_with_loops import \ - my_pipeline as pipeline_with_loops from test_data.sdk_compiled_pipelines.valid.parameter import \ crust as parameter_pipeline from test_data.sdk_compiled_pipelines.valid.pipeline_with_parallelfor_list_artifacts import \ my_pipeline as pipeline_with_parallelfor_list_artifacts from test_data.sdk_compiled_pipelines.valid.pipeline_with_parallelfor_parallelism import \ my_pipeline as pipeline_with_parallelfor_parallelism -from test_data.sdk_compiled_pipelines.valid.pipeline_with_parallelfor_pipeline_param import \ - my_pipeline as pipeline_with_parallelfor_pipeline_param from test_data.sdk_compiled_pipelines.valid.sequential_v1 import sequential @@ -198,15 +195,6 @@ def idfn(val): pipeline_func_args={'loop_parameter': ['item1', 'item2', 'item3']}, expected_output=None, ), - TestData( - name='Pipeline with ParallelFor Pipeline Param', - pipeline_func=pipeline_with_parallelfor_pipeline_param, - pipeline_func_args={ - 'prefix': 'test-prefix', - 'loop_parameter': ['a', 'b'], - }, - expected_output=None, - ), ] docker_specific_pipeline_funcs = [ @@ -266,9 +254,9 @@ def idfn(val): class TestDockerRunner: @pytest.fixture(autouse=True) - def setup_and_teardown(self, worker_id): - ws_root = f'{ws_root_base}_docker_{worker_id}' - pipeline_root = f'{pipeline_root_base}_docker_{worker_id}' + def setup_and_teardown(self): + ws_root = f'{ws_root_base}_docker' + pipeline_root = f'{pipeline_root_base}_docker' # Create directories with proper permissions Path(ws_root).mkdir(exist_ok=True) @@ -316,9 +304,9 @@ def test_execution(self, test_data: TestData): class TestSubProcessRunner: @pytest.fixture(scope="class", autouse=True) - def setup_and_teardown(self, worker_id): - ws_root = f'{ws_root_base}_subprocess_{worker_id}' - pipeline_root = f'{pipeline_root_base}_subprocess_{worker_id}' + def setup_and_teardown(self): + ws_root = f'{ws_root_base}_subprocess' + pipeline_root = f'{pipeline_root_base}_subprocess' Path(ws_root).mkdir(exist_ok=True) Path(pipeline_root).mkdir(exist_ok=True) local.init( diff --git a/test/deploy-pipeline-lite.sh b/test/deploy-pipeline-lite.sh index ab91ecf17c7..d7f6b42a213 100755 --- a/test/deploy-pipeline-lite.sh +++ b/test/deploy-pipeline-lite.sh @@ -20,8 +20,9 @@ set -ex # * $GCR_IMAGE_BASE_DIR # * $GCR_IMAGE_TAG # * $KFP_DEPLOY_RELEASE - +# * $ENABLE_WORKLOAD_IDENTITY GCR_IMAGE_TAG=${GCR_IMAGE_TAG:-latest} +ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false} KFP_MANIFEST_DIR="${DIR}/manifests" pushd ${KFP_MANIFEST_DIR} @@ -103,4 +104,36 @@ done echo "Status of pods after rollouts are successful" kubectl get pods -n ${NAMESPACE} +if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then + # Use static GSAs for testing, so we don't need to GC them. + export SYSTEM_GSA="test-kfp-system" + export USER_GSA="test-kfp-user" + source "${DIR}/scripts/retry.sh" + + function setup_workload_identity { + # Workaround for flakiness from gcp-workload-identity-setup.sh: + # When two tests add iam policy bindings at the same time, one will fail because + # there could be two concurrent changes. + # Wait here randomly to reduce chance both scripts are run at the same time + # between tests. Unless for testing scenario like this, it won't + # meet the concurrent change issue. + sleep $((RANDOM%30)) + yes | PROJECT_ID=$PROJECT RESOURCE_PREFIX=$TEST_CLUSTER NAMESPACE=$NAMESPACE \ + ${DIR}/../manifests/kustomize/gcp-workload-identity-setup.sh + } + retry setup_workload_identity + + retry gcloud projects add-iam-policy-binding $PROJECT \ + --member="serviceAccount:$SYSTEM_GSA@$PROJECT.iam.gserviceaccount.com" \ + --role="roles/editor" + retry gcloud projects add-iam-policy-binding $PROJECT \ + --member="serviceAccount:$USER_GSA@$PROJECT.iam.gserviceaccount.com" \ + --role="roles/editor" + + source "$DIR/../manifests/kustomize/wi-utils.sh" + # TODO(Bobgy): re-enable this after temporary flakiness is resolved. + # verify_workload_identity_binding "pipeline-runner" $NAMESPACE + sleep 30 +fi + popd diff --git a/test/frontend-integration-test/helloworld.spec.js b/test/frontend-integration-test/helloworld.spec.js index fdd7b9e417b..52657057fee 100644 --- a/test/frontend-integration-test/helloworld.spec.js +++ b/test/frontend-integration-test/helloworld.spec.js @@ -19,7 +19,6 @@ const { clearDefaultInput, getValueFromDetailsTable, saveDebugScreenshot, - selectPipelineForRun, waitForCondition, waitForGraphNodeCount, waitForHashPrefix, @@ -39,6 +38,41 @@ const runStartTimeout = 30000; const runCompletionTimeout = 60000; const outputParameterValue = 'Hello world in test'; +async function selectPipelineForRun() { + await $('#choosePipelineBtn').waitForDisplayed({ timeout: uiTimeout }); + await $('#choosePipelineBtn').click(); + + await $('#pipelineSelectorDialog').waitForDisplayed({ timeout: uiTimeout }); + const pipelineRowSelector = buildTableRowSelector(pipelineName, { + containerXPath: '//*[@id="pipelineSelectorDialog"]', + }); + + try { + await waitForCondition( + async () => (await $(pipelineRowSelector).isExisting()), + { + timeout: uiTimeout, + timeoutMsg: `expected pipeline row for ${pipelineName} to appear`, + }, + ); + } catch (error) { + const rowCount = await browser.execute(() => document.querySelectorAll('[data-testid="table-row"]').length); + const emptyMessage = await browser.execute(() => { + const emptyEl = document.querySelector('.emptyMessage'); + return emptyEl ? emptyEl.textContent : null; + }); + console.log('PIPELINE_SELECTOR_ROW_COUNT', rowCount); + console.log('PIPELINE_SELECTOR_EMPTY_MESSAGE', emptyMessage); + await saveDebugScreenshot('pipeline-selector'); + throw error; + } + + await $(pipelineRowSelector).click(); + await $('#usePipelineBtn').waitForEnabled({ timeout: uiTimeout }); + await $('#usePipelineBtn').click(); + await $('#pipelineSelectorDialog').waitForDisplayed({ timeout: uiTimeout, reverse: true }); +} + async function waitForRunParameterField(selector) { try { await $(selector).waitForDisplayed({ timeout: runStartTimeout }); @@ -125,7 +159,7 @@ describe('deploy helloworld sample run', () => { }); it('creates a new run in the experiment', async () => { - await selectPipelineForRun(pipelineName, { timeout: uiTimeout }); + await selectPipelineForRun(); await fillRunForm({ description: runDescription, @@ -225,7 +259,7 @@ describe('deploy helloworld sample run', () => { await $('#createNewRunBtn').waitForDisplayed({ timeout: uiTimeout }); await $('#createNewRunBtn').click(); - await selectPipelineForRun(pipelineName, { timeout: uiTimeout }); + await selectPipelineForRun(); await fillRunForm({ description: runWithoutExperimentDescription, diff --git a/test/frontend-integration-test/literal-input.spec.js b/test/frontend-integration-test/literal-input.spec.js deleted file mode 100644 index e5f9ca168a3..00000000000 --- a/test/frontend-integration-test/literal-input.spec.js +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright 2018-2026 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const assert = require('assert'); -const URL = require('url').URL; -const { - buildTableRowSelector, - clearDefaultInput, - saveDebugScreenshot, - selectPipelineForRun, - waitForCondition, - waitForHashPrefix, - waitForRunPageReady, -} = require('./test-helpers'); - -const pipelineName = 'literal-input-pipeline-' + Date.now(); -const runName = 'literal-input-run-' + Date.now(); -const selectedLiteral = 'staging'; -const uiTimeout = 10000; -const runStartTimeout = 30000; - -let createdRunId = ''; -let pipelineUploaded = false; - -async function waitForCreatedRunId() { - const currentHash = new URL(await browser.getUrl()).hash; - if (currentHash.startsWith('#/runs/details/')) { - return currentHash.replace('#/runs/details/', '').split('?')[0]; - } - - const runLinkSelector = `[data-testid="run-name-link"][data-run-name="${runName}"]`; - let runId = ''; - - await $('#refreshBtn').waitForDisplayed({ timeout: uiTimeout }); - await waitForCondition( - async () => { - const runLink = await $(runLinkSelector); - if (await runLink.isExisting()) { - runId = (await runLink.getAttribute('data-run-id')) || ''; - return !!runId; - } - await $('#refreshBtn').click(); - return false; - }, - { - timeout: runStartTimeout, - interval: 1000, - timeoutMsg: `waited ${runStartTimeout / 1000} seconds but run ${runName} did not start`, - }, - ); - - return runId; -} - -async function fetchRunById(runId) { - return browser.execute(async (currentRunId) => { - const response = await fetch(`/apis/v2beta1/runs/${currentRunId}`); - const responseText = await response.text(); - let responseBody; - try { - responseBody = responseText ? JSON.parse(responseText) : null; - } catch (error) { - responseBody = responseText; - } - return { - ok: response.ok, - run: responseBody, - status: response.status, - }; - }, runId); -} - -async function deleteCreatedRun() { - if (!createdRunId) { - return; - } - - try { - const deleteResponse = await browser.execute(async (currentRunId) => { - const response = await fetch(`/apis/v2beta1/runs/${currentRunId}`, { - method: 'DELETE', - }); - return { - ok: response.ok, - responseText: await response.text(), - status: response.status, - }; - }, createdRunId); - - if (!deleteResponse.ok && deleteResponse.status !== 404) { - throw new Error( - `run delete returned ${deleteResponse.status}: ${deleteResponse.responseText || '(empty body)'}`, - ); - } - } catch (error) { - console.log('RUN_CLEANUP_FAILED', error.message); - try { - await saveDebugScreenshot('run-cleanup'); - } catch (screenshotError) { - console.log('RUN_CLEANUP_SCREENSHOT_FAILED', screenshotError.message); - } - } -} - -async function deleteUploadedPipeline() { - if (!pipelineUploaded) { - return; - } - - try { - await $('#pipelinesBtn').waitForDisplayed({ timeout: uiTimeout }); - await $('#pipelinesBtn').click(); - await waitForHashPrefix('#/pipelines', { timeout: uiTimeout }); - - await $('#tableFilterBox').waitForDisplayed({ timeout: uiTimeout }); - await $('#tableFilterBox').click(); - await clearDefaultInput(); - await browser.keys(pipelineName); - - const pipelineRowSelector = buildTableRowSelector(pipelineName); - await waitForCondition( - async () => (await $(pipelineRowSelector).isExisting()), - { - timeout: uiTimeout, - timeoutMsg: `expected pipeline row for ${pipelineName} after filtering`, - }, - ); - - await $(pipelineRowSelector).click(); - await $('#deletePipelinesAndPipelineVersionsBtn').waitForDisplayed({ timeout: uiTimeout }); - await $('#deletePipelinesAndPipelineVersionsBtn').click(); - - const dialog = await $('[role="dialog"]'); - await dialog.waitForDisplayed({ timeout: uiTimeout }); - await dialog.$('button=Delete All').click(); - await dialog.waitForDisplayed({ timeout: uiTimeout, reverse: true }); - } catch (error) { - console.log('PIPELINE_CLEANUP_FAILED', error.message); - try { - await saveDebugScreenshot('pipeline-cleanup'); - } catch (screenshotError) { - console.log('PIPELINE_CLEANUP_SCREENSHOT_FAILED', screenshotError.message); - } - } -} - -describe('literal input parameter integration', () => { - before(async () => { - await browser.url('/'); - }); - - after(async () => { - await deleteCreatedRun(); - await deleteUploadedPipeline(); - }); - - it('uploads the literal-input pipeline', async () => { - await $('#createPipelineVersionBtn').waitForDisplayed({ timeout: uiTimeout }); - await $('#createPipelineVersionBtn').click(); - await waitForHashPrefix('#/pipeline_versions/new', { timeout: uiTimeout }); - - await $('#localPackageBtn').click(); - const remoteFilePath = await browser.uploadFile('./literal-input.yaml'); - await $('#dropZone input[type="file"]').addValue(remoteFilePath); - await $('#newPipelineName').click(); - await clearDefaultInput(); - await browser.keys(pipelineName); - await $('#createNewPipelineOrVersionBtn').click(); - - await waitForHashPrefix('#/pipelines/details', { timeout: uiTimeout }); - pipelineUploaded = true; - }); - - it('opens the new run page for the uploaded pipeline', async () => { - await $('#runsBtn').click(); - await waitForHashPrefix('#/runs', { timeout: uiTimeout }); - - await $('#createNewRunBtn').waitForDisplayed({ timeout: uiTimeout }); - await $('#createNewRunBtn').click(); - await waitForHashPrefix('#/runs/new', { timeout: uiTimeout }); - - await selectPipelineForRun(pipelineName, { timeout: uiTimeout }); - }); - - it('renders the literal parameter as a dropdown and requires a selection before start', async () => { - const runFormVariant = await waitForRunPageReady({ timeout: runStartTimeout }); - assert.equal( - runFormVariant.name, - 'v2', - 'compiled literal-input pipeline should open the v2 run form', - ); - const selectors = runFormVariant.selectors; - - await $(selectors.runName).click(); - await clearDefaultInput(); - await browser.keys(runName); - - const literalSelect = await $('//*[@role="combobox" and @id="environment"]'); - await literalSelect.waitForDisplayed({ timeout: uiTimeout }); - assert.equal( - (await literalSelect.getText()).trim(), - 'Select a value', - 'literal dropdown should start without a selected value', - ); - - const startButton = await $('#startNewRunBtn'); - await startButton.waitForDisplayed({ timeout: uiTimeout }); - assert.equal(await startButton.isEnabled(), false, 'start should stay disabled before selection'); - - await literalSelect.click(); - await $('[role="listbox"]').waitForDisplayed({ timeout: uiTimeout }); - await browser.keys('qa'); - await browser.keys('Escape'); - await $('[role="listbox"]').waitForDisplayed({ timeout: uiTimeout, reverse: true }); - assert.equal( - (await literalSelect.getText()).trim(), - 'Select a value', - 'literal dropdown should reject arbitrary typed values', - ); - assert.equal( - await startButton.isEnabled(), - false, - 'start should remain disabled after an invalid typed value attempt', - ); - - await literalSelect.click(); - await $('[role="listbox"]').waitForDisplayed({ timeout: uiTimeout }); - const optionElements = await $$('[role="option"]'); - const optionTexts = []; - for (const optionElement of optionElements) { - optionTexts.push(await optionElement.getText()); - } - assert.deepEqual(optionTexts, ['dev', 'staging', 'prod'], 'literal dropdown options mismatch'); - - await $(`li=${selectedLiteral}`).click(); - await waitForCondition( - async () => (await literalSelect.getText()).trim() === selectedLiteral && (await startButton.isEnabled()), - { - timeout: uiTimeout, - timeoutMsg: 'literal dropdown did not preserve the selected value or enable start', - }, - ); - }); - - it('creates a run with the selected literal value', async () => { - await $('#startNewRunBtn').click(); - - await waitForCondition( - async () => { - const hash = new URL(await browser.getUrl()).hash; - return hash === '#/runs' || hash.startsWith('#/runs/details/'); - }, - { - timeout: uiTimeout, - timeoutMsg: 'expected run creation to land on the runs list or run details page', - }, - ); - - createdRunId = await waitForCreatedRunId(); - let fetchedRun; - await waitForCondition( - async () => { - fetchedRun = await fetchRunById(createdRunId); - return fetchedRun.ok && !!fetchedRun.run.runtime_config?.parameters; - }, - { - timeout: runStartTimeout, - interval: 1000, - timeoutMsg: 'created run did not expose runtime parameters in time', - }, - ); - assert.equal( - fetchedRun.run.runtime_config?.parameters?.environment, - selectedLiteral, - 'selected literal value was not propagated to the run', - ); - }); -}); diff --git a/test/frontend-integration-test/literal-input.yaml b/test/frontend-integration-test/literal-input.yaml deleted file mode 100644 index 5be66ab3b3f..00000000000 --- a/test/frontend-integration-test/literal-input.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2026 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# PIPELINE DEFINITION -# Name: literal-input-pipeline -# Inputs: -# environment: str -components: - comp-consumer: - executorLabel: exec-consumer - inputDefinitions: - parameters: - input_value: - parameterType: STRING - comp-producer: - executorLabel: exec-producer - inputDefinitions: - parameters: - input_text: - parameterType: STRING - outputDefinitions: - parameters: - output_value: - parameterType: STRING -deploymentSpec: - executors: - exec-consumer: - container: - command: - - sh - - -c - - 'set -e -x - - echo "Read from an input parameter: " && echo "$0" - - ' - - '{{$.inputs.parameters[''input_value'']}}' - image: python:alpine3.9 - exec-producer: - container: - command: - - sh - - -c - - 'set -e -x - - mkdir -p "$(dirname "$1")" - - echo "$0, this is an output parameter" > "$1" - - ' - - '{{$.inputs.parameters[''input_text'']}}' - - '{{$.outputs.parameters[''output_value''].output_file}}' - image: python:alpine3.9 -pipelineInfo: - name: literal-input-pipeline -root: - dag: - tasks: - consumer: - cachingOptions: - enableCache: true - componentRef: - name: comp-consumer - dependentTasks: - - producer - inputs: - parameters: - input_value: - taskOutputParameter: - outputParameterKey: output_value - producerTask: producer - taskInfo: - name: consumer - producer: - cachingOptions: - enableCache: true - componentRef: - name: comp-producer - inputs: - parameters: - input_text: - componentInputParameter: environment - taskInfo: - name: producer - inputDefinitions: - parameters: - environment: - literals: - - dev - - staging - - prod - parameterType: STRING -schemaVersion: 2.1.0 -sdkVersion: kfp-2.14.6 diff --git a/test/frontend-integration-test/tensorboard-example.spec.js b/test/frontend-integration-test/tensorboard-example.spec.js index b5adf305682..f45eeca5b8a 100644 --- a/test/frontend-integration-test/tensorboard-example.spec.js +++ b/test/frontend-integration-test/tensorboard-example.spec.js @@ -20,7 +20,6 @@ const { isSelectorDisplayed, runPhase, saveDebugScreenshot, - selectPipelineForRun, waitForCondition, waitForGraphNodeCount, waitForHashPrefix, @@ -59,6 +58,34 @@ async function waitForTensorboardControls() { } } +async function selectPipelineForRun() { + await $('#choosePipelineBtn').waitForDisplayed({ timeout: uiTimeout }); + await $('#choosePipelineBtn').click(); + + await $('#pipelineSelectorDialog').waitForDisplayed({ timeout: uiTimeout }); + const pipelineRowSelector = buildTableRowSelector(pipelineName, { + containerXPath: '//*[@id="pipelineSelectorDialog"]', + }); + + try { + await waitForCondition( + async () => (await $(pipelineRowSelector).isExisting()), + { + timeout: uiTimeout, + timeoutMsg: `expected pipeline row for ${pipelineName} to appear`, + }, + ); + } catch (error) { + await saveDebugScreenshot('pipeline-selector'); + throw error; + } + + await $(pipelineRowSelector).click(); + await $('#usePipelineBtn').waitForEnabled({ timeout: uiTimeout }); + await $('#usePipelineBtn').click(); + await $('#pipelineSelectorDialog').waitForDisplayed({ timeout: uiTimeout, reverse: true }); +} + async function openNewRunDetails() { const runLinkSelector = `[data-testid="run-name-link"][data-run-name="${runName}"]`; @@ -287,7 +314,7 @@ describe('deploy tensorboard example run', () => { await runPhase('create run', async () => { await $('#choosePipelineBtn').waitForDisplayed({ timeout: uiTimeout }); - await selectPipelineForRun(pipelineName, { timeout: uiTimeout }); + await selectPipelineForRun(); const runFormVariant = await waitForRunPageReady({ timeout: runStartTimeout, timeoutMsg: 'expected a run creation form to load', diff --git a/test/frontend-integration-test/test-helpers.js b/test/frontend-integration-test/test-helpers.js index 42a4e89bc10..7e69f7a9116 100644 --- a/test/frontend-integration-test/test-helpers.js +++ b/test/frontend-integration-test/test-helpers.js @@ -58,15 +58,11 @@ async function waitForHashPrefix(prefix, { timeout = defaultTimeout } = {}) { ); } -async function pageContainsText(text) { - return browser.execute((pageTextToFind) => { - const pageText = document.body ? document.body.innerText : ''; - return pageText.includes(pageTextToFind); - }, text); -} - async function isRunPageLoading() { - return pageContainsText(runPageLoadingText); + return browser.execute((loadingText) => { + const pageText = document.body ? document.body.innerText : ''; + return pageText.includes(loadingText); + }, runPageLoadingText); } async function waitForRunPageReady({ @@ -118,6 +114,7 @@ async function waitForRunPageReady({ return matchedVariant; } + async function getValueFromDetailsTable(key) { // Find the span that shows the key, get its parent div (the row), then // get that row's inner text, and remove the key. @@ -145,48 +142,6 @@ async function saveDebugScreenshot(name) { return screenshotPath; } -async function logPipelineSelectorDiagnostics() { - const rowCount = await browser.execute(() => document.querySelectorAll('[data-testid="table-row"]').length); - const emptyMessage = await browser.execute(() => { - const emptyEl = document.querySelector('.emptyMessage'); - return emptyEl ? emptyEl.textContent : null; - }); - console.log('PIPELINE_SELECTOR_ROW_COUNT', rowCount); - console.log('PIPELINE_SELECTOR_EMPTY_MESSAGE', emptyMessage); -} - -async function selectPipelineForRun( - pipelineName, - { timeout = defaultTimeout, screenshotName = 'pipeline-selector' } = {}, -) { - await $('#choosePipelineBtn').waitForDisplayed({ timeout }); - await $('#choosePipelineBtn').click(); - - await $('#pipelineSelectorDialog').waitForDisplayed({ timeout }); - const pipelineRowSelector = buildTableRowSelector(pipelineName, { - containerXPath: '//*[@id="pipelineSelectorDialog"]', - }); - - try { - await waitForCondition( - async () => (await $(pipelineRowSelector).isExisting()), - { - timeout, - timeoutMsg: `expected pipeline row for ${pipelineName} to appear`, - }, - ); - } catch (error) { - await logPipelineSelectorDiagnostics(); - await saveDebugScreenshot(screenshotName); - throw error; - } - - await $(pipelineRowSelector).click(); - await $('#usePipelineBtn').waitForEnabled({ timeout }); - await $('#usePipelineBtn').click(); - await $('#pipelineSelectorDialog').waitForDisplayed({ timeout, reverse: true }); -} - async function isSelectorDisplayed(selector) { const element = await $(selector); return (await element.isExisting()) && (await element.isDisplayed()); @@ -256,7 +211,6 @@ module.exports = { isSelectorDisplayed, runPhase, saveDebugScreenshot, - selectPipelineForRun, waitForCondition, waitForGraphNodeCount, waitForHashPrefix, diff --git a/test/frontend-integration-test/wdio.conf.js b/test/frontend-integration-test/wdio.conf.js index 73de1d75597..15d36d554ce 100644 --- a/test/frontend-integration-test/wdio.conf.js +++ b/test/frontend-integration-test/wdio.conf.js @@ -26,7 +26,7 @@ const specsFromEnv = process.env.WDIO_SPECS : []; const specs = specsFromEnv.length ? specsFromEnv - : ['./helloworld.spec.js', './literal-input.spec.js', './tensorboard-example.spec.js']; + : ['./helloworld.spec.js', './tensorboard-example.spec.js']; const chromeArgs = ['--disable-gpu', '--no-sandbox', '--disable-dev-shm-usage']; if (headless) { chromeArgs.unshift('--headless'); diff --git a/test/install-argo.sh b/test/install-argo.sh index 7809ba5290f..2c8a0f5c40f 100755 --- a/test/install-argo.sh +++ b/test/install-argo.sh @@ -16,6 +16,8 @@ set -ex +ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false} + # Tests work without these lines. TODO: Verify and remove these lines kubectl config set-context $(kubectl config current-context) --namespace=default echo "Add necessary cluster role bindings" @@ -32,3 +34,20 @@ ARGO_KSA="test-runner" echo "add service account for running the test workflow" kubectl create serviceaccount ${ARGO_KSA} -n ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - kubectl create clusterrolebinding test-admin-binding --clusterrole=cluster-admin --serviceaccount=${NAMESPACE}:${ARGO_KSA} --dry-run=client -o yaml | kubectl apply -f - + +if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then + ARGO_GSA="test-argo" + # Util library including create_gsa_if_not_present and bind_gsa_and_ksa functions. + source "$DIR/../manifests/kustomize/wi-utils.sh" + create_gsa_if_not_present $ARGO_GSA + + source "${DIR}/scripts/retry.sh" + retry gcloud projects add-iam-policy-binding $PROJECT \ + --member="serviceAccount:$ARGO_GSA@$PROJECT.iam.gserviceaccount.com" \ + --role="roles/editor" \ + > /dev/null # hide verbose output + retry bind_gsa_and_ksa $ARGO_GSA $ARGO_KSA $PROJECT $NAMESPACE + + # TODO(Bobgy): re-enable this after temporary flakiness is resolved. + # verify_workload_identity_binding $ARGO_KSA $NAMESPACE +fi diff --git a/test/presubmit-tests-sdk.sh b/test/presubmit-tests-sdk.sh index 59d09b9176a..147115c79a8 100755 --- a/test/presubmit-tests-sdk.sh +++ b/test/presubmit-tests-sdk.sh @@ -15,7 +15,6 @@ source_root=$(pwd) SETUP_ENV="${SETUP_ENV:-true}" -PYTEST_PARALLEL_WORKERS="${PYTEST_PARALLEL_WORKERS:-2}" if [ "${SETUP_ENV}" = "true" ]; then # Create a virtual environment and activate it @@ -28,7 +27,6 @@ if [ "${SETUP_ENV}" = "true" ]; then python3 -m pip install setuptools python3 -m pip install wheel==0.42.0 python3 -m pip install pytest-cov - python3 -m pip install pytest-xdist python3 -m pip install pytest python3 -m pip install google_cloud_pipeline_components python3 -m pip install docker @@ -49,7 +47,7 @@ else export KFP_PACKAGE_PATH="git+https://github.com/${REPO_NAME}@refs/pull/${PULL_NUMBER}/merge#egg=kfp&subdirectory=sdk/python" fi -python -m pytest sdk/python/test -v -m regression --cov=kfp -n "${PYTEST_PARALLEL_WORKERS}" +python -m pytest sdk/python/test -v -s -m regression --cov=kfp if [ "${SETUP_ENV}" = "true" ]; then # Deactivate the virtual environment diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index de6ffdefd65..d08822ebc5d 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"add-numbers","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -265,10 +183,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-numbers-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-numbers-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-numbers-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-numbers-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-numbers-driver.Succeeded name: add-numbers template: system-container-executor @@ -278,69 +196,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"add-numbers","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -358,23 +222,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -390,7 +238,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/arguments_parameters.yaml b/test_data/compiled-workflows/arguments_parameters.yaml index 6e33503119a..2b58615cfa0 100644 --- a/test_data/compiled-workflows/arguments_parameters.yaml +++ b/test_data/compiled-workflows/arguments_parameters.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -269,69 +187,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +213,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +229,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index 401b4d7d9d8..513dd3dbd04 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,80 +52,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"artifact-cache-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -138,25 +74,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -284,10 +202,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: core-comp-driver.Succeeded name: core-comp template: system-container-executor @@ -297,69 +215,15 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"artifact-cache-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -377,23 +241,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -411,9 +259,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''condition'']")}}' depends: core-driver.Succeeded name: core template: comp-core @@ -443,10 +291,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: crust-comp-driver.Succeeded name: crust-comp template: system-container-executor @@ -465,9 +313,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''condition'']")}}' depends: mantle-driver.Succeeded name: mantle template: comp-mantle @@ -492,7 +340,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index 1e55ef9d2a9..425a14ba908 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,80 +52,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"artifact-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -138,25 +74,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -284,10 +202,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: core-comp-driver.Succeeded name: core-comp template: system-container-executor @@ -297,69 +215,15 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"artifact-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -377,23 +241,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -411,9 +259,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''condition'']")}}' depends: core-driver.Succeeded name: core template: comp-core @@ -443,10 +291,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: crust-comp-driver.Succeeded name: crust-comp template: system-container-executor @@ -465,9 +313,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''condition'']")}}' depends: mantle-driver.Succeeded name: mantle template: comp-mantle @@ -492,7 +340,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index 23f3c90176a..d9a3887b6c6 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -69,6 +69,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -77,80 +78,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -164,25 +100,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -310,10 +228,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-driver.Succeeded name: add template: system-container-executor @@ -334,10 +252,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-2-driver.Succeeded name: add-2 template: system-container-executor @@ -359,10 +277,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-two-ints-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-two-ints-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-two-ints-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-two-ints-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-two-ints-driver.Succeeded name: add-two-ints template: system-container-executor @@ -391,10 +309,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-2-driver.Succeeded name: double-2 template: system-container-executor @@ -404,69 +322,15 @@ spec: metadata: {} name: comp-condition-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -484,23 +348,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -519,13 +367,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-5-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-5-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}}' depends: condition-5-driver.Succeeded name: condition-5 template: comp-condition-5 - when: '{{tasks.condition-5-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -552,9 +401,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -583,14 +432,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -616,10 +465,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-driver.Succeeded name: double template: system-container-executor @@ -655,9 +504,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -686,14 +535,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -718,9 +567,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.add-two-lists-of-datasets-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''add-two-lists-of-datasets-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.add-two-lists-of-datasets-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''add-two-lists-of-datasets-driver''].outputs.result, + "$[''condition'']")}}' depends: add-two-lists-of-datasets-driver.Succeeded name: add-two-lists-of-datasets template: comp-add-two-lists-of-datasets @@ -751,7 +602,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index 980c19b8ec0..d9cddf7110e 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -54,6 +54,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -62,80 +63,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -149,25 +85,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -295,10 +213,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-driver.Succeeded name: double template: system-container-executor @@ -308,69 +226,15 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -388,23 +252,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -425,9 +273,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -456,14 +304,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -490,10 +338,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-driver.Succeeded name: add template: system-container-executor @@ -515,10 +363,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-container-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-container-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-container-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-container-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-container-driver.Succeeded name: add-container template: system-container-executor @@ -549,7 +397,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index 92272c9cd13..0ce2e5de2e9 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -135,6 +135,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -143,80 +144,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"collected-artifact-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -230,25 +166,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -376,10 +294,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.create-dataset-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''create-dataset-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.create-dataset-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''create-dataset-driver''].outputs.result, "$[''cached-decision'']")}}' depends: create-dataset-driver.Succeeded name: create-dataset template: system-container-executor @@ -389,69 +307,15 @@ spec: metadata: {} name: comp-single-node-dag outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"collected-artifact-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -469,23 +333,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -506,10 +354,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-single-dataset-generate-model-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-single-dataset-generate-model-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-single-dataset-generate-model-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-single-dataset-generate-model-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: read-single-dataset-generate-model-driver.Succeeded name: read-single-dataset-generate-model template: system-container-executor @@ -528,9 +378,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.single-node-dag-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''single-node-dag-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.single-node-dag-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''single-node-dag-driver''].outputs.result, + "$[''condition'']")}}' depends: single-node-dag-driver.Succeeded name: single-node-dag template: comp-single-node-dag @@ -559,9 +411,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -589,14 +441,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -622,10 +474,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.create-file-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''create-file-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.create-file-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''create-file-driver''].outputs.result, "$[''cached-decision'']")}}' depends: create-file-driver.Succeeded name: create-file template: system-container-executor @@ -653,10 +505,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-datasets-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-datasets-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-datasets-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-datasets-driver''].outputs.result, "$[''cached-decision'']")}}' depends: read-datasets-driver.Succeeded name: read-datasets template: system-container-executor @@ -678,10 +530,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-models-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-models-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-models-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-models-driver''].outputs.result, "$[''cached-decision'']")}}' depends: read-models-driver.Succeeded name: read-models template: system-container-executor @@ -703,10 +555,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-single-file-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-single-file-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-single-file-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-single-file-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: read-single-file-driver.Succeeded name: read-single-file template: system-container-executor @@ -735,9 +589,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -765,14 +619,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -805,10 +659,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.split-chars-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''split-chars-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.split-chars-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''split-chars-driver''].outputs.result, "$[''cached-decision'']")}}' depends: split-chars-driver.Succeeded name: split-chars template: system-container-executor @@ -829,10 +683,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.split-ids-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''split-ids-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.split-ids-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''split-ids-driver''].outputs.result, "$[''cached-decision'']")}}' depends: split-ids-driver.Succeeded name: split-ids template: system-container-executor @@ -859,9 +713,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.collecting-artifacts-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''collecting-artifacts-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.collecting-artifacts-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''collecting-artifacts-driver''].outputs.result, + "$[''condition'']")}}' depends: collecting-artifacts-driver.Succeeded name: collecting-artifacts template: comp-collecting-artifacts @@ -883,10 +739,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-models-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-models-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-models-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-models-driver''].outputs.result, "$[''cached-decision'']")}}' depends: read-models-driver.Succeeded name: read-models template: system-container-executor @@ -911,7 +767,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index 0dcc4c50b89..fee6ceec049 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -70,6 +70,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -78,80 +79,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"collected-param-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -165,25 +101,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -312,10 +230,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consume-single-id-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consume-single-id-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consume-single-id-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consume-single-id-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: consume-single-id-driver.Succeeded name: consume-single-id template: system-container-executor @@ -336,10 +256,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.prepend-id-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''prepend-id-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.prepend-id-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''prepend-id-driver''].outputs.result, "$[''cached-decision'']")}}' depends: prepend-id-driver.Succeeded name: prepend-id template: system-container-executor @@ -349,69 +269,15 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"collected-param-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -429,23 +295,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -465,9 +315,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -495,14 +345,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -529,10 +379,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consume-ids-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consume-ids-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consume-ids-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consume-ids-driver''].outputs.result, "$[''cached-decision'']")}}' depends: consume-ids-driver.Succeeded name: consume-ids template: system-container-executor @@ -560,10 +410,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.split-ids-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''split-ids-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.split-ids-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''split-ids-driver''].outputs.result, "$[''cached-decision'']")}}' depends: split-ids-driver.Succeeded name: split-ids template: system-container-executor @@ -590,9 +440,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.collecting-parameters-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''collecting-parameters-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.collecting-parameters-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''collecting-parameters-driver''].outputs.result, + "$[''condition'']")}}' depends: collecting-parameters-driver.Succeeded name: collecting-parameters template: comp-collecting-parameters @@ -614,10 +466,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consume-ids-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consume-ids-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consume-ids-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consume-ids-driver''].outputs.result, "$[''cached-decision'']")}}' depends: consume-ids-driver.Succeeded name: consume-ids template: system-container-executor @@ -642,7 +494,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index 8a379d4acaa..f882c7452b7 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -39,6 +39,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -47,80 +48,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"dataset-joiner","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -134,25 +70,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -280,10 +198,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.dataset-joiner-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''dataset-joiner-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.dataset-joiner-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''dataset-joiner-driver''].outputs.result, "$[''cached-decision'']")}}' depends: dataset-joiner-driver.Succeeded name: dataset-joiner template: system-container-executor @@ -293,69 +211,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"dataset-joiner","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -373,23 +237,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -405,7 +253,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index f8e8d63a33b..4cfcbc9abcd 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -28,6 +28,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -36,80 +37,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"v2-component-optional-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -123,25 +59,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -269,10 +187,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: component-op-driver.Succeeded name: component-op template: system-container-executor @@ -282,69 +200,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"v2-component-optional-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -362,23 +226,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -394,7 +242,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index cb5e4cf00a7..aa7faed7d8f 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,80 +35,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"v2-component-pip-index-urls","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -121,25 +57,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -267,10 +185,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: component-op-driver.Succeeded name: component-op template: system-container-executor @@ -280,69 +198,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"v2-component-pip-index-urls","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -360,23 +224,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -392,7 +240,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index bc771212680..866b8a62c07 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,80 +35,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"component-with-pip-install","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -121,25 +57,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -267,10 +185,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-with-pip-install-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-with-pip-install-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-with-pip-install-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-with-pip-install-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-with-pip-install-driver.Succeeded name: component-with-pip-install template: system-container-executor @@ -280,69 +200,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"component-with-pip-install","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -360,23 +226,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -392,7 +242,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index 4f6c6ef817b..f2e4a1dd285 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -27,6 +27,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -35,80 +36,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"component-with-pip-install","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -122,25 +58,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -268,10 +186,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-with-pip-install-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-with-pip-install-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-with-pip-install-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-with-pip-install-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-with-pip-install-driver.Succeeded name: component-with-pip-install template: system-container-executor @@ -281,69 +201,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"component-with-pip-install","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -361,23 +227,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -393,7 +243,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index 304042d1a4c..76e0cf32d14 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -48,80 +49,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"optional-artifact-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -135,25 +71,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -281,10 +199,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.python-artifact-printer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''python-artifact-printer-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.python-artifact-printer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''python-artifact-printer-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: python-artifact-printer-driver.Succeeded name: python-artifact-printer template: system-container-executor @@ -313,10 +233,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.python-artifact-printer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''python-artifact-printer-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.python-artifact-printer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''python-artifact-printer-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: python-artifact-printer-driver.Succeeded name: python-artifact-printer template: system-container-executor @@ -395,69 +317,15 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"optional-artifact-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -475,23 +343,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -511,10 +363,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.custom-artifact-printer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''custom-artifact-printer-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.custom-artifact-printer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''custom-artifact-printer-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: custom-artifact-printer-driver.Succeeded name: custom-artifact-printer template: system-container-executor @@ -535,10 +389,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.custom-artifact-printer-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''custom-artifact-printer-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.custom-artifact-printer-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''custom-artifact-printer-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: custom-artifact-printer-2-driver.Succeeded name: custom-artifact-printer-2 template: system-container-executor @@ -570,9 +426,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.inner-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.inner-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''condition'']")}}' depends: inner-pipeline-driver.Succeeded name: inner-pipeline template: comp-inner-pipeline @@ -591,9 +447,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.inner-pipeline-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''inner-pipeline-2-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.inner-pipeline-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''inner-pipeline-2-driver''].outputs.result, + "$[''condition'']")}}' depends: inner-pipeline-2-driver.Succeeded name: inner-pipeline-2 template: comp-inner-pipeline-2 @@ -618,7 +476,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index 83347d58281..80b2a0508ac 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -25,6 +25,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -33,80 +34,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"concat-message","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -120,25 +56,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -266,10 +184,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.concat-message-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''concat-message-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.concat-message-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''concat-message-driver''].outputs.result, "$[''cached-decision'']")}}' depends: concat-message-driver.Succeeded name: concat-message template: system-container-executor @@ -279,69 +197,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"concat-message","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -359,23 +223,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -391,7 +239,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index eeef66b184b..b6bb4993049 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,80 +55,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -141,25 +77,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -287,10 +205,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-driver.Succeeded name: add template: system-container-executor @@ -319,10 +237,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-driver.Succeeded name: double template: system-container-executor @@ -332,69 +250,15 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -412,23 +276,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -447,13 +295,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -480,9 +329,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -511,14 +360,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -544,13 +393,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-4-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-4-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}}' depends: condition-4-driver.Succeeded name: condition-4 template: comp-condition-4 - when: '{{tasks.condition-4-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: parent-dag-id @@ -578,7 +428,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index 9c2081fe84e..58ffa12ef56 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"v2-container-component-no-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.hello-world-container-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''hello-world-container-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.hello-world-container-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''hello-world-container-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: hello-world-container-driver.Succeeded name: hello-world-container template: system-container-executor @@ -269,69 +189,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"v2-container-component-no-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +215,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +231,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index d22e741c5c0..be51e15359a 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-io","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.container-io-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''container-io-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.container-io-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''container-io-driver''].outputs.result, "$[''cached-decision'']")}}' depends: container-io-driver.Succeeded name: container-io template: system-container-executor @@ -269,69 +187,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-io","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +213,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +229,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 923a04b6e85..810c7f613c2 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-no-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.container-no-input-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''container-no-input-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.container-no-input-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''container-no-input-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: container-no-input-driver.Succeeded name: container-no-input template: system-container-executor @@ -269,69 +189,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-no-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +215,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +231,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/container_with_artifact_output.yaml b/test_data/compiled-workflows/container_with_artifact_output.yaml index 4f858708083..03b1543f673 100644 --- a/test_data/compiled-workflows/container_with_artifact_output.yaml +++ b/test_data/compiled-workflows/container_with_artifact_output.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-artifact-output","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.container-with-artifact-output-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''container-with-artifact-output-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.container-with-artifact-output-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''container-with-artifact-output-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: container-with-artifact-output-driver.Succeeded name: container-with-artifact-output template: system-container-executor @@ -269,69 +189,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-artifact-output","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +215,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +231,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index 89f38c22455..d852032e404 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -16,6 +16,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -24,80 +25,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-concat-placeholder","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -111,25 +47,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -257,10 +175,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.container-with-concat-placeholder-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''container-with-concat-placeholder-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.container-with-concat-placeholder-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''container-with-concat-placeholder-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: container-with-concat-placeholder-driver.Succeeded name: container-with-concat-placeholder template: system-container-executor @@ -270,69 +190,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-concat-placeholder","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -350,23 +216,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -382,7 +232,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index 0df3f24826e..27561794064 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -18,6 +18,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -26,80 +27,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-if-placeholder","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -113,25 +49,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -259,10 +177,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.container-with-if-placeholder-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''container-with-if-placeholder-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.container-with-if-placeholder-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''container-with-if-placeholder-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: container-with-if-placeholder-driver.Succeeded name: container-with-if-placeholder template: system-container-executor @@ -272,69 +192,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-if-placeholder","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -352,23 +218,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -384,7 +234,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 996046e0c58..55c8bbec880 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-placeholder-in-fstring","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.container-with-placeholder-in-fstring-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''container-with-placeholder-in-fstring-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.container-with-placeholder-in-fstring-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''container-with-placeholder-in-fstring-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: container-with-placeholder-in-fstring-driver.Succeeded name: container-with-placeholder-in-fstring template: system-container-executor @@ -269,69 +189,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"container-with-placeholder-in-fstring","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +215,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +231,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/containerized_python_component.yaml b/test_data/compiled-workflows/containerized_python_component.yaml index c4f3550b4ac..a0fb8a8d103 100644 --- a/test_data/compiled-workflows/containerized_python_component.yaml +++ b/test_data/compiled-workflows/containerized_python_component.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"concat-message","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.concat-message-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''concat-message-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.concat-message-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''concat-message-driver''].outputs.result, "$[''cached-decision'']")}}' depends: concat-message-driver.Succeeded name: concat-message template: system-container-executor @@ -269,69 +187,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"concat-message","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +213,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +229,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index cbac80c4550..ceb6f1fc31c 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -68,6 +68,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -76,80 +77,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-pod-metadata","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -163,25 +99,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -573,10 +491,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-no-pod-metadata-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-no-pod-metadata-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-no-pod-metadata-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-no-pod-metadata-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: validate-no-pod-metadata-driver.Succeeded name: validate-no-pod-metadata template: system-container-executor @@ -599,10 +519,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-pod-metadata-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-pod-metadata-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-driver''].outputs.result, + "$[''cached-decision'']")}}' - name: pod-metadata-annotation-key value: task-annotation - name: pod-metadata-annotation-val @@ -637,10 +559,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-pod-metadata-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-pod-metadata-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-2-driver''].outputs.result, + "$[''cached-decision'']")}}' - name: pod-metadata-annotation-key-1 value: task-annotation-1 - name: pod-metadata-annotation-val-1 @@ -658,69 +582,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-pod-metadata","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -738,23 +608,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -770,7 +624,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index a53b89b94a9..c2730432125 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,80 +54,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"my-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -140,25 +76,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-5-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-5-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-5-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-5-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-5-driver.Succeeded name: print-op-5 template: system-container-executor @@ -318,10 +236,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-7-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-7-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-7-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-7-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-7-driver.Succeeded name: print-op-7 template: system-container-executor @@ -331,69 +249,15 @@ spec: metadata: {} name: comp-for-loop-14 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"my-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -411,23 +275,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -448,9 +296,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-14 @@ -479,14 +327,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-14-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -519,10 +367,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-8-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-8-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-8-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-8-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-8-driver.Succeeded name: print-op-8 template: system-container-executor @@ -551,10 +399,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -583,10 +431,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -615,10 +463,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-4-driver.Succeeded name: print-op-4 template: system-container-executor @@ -648,9 +496,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-8 @@ -679,14 +527,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-8-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -719,10 +567,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-3-driver.Succeeded name: print-op-3 template: system-container-executor @@ -752,9 +600,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-10 @@ -783,14 +631,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-10-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -817,9 +665,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-12 @@ -848,14 +696,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-12-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -882,9 +730,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -913,14 +761,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -947,9 +795,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -978,14 +826,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1012,9 +860,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-6 @@ -1043,14 +891,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-6-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1108,10 +956,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-6-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-6-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-6-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-6-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-6-driver.Succeeded name: print-op-6 template: system-container-executor @@ -1136,7 +984,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index 06d27d21937..0170cb7619a 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"dict-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -265,10 +183,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.dict-input-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''dict-input-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.dict-input-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''dict-input-driver''].outputs.result, "$[''cached-decision'']")}}' depends: dict-input-driver.Succeeded name: dict-input template: system-container-executor @@ -278,69 +196,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"dict-input","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -358,23 +222,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -390,7 +238,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/embedded_artifact.yaml b/test_data/compiled-workflows/embedded_artifact.yaml index 5275df4d526..c134af12ae5 100644 --- a/test_data/compiled-workflows/embedded_artifact.yaml +++ b/test_data/compiled-workflows/embedded_artifact.yaml @@ -65,6 +65,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -73,80 +74,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nb-simple","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -160,25 +96,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -306,10 +224,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-embedded-artifact-dir-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-embedded-artifact-dir-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-embedded-artifact-dir-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-embedded-artifact-dir-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: read-embedded-artifact-dir-driver.Succeeded name: read-embedded-artifact-dir template: system-container-executor @@ -330,10 +250,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-embedded-artifact-file-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-embedded-artifact-file-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-embedded-artifact-file-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-embedded-artifact-file-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: read-embedded-artifact-file-driver.Succeeded name: read-embedded-artifact-file template: system-container-executor @@ -343,69 +265,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nb-simple","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -423,23 +291,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -455,7 +307,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/env-var.yaml b/test_data/compiled-workflows/env-var.yaml index 0177ac3b3ed..3134a4ab01a 100644 --- a/test_data/compiled-workflows/env-var.yaml +++ b/test_data/compiled-workflows/env-var.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,80 +35,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"test-env-exists","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -121,25 +57,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -267,10 +185,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: comp-driver.Succeeded name: comp template: system-container-executor @@ -280,69 +198,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"test-env-exists","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -360,23 +224,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -392,7 +240,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/fail_v2.yaml b/test_data/compiled-workflows/fail_v2.yaml index 49cfe93539c..26d4e3a3cb9 100644 --- a/test_data/compiled-workflows/fail_v2.yaml +++ b/test_data/compiled-workflows/fail_v2.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"fail-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -265,10 +183,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.fail-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''fail-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.fail-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''fail-driver''].outputs.result, "$[''cached-decision'']")}}' depends: fail-driver.Succeeded name: fail template: system-container-executor @@ -278,69 +196,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"fail-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -358,23 +222,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -390,7 +238,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/flip_coin.yaml b/test_data/compiled-workflows/flip_coin.yaml index be8efc3a924..5a7397e3c2c 100644 --- a/test_data/compiled-workflows/flip_coin.yaml +++ b/test_data/compiled-workflows/flip_coin.yaml @@ -81,6 +81,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -89,80 +90,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"conditional-execution-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -176,25 +112,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -324,10 +242,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-msg-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-msg-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-msg-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-msg-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-msg-driver.Succeeded name: print-msg template: system-container-executor @@ -358,10 +276,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-msg-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-msg-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-msg-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-msg-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-msg-2-driver.Succeeded name: print-msg-2 template: system-container-executor @@ -371,69 +289,15 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"conditional-execution-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -451,23 +315,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -487,13 +335,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -511,13 +360,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -535,10 +385,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.random-num-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''random-num-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.random-num-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''random-num-driver''].outputs.result, "$[''cached-decision'']")}}' depends: random-num-driver.Succeeded name: random-num template: system-container-executor @@ -569,10 +419,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-msg-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-msg-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-msg-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-msg-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-msg-3-driver.Succeeded name: print-msg-3 template: system-container-executor @@ -603,10 +453,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-msg-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-msg-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-msg-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-msg-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-msg-4-driver.Succeeded name: print-msg-4 template: system-container-executor @@ -635,13 +485,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-5-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-5-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}}' depends: condition-5-driver.Succeeded name: condition-5 template: comp-condition-5 - when: '{{tasks.condition-5-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -659,13 +510,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-6-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-6-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''condition'']")}}' depends: condition-6-driver.Succeeded name: condition-6 template: comp-condition-6 - when: '{{tasks.condition-6-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -683,10 +535,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.random-num-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''random-num-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.random-num-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''random-num-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: random-num-2-driver.Succeeded name: random-num-2 template: system-container-executor @@ -715,13 +567,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -739,13 +592,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-4-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-4-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}}' depends: condition-4-driver.Succeeded name: condition-4 template: comp-condition-4 - when: '{{tasks.condition-4-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -763,10 +617,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-driver.Succeeded name: flip-coin template: system-container-executor @@ -791,7 +645,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/hello_world.yaml b/test_data/compiled-workflows/hello_world.yaml index 497c09c20b9..6ba8ad34ef2 100644 --- a/test_data/compiled-workflows/hello_world.yaml +++ b/test_data/compiled-workflows/hello_world.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -269,69 +187,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +213,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +229,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index e35b003f95a..349cc16545c 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"identity","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -265,10 +183,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.identity-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''identity-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.identity-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''identity-driver''].outputs.result, "$[''cached-decision'']")}}' depends: identity-driver.Succeeded name: identity template: system-container-executor @@ -278,69 +196,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"identity","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -358,23 +222,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -390,7 +238,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index 541abc3533f..502e58e80dd 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -115,6 +115,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -123,80 +124,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"lucky-number-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -210,25 +146,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -357,10 +275,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-driver.Succeeded name: print-and-return template: system-container-executor @@ -370,69 +290,15 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"lucky-number-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -450,23 +316,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -485,13 +335,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -518,10 +369,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-9-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-9-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-9-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-9-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-9-driver.Succeeded name: print-and-return-9 template: system-container-executor @@ -551,9 +404,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-16 @@ -582,14 +435,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-16-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -628,13 +481,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-14-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-14-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-14-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-14-driver''].outputs.result, "$[''condition'']")}}' depends: condition-14-driver.Succeeded name: condition-14 template: comp-condition-14 - when: '{{tasks.condition-14-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-14-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -653,10 +507,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-8-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-8-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-8-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-8-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-8-driver.Succeeded name: print-and-return-8 template: system-container-executor @@ -686,10 +542,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-2-driver.Succeeded name: print-and-return-2 template: system-container-executor @@ -719,10 +577,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-3-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-3-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-3-driver.Succeeded name: print-and-return-3 template: system-container-executor @@ -750,13 +610,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-6-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-6-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''condition'']")}}' depends: condition-6-driver.Succeeded name: condition-6 template: comp-condition-6 - when: '{{tasks.condition-6-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -773,13 +634,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-7-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-7-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-7-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-7-driver''].outputs.result, "$[''condition'']")}}' depends: condition-7-driver.Succeeded name: condition-7 template: comp-condition-7 - when: '{{tasks.condition-7-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-7-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -804,9 +666,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-branches-5-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-branches-5-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-branches-5-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-branches-5-driver''].outputs.result, + "$[''condition'']")}}' depends: condition-branches-5-driver.Succeeded name: condition-branches-5 template: comp-condition-branches-5 @@ -827,10 +691,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.is-even-or-odd-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''is-even-or-odd-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.is-even-or-odd-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''is-even-or-odd-driver''].outputs.result, "$[''cached-decision'']")}}' depends: is-even-or-odd-driver.Succeeded name: is-even-or-odd template: system-container-executor @@ -852,10 +716,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-4-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-4-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-4-driver.Succeeded name: print-and-return-4 template: system-container-executor @@ -885,10 +751,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-5-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-5-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-5-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-5-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-5-driver.Succeeded name: print-and-return-5 template: system-container-executor @@ -918,10 +786,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-6-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-6-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-6-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-6-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-6-driver.Succeeded name: print-and-return-6 template: system-container-executor @@ -949,13 +819,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-11-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-11-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-11-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-11-driver''].outputs.result, "$[''condition'']")}}' depends: condition-11-driver.Succeeded name: condition-11 template: comp-condition-11 - when: '{{tasks.condition-11-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-11-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -972,13 +843,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-12-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-12-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-12-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-12-driver''].outputs.result, "$[''condition'']")}}' depends: condition-12-driver.Succeeded name: condition-12 template: comp-condition-12 - when: '{{tasks.condition-12-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-12-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -1003,9 +875,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-branches-10-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-branches-10-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-branches-10-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-branches-10-driver''].outputs.result, + "$[''condition'']")}}' depends: condition-branches-10-driver.Succeeded name: condition-branches-10 template: comp-condition-branches-10 @@ -1026,10 +900,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.is-even-or-odd-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''is-even-or-odd-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.is-even-or-odd-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''is-even-or-odd-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: is-even-or-odd-2-driver.Succeeded name: is-even-or-odd-2 template: system-container-executor @@ -1051,10 +927,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-7-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-7-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-7-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-7-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-7-driver.Succeeded name: print-and-return-7 template: system-container-executor @@ -1083,13 +961,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-13-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-13-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-13-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-13-driver''].outputs.result, "$[''condition'']")}}' depends: condition-13-driver.Succeeded name: condition-13 template: comp-condition-13 - when: '{{tasks.condition-13-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-13-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -1106,13 +985,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-8-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-8-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-8-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-8-driver''].outputs.result, "$[''condition'']")}}' depends: condition-8-driver.Succeeded name: condition-8 template: comp-condition-8 - when: '{{tasks.condition-8-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-8-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -1130,13 +1010,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-9-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-9-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-9-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-9-driver''].outputs.result, "$[''condition'']")}}' depends: condition-9-driver.Succeeded name: condition-9 template: comp-condition-9 - when: '{{tasks.condition-9-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-9-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -1161,13 +1042,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -1184,9 +1066,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-branches-4-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-branches-4-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-branches-4-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-branches-4-driver''].outputs.result, + "$[''condition'']")}}' depends: condition-branches-4-driver.Succeeded name: condition-branches-4 template: comp-condition-branches-4 @@ -1207,10 +1091,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.int-0-to-9999-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''int-0-to-9999-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.int-0-to-9999-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''int-0-to-9999-driver''].outputs.result, "$[''cached-decision'']")}}' depends: int-0-to-9999-driver.Succeeded name: int-0-to-9999 template: system-container-executor @@ -1239,9 +1123,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -1269,14 +1153,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1309,10 +1193,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-ints-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-ints-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-ints-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-ints-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-ints-driver.Succeeded name: print-ints template: system-container-executor @@ -1337,7 +1221,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index eae6957a429..0ce23e5cbbe 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -70,6 +70,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -78,80 +79,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"outer-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -165,25 +101,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -312,10 +230,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-driver.Succeeded name: print-and-return template: system-container-executor @@ -345,10 +265,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-2-driver.Succeeded name: print-and-return-2 template: system-container-executor @@ -377,10 +299,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.special-print-and-return-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''special-print-and-return-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.special-print-and-return-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''special-print-and-return-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: special-print-and-return-driver.Succeeded name: special-print-and-return template: system-container-executor @@ -390,69 +314,15 @@ spec: metadata: {} name: comp-condition-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"outer-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -470,23 +340,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -505,13 +359,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -529,13 +384,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -553,13 +409,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-4-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-4-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}}' depends: condition-4-driver.Succeeded name: condition-4 template: comp-condition-4 - when: '{{tasks.condition-4-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -584,9 +441,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-branches-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-branches-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''condition'']")}}' depends: condition-branches-1-driver.Succeeded name: condition-branches-1 template: comp-condition-branches-1 @@ -607,10 +466,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-three-sided-die-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-three-sided-die-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-three-sided-die-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-three-sided-die-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: flip-three-sided-die-driver.Succeeded name: flip-three-sided-die template: system-container-executor @@ -640,10 +501,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-driver.Succeeded name: print-and-return template: system-container-executor @@ -662,9 +525,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.roll-die-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''roll-die-pipeline-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.roll-die-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''roll-die-pipeline-driver''].outputs.result, + "$[''condition'']")}}' depends: roll-die-pipeline-driver.Succeeded name: roll-die-pipeline template: comp-roll-die-pipeline @@ -689,7 +554,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index 48ac299fe70..b1a4d853ba3 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -62,6 +62,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -70,80 +71,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"outer-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -157,25 +93,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -303,10 +221,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.param-to-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''param-to-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.param-to-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''param-to-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: param-to-artifact-driver.Succeeded name: param-to-artifact template: system-container-executor @@ -335,10 +255,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.param-to-artifact-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''param-to-artifact-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.param-to-artifact-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''param-to-artifact-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: param-to-artifact-2-driver.Succeeded name: param-to-artifact-2 template: system-container-executor @@ -348,69 +270,15 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"outer-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -428,23 +296,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -463,13 +315,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -486,13 +339,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -517,9 +371,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-branches-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-branches-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''condition'']")}}' depends: condition-branches-1-driver.Succeeded name: condition-branches-1 template: comp-condition-branches-1 @@ -540,10 +396,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-driver.Succeeded name: flip-coin template: system-container-executor @@ -565,10 +421,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-artifact-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-artifact-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-artifact-driver.Succeeded name: print-artifact template: system-container-executor @@ -595,9 +451,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.flip-coin-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''flip-coin-pipeline-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.flip-coin-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''flip-coin-pipeline-driver''].outputs.result, + "$[''condition'']")}}' depends: flip-coin-pipeline-driver.Succeeded name: flip-coin-pipeline template: comp-flip-coin-pipeline @@ -619,10 +477,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-artifact-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-artifact-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-artifact-driver.Succeeded name: print-artifact template: system-container-executor @@ -647,7 +505,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index bac42bafaf2..a3a6e45f243 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -48,6 +48,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -56,80 +57,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"flip-coin-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -143,25 +79,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -290,10 +208,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-driver.Succeeded name: print-and-return template: system-container-executor @@ -323,10 +243,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-2-driver.Succeeded name: print-and-return-2 template: system-container-executor @@ -336,69 +258,15 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"flip-coin-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -416,23 +284,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -451,13 +303,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -474,13 +327,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -505,9 +359,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-branches-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-branches-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''condition'']")}}' depends: condition-branches-1-driver.Succeeded name: condition-branches-1 template: comp-condition-branches-1 @@ -528,10 +384,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-driver.Succeeded name: flip-coin template: system-container-executor @@ -553,10 +409,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-and-return-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-and-return-3-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-and-return-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-and-return-3-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-and-return-3-driver.Succeeded name: print-and-return-3 template: system-container-executor @@ -581,7 +439,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index f7f824e1002..3c1cb5ccbe4 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"input-artifact","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -265,10 +183,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.input-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''input-artifact-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.input-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''input-artifact-driver''].outputs.result, "$[''cached-decision'']")}}' depends: input-artifact-driver.Succeeded name: input-artifact template: system-container-executor @@ -278,69 +196,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"input-artifact","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -358,23 +222,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -390,7 +238,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index 5fbe07a89f1..5cc35484b80 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -77,6 +77,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -85,80 +86,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"iris-training-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -172,25 +108,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -318,10 +236,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.create-dataset-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''create-dataset-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.create-dataset-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''create-dataset-driver''].outputs.result, "$[''cached-decision'']")}}' depends: create-dataset-driver.Succeeded name: create-dataset template: system-container-executor @@ -343,10 +261,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.normalize-dataset-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''normalize-dataset-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.normalize-dataset-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''normalize-dataset-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: normalize-dataset-driver.Succeeded name: normalize-dataset template: system-container-executor @@ -368,10 +288,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.train-model-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''train-model-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.train-model-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''train-model-driver''].outputs.result, "$[''cached-decision'']")}}' depends: train-model-driver.Succeeded name: train-model template: system-container-executor @@ -381,69 +301,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"iris-training-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -461,23 +327,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -493,7 +343,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index f3a185613bb..0ea9e5d0207 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -79,6 +79,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -87,80 +88,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"my-test-pipeline-beta","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -174,25 +110,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -320,10 +238,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.preprocess-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''preprocess-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.preprocess-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''preprocess-driver''].outputs.result, "$[''cached-decision'']")}}' depends: preprocess-driver.Succeeded name: preprocess template: system-container-executor @@ -345,10 +263,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.train-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''train-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.train-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''train-driver''].outputs.result, "$[''cached-decision'']")}}' depends: train-driver.Succeeded name: train template: system-container-executor @@ -358,69 +276,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"my-test-pipeline-beta","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -438,23 +302,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -470,7 +318,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index ec74f4ca372..6e479994fdf 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -71,6 +71,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -79,80 +80,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"functions-with-outputs","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -166,25 +102,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -312,10 +230,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-numbers-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-numbers-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-numbers-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-numbers-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-numbers-driver.Succeeded name: add-numbers template: system-container-executor @@ -336,10 +254,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.concat-message-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''concat-message-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.concat-message-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''concat-message-driver''].outputs.result, "$[''cached-decision'']")}}' depends: concat-message-driver.Succeeded name: concat-message template: system-container-executor @@ -361,10 +279,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.output-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''output-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.output-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''output-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: output-artifact-driver.Succeeded name: output-artifact template: system-container-executor @@ -386,10 +306,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.output-named-tuple-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''output-named-tuple-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.output-named-tuple-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''output-named-tuple-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: output-named-tuple-driver.Succeeded name: output-named-tuple template: system-container-executor @@ -399,69 +321,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"functions-with-outputs","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -479,23 +347,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -511,7 +363,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index ebb532ced27..444633f65ee 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -28,6 +28,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -36,80 +37,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"log-streaming-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -123,25 +59,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -269,10 +187,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-message-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-message-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-message-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-message-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-message-driver.Succeeded name: print-message template: system-container-executor @@ -282,69 +200,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"log-streaming-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -362,23 +226,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -394,7 +242,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/long-running.yaml b/test_data/compiled-workflows/long-running.yaml index f20fe9eb81e..7dd3b91c7cc 100644 --- a/test_data/compiled-workflows/long-running.yaml +++ b/test_data/compiled-workflows/long-running.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"wait-awhile","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.wait-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''wait-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.wait-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''wait-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: wait-op-driver.Succeeded name: wait-op template: system-container-executor @@ -281,10 +199,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.wait-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''wait-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.wait-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''wait-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: wait-op-2-driver.Succeeded name: wait-op-2 template: system-container-executor @@ -294,69 +212,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"wait-awhile","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -374,23 +238,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -406,7 +254,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index dd511ef15d2..15211a18a2e 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -72,6 +72,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -80,80 +81,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"loop-consume-upstream","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -167,25 +103,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -314,10 +232,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.create-file-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''create-file-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.create-file-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''create-file-driver''].outputs.result, "$[''cached-decision'']")}}' depends: create-file-driver.Succeeded name: create-file template: system-container-executor @@ -340,10 +258,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-file-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-file-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-file-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-file-driver''].outputs.result, "$[''cached-decision'']")}}' depends: read-file-driver.Succeeded name: read-file template: system-container-executor @@ -353,69 +271,15 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"loop-consume-upstream","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -433,23 +297,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -469,9 +317,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -499,14 +347,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -541,10 +389,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-input-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-input-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-input-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-input-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-input-driver.Succeeded name: print-input template: system-container-executor @@ -566,10 +414,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.split-input-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''split-input-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.split-input-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''split-input-driver''].outputs.result, "$[''cached-decision'']")}}' depends: split-input-driver.Succeeded name: split-input template: system-container-executor @@ -594,7 +442,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index 908aee03a83..b1d030aade7 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -116,6 +116,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -124,80 +125,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"metrics-visualization-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -211,25 +147,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -357,10 +275,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.digit-classification-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''digit-classification-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.digit-classification-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''digit-classification-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: digit-classification-driver.Succeeded name: digit-classification template: system-container-executor @@ -381,10 +301,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.html-visualization-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''html-visualization-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.html-visualization-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''html-visualization-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: html-visualization-driver.Succeeded name: html-visualization template: system-container-executor @@ -405,10 +327,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.iris-sgdclassifier-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''iris-sgdclassifier-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.iris-sgdclassifier-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''iris-sgdclassifier-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: iris-sgdclassifier-driver.Succeeded name: iris-sgdclassifier template: system-container-executor @@ -429,10 +353,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.markdown-visualization-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''markdown-visualization-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.markdown-visualization-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''markdown-visualization-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: markdown-visualization-driver.Succeeded name: markdown-visualization template: system-container-executor @@ -453,10 +379,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.wine-classification-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''wine-classification-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.wine-classification-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''wine-classification-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: wine-classification-driver.Succeeded name: wine-classification template: system-container-executor @@ -466,69 +394,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"metrics-visualization-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -546,23 +420,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -578,7 +436,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml index bdfccd231af..364fffbfe27 100644 --- a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -27,6 +27,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -35,80 +36,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - missing-kubernetes-optional-inputs-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"missing-kubernetes-optional-inputs-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -122,25 +58,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -271,10 +189,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.log-message-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''log-message-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.log-message-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''log-message-driver''].outputs.result, "$[''cached-decision'']")}}' depends: log-message-driver.Succeeded name: log-message template: system-container-executor @@ -284,69 +202,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - missing-kubernetes-optional-inputs-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"missing-kubernetes-optional-inputs-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -364,23 +228,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -396,7 +244,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index 759de56d366..0614fb3b93f 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -41,6 +41,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -49,80 +50,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"mixed-parameters-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -136,25 +72,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -282,10 +200,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: core-comp-driver.Succeeded name: core-comp template: system-container-executor @@ -295,69 +213,15 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"mixed-parameters-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -375,23 +239,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -409,9 +257,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''condition'']")}}' depends: core-driver.Succeeded name: core template: comp-core @@ -441,10 +289,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: crust-comp-driver.Succeeded name: crust-comp template: system-container-executor @@ -463,9 +311,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''condition'']")}}' depends: mantle-driver.Succeeded name: mantle template: comp-mantle @@ -490,7 +338,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index 5f85063aec6..5e475f209ac 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -49,6 +49,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -57,80 +58,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-modelcar-model","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -144,25 +80,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -359,10 +277,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.build-model-car-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''build-model-car-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.build-model-car-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''build-model-car-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: build-model-car-driver.Succeeded name: build-model-car template: system-container-executor @@ -384,10 +304,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.get-model-files-list-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''get-model-files-list-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.get-model-files-list-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''get-model-files-list-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: get-model-files-list-driver.Succeeded name: get-model-files-list template: system-container-executor @@ -409,69 +331,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-modelcar-model","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -489,23 +357,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -521,7 +373,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml index af20a1ee82a..1893fe5b103 100644 --- a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,85 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca + - http: + body: '{"ca_cert_path":"/kfp/certs/ca.crt","cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -115,32 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumes: - - configMap: - items: - - key: test-configmap-key - path: ca.crt - name: test-configmap-name - name: custom-ca + outputs: {} - dag: tasks: - arguments: @@ -276,10 +182,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -289,74 +195,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca + - http: + body: '{"ca_cert_path":"/kfp/certs/ca.crt","cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -374,30 +221,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumes: - - configMap: - items: - - key: test-configmap-key - path: ca.crt - name: test-configmap-name - name: custom-ca + outputs: {} - dag: tasks: - arguments: @@ -413,7 +237,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/mounted_cabundle_secret.yaml b/test_data/compiled-workflows/mounted_cabundle_secret.yaml index 191b1a9e0e8..adf155036cb 100644 --- a/test_data/compiled-workflows/mounted_cabundle_secret.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_secret.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,85 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca + - http: + body: '{"ca_cert_path":"/kfp/certs/ca.crt","cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -115,32 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumes: - - name: custom-ca - secret: - items: - - key: ca.crt - path: ca.crt - secretName: test-secret-name + outputs: {} - dag: tasks: - arguments: @@ -276,10 +182,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -289,74 +195,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca + - http: + body: '{"ca_cert_path":"/kfp/certs/ca.crt","cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -374,30 +221,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumes: - - name: custom-ca - secret: - items: - - key: ca.crt - path: ca.crt - secretName: test-secret-name + outputs: {} - dag: tasks: - arguments: @@ -413,7 +237,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index 0f7888c032e..686ccb812a0 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,80 +54,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"multiple-artifacts-namedtuple-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -140,25 +76,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: core-comp-driver.Succeeded name: core-comp template: system-container-executor @@ -299,69 +217,15 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"multiple-artifacts-namedtuple-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -379,23 +243,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -413,9 +261,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''condition'']")}}' depends: core-driver.Succeeded name: core template: comp-core @@ -445,10 +293,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: crust-comp-driver.Succeeded name: crust-comp template: system-container-executor @@ -467,9 +315,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''condition'']")}}' depends: mantle-driver.Succeeded name: mantle template: comp-mantle @@ -494,7 +342,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index 80250a01eb4..7140e114e66 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -44,6 +44,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -52,80 +53,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"multiple-parameters-namedtuple-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -139,25 +75,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -285,10 +203,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: core-comp-driver.Succeeded name: core-comp template: system-container-executor @@ -298,69 +216,15 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"multiple-parameters-namedtuple-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -378,23 +242,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -412,9 +260,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''condition'']")}}' depends: core-driver.Succeeded name: core template: comp-core @@ -444,10 +292,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: crust-comp-driver.Succeeded name: crust-comp template: system-container-executor @@ -466,9 +314,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''condition'']")}}' depends: mantle-driver.Succeeded name: mantle template: comp-mantle @@ -493,7 +341,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/nested_parallel_for_secret.yaml b/test_data/compiled-workflows/nested_parallel_for_secret.yaml deleted file mode 100644 index cafdaafea00..00000000000 --- a/test_data/compiled-workflows/nested_parallel_for_secret.yaml +++ /dev/null @@ -1,533 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: pipeline-4-nested-parallel-for-secret- -spec: - arguments: - parameters: - - name: components-c113b206fedb705aa292f24853d4c2cb9fad687b038e8b2a25cdd920c02b96cd - value: '{"executorLabel":"exec-emit-secret-name","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-c113b206fedb705aa292f24853d4c2cb9fad687b038e8b2a25cdd920c02b96cd - value: '{"args":["--executor_input","{{$}}","--function_to_execute","emit_secret_name"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 - -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 - -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - emit_secret_name() -\u003e str:\n \"\"\"Emits the secret name dynamically.\"\"\"\n return - \"test-secret-1\"\n\n"],"image":"python:3.11"}' - - name: kubernetes-comp-worker-component - value: '{"secretAsEnv":[{"keyToEnv":[{"envVar":"MY_SECRET_KEY","secretKey":"username"}],"optional":false,"secretNameParameter":{"componentInputParameter":"pipelinechannel--emit-secret-name-Output"}}]}' - - name: components-3b4506509514761c798c91195d4caf336df6411cbc2327c68dd7902b1bcad98c - value: '{"executorLabel":"exec-worker-component","inputDefinitions":{"parameters":{"item":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-3b4506509514761c798c91195d4caf336df6411cbc2327c68dd7902b1bcad98c - value: '{"args":["--executor_input","{{$}}","--function_to_execute","worker_component"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 - -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 - -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - worker_component(item: str) -\u003e str:\n import os\n secret_val = - os.environ.get(\"MY_SECRET_KEY\", \"not-set\")\n print(f\"Item: {item}, - Secret set: {secret_val != ''not-set''}\")\n return secret_val\n\n"],"image":"python:3.11"}' - - name: components-comp-for-loop-2 - value: '{"dag":{"tasks":{"worker-component":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-worker-component"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"worker-component"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--emit-secret-name-Output":{"parameterType":"STRING"},"pipelinechannel--loop-item-param-1":{"parameterType":"STRING"}}}}' - - name: components-root - value: '{"dag":{"tasks":{"emit-secret-name":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-emit-secret-name"},"taskInfo":{"name":"emit-secret-name"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["emit-secret-name"],"inputs":{"parameters":{"pipelinechannel--emit-secret-name-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"emit-secret-name"}}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"x\", - \"y\"]"}},"taskInfo":{"name":"for-loop-2"}}}}}' - entrypoint: entrypoint - podMetadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - labels: - pipelines.kubeflow.org/v2_component: "true" - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-4-nested-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 256Mi - requests: - cpu: 100m - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - securityContext: - seccompProfile: - type: RuntimeDefault - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-3b4506509514761c798c91195d4caf336df6411cbc2327c68dd7902b1bcad98c}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-worker-component"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"worker-component"}}' - - name: container - value: '{{workflow.parameters.implementations-3b4506509514761c798c91195d4caf336df6411cbc2327c68dd7902b1bcad98c}}' - - name: task-name - value: worker-component - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-worker-component}}' - name: worker-component-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.worker-component-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.worker-component-driver.outputs.parameters.cached-decision}}' - depends: worker-component-driver.Succeeded - name: worker-component - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2 - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-4-nested-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["emit-secret-name"],"inputs":{"parameters":{"pipelinechannel--emit-secret-name-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"emit-secret-name"}}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"x\", - \"y\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-2 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-2-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["emit-secret-name"],"inputs":{"parameters":{"pipelinechannel--emit-secret-name-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"emit-secret-name"}}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"x\", - \"y\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-2-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2-for-loop-2-iterator - outputs: {} - parallelism: 1 - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c113b206fedb705aa292f24853d4c2cb9fad687b038e8b2a25cdd920c02b96cd}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-emit-secret-name"},"taskInfo":{"name":"emit-secret-name"}}' - - name: container - value: '{{workflow.parameters.implementations-c113b206fedb705aa292f24853d4c2cb9fad687b038e8b2a25cdd920c02b96cd}}' - - name: task-name - value: emit-secret-name - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: emit-secret-name-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.emit-secret-name-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.emit-secret-name-driver.outputs.parameters.cached-decision}}' - depends: emit-secret-name-driver.Succeeded - name: emit-secret-name - template: system-container-executor - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - depends: emit-secret-name.Succeeded - name: for-loop-2 - template: comp-for-loop-2-for-loop-2-iterator - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} -status: - finishedAt: null - startedAt: null diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index ea49f9f4ad1..3b95502cb63 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -104,6 +104,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -112,80 +113,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-pipeline-opt-input-child-level","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -199,25 +135,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -345,10 +263,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-a-bool-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-a-bool-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-a-bool-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-a-bool-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-a-bool-driver.Succeeded name: component-a-bool template: system-container-executor @@ -369,10 +289,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-a-int-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-a-int-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-a-int-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-a-int-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-a-int-driver.Succeeded name: component-a-int template: system-container-executor @@ -393,10 +315,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-a-str-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-a-str-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-a-str-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-a-str-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-a-str-driver.Succeeded name: component-a-str template: system-container-executor @@ -417,10 +341,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-b-bool-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-b-bool-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-b-bool-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-b-bool-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-b-bool-driver.Succeeded name: component-b-bool template: system-container-executor @@ -441,10 +367,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-b-int-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-b-int-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-b-int-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-b-int-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-b-int-driver.Succeeded name: component-b-int template: system-container-executor @@ -465,10 +393,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-b-str-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-b-str-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-b-str-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-b-str-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-b-str-driver.Succeeded name: component-b-str template: system-container-executor @@ -478,69 +408,15 @@ spec: metadata: {} name: comp-nested-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-pipeline-opt-input-child-level","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -558,23 +434,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -593,9 +453,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''nested-pipeline-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''nested-pipeline-driver''].outputs.result, + "$[''condition'']")}}' depends: nested-pipeline-driver.Succeeded name: nested-pipeline template: comp-nested-pipeline @@ -620,7 +482,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index 0990d07101f..a5a306fd251 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -55,6 +55,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -63,80 +64,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-pipeline-opt-inputs-nil","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -150,25 +86,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -296,10 +214,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-bool-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-bool-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-bool-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-bool-driver''].outputs.result, "$[''cached-decision'']")}}' depends: component-bool-driver.Succeeded name: component-bool template: system-container-executor @@ -320,10 +238,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-int-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-int-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-int-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-int-driver''].outputs.result, "$[''cached-decision'']")}}' depends: component-int-driver.Succeeded name: component-int template: system-container-executor @@ -344,10 +262,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-str-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-str-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-str-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-str-driver''].outputs.result, "$[''cached-decision'']")}}' depends: component-str-driver.Succeeded name: component-str template: system-container-executor @@ -357,69 +275,15 @@ spec: metadata: {} name: comp-nested-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-pipeline-opt-inputs-nil","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -437,23 +301,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -471,9 +319,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''nested-pipeline-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''nested-pipeline-driver''].outputs.result, + "$[''condition'']")}}' depends: nested-pipeline-driver.Succeeded name: nested-pipeline template: comp-nested-pipeline @@ -498,7 +348,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index ba852795df5..005c99b7bb6 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -107,6 +107,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -115,80 +116,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-pipeline-opt-inputs-parent-level","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -202,25 +138,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -348,10 +266,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-nil-bool-default-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-nil-bool-default-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-nil-bool-default-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-nil-bool-default-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-nil-bool-default-driver.Succeeded name: component-nil-bool-default template: system-container-executor @@ -372,10 +292,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-nil-int-default-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-nil-int-default-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-nil-int-default-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-nil-int-default-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-nil-int-default-driver.Succeeded name: component-nil-int-default template: system-container-executor @@ -396,10 +318,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-nil-str-default-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-nil-str-default-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-nil-str-default-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-nil-str-default-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-nil-str-default-driver.Succeeded name: component-nil-str-default template: system-container-executor @@ -428,10 +352,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-bool-default-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-bool-default-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-bool-default-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-bool-default-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-bool-default-driver.Succeeded name: component-bool-default template: system-container-executor @@ -452,10 +378,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-int-default-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-int-default-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-int-default-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-int-default-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-int-default-driver.Succeeded name: component-int-default template: system-container-executor @@ -476,10 +404,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-str-default-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-str-default-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-str-default-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-str-default-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-str-default-driver.Succeeded name: component-str-default template: system-container-executor @@ -489,69 +419,15 @@ spec: metadata: {} name: comp-nested-pipeline-non-nil-defaults outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-pipeline-opt-inputs-parent-level","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -569,23 +445,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -603,9 +463,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.nested-pipeline-nil-defaults-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''nested-pipeline-nil-defaults-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.nested-pipeline-nil-defaults-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''nested-pipeline-nil-defaults-driver''].outputs.result, + "$[''condition'']")}}' depends: nested-pipeline-nil-defaults-driver.Succeeded name: nested-pipeline-nil-defaults template: comp-nested-pipeline-nil-defaults @@ -624,9 +486,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.nested-pipeline-non-nil-defaults-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''nested-pipeline-non-nil-defaults-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.nested-pipeline-non-nil-defaults-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''nested-pipeline-non-nil-defaults-driver''].outputs.result, + "$[''condition'']")}}' depends: nested-pipeline-non-nil-defaults-driver.Succeeded name: nested-pipeline-non-nil-defaults template: comp-nested-pipeline-non-nil-defaults @@ -652,7 +516,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index afefd925131..2a5bb5fa6d6 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -25,6 +25,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -33,80 +34,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-return","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -120,25 +56,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -266,10 +184,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.nested-return-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''nested-return-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.nested-return-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''nested-return-driver''].outputs.result, "$[''cached-decision'']")}}' depends: nested-return-driver.Succeeded name: nested-return template: system-container-executor @@ -279,69 +197,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-return","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -359,23 +223,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -391,7 +239,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index 033f1a45129..91228ab6a4f 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -57,6 +57,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -65,80 +66,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -152,25 +88,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -299,10 +217,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-two-nums-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-two-nums-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-two-nums-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-two-nums-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-two-nums-driver.Succeeded name: add-two-nums template: system-container-executor @@ -323,10 +241,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-driver.Succeeded name: double template: system-container-executor @@ -347,10 +265,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-2-driver.Succeeded name: double-2 template: system-container-executor @@ -360,69 +278,15 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -440,23 +304,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -477,9 +325,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -508,14 +356,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -556,9 +404,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -587,14 +435,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -621,10 +469,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-driver.Succeeded name: add template: system-container-executor @@ -655,7 +503,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/notebook_component_mixed.yaml b/test_data/compiled-workflows/notebook_component_mixed.yaml index f20b92ce2ec..e6ce3e211b4 100644 --- a/test_data/compiled-workflows/notebook_component_mixed.yaml +++ b/test_data/compiled-workflows/notebook_component_mixed.yaml @@ -253,6 +253,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -261,80 +262,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-mixed - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nb-mixed","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -348,25 +284,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -495,10 +413,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.evaluate-model-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''evaluate-model-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.evaluate-model-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''evaluate-model-driver''].outputs.result, "$[''cached-decision'']")}}' depends: evaluate-model-driver.Succeeded name: evaluate-model template: system-container-executor @@ -519,10 +437,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.preprocess-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''preprocess-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.preprocess-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''preprocess-driver''].outputs.result, "$[''cached-decision'']")}}' depends: preprocess-driver.Succeeded name: preprocess template: system-container-executor @@ -544,10 +462,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.train-model-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''train-model-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.train-model-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''train-model-driver''].outputs.result, "$[''cached-decision'']")}}' depends: train-model-driver.Succeeded name: train-model template: system-container-executor @@ -557,69 +475,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-mixed - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nb-mixed","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -637,23 +501,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -669,7 +517,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/notebook_component_simple.yaml b/test_data/compiled-workflows/notebook_component_simple.yaml index 8635ad128f7..6483e83f420 100644 --- a/test_data/compiled-workflows/notebook_component_simple.yaml +++ b/test_data/compiled-workflows/notebook_component_simple.yaml @@ -122,6 +122,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -130,80 +131,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nb-simple","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -217,25 +153,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -363,10 +281,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.run-train-notebook-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''run-train-notebook-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.run-train-notebook-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''run-train-notebook-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: run-train-notebook-driver.Succeeded name: run-train-notebook template: system-container-executor @@ -376,69 +296,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nb-simple","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -456,23 +322,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -488,7 +338,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index a62a2af6f91..e83fa15a196 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,80 +35,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"output-metrics","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -121,25 +57,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -267,10 +185,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.output-metrics-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''output-metrics-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.output-metrics-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''output-metrics-driver''].outputs.result, "$[''cached-decision'']")}}' depends: output-metrics-driver.Succeeded name: output-metrics template: system-container-executor @@ -280,69 +198,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"output-metrics","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -360,23 +224,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -392,7 +240,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index 5776ce6f02c..03dd9db124e 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -27,6 +27,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -35,80 +36,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"loop-with-after-dependency-set","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -122,25 +58,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -268,10 +186,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -281,69 +199,15 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"loop-with-after-dependency-set","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -361,23 +225,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -398,9 +246,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -429,14 +277,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -469,10 +317,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -494,10 +342,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-3-driver.Succeeded name: print-op-3 template: system-container-executor @@ -522,7 +370,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/parallel_for_mount_pvc.yaml b/test_data/compiled-workflows/parallel_for_mount_pvc.yaml deleted file mode 100644 index 811aa3b9dab..00000000000 --- a/test_data/compiled-workflows/parallel_for_mount_pvc.yaml +++ /dev/null @@ -1,526 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: pipeline-parallel-for-mount-pvc- -spec: - arguments: - parameters: - - name: components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 - value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"description":"AccessModes - to request for the provisioned PVC. May\nbe one or more of ``''ReadWriteOnce''``, - ``''ReadOnlyMany''``, ``''ReadWriteMany''``, or\n``''ReadWriteOncePod''``. - Corresponds to `PersistentVolumeClaim.spec.accessModes \u003chttps://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes\u003e`_.","parameterType":"LIST"},"annotations":{"description":"Annotations - for the PVC''s metadata. Corresponds to `PersistentVolumeClaim.metadata.annotations - \u003chttps://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/persistent-volume-claim-v1/#PersistentVolumeClaim\u003e`_.","isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"description":"Name - of the PVC. Corresponds to `PersistentVolumeClaim.metadata.name \u003chttps://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/persistent-volume-claim-v1/#PersistentVolumeClaim\u003e`_. - Only one of ``pvc_name`` and ``pvc_name_suffix`` can\nbe provided.","isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"description":"Prefix - to use for a dynamically generated name, which\nwill take the form ``\u003cargo-workflow-name\u003e-\u003cpvc_name_suffix\u003e``. - Only one\nof ``pvc_name`` and ``pvc_name_suffix`` can be provided.","isOptional":true,"parameterType":"STRING"},"size":{"description":"The - size of storage requested by the PVC that will be provisioned. For example, - ``''5Gi''``. Corresponds to `PersistentVolumeClaim.spec.resources.requests.storage - \u003chttps://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/persistent-volume-claim-v1/#PersistentVolumeClaimSpec\u003e`_.","parameterType":"STRING"},"storage_class_name":{"defaultValue":"","description":"Name - of StorageClass from which to provision the PV\nto back the PVC. ``None`` - indicates to use the cluster''s default\nstorage_class_name. Set to ``''''`` - for a statically specified PVC.","isOptional":true,"parameterType":"STRING"},"volume_name":{"description":"Pre-existing - PersistentVolume that should back the\nprovisioned PersistentVolumeClaim. - Used for statically\nspecified PV only. Corresponds to `PersistentVolumeClaim.spec.volumeName - \u003chttps://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/persistent-volume-claim-v1/#PersistentVolumeClaimSpec\u003e`_.","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}' - - name: implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 - value: '{"image":"argostub/createpvc"}' - - name: kubernetes-comp-example-component - value: '{"pvcMount":[{"componentInputParameter":"pipelinechannel--createpvc-name","mountPath":"/data","pvcNameParameter":{"componentInputParameter":"pipelinechannel--createpvc-name"}}]}' - - name: components-1e6d3fc350583cb591281c08420927b09565ea68a8ea52650e820b1ed008f913 - value: '{"executorLabel":"exec-example-component","inputDefinitions":{"parameters":{"item":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-1e6d3fc350583cb591281c08420927b09565ea68a8ea52650e820b1ed008f913 - value: '{"args":["--executor_input","{{$}}","--function_to_execute","example_component"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 - -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 - -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - example_component(item: str) -\u003e str:\n print(f\"Item: {item}\")\n return - item\n\n"],"image":"python:3.11"}' - - name: components-comp-for-loop-2 - value: '{"dag":{"tasks":{"example-component":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-example-component"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"example-component"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--createpvc-name":{"parameterType":"STRING"},"pipelinechannel--loop-item-param-1":{"parameterType":"STRING"}}}}' - - name: components-root - value: '{"dag":{"tasks":{"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Mi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["createpvc"],"inputs":{"parameters":{"pipelinechannel--createpvc-name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"a\"]"}},"taskInfo":{"name":"for-loop-2"}}}},"inputDefinitions":{"parameters":{"my_pvc_name_suffix":{"defaultValue":"test-pvc","isOptional":true,"parameterType":"STRING"}}}}' - entrypoint: entrypoint - podMetadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - labels: - pipelines.kubeflow.org/v2_component: "true" - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-parallel-for-mount-pvc - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 256Mi - requests: - cpu: 100m - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - securityContext: - seccompProfile: - type: RuntimeDefault - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-1e6d3fc350583cb591281c08420927b09565ea68a8ea52650e820b1ed008f913}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-example-component"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"example-component"}}' - - name: container - value: '{{workflow.parameters.implementations-1e6d3fc350583cb591281c08420927b09565ea68a8ea52650e820b1ed008f913}}' - - name: task-name - value: example-component - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-example-component}}' - name: example-component-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.example-component-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.example-component-driver.outputs.parameters.cached-decision}}' - depends: example-component-driver.Succeeded - name: example-component - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2 - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-parallel-for-mount-pvc - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["createpvc"],"inputs":{"parameters":{"pipelinechannel--createpvc-name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"a\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-2 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-2-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["createpvc"],"inputs":{"parameters":{"pipelinechannel--createpvc-name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"a\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-2-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2-for-loop-2-iterator - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Mi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}' - - name: container - value: '{{workflow.parameters.implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - - name: task-name - value: createpvc - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: createpvc - template: system-container-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - depends: createpvc.Succeeded - name: for-loop-2 - template: comp-for-loop-2-for-loop-2-iterator - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"my_pvc_name_suffix":"test-pvc"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} -status: - finishedAt: null - startedAt: null diff --git a/test_data/compiled-workflows/parallel_for_secret.yaml b/test_data/compiled-workflows/parallel_for_secret.yaml deleted file mode 100644 index 95c5ce528e6..00000000000 --- a/test_data/compiled-workflows/parallel_for_secret.yaml +++ /dev/null @@ -1,494 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: pipeline-2-parallel-for-secret- -spec: - arguments: - parameters: - - name: kubernetes-comp-example-component - value: '{"secretAsEnv":[{"keyToEnv":[{"envVar":"MY_SECRET_KEY","secretKey":"username"}],"optional":false,"secretNameParameter":{"componentInputParameter":"pipelinechannel--my_secret_name"}}]}' - - name: components-35371eb3fea5c3873b2f3559dbdf0f49023f73012ebba08dfbaf2d08f55a8c62 - value: '{"executorLabel":"exec-example-component","inputDefinitions":{"parameters":{"item":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-35371eb3fea5c3873b2f3559dbdf0f49023f73012ebba08dfbaf2d08f55a8c62 - value: '{"args":["--executor_input","{{$}}","--function_to_execute","example_component"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 - -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 - -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - example_component(item: str) -\u003e str:\n import os\n secret_val = - os.environ.get(\"MY_SECRET_KEY\", \"not-set\")\n print(f\"Item: {item}, - Secret set: {secret_val != ''not-set''}\")\n return secret_val\n\n"],"image":"python:3.11"}' - - name: components-comp-for-loop-2 - value: '{"dag":{"tasks":{"example-component":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-example-component"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"example-component"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"STRING"},"pipelinechannel--my_secret_name":{"parameterType":"STRING"}}}}' - - name: components-root - value: '{"dag":{"tasks":{"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--my_secret_name":{"componentInputParameter":"my_secret_name"}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"a\", - \"b\", \"c\"]"}},"taskInfo":{"name":"for-loop-2"}}}},"inputDefinitions":{"parameters":{"my_secret_name":{"defaultValue":"test-secret-1","isOptional":true,"parameterType":"STRING"}}}}' - entrypoint: entrypoint - podMetadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - labels: - pipelines.kubeflow.org/v2_component: "true" - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-2-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 256Mi - requests: - cpu: 100m - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - securityContext: - seccompProfile: - type: RuntimeDefault - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-35371eb3fea5c3873b2f3559dbdf0f49023f73012ebba08dfbaf2d08f55a8c62}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-example-component"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"example-component"}}' - - name: container - value: '{{workflow.parameters.implementations-35371eb3fea5c3873b2f3559dbdf0f49023f73012ebba08dfbaf2d08f55a8c62}}' - - name: task-name - value: example-component - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-example-component}}' - name: example-component-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.example-component-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.example-component-driver.outputs.parameters.cached-decision}}' - depends: example-component-driver.Succeeded - name: example-component - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2 - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-2-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--my_secret_name":{"componentInputParameter":"my_secret_name"}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"a\", - \"b\", \"c\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-2 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-2-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--my_secret_name":{"componentInputParameter":"my_secret_name"}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"a\", - \"b\", \"c\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-2-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2-for-loop-2-iterator - outputs: {} - parallelism: 1 - - dag: - tasks: - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: for-loop-2 - template: comp-for-loop-2-for-loop-2-iterator - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"my_secret_name":"test-secret-1"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} -status: - finishedAt: null - startedAt: null diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index 0d874d1e1ab..c782d649cc3 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -41,6 +41,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -49,80 +50,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"parameter-cache-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -136,25 +72,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -282,10 +200,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: core-comp-driver.Succeeded name: core-comp template: system-container-executor @@ -295,69 +213,15 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"parameter-cache-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -375,23 +239,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -409,9 +257,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''condition'']")}}' depends: core-driver.Succeeded name: core template: comp-core @@ -441,10 +289,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: crust-comp-driver.Succeeded name: crust-comp template: system-container-executor @@ -463,9 +311,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''condition'']")}}' depends: mantle-driver.Succeeded name: mantle template: comp-mantle @@ -490,7 +338,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index df9c5ac6f90..cb08b5bd18a 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -80,6 +80,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -88,80 +89,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"parameter-oneof-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -175,25 +111,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -322,10 +240,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: core-comp-driver.Succeeded name: core-comp template: system-container-executor @@ -355,10 +273,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.core-output-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''core-output-comp-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.core-output-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''core-output-comp-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: core-output-comp-driver.Succeeded name: core-output-comp template: system-container-executor @@ -368,69 +288,15 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"parameter-oneof-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -448,23 +314,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -483,13 +333,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -506,13 +357,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -537,9 +389,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-branches-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-branches-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-branches-1-driver''].outputs.result, + "$[''condition'']")}}' depends: condition-branches-1-driver.Succeeded name: condition-branches-1 template: comp-condition-branches-1 @@ -560,10 +414,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-driver.Succeeded name: flip-coin template: system-container-executor @@ -590,9 +444,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''core-driver''].outputs.result, "$[''condition'']")}}' depends: core-driver.Succeeded name: core template: comp-core @@ -622,10 +476,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''crust-comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: crust-comp-driver.Succeeded name: crust-comp template: system-container-executor @@ -644,9 +498,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''mantle-driver''].outputs.result, "$[''condition'']")}}' depends: mantle-driver.Succeeded name: mantle template: comp-mantle @@ -671,7 +525,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index ba547b6933e..977fa0b0294 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -74,6 +74,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -82,80 +83,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -169,25 +105,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -315,10 +233,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-2-driver.Succeeded name: double-2 template: system-container-executor @@ -328,69 +246,15 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -408,23 +272,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -445,9 +293,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -476,14 +324,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -509,10 +357,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-driver.Succeeded name: double template: system-container-executor @@ -540,10 +388,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.simple-add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''simple-add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.simple-add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''simple-add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: simple-add-driver.Succeeded name: simple-add template: system-container-executor @@ -572,10 +420,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.nested-add-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''nested-add-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.nested-add-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''nested-add-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: nested-add-2-driver.Succeeded name: nested-add-2 template: system-container-executor @@ -596,10 +444,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.simple-add-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''simple-add-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.simple-add-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''simple-add-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: simple-add-2-driver.Succeeded name: simple-add-2 template: system-container-executor @@ -629,9 +477,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -660,14 +508,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -694,9 +542,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-6 @@ -725,14 +573,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-6-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -759,10 +607,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-two-numbers-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-two-numbers-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-two-numbers-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-two-numbers-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: add-two-numbers-driver.Succeeded name: add-two-numbers template: system-container-executor @@ -797,10 +647,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.nested-add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''nested-add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.nested-add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''nested-add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: nested-add-driver.Succeeded name: nested-add template: system-container-executor @@ -825,7 +675,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index 5598d7ca741..898fec6c9b3 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,80 +54,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -140,25 +76,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-driver.Succeeded name: double template: system-container-executor @@ -299,69 +217,15 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -379,23 +243,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -416,9 +264,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -447,14 +295,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -481,10 +329,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-driver.Succeeded name: add template: system-container-executor @@ -506,10 +354,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-container-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-container-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-container-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-container-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-container-driver.Succeeded name: add-container template: system-container-executor @@ -540,7 +388,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index c54aa3afcf3..f5cee191074 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -61,6 +61,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -69,80 +70,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-task-final-status-conditional","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -156,25 +92,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -303,10 +221,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -316,69 +234,15 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-task-final-status-conditional","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -396,23 +260,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -432,13 +280,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -456,10 +305,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.get-run-state-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''get-run-state-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.get-run-state-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''get-run-state-driver''].outputs.result, "$[''cached-decision'']")}}' depends: get-run-state-driver.Succeeded name: get-run-state template: system-container-executor @@ -489,10 +338,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: fail-op-driver.Succeeded name: fail-op template: system-container-executor @@ -513,10 +362,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -543,9 +392,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.conditional-notification-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''conditional-notification-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.conditional-notification-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''conditional-notification-driver''].outputs.result, + "$[''condition'']")}}' depends: conditional-notification-driver.Succeeded name: conditional-notification template: comp-conditional-notification @@ -572,9 +423,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-1-driver.Succeeded hooks: exit: @@ -606,7 +457,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index 2042a6496cb..521c42f1fad 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -30,6 +30,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -38,80 +39,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -125,25 +61,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -271,10 +189,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -296,10 +214,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op2-driver.Succeeded name: print-op2 template: system-container-executor @@ -309,69 +227,15 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -389,23 +253,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -423,9 +271,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.inner-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.inner-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''condition'']")}}' depends: inner-pipeline-driver.Succeeded name: inner-pipeline template: comp-inner-pipeline @@ -446,10 +294,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -474,7 +322,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index 81798ae1c4c..e046fb8ea7e 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -39,6 +39,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -47,80 +48,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline-complex","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -134,25 +70,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -280,10 +198,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op2-driver.Succeeded name: print-op2 template: system-container-executor @@ -312,10 +230,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -325,69 +243,15 @@ spec: metadata: {} name: comp-condition-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline-complex","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -405,23 +269,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -441,13 +289,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -465,13 +314,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -489,10 +339,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -519,9 +369,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.inner-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.inner-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''condition'']")}}' depends: inner-pipeline-driver.Succeeded name: inner-pipeline template: comp-inner-pipeline @@ -551,9 +401,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -582,14 +432,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -621,10 +471,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -649,7 +499,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index 50a866f8c90..4b5837da1cb 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,80 +55,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -141,25 +77,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -287,10 +205,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -312,10 +230,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op2-driver.Succeeded name: print-op2 template: system-container-executor @@ -325,69 +243,15 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -405,23 +269,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -439,9 +287,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.inner-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.inner-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''condition'']")}}' depends: inner-pipeline-driver.Succeeded name: inner-pipeline template: comp-inner-pipeline @@ -462,10 +310,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -492,9 +340,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.pipeline-in-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''pipeline-in-pipeline-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.pipeline-in-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''pipeline-in-pipeline-driver''].outputs.result, + "$[''condition'']")}}' depends: pipeline-in-pipeline-driver.Succeeded name: pipeline-in-pipeline template: comp-pipeline-in-pipeline @@ -516,10 +366,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -544,7 +394,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index d8e486cc89b..9b6d4de2bd0 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -77,6 +77,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -85,80 +86,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -172,25 +108,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -318,10 +236,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-and-return-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-and-return-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-and-return-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-and-return-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: echo-and-return-driver.Succeeded name: echo-and-return template: system-container-executor @@ -331,69 +251,15 @@ spec: metadata: {} name: comp-for-loop-2-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"math-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -411,23 +277,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -448,9 +298,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2-2 @@ -479,14 +329,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -512,10 +362,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-driver.Succeeded name: add template: system-container-executor @@ -550,10 +400,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.double-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.double-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''double-driver''].outputs.result, "$[''cached-decision'']")}}' depends: double-driver.Succeeded name: double template: system-container-executor @@ -580,9 +430,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.double-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''double-pipeline-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.double-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''double-pipeline-driver''].outputs.result, + "$[''condition'']")}}' depends: double-pipeline-driver.Succeeded name: double-pipeline template: comp-double-pipeline @@ -612,9 +464,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -643,14 +495,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -691,9 +543,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -722,14 +574,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -754,9 +606,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.add-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''add-pipeline-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.add-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''add-pipeline-driver''].outputs.result, "$[''condition'']")}}' depends: add-pipeline-driver.Succeeded name: add-pipeline template: comp-add-pipeline @@ -784,10 +636,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.join-and-print-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''join-and-print-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.join-and-print-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''join-and-print-driver''].outputs.result, "$[''cached-decision'']")}}' depends: join-and-print-driver.Succeeded name: join-and-print template: system-container-executor @@ -812,7 +664,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_after.yaml b/test_data/compiled-workflows/pipeline_with_after.yaml index 3912ec5fcbb..def9da1be8e 100644 --- a/test_data/compiled-workflows/pipeline_with_after.yaml +++ b/test_data/compiled-workflows/pipeline_with_after.yaml @@ -18,6 +18,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -26,80 +27,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-after","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -113,25 +49,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -260,10 +178,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-driver.Succeeded name: print-text template: system-container-executor @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-2-driver.Succeeded name: print-text-2 template: system-container-executor @@ -312,10 +230,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-3-driver.Succeeded name: print-text-3 template: system-container-executor @@ -325,69 +243,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-after","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -405,23 +269,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -437,7 +285,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml index 98939fb15d0..e35a5380455 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,80 +55,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-custom-path-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-custom-path-artifact","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -141,25 +77,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -287,10 +205,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-output-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-output-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-output-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-output-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-output-artifact-driver.Succeeded name: component-output-artifact template: system-container-executor @@ -312,10 +232,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-input-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-input-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-input-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-input-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: validate-input-artifact-driver.Succeeded name: validate-input-artifact template: system-container-executor @@ -325,69 +247,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-custom-path-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-custom-path-artifact","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -405,23 +273,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -437,7 +289,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index 3682568eb4c..e810bf5883d 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,80 +54,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-datasets","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -140,25 +76,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -286,10 +204,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.download-dataset-and-upload-as-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''download-dataset-and-upload-as-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.download-dataset-and-upload-as-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''download-dataset-and-upload-as-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: download-dataset-and-upload-as-artifact-driver.Succeeded name: download-dataset-and-upload-as-artifact template: system-container-executor @@ -311,10 +231,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-dataset-info-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-dataset-info-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-dataset-info-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-dataset-info-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-dataset-info-driver.Succeeded name: print-dataset-info template: system-container-executor @@ -324,69 +246,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-datasets","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -404,23 +272,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -436,7 +288,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml index 17fa857645b..3e2473d69d2 100644 --- a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml @@ -17,6 +17,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -25,80 +26,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"one-step-pipeline-with-concat-placeholder","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -112,25 +48,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -258,10 +176,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-with-concat-placeholder-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-with-concat-placeholder-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-with-concat-placeholder-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-with-concat-placeholder-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-with-concat-placeholder-driver.Succeeded name: component-with-concat-placeholder template: system-container-executor @@ -271,69 +191,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"one-step-pipeline-with-concat-placeholder","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -351,23 +217,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -383,7 +233,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index b4bf744556d..e3da7dbd4cf 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,80 +52,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"single-condition-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -138,25 +74,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -284,10 +202,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-op-2-driver.Succeeded name: flip-coin-op-2 template: system-container-executor @@ -309,10 +227,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -333,10 +251,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-3-driver.Succeeded name: print-op-3 template: system-container-executor @@ -346,69 +264,15 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"single-condition-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -426,23 +290,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -462,13 +310,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -486,10 +335,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-op-driver.Succeeded name: flip-coin-op template: system-container-executor @@ -511,10 +360,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -539,7 +388,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index f8f8556a811..52aa562dff4 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -140,6 +140,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -148,80 +149,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-dynamic-condition-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-dynamic-condition-output","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -235,25 +171,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,10 +299,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.custom-training-job-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''custom-training-job-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.custom-training-job-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''custom-training-job-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: custom-training-job-driver.Succeeded name: custom-training-job template: system-container-executor @@ -394,69 +314,15 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-dynamic-condition-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-dynamic-condition-output","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -474,23 +340,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -510,10 +360,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.accelerator-count-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''accelerator-count-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.accelerator-count-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''accelerator-count-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: accelerator-count-driver.Succeeded name: accelerator-count template: system-container-executor @@ -534,10 +386,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.accelerator-type-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''accelerator-type-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.accelerator-type-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''accelerator-type-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: accelerator-type-driver.Succeeded name: accelerator-type template: system-container-executor @@ -559,13 +413,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -583,10 +438,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-biased-coin-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-biased-coin-op-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-biased-coin-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-biased-coin-op-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: flip-biased-coin-op-driver.Succeeded name: flip-biased-coin-op template: system-container-executor @@ -607,10 +464,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.machine-type-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''machine-type-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.machine-type-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''machine-type-driver''].outputs.result, "$[''cached-decision'']")}}' depends: machine-type-driver.Succeeded name: machine-type template: system-container-executor @@ -635,7 +492,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index d561b789414..6163631112b 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -32,6 +32,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -109,80 +110,15 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -196,25 +132,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -366,10 +284,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.make-name-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''make-name-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.make-name-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''make-name-driver''].outputs.result, "$[''cached-decision'']")}}' depends: make-name-driver.Succeeded name: make-name template: system-container-executor @@ -379,69 +297,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -459,23 +323,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -491,7 +339,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index cf68933f943..372df73d261 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -102,6 +102,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -110,80 +111,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -197,25 +133,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -343,10 +261,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.accelerator-count-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''accelerator-count-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.accelerator-count-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''accelerator-count-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: accelerator-count-driver.Succeeded name: accelerator-count template: system-container-executor @@ -367,10 +287,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.accelerator-type-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''accelerator-type-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.accelerator-type-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''accelerator-type-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: accelerator-type-driver.Succeeded name: accelerator-type template: system-container-executor @@ -392,10 +314,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.custom-training-job-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''custom-training-job-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.custom-training-job-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''custom-training-job-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: custom-training-job-driver.Succeeded name: custom-training-job template: system-container-executor @@ -416,10 +340,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.machine-type-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''machine-type-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.machine-type-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''machine-type-driver''].outputs.result, "$[''cached-decision'']")}}' depends: machine-type-driver.Succeeded name: machine-type template: system-container-executor @@ -429,69 +353,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -509,23 +379,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -541,7 +395,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index fde5ffa35d5..934f7111769 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -30,6 +30,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -38,80 +39,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-env","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -125,25 +61,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -271,10 +189,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-env-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-env-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-env-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-env-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-env-driver.Succeeded name: print-env template: system-container-executor @@ -295,10 +213,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-env-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-env-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-env-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-env-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-env-op-driver.Succeeded name: print-env-op template: system-container-executor @@ -308,69 +226,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-env","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -388,23 +252,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -420,7 +268,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index f7cbdb42a3f..6f9195aac86 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -42,6 +42,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -50,80 +51,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-exit-handler","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -137,25 +73,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -284,10 +202,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: fail-op-driver.Succeeded name: fail-op template: system-container-executor @@ -308,10 +226,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -341,10 +259,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -354,69 +272,15 @@ spec: metadata: {} name: exit-hook-root-print-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-exit-handler","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -434,23 +298,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -468,9 +316,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-1-driver.Succeeded hooks: exit: @@ -502,7 +350,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index aef085eeaf5..ea2f532b4e4 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -50,6 +50,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -127,80 +128,15 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-google-types","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -214,25 +150,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -373,10 +291,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.model-consumer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''model-consumer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.model-consumer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''model-consumer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: model-consumer-driver.Succeeded name: model-consumer template: system-container-executor @@ -397,10 +315,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.model-producer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''model-producer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.model-producer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''model-producer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: model-producer-driver.Succeeded name: model-producer template: system-container-executor @@ -410,69 +328,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-google-types","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -490,23 +354,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -522,7 +370,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index 2c1634e9548..dbc344fc6cd 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -117,80 +118,15 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -204,25 +140,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -363,10 +281,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.train-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''train-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.train-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''train-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: train-2-driver.Succeeded name: train-2 template: system-container-executor @@ -376,69 +294,15 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -456,23 +320,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -492,13 +340,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: task @@ -529,10 +378,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.train-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''train-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.train-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''train-driver''].outputs.result, "$[''cached-decision'']")}}' depends: train-driver.Succeeded name: train template: system-container-executor @@ -557,7 +406,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml index f230897198d..230d44eb9cc 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml @@ -19,6 +19,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -27,80 +28,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer-and-gcpc-type","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -114,25 +50,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -330,10 +248,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consumer-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consumer-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consumer-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consumer-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: consumer-op-driver.Succeeded name: consumer-op template: system-container-executor @@ -355,69 +273,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer-and-gcpc-type","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -435,23 +299,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -467,7 +315,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml index 81dd2fa7c49..3ec9deb53b7 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml @@ -107,6 +107,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -191,80 +192,15 @@ spec: - name: kfp-workspace persistentVolumeClaim: claimName: '{{workflow.name}}-kfp-workspace' - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer-workspace","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -278,25 +214,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -449,10 +367,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-dir-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-dir-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-dir-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-dir-driver''].outputs.result, "$[''cached-decision'']")}}' depends: read-dir-driver.Succeeded name: read-dir template: system-container-executor @@ -474,10 +392,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.train-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''train-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.train-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''train-driver''].outputs.result, "$[''cached-decision'']")}}' depends: train-driver.Succeeded name: train template: system-container-executor @@ -487,69 +405,15 @@ spec: metadata: {} name: comp-import-stage outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-importer-workspace","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -567,23 +431,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -604,10 +452,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.get-uri-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''get-uri-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.get-uri-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''get-uri-driver''].outputs.result, "$[''cached-decision'']")}}' depends: get-uri-driver.Succeeded name: get-uri template: system-container-executor @@ -629,10 +477,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.get-uri-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''get-uri-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.get-uri-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''get-uri-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: get-uri-2-driver.Succeeded name: get-uri-2 template: system-container-executor @@ -652,9 +500,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.import-stage-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''import-stage-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.import-stage-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''import-stage-driver''].outputs.result, "$[''condition'']")}}' depends: import-stage-driver.Succeeded name: import-stage template: comp-import-stage @@ -675,10 +523,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.write-dir-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''write-dir-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.write-dir-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''write-dir-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: write-dir-artifact-driver.Succeeded name: write-dir-artifact template: system-container-executor @@ -699,10 +549,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.write-file-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''write-file-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.write-file-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''write-file-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: write-file-artifact-driver.Succeeded name: write-file-artifact template: system-container-executor @@ -727,7 +579,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index 1143ae8d57b..01cba1eb3a9 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -42,6 +42,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -50,80 +51,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"status-state-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -137,25 +73,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -283,10 +201,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.some-task-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''some-task-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.some-task-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''some-task-driver''].outputs.result, "$[''cached-decision'']")}}' depends: some-task-driver.Succeeded name: some-task template: system-container-executor @@ -315,10 +233,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-state-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-state-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-state-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-state-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-state-driver.Succeeded name: echo-state template: system-container-executor @@ -328,69 +246,15 @@ spec: metadata: {} name: exit-hook-root-echo-state outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"status-state-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -408,23 +272,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -442,9 +290,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-1-driver.Succeeded hooks: exit: @@ -476,7 +324,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index da73914e125..1f79d451503 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -5,13 +5,13 @@ metadata: spec: arguments: parameters: - - name: components-fa52219045a80d72f6f8f40492570e94942c3f25927d7cbc4e112c1ac54e4e04 + - name: components-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530 value: '{"executorLabel":"exec-args-generator-op","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-fa52219045a80d72f6f8f40492570e94942c3f25927d7cbc4e112c1ac54e4e04 + - name: implementations-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530 value: '{"args":["--executor_input","{{$}}","--function_to_execute","args_generator_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps'' ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 @@ -19,13 +19,13 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -\u003e List[Dict[str, str]]:\n return [{''A_a'': ''1'', ''B_b'': ''2''}, {''A_a'': ''10'', ''B_b'': ''20''}]\n\n"],"image":"python:3.11"}' - - name: components-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d + - name: components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d value: '{"executorLabel":"exec-print-text","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - - name: implementations-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d + - name: implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_text"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps'' ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 @@ -34,13 +34,13 @@ spec: print_text(msg: str):\n print(msg)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-1 value: '{"dag":{"tasks":{"print-text":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"}}}}' - - name: components-b550be7b0684fb8004fa1b01d3b9533640ab67f998151e93e27513fdd1aa41c0 + - name: components-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f value: '{"executorLabel":"exec-print-struct","inputDefinitions":{"parameters":{"struct":{"parameterType":"STRUCT"}}}}' - - name: implementations-b550be7b0684fb8004fa1b01d3b9533640ab67f998151e93e27513fdd1aa41c0 + - name: implementations-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_struct"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps'' ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 @@ -53,11 +53,12 @@ spec: value: '{"dag":{"tasks":{"print-struct-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct-2"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"print-struct-2"}},"print-text-4":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-4"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-4"}},"print-text-5":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-5"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-5"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-3":{"parameterType":"STRUCT"}}}}' - name: components-root value: '{"dag":{"tasks":{"args-generator-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-args-generator-op"},"taskInfo":{"name":"args-generator-op"}},"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["args-generator-op"],"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"args-generator-op"}}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output"}},"taskInfo":{"name":"for-loop-2"}},"for-loop-4":{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": - \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-4"}}}},"inputDefinitions":{"parameters":{"loop_parameter":{"defaultValue":["a","b"],"isOptional":true,"parameterType":"LIST"}}}}' + \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-4"}}}},"inputDefinitions":{"parameters":{"loop_parameter":{"parameterType":"LIST"}}}}' entrypoint: entrypoint podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -66,80 +67,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-loops","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -153,25 +89,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -285,11 +203,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text"}}' - name: container - value: '{{workflow.parameters.implementations-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text - name: parent-dag-id @@ -299,10 +217,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-driver.Succeeded name: print-text template: system-container-executor @@ -317,11 +235,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b550be7b0684fb8004fa1b01d3b9533640ab67f998151e93e27513fdd1aa41c0}}' + value: '{{workflow.parameters.components-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"}}},"taskInfo":{"name":"print-struct"}}' - name: container - value: '{{workflow.parameters.implementations-b550be7b0684fb8004fa1b01d3b9533640ab67f998151e93e27513fdd1aa41c0}}' + value: '{{workflow.parameters.implementations-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task-name value: print-struct - name: parent-dag-id @@ -331,21 +249,21 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-struct-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-struct-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-struct-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-struct-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-struct-driver.Succeeded name: print-struct template: system-container-executor - arguments: parameters: - name: component - value: '{{workflow.parameters.components-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-2"}}' - name: container - value: '{{workflow.parameters.implementations-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-2 - name: parent-dag-id @@ -355,21 +273,21 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-2-driver.Succeeded name: print-text-2 template: system-container-executor - arguments: parameters: - name: component - value: '{{workflow.parameters.components-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-3"}}' - name: container - value: '{{workflow.parameters.implementations-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-3 - name: parent-dag-id @@ -379,10 +297,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-3-driver.Succeeded name: print-text-3 template: system-container-executor @@ -397,11 +315,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b550be7b0684fb8004fa1b01d3b9533640ab67f998151e93e27513fdd1aa41c0}}' + value: '{{workflow.parameters.components-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct-2"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"print-struct-2"}}' - name: container - value: '{{workflow.parameters.implementations-b550be7b0684fb8004fa1b01d3b9533640ab67f998151e93e27513fdd1aa41c0}}' + value: '{{workflow.parameters.implementations-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task-name value: print-struct-2 - name: parent-dag-id @@ -411,21 +329,21 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-struct-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-struct-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-struct-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-struct-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-struct-2-driver.Succeeded name: print-struct-2 template: system-container-executor - arguments: parameters: - name: component - value: '{{workflow.parameters.components-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-4"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-4"}}' - name: container - value: '{{workflow.parameters.implementations-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-4 - name: parent-dag-id @@ -435,21 +353,21 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-4-driver.Succeeded name: print-text-4 template: system-container-executor - arguments: parameters: - name: component - value: '{{workflow.parameters.components-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-5"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-5"}}' - name: container - value: '{{workflow.parameters.implementations-317c58411818b2658547854882ddd2335912f90f03549c22a3ed7031bb45a11d}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-5 - name: parent-dag-id @@ -459,10 +377,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-5-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-5-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-5-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-5-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-5-driver.Succeeded name: print-text-5 template: system-container-executor @@ -472,69 +390,15 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-loops","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -552,23 +416,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -588,9 +436,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -618,14 +466,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -651,9 +499,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -681,14 +529,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -715,9 +563,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -746,14 +594,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -765,11 +613,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-fa52219045a80d72f6f8f40492570e94942c3f25927d7cbc4e112c1ac54e4e04}}' + value: '{{workflow.parameters.components-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-args-generator-op"},"taskInfo":{"name":"args-generator-op"}}' - name: container - value: '{{workflow.parameters.implementations-fa52219045a80d72f6f8f40492570e94942c3f25927d7cbc4e112c1ac54e4e04}}' + value: '{{workflow.parameters.implementations-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530}}' - name: task-name value: args-generator-op - name: parent-dag-id @@ -779,10 +627,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.args-generator-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''args-generator-op-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.args-generator-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''args-generator-op-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: args-generator-op-driver.Succeeded name: args-generator-op template: system-container-executor @@ -818,7 +668,7 @@ spec: - name: component value: '{{workflow.parameters.components-root}}' - name: runtime-config - value: '{"parameterValues":{"loop_parameter":["a","b"]}}' + value: '{}' - name: driver-type value: ROOT_DAG name: root-driver @@ -826,7 +676,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index 70bb6366691..e7132bff151 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -104,6 +104,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -112,80 +113,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-loops-and-conditions-multi-layers","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -199,25 +135,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -345,10 +263,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-2-driver.Succeeded name: print-text-2 template: system-container-executor @@ -377,10 +295,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-3-driver.Succeeded name: print-text-3 template: system-container-executor @@ -409,10 +327,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-struct-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-struct-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-struct-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-struct-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-struct-driver.Succeeded name: print-struct template: system-container-executor @@ -422,69 +340,15 @@ spec: metadata: {} name: comp-for-loop-7 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-loops-and-conditions-multi-layers","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -502,23 +366,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -539,9 +387,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-7 @@ -570,14 +418,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-7-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -617,10 +465,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-8-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-8-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-8-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-8-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-8-driver.Succeeded name: print-text-8 template: system-container-executor @@ -648,13 +496,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-13-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-13-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-13-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-13-driver''].outputs.result, "$[''condition'']")}}' depends: condition-13-driver.Succeeded name: condition-13 template: comp-condition-13 - when: '{{tasks.condition-13-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-13-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -672,10 +521,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-7-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-7-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-7-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-7-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-7-driver.Succeeded name: print-text-7 template: system-container-executor @@ -704,10 +553,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-4-driver.Succeeded name: print-text-4 template: system-container-executor @@ -736,10 +585,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-6-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-6-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-6-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-6-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-6-driver.Succeeded name: print-text-6 template: system-container-executor @@ -768,9 +617,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-10 @@ -798,14 +647,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-10-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -837,10 +686,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-5-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-5-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-5-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-5-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-5-driver.Succeeded name: print-text-5 template: system-container-executor @@ -870,9 +719,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-12 @@ -901,14 +750,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-12-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -934,9 +783,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-8 @@ -964,14 +813,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-8-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -997,9 +846,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-9 @@ -1027,14 +876,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-9-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1059,13 +908,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -1082,13 +932,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-4-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-4-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}}' depends: condition-4-driver.Succeeded name: condition-4 template: comp-condition-4 - when: '{{tasks.condition-4-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -1105,13 +956,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-5-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-5-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}}' depends: condition-5-driver.Succeeded name: condition-5 template: comp-condition-5 - when: '{{tasks.condition-5-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: parent-dag-id @@ -1147,10 +999,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-driver.Succeeded name: print-text template: system-container-executor @@ -1179,9 +1031,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -1209,14 +1061,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1242,10 +1094,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.args-generator-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''args-generator-op-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.args-generator-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''args-generator-op-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: args-generator-op-2-driver.Succeeded name: args-generator-op-2 template: system-container-executor @@ -1281,10 +1135,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-9-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-9-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-9-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-9-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-9-driver.Succeeded name: print-text-9 template: system-container-executor @@ -1313,9 +1167,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-16 @@ -1343,14 +1197,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-16-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1389,13 +1243,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-15-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-15-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-15-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-15-driver''].outputs.result, "$[''condition'']")}}' depends: condition-15-driver.Succeeded name: condition-15 template: comp-condition-15 - when: '{{tasks.condition-15-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-15-driver''].outputs.result, "$[''condition'']")}} + != false' inputs: parameters: - name: parent-dag-id @@ -1421,9 +1276,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-14 @@ -1451,14 +1306,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-14-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1484,10 +1339,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.args-generator-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''args-generator-op-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.args-generator-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''args-generator-op-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: args-generator-op-driver.Succeeded name: args-generator-op template: system-container-executor @@ -1508,13 +1365,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -1532,10 +1390,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-op-driver.Succeeded name: flip-coin-op template: system-container-executor @@ -1566,7 +1424,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index a470d72ca21..f6ee8c5d695 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -55,6 +55,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -63,80 +64,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"dataset-concatenator","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -150,25 +86,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -297,10 +215,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.dataset-joiner-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''dataset-joiner-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.dataset-joiner-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''dataset-joiner-driver''].outputs.result, "$[''cached-decision'']")}}' depends: dataset-joiner-driver.Succeeded name: dataset-joiner template: system-container-executor @@ -321,10 +239,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.str-to-dataset-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''str-to-dataset-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.str-to-dataset-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''str-to-dataset-driver''].outputs.result, "$[''cached-decision'']")}}' depends: str-to-dataset-driver.Succeeded name: str-to-dataset template: system-container-executor @@ -334,69 +252,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"dataset-concatenator","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -414,23 +278,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -446,7 +294,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index 5154c2415be..34db04a86f4 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -29,6 +29,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -37,80 +38,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-metrics-outputs","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -124,25 +60,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -270,10 +188,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.output-metrics-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''output-metrics-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.output-metrics-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''output-metrics-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: output-metrics-2-driver.Succeeded name: output-metrics-2 template: system-container-executor @@ -283,69 +203,15 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-metrics-outputs","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -363,23 +229,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -400,9 +250,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -431,14 +281,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -470,10 +320,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.output-metrics-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''output-metrics-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.output-metrics-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''output-metrics-driver''].outputs.result, "$[''cached-decision'']")}}' depends: output-metrics-driver.Succeeded name: output-metrics template: system-container-executor @@ -498,7 +348,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index f73f7d0a3f9..1a11726e3a2 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -48,6 +48,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -56,80 +57,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-multiple-exit-handlers","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -143,25 +79,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -290,10 +208,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: fail-op-driver.Succeeded name: fail-op template: system-container-executor @@ -314,10 +232,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -346,10 +264,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-4-driver.Succeeded name: print-op-4 template: system-container-executor @@ -378,10 +296,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-6-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-6-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-6-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-6-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-6-driver.Succeeded name: print-op-6 template: system-container-executor @@ -411,10 +329,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -444,10 +362,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-3-driver.Succeeded name: print-op-3 template: system-container-executor @@ -477,10 +395,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-5-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-5-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-5-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-5-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-5-driver.Succeeded name: print-op-5 template: system-container-executor @@ -490,69 +408,15 @@ spec: metadata: {} name: exit-hook-root-print-op-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-multiple-exit-handlers","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -570,23 +434,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -604,9 +452,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-1-driver.Succeeded hooks: exit: @@ -632,9 +480,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-2-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-2-driver.Succeeded hooks: exit: @@ -660,9 +508,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-3-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-3-driver.Succeeded hooks: exit: @@ -694,7 +542,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index 21b88b2684f..e5d259d6f1d 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,80 +54,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-conditions-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -140,25 +76,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-op-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-op-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-op-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-op-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-op-4-driver.Succeeded name: flip-coin-op-4 template: system-container-executor @@ -311,10 +229,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-4-driver.Succeeded name: print-op-4 template: system-container-executor @@ -324,69 +242,15 @@ spec: metadata: {} name: comp-condition-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"nested-conditions-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -404,23 +268,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -440,13 +288,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -464,10 +313,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-op-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-op-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-op-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-op-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-op-3-driver.Succeeded name: flip-coin-op-3 template: system-container-executor @@ -489,10 +338,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-3-driver.Succeeded name: print-op-3 template: system-container-executor @@ -521,13 +370,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -545,10 +395,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-op-driver.Succeeded name: flip-coin-op template: system-container-executor @@ -569,10 +419,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-op-2-driver.Succeeded name: flip-coin-op-2 template: system-container-executor @@ -594,10 +444,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -619,10 +469,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -647,7 +497,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml index 2d7853adffb..813af8f40ec 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml @@ -57,6 +57,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -65,80 +66,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"conditional-execution-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -152,25 +88,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -300,10 +218,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-driver.Succeeded name: print template: system-container-executor @@ -334,10 +252,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-2-driver.Succeeded name: print-2 template: system-container-executor @@ -347,69 +265,15 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"conditional-execution-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -427,23 +291,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -463,13 +311,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}}' depends: condition-2-driver.Succeeded name: condition-2 template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-2-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -487,13 +336,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}}' depends: condition-3-driver.Succeeded name: condition-3 template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-3-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -511,10 +361,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.generate-random-number-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''generate-random-number-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.generate-random-number-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''generate-random-number-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: generate-random-number-driver.Succeeded name: generate-random-number template: system-container-executor @@ -545,10 +397,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-3-driver.Succeeded name: print-3 template: system-container-executor @@ -579,10 +431,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-4-driver.Succeeded name: print-4 template: system-container-executor @@ -611,13 +463,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-5-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-5-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}}' depends: condition-5-driver.Succeeded name: condition-5 template: comp-condition-5 - when: '{{tasks.condition-5-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-5-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -635,13 +488,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-6-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-6-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''condition'']")}}' depends: condition-6-driver.Succeeded name: condition-6 template: comp-condition-6 - when: '{{tasks.condition-6-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-6-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -659,10 +513,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.generate-random-number-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''generate-random-number-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.generate-random-number-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''generate-random-number-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: generate-random-number-2-driver.Succeeded name: generate-random-number-2 template: system-container-executor @@ -691,13 +547,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}}' depends: condition-1-driver.Succeeded name: condition-1 template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-1-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -715,13 +572,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.condition-4-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.condition-4-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}}' depends: condition-4-driver.Succeeded name: condition-4 template: comp-condition-4 - when: '{{tasks.condition-4-driver.outputs.parameters.condition}} != false' + when: '{{=jsonpath(tasks[''condition-4-driver''].outputs.result, "$[''condition'']")}} + != false' - arguments: parameters: - name: component @@ -739,10 +597,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.flip-coin-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.flip-coin-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''flip-coin-driver''].outputs.result, "$[''cached-decision'']")}}' depends: flip-coin-driver.Succeeded name: flip-coin template: system-container-executor @@ -767,7 +625,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index 57b1bfbf194..3a26ab7f780 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -35,6 +35,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -43,80 +44,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-nested-loops","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -130,25 +66,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -276,10 +194,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -289,69 +207,15 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-nested-loops","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -369,23 +233,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -405,9 +253,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -435,14 +283,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -482,10 +330,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-3-driver.Succeeded name: print-op-3 template: system-container-executor @@ -515,9 +363,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-6 @@ -546,14 +394,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-6-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -585,10 +433,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -617,9 +465,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -647,14 +495,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -681,9 +529,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -712,14 +560,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -761,7 +609,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml index fec2b482f4f..a467f280bee 100644 --- a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo-name","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -269,69 +187,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo-name","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -349,23 +213,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -381,7 +229,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index 099c71c3d2a..8a688849dba 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -31,6 +31,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -39,80 +40,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -126,25 +62,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -272,10 +190,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -297,10 +215,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op2-driver.Succeeded name: print-op2 template: system-container-executor @@ -310,69 +228,15 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-in-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -390,23 +254,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -424,9 +272,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.inner-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.inner-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''inner-pipeline-driver''].outputs.result, "$[''condition'']")}}' depends: inner-pipeline-driver.Succeeded name: inner-pipeline template: comp-inner-pipeline @@ -447,10 +295,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op1-driver.Succeeded name: print-op1 template: system-container-executor @@ -475,7 +323,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index f3750fa4972..f14513e7418 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -122,6 +122,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -130,80 +131,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-loops","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -217,25 +153,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -363,10 +281,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-2-driver.Succeeded name: print-text-2 template: system-container-executor @@ -376,69 +294,15 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-loops","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -456,23 +320,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -492,9 +340,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -522,14 +370,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -561,10 +409,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-driver.Succeeded name: print-text template: system-container-executor @@ -593,10 +441,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-int-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-int-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-int-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-int-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-int-3-driver.Succeeded name: print-int-3 template: system-container-executor @@ -625,10 +473,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-int-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-int-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-int-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-int-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-int-4-driver.Succeeded name: print-int-4 template: system-container-executor @@ -657,10 +505,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-int-5-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-int-5-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-int-5-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-int-5-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-int-5-driver.Succeeded name: print-int-5 template: system-container-executor @@ -689,10 +537,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-int-6-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-int-6-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-int-6-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-int-6-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-int-6-driver.Succeeded name: print-int-6 template: system-container-executor @@ -721,10 +569,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-5-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-5-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-5-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-5-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-5-driver.Succeeded name: print-text-5 template: system-container-executor @@ -745,10 +593,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-6-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-6-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-6-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-6-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-6-driver.Succeeded name: print-text-6 template: system-container-executor @@ -778,9 +626,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-6 @@ -809,14 +657,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-6-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -849,10 +697,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-3-driver.Succeeded name: print-text-3 template: system-container-executor @@ -873,10 +721,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-text-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-text-4-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-text-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-text-4-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-text-4-driver.Succeeded name: print-text-4 template: system-container-executor @@ -905,10 +753,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-int-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-int-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-int-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-int-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-int-driver.Succeeded name: print-int template: system-container-executor @@ -937,10 +785,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-int-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-int-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-int-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-int-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-int-2-driver.Succeeded name: print-int-2 template: system-container-executor @@ -969,9 +817,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -999,14 +847,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1033,9 +881,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-10 @@ -1063,14 +911,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-10-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1096,9 +944,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-11 @@ -1126,14 +974,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-11-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1159,9 +1007,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-12 @@ -1189,14 +1037,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-12-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1222,9 +1070,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-13 @@ -1252,14 +1100,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-13-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1286,9 +1134,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-4 @@ -1317,14 +1165,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-4-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1351,9 +1199,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-8 @@ -1382,14 +1230,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-8-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1416,9 +1264,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-9 @@ -1446,14 +1294,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-9-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -1532,10 +1380,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.list-dict-maker-0-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''list-dict-maker-0-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.list-dict-maker-0-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''list-dict-maker-0-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: list-dict-maker-0-driver.Succeeded name: list-dict-maker-0 template: system-container-executor @@ -1556,10 +1406,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.list-dict-maker-1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''list-dict-maker-1-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.list-dict-maker-1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''list-dict-maker-1-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: list-dict-maker-1-driver.Succeeded name: list-dict-maker-1 template: system-container-executor @@ -1580,10 +1432,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.list-dict-maker-1-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''list-dict-maker-1-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.list-dict-maker-1-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''list-dict-maker-1-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: list-dict-maker-1-2-driver.Succeeded name: list-dict-maker-1-2 template: system-container-executor @@ -1604,10 +1458,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.list-dict-maker-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''list-dict-maker-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.list-dict-maker-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''list-dict-maker-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: list-dict-maker-2-driver.Succeeded name: list-dict-maker-2 template: system-container-executor @@ -1628,10 +1484,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.list-dict-maker-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''list-dict-maker-3-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.list-dict-maker-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''list-dict-maker-3-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: list-dict-maker-3-driver.Succeeded name: list-dict-maker-3 template: system-container-executor @@ -1656,7 +1514,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml deleted file mode 100644 index 590070fafb2..00000000000 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml +++ /dev/null @@ -1,813 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: pipeline-parallelfor-pipeline-param- -spec: - arguments: - parameters: - - name: components-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f - value: '{"executorLabel":"exec-print-with-prefix","inputDefinitions":{"parameters":{"item":{"parameterType":"STRING"},"prefix":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f - value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_with_prefix"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 - -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 - -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_with_prefix(prefix: str, item: str) -\u003e str:\n result = f''{prefix}: - {item}''\n print(result)\n return result\n\n"],"image":"python:3.11"}' - - name: components-comp-for-loop-1 - value: '{"dag":{"tasks":{"print-with-prefix":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-with-prefix"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"prefix":{"componentInputParameter":"pipelinechannel--prefix"}}},"taskInfo":{"name":"print-with-prefix"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"},"pipelinechannel--prefix":{"parameterType":"STRING"}}}}' - - name: components-comp-for-loop-2 - value: '{"dag":{"tasks":{"print-with-prefix-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-with-prefix-2"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"prefix":{"componentInputParameter":"pipelinechannel--make-prefix-Output"}}},"taskInfo":{"name":"print-with-prefix-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"},"pipelinechannel--make-prefix-Output":{"parameterType":"STRING"}}}}' - - name: components-comp-for-loop-5 - value: '{"dag":{"tasks":{"print-with-prefix-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-with-prefix-3"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"prefix":{"componentInputParameter":"pipelinechannel--prefix"}}},"taskInfo":{"name":"print-with-prefix-3"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"},"pipelinechannel--prefix":{"parameterType":"STRING"}}}}' - - name: components-comp-for-loop-4 - value: '{"dag":{"tasks":{"for-loop-5":{"componentRef":{"name":"comp-for-loop-5"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"pipelinechannel--prefix"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-5"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-3":{"parameterType":"STRING"},"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--prefix":{"parameterType":"STRING"}}}}' - - name: components-ef03c5436a34be5b731ae28293fe4b6509e540c6f27ed14e4a418fced6800550 - value: '{"executorLabel":"exec-make-prefix","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-ef03c5436a34be5b731ae28293fe4b6509e540c6f27ed14e4a418fced6800550 - value: '{"args":["--executor_input","{{$}}","--function_to_execute","make_prefix"],"command":["sh","-c","\nif - ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 - -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.15.2'' ''--no-deps'' - ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 - \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" - \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 - -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport - kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - make_prefix() -\u003e str:\n return ''dynamic-prefix''\n\n"],"image":"python:3.11"}' - - name: components-root - value: '{"dag":{"tasks":{"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"prefix"}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["make-prefix"],"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--make-prefix-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"make-prefix"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-2"}},"for-loop-4":{"componentRef":{"name":"comp-for-loop-4"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"prefix"}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[\"x\", - \"y\"]"}},"taskInfo":{"name":"for-loop-4"}},"make-prefix":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-prefix"},"taskInfo":{"name":"make-prefix"}}}},"inputDefinitions":{"parameters":{"loop_parameter":{"parameterType":"LIST"},"prefix":{"parameterType":"STRING"}}}}' - entrypoint: entrypoint - podMetadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - labels: - pipelines.kubeflow.org/v2_component: "true" - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-parallelfor-pipeline-param - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 256Mi - requests: - cpu: 100m - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - securityContext: - seccompProfile: - type: RuntimeDefault - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-with-prefix"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"prefix":{"componentInputParameter":"pipelinechannel--prefix"}}},"taskInfo":{"name":"print-with-prefix"}}' - - name: container - value: '{{workflow.parameters.implementations-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f}}' - - name: task-name - value: print-with-prefix - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-with-prefix-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-with-prefix-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-with-prefix-driver.outputs.parameters.cached-decision}}' - depends: print-with-prefix-driver.Succeeded - name: print-with-prefix - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-1 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-with-prefix-2"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"prefix":{"componentInputParameter":"pipelinechannel--make-prefix-Output"}}},"taskInfo":{"name":"print-with-prefix-2"}}' - - name: container - value: '{{workflow.parameters.implementations-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f}}' - - name: task-name - value: print-with-prefix-2 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-with-prefix-2-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-with-prefix-2-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-with-prefix-2-driver.outputs.parameters.cached-decision}}' - depends: print-with-prefix-2-driver.Succeeded - name: print-with-prefix-2 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-with-prefix-3"},"inputs":{"parameters":{"item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"prefix":{"componentInputParameter":"pipelinechannel--prefix"}}},"taskInfo":{"name":"print-with-prefix-3"}}' - - name: container - value: '{{workflow.parameters.implementations-52f5712c05b2f7d5409bb6666d49a6c40f3a31a55f80593ddff677822f25db8f}}' - - name: task-name - value: print-with-prefix-3 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-with-prefix-3-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-with-prefix-3-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-with-prefix-3-driver.outputs.parameters.cached-decision}}' - depends: print-with-prefix-3-driver.Succeeded - name: print-with-prefix-3 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-5 - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-parallelfor-pipeline-param - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-5}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-5"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"pipelinechannel--prefix"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-5"}}' - - name: task-name - value: for-loop-5 - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-5 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-5-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-5}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-5"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"pipelinechannel--prefix"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-5"}}' - - name: task-name - value: for-loop-5 - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-5-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-5-for-loop-5-iterator - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: for-loop-5 - template: comp-for-loop-5-for-loop-5-iterator - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-4 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-1}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"prefix"}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' - - name: task-name - value: for-loop-1 - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-1 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-1-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-1}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"prefix"}}},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' - - name: task-name - value: for-loop-1 - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-1-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-1-for-loop-1-iterator - outputs: {} - parallelism: 1 - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["make-prefix"],"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--make-prefix-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"make-prefix"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-2 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-2-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["make-prefix"],"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--make-prefix-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"make-prefix"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-2"}}' - - name: task-name - value: for-loop-2 - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-2-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2-for-loop-2-iterator - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-4}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-4"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"prefix"}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[\"x\", - \"y\"]"}},"taskInfo":{"name":"for-loop-4"}}' - - name: task-name - value: for-loop-4 - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-4 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-4-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-4}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-4"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"},"pipelinechannel--prefix":{"componentInputParameter":"prefix"}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[\"x\", - \"y\"]"}},"taskInfo":{"name":"for-loop-4"}}' - - name: task-name - value: for-loop-4 - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-4-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-4-for-loop-4-iterator - outputs: {} - parallelism: 2 - - dag: - tasks: - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: for-loop-1 - template: comp-for-loop-1-for-loop-1-iterator - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - depends: make-prefix.Succeeded - name: for-loop-2 - template: comp-for-loop-2-for-loop-2-iterator - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: for-loop-4 - template: comp-for-loop-4-for-loop-4-iterator - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-ef03c5436a34be5b731ae28293fe4b6509e540c6f27ed14e4a418fced6800550}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-prefix"},"taskInfo":{"name":"make-prefix"}}' - - name: container - value: '{{workflow.parameters.implementations-ef03c5436a34be5b731ae28293fe4b6509e540c6f27ed14e4a418fced6800550}}' - - name: task-name - value: make-prefix - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: make-prefix-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.make-prefix-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.make-prefix-driver.outputs.parameters.cached-decision}}' - depends: make-prefix-driver.Succeeded - name: make-prefix - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} -status: - finishedAt: null - startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index 2c29a52640a..e4f153f6080 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -44,6 +44,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -52,80 +53,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-pipelineparam-containing-format","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -139,25 +75,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op2-driver.Succeeded name: print-op2 template: system-container-executor @@ -299,69 +217,15 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-pipelineparam-containing-format","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -379,23 +243,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -416,9 +264,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-2 @@ -447,14 +295,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-2-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -487,10 +335,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -513,10 +361,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-2-driver.Succeeded name: print-op-2 template: system-container-executor @@ -541,7 +389,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index 7cec62f4b53..821e10a9461 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -31,6 +31,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -39,80 +40,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-placeholders","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -126,25 +62,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -272,10 +190,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-all-placeholders-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-all-placeholders-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-all-placeholders-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-all-placeholders-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: print-all-placeholders-driver.Succeeded name: print-all-placeholders template: system-container-executor @@ -285,69 +205,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-placeholders","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -365,23 +231,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -397,7 +247,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index 47bed1a657a..aa2171f400f 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -146,6 +146,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -154,80 +155,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-pod-metadata","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -241,25 +177,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -962,10 +880,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-no-pod-metadata-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-no-pod-metadata-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-no-pod-metadata-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-no-pod-metadata-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: validate-no-pod-metadata-driver.Succeeded name: validate-no-pod-metadata template: system-container-executor @@ -988,10 +908,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-pod-metadata-task-a-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-a-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-pod-metadata-task-a-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-a-driver''].outputs.result, + "$[''cached-decision'']")}}' - name: pod-metadata-annotation-key value: task-annotation - name: pod-metadata-annotation-val @@ -1026,10 +948,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-pod-metadata-task-b-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-b-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-pod-metadata-task-b-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-b-driver''].outputs.result, + "$[''cached-decision'']")}}' - name: pod-metadata-annotation-key-1 value: task-annotation-1 - name: pod-metadata-annotation-val-1 @@ -1080,10 +1004,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-pod-metadata-task-c-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-c-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-pod-metadata-task-c-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-c-driver''].outputs.result, + "$[''cached-decision'']")}}' - name: pod-metadata-annotation-key-1 value: task-annotation-1 - name: pod-metadata-annotation-val-1 @@ -1114,10 +1040,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.validate-pod-metadata-task-d-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-d-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.validate-pod-metadata-task-d-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''validate-pod-metadata-task-d-driver''].outputs.result, + "$[''cached-decision'']")}}' - name: pod-metadata-label-key-1 value: task-label-1 - name: pod-metadata-label-val-1 @@ -1139,69 +1067,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-pod-metadata","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -1219,23 +1093,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -1251,7 +1109,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index 11f55866a23..ff96fa8bf26 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"test-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -183,8 +101,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid - - name: KFP_RETRY_INDEX - value: '{{retries}}' envFrom: - configMapRef: name: metadata-grpc-configmap @@ -293,10 +209,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-driver''].outputs.result, "$[''cached-decision'']")}}' - name: retry-max-count value: "3" - name: retry-backoff-duration @@ -314,69 +230,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"test-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -394,23 +256,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -426,7 +272,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index 0ff9b9cedba..b02ea6aca7b 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-reused-component - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-reused-component","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -265,10 +183,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-numbers-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-numbers-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-numbers-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-numbers-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-numbers-driver.Succeeded name: add-numbers template: system-container-executor @@ -290,10 +208,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-numbers-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-numbers-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-numbers-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-numbers-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-numbers-2-driver.Succeeded name: add-numbers-2 template: system-container-executor @@ -315,10 +233,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.add-numbers-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''add-numbers-3-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.add-numbers-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''add-numbers-3-driver''].outputs.result, "$[''cached-decision'']")}}' depends: add-numbers-3-driver.Succeeded name: add-numbers-3 template: system-container-executor @@ -328,69 +246,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-reused-component - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-reused-component","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -408,23 +272,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -440,7 +288,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index 7e2f8625519..e8166a0e8c0 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,80 +52,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-secret-env","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -138,25 +74,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -287,10 +205,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: comp-driver.Succeeded name: comp template: system-container-executor @@ -311,10 +229,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.generate-secret-name-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''generate-secret-name-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.generate-secret-name-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''generate-secret-name-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: generate-secret-name-driver.Succeeded name: generate-secret-name template: system-container-executor @@ -324,69 +244,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-secret-env","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -404,23 +270,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -436,7 +286,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index eb023f22197..ac194dc9d97 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -31,6 +31,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -39,80 +40,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-secret-volume","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -126,25 +62,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -274,10 +192,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''comp-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''comp-driver''].outputs.result, "$[''cached-decision'']")}}' depends: comp-driver.Succeeded name: comp template: system-container-executor @@ -287,69 +205,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-secret-volume","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -367,23 +231,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -399,7 +247,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index 5ac3a9a795b..304f1315deb 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,80 +33,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -119,25 +55,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -265,10 +183,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.sum-numbers-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''sum-numbers-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.sum-numbers-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''sum-numbers-driver''].outputs.result, "$[''cached-decision'']")}}' depends: sum-numbers-driver.Succeeded name: sum-numbers template: system-container-executor @@ -278,69 +196,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -358,23 +222,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -390,7 +238,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 86cee35bd46..7fd3e218db4 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -71,11 +71,12 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["cpu-limit","memory-limit","accelerator-limit","accelerator-type"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -84,80 +85,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -171,25 +107,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -317,10 +235,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.accelerator-limit-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''accelerator-limit-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.accelerator-limit-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''accelerator-limit-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: accelerator-limit-driver.Succeeded name: accelerator-limit template: system-container-executor @@ -341,10 +261,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.accelerator-type-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''accelerator-type-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.accelerator-type-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''accelerator-type-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: accelerator-type-driver.Succeeded name: accelerator-type template: system-container-executor @@ -365,10 +287,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.cpu-limit-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''cpu-limit-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.cpu-limit-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''cpu-limit-driver''].outputs.result, "$[''cached-decision'']")}}' depends: cpu-limit-driver.Succeeded name: cpu-limit template: system-container-executor @@ -389,10 +311,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.memory-limit-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''memory-limit-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.memory-limit-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''memory-limit-driver''].outputs.result, "$[''cached-decision'']")}}' depends: memory-limit-driver.Succeeded name: memory-limit template: system-container-executor @@ -401,24 +323,24 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["cpu-limit","memory-limit","accelerator-limit","accelerator-type"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: cpu-limit.Succeeded && memory-limit.Succeeded && accelerator-limit.Succeeded - && accelerator-type.Succeeded + depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && cpu-limit.Succeeded + && memory-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.sum-numbers-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''sum-numbers-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.sum-numbers-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''sum-numbers-driver''].outputs.result, "$[''cached-decision'']")}}' depends: sum-numbers-driver.Succeeded name: sum-numbers template: system-container-executor @@ -428,69 +350,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -508,23 +376,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -540,7 +392,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_submit_request.yaml b/test_data/compiled-workflows/pipeline_with_submit_request.yaml index 64dfa4657af..5bf22ace768 100644 --- a/test_data/compiled-workflows/pipeline_with_submit_request.yaml +++ b/test_data/compiled-workflows/pipeline_with_submit_request.yaml @@ -35,6 +35,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -43,80 +44,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-external-request - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-external-request","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -130,25 +66,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -276,10 +194,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.submit-request-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''submit-request-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.submit-request-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''submit-request-driver''].outputs.result, "$[''cached-decision'']")}}' depends: submit-request-driver.Succeeded name: submit-request template: system-container-executor @@ -300,10 +218,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.submit-request-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''submit-request-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.submit-request-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''submit-request-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: submit-request-2-driver.Succeeded name: submit-request-2 template: system-container-executor @@ -313,69 +233,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-external-request - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-external-request","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -393,23 +259,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -425,7 +275,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml index 72a58330fd2..dd075a5f95b 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml @@ -58,6 +58,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -66,80 +67,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-task-final-status","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -153,25 +89,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -300,10 +218,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: fail-op-driver.Succeeded name: fail-op template: system-container-executor @@ -324,10 +242,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -356,10 +274,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.exit-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''exit-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.exit-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''exit-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: exit-op-driver.Succeeded name: exit-op template: system-container-executor @@ -369,69 +287,15 @@ spec: metadata: {} name: exit-hook-root-exit-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-task-final-status","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -449,23 +313,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -483,9 +331,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-1-driver.Succeeded hooks: exit: @@ -517,7 +365,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml index e2ec3a77b22..4cd2d1c7acf 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml @@ -23,6 +23,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -31,80 +32,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status-yaml - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-task-final-status-yaml","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -118,25 +54,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -264,10 +182,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -296,10 +214,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.exit-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''exit-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.exit-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''exit-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: exit-op-driver.Succeeded name: exit-op template: system-container-executor @@ -309,69 +227,15 @@ spec: metadata: {} name: exit-hook-root-exit-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status-yaml - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-task-final-status-yaml","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -389,23 +253,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -423,9 +271,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''exit-handler-1-driver''].outputs.result, "$[''condition'']")}}' depends: exit-handler-1-driver.Succeeded hooks: exit: @@ -457,7 +305,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index 1b4856bbf25..e277de91c7e 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -38,6 +38,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -46,80 +47,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"my-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -133,25 +69,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -279,10 +197,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''print-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: print-op-driver.Succeeded name: print-op template: system-container-executor @@ -311,10 +229,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''fail-op-driver''].outputs.result, "$[''cached-decision'']")}}' depends: fail-op-driver.Succeeded hooks: exit: @@ -331,69 +249,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"my-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -411,23 +275,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -443,7 +291,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_utils.yaml b/test_data/compiled-workflows/pipeline_with_utils.yaml index e7fb28ee7f9..67c86712d11 100644 --- a/test_data/compiled-workflows/pipeline_with_utils.yaml +++ b/test_data/compiled-workflows/pipeline_with_utils.yaml @@ -28,6 +28,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -36,80 +37,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-utils - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-utils","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -123,25 +59,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -269,10 +187,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -282,69 +200,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-utils - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-utils","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -362,23 +226,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -394,7 +242,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml index 799eb7d5f2c..cfa536d4415 100644 --- a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml @@ -19,6 +19,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -27,80 +28,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-various-types","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -114,25 +50,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -261,10 +179,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.downstream-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''downstream-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.downstream-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''downstream-driver''].outputs.result, "$[''cached-decision'']")}}' depends: downstream-driver.Succeeded name: downstream template: system-container-executor @@ -285,10 +203,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.upstream-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''upstream-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.upstream-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''upstream-driver''].outputs.result, "$[''cached-decision'']")}}' depends: upstream-driver.Succeeded name: upstream template: system-container-executor @@ -298,69 +216,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-various-types","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -378,23 +242,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -410,7 +258,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index 4dafff22b1e..99cc931d41e 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -72,6 +72,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -80,80 +81,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-volume","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -167,25 +103,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -316,10 +234,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consumer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consumer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: consumer-driver.Succeeded name: consumer template: system-container-executor @@ -372,10 +290,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.producer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.producer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: producer-driver.Succeeded name: producer template: system-container-executor @@ -385,69 +303,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-volume","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -465,23 +329,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -497,7 +345,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml b/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml index a656b0a7479..0b0564fcc31 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,80 +55,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - here-is-a-pipeline-very-long-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"here-is-a-pipeline-very-long-name","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -141,25 +77,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -301,10 +219,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.dummy-task-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''dummy-task-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.dummy-task-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''dummy-task-driver''].outputs.result, "$[''cached-decision'']")}}' depends: dummy-task-driver.Succeeded name: dummy-task template: system-container-executor @@ -314,69 +232,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - here-is-a-pipeline-very-long-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"here-is-a-pipeline-very-long-name","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -394,23 +258,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -426,7 +274,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index d8d96839d16..112deec0cdb 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -72,6 +72,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -80,80 +81,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-volume-no-cache","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -167,25 +103,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -316,10 +234,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consumer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consumer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: consumer-driver.Succeeded name: consumer template: system-container-executor @@ -372,10 +290,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.producer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.producer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: producer-driver.Succeeded name: producer template: system-container-executor @@ -385,69 +303,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-volume-no-cache","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -465,23 +329,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -497,7 +345,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 34a3dd49915..52510e23d5d 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -48,6 +48,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -56,80 +57,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-workspace","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -143,25 +79,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -290,10 +208,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-from-workspace-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-from-workspace-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-from-workspace-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-from-workspace-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: read-from-workspace-driver.Succeeded name: read-from-workspace template: system-container-executor @@ -314,10 +234,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.write-to-workspace-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''write-to-workspace-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.write-to-workspace-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''write-to-workspace-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: write-to-workspace-driver.Succeeded name: write-to-workspace template: system-container-executor @@ -327,69 +249,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pipeline-with-workspace","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -407,23 +275,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -439,7 +291,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml index f2bb06458c2..b1e2beced2d 100644 --- a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml +++ b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml @@ -18,6 +18,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -26,80 +27,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"one-step-pipeline-with-if-placeholder-supply-none","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -113,25 +49,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -259,10 +177,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component-with-optional-inputs-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component-with-optional-inputs-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component-with-optional-inputs-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component-with-optional-inputs-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: component-with-optional-inputs-driver.Succeeded name: component-with-optional-inputs template: system-container-executor @@ -272,69 +192,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"one-step-pipeline-with-if-placeholder-supply-none","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -352,23 +218,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -384,7 +234,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index 2165b01e6b0..f8ba158df93 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,80 +55,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"preprocess","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -141,25 +77,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -287,10 +205,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.preprocess-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''preprocess-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.preprocess-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''preprocess-driver''].outputs.result, "$[''cached-decision'']")}}' depends: preprocess-driver.Succeeded name: preprocess template: system-container-executor @@ -300,69 +218,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"preprocess","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -380,23 +244,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -412,7 +260,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml index ce71e5ff290..61fd61a8d32 100644 --- a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml +++ b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml @@ -22,6 +22,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -30,80 +31,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"producer-consumer-param-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -117,25 +53,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -264,10 +182,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consumer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consumer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: consumer-driver.Succeeded name: consumer template: system-container-executor @@ -288,10 +206,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.producer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.producer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: producer-driver.Succeeded name: producer template: system-container-executor @@ -301,69 +219,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"producer-consumer-param-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -381,23 +245,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -413,7 +261,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pvc_mount.yaml b/test_data/compiled-workflows/pvc_mount.yaml index 50fa4a99b05..9c4ac7b199d 100644 --- a/test_data/compiled-workflows/pvc_mount.yaml +++ b/test_data/compiled-workflows/pvc_mount.yaml @@ -42,6 +42,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -50,80 +51,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pvc-mount-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pvc-mount-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -137,25 +73,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.consumer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.consumer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''consumer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: consumer-driver.Succeeded name: consumer template: system-container-executor @@ -312,10 +230,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.producer-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.producer-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''producer-driver''].outputs.result, "$[''cached-decision'']")}}' depends: producer-driver.Succeeded name: producer template: system-container-executor @@ -325,69 +243,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pvc-mount-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pvc-mount-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -405,23 +269,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -437,7 +285,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index 4019446da75..c65caf49217 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -35,6 +35,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -112,80 +113,15 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - make-language-model-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"make-language-model-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -199,25 +135,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -358,10 +276,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.make-language-model-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''make-language-model-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.make-language-model-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''make-language-model-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: make-language-model-driver.Succeeded name: make-language-model template: system-container-executor @@ -371,69 +291,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - make-language-model-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"make-language-model-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -451,23 +317,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -483,7 +333,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index 8d1a56969ff..2118f76ecd9 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,80 +52,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pythonic-artifacts-test","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -138,25 +74,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -284,10 +202,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.gen-data-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''gen-data-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.gen-data-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''gen-data-driver''].outputs.result, "$[''cached-decision'']")}}' depends: gen-data-driver.Succeeded name: gen-data template: system-container-executor @@ -309,10 +227,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.train-model-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''train-model-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.train-model-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''train-model-driver''].outputs.result, "$[''cached-decision'']")}}' depends: train-model-driver.Succeeded name: train-model template: system-container-executor @@ -322,69 +240,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"pythonic-artifacts-test","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -402,23 +266,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -434,7 +282,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index 7cbebbad713..106ae62e27f 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -44,6 +44,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -52,80 +53,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"make-and-join-datasets","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -139,25 +75,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -285,10 +203,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.make-dataset-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''make-dataset-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.make-dataset-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''make-dataset-driver''].outputs.result, "$[''cached-decision'']")}}' depends: make-dataset-driver.Succeeded name: make-dataset template: system-container-executor @@ -298,69 +216,15 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"make-and-join-datasets","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -378,23 +242,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -414,9 +262,9 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''iteration-item-driver''].outputs.result, "$[''condition'']")}}' depends: iteration-item-driver.Succeeded name: iteration-item template: comp-for-loop-1 @@ -444,14 +292,14 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''execution-id'']")}}' - name: iteration-index value: '{{item}}' depends: iteration-driver.Succeeded name: iteration-iterations template: comp-for-loop-1-iteration withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + count: '{{=jsonpath(tasks[''iteration-driver''].outputs.result, "$[''iteration-count'']")}}' inputs: parameters: - name: parent-dag-id @@ -484,10 +332,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.join-datasets-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''join-datasets-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.join-datasets-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''join-datasets-driver''].outputs.result, "$[''cached-decision'']")}}' depends: join-datasets-driver.Succeeded name: join-datasets template: system-container-executor @@ -512,7 +360,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index 1952102bdc9..9c2bce2647c 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -49,6 +49,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -57,80 +58,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"split-datasets-and-return-first","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -144,25 +80,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -290,10 +208,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.dataset-splitter-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''dataset-splitter-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.dataset-splitter-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''dataset-splitter-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: dataset-splitter-driver.Succeeded name: dataset-splitter template: system-container-executor @@ -303,69 +223,15 @@ spec: metadata: {} name: comp-splitter-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"split-datasets-and-return-first","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -383,23 +249,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -419,10 +269,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.make-dataset-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''make-dataset-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.make-dataset-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''make-dataset-driver''].outputs.result, "$[''cached-decision'']")}}' depends: make-dataset-driver.Succeeded name: make-dataset template: system-container-executor @@ -442,9 +292,11 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.splitter-pipeline-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''splitter-pipeline-driver''].outputs.result, + "$[''execution-id'']")}}' - name: condition - value: '{{tasks.splitter-pipeline-driver.outputs.parameters.condition}}' + value: '{{=jsonpath(tasks[''splitter-pipeline-driver''].outputs.result, + "$[''condition'']")}}' depends: splitter-pipeline-driver.Succeeded name: splitter-pipeline template: comp-splitter-pipeline @@ -469,7 +321,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/ray_integration_compiled.yaml b/test_data/compiled-workflows/ray_integration_compiled.yaml index d495579ff62..9447b174b09 100644 --- a/test_data/compiled-workflows/ray_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_integration_compiled.yaml @@ -89,6 +89,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -97,80 +98,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"ray-integration-test","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -184,25 +120,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -330,10 +248,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.ray-fn-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''ray-fn-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.ray-fn-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''ray-fn-driver''].outputs.result, "$[''cached-decision'']")}}' depends: ray-fn-driver.Succeeded name: ray-fn template: system-container-executor @@ -343,69 +261,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"ray-integration-test","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -423,23 +287,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -455,7 +303,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml index 2ed37911c49..b77158a024b 100644 --- a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -24,82 +25,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","log_level":"3","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -113,25 +47,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -261,10 +177,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -274,71 +190,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","log_level":"3","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -356,23 +216,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -388,7 +232,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml index 2ed37911c49..b77158a024b 100644 --- a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -24,82 +25,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","log_level":"3","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -113,25 +47,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -261,10 +177,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -274,71 +190,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","log_level":"3","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"echo","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -356,23 +216,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -388,7 +232,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/sequential_v1.yaml b/test_data/compiled-workflows/sequential_v1.yaml index aa05a6bd410..9b707045323 100644 --- a/test_data/compiled-workflows/sequential_v1.yaml +++ b/test_data/compiled-workflows/sequential_v1.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,80 +24,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"sequential","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -110,25 +46,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -256,10 +174,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -280,10 +198,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-2-driver.Succeeded name: echo-2 template: system-container-executor @@ -293,69 +211,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"sequential","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -373,23 +237,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -405,7 +253,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/sequential_v2.yaml b/test_data/compiled-workflows/sequential_v2.yaml index d5dd44ca770..725f818ee38 100644 --- a/test_data/compiled-workflows/sequential_v2.yaml +++ b/test_data/compiled-workflows/sequential_v2.yaml @@ -19,6 +19,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -27,80 +28,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"sequential","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -114,25 +50,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -260,10 +178,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.download-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''download-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.download-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''download-driver''].outputs.result, "$[''cached-decision'']")}}' depends: download-driver.Succeeded name: download template: system-container-executor @@ -285,10 +203,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''echo-driver''].outputs.result, "$[''cached-decision'']")}}' depends: echo-driver.Succeeded name: echo template: system-container-executor @@ -298,69 +216,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"sequential","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -378,23 +242,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -410,7 +258,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index 0f94165371c..dc52288a10b 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -48,80 +49,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"take-nap-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -135,25 +71,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -281,10 +199,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.take-nap-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''take-nap-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.take-nap-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''take-nap-driver''].outputs.result, "$[''cached-decision'']")}}' depends: take-nap-driver.Succeeded name: take-nap template: system-container-executor @@ -306,10 +224,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.wake-up-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''wake-up-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.wake-up-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''wake-up-driver''].outputs.result, "$[''cached-decision'']")}}' depends: wake-up-driver.Succeeded name: wake-up template: system-container-executor @@ -319,69 +237,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"take-nap-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -399,23 +263,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -431,7 +279,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index 0f94165371c..dc52288a10b 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -48,80 +49,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"take-nap-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -135,25 +71,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -281,10 +199,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.take-nap-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''take-nap-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.take-nap-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''take-nap-driver''].outputs.result, "$[''cached-decision'']")}}' depends: take-nap-driver.Succeeded name: take-nap template: system-container-executor @@ -306,10 +224,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.wake-up-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''wake-up-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.wake-up-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''wake-up-driver''].outputs.result, "$[''cached-decision'']")}}' depends: wake-up-driver.Succeeded name: wake-up template: system-container-executor @@ -319,69 +237,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"take-nap-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -399,23 +263,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -431,7 +279,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/two_step_pipeline.yaml b/test_data/compiled-workflows/two_step_pipeline.yaml index f6b9c888276..6818b696a74 100644 --- a/test_data/compiled-workflows/two_step_pipeline.yaml +++ b/test_data/compiled-workflows/two_step_pipeline.yaml @@ -20,6 +20,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -28,80 +29,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - simple-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"simple-two-step-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -115,25 +51,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -262,10 +180,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.read-from-gcs-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''read-from-gcs-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.read-from-gcs-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''read-from-gcs-driver''].outputs.result, "$[''cached-decision'']")}}' depends: read-from-gcs-driver.Succeeded name: read-from-gcs template: system-container-executor @@ -286,10 +204,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.write-to-gcs-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''write-to-gcs-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.write-to-gcs-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''write-to-gcs-driver''].outputs.result, "$[''cached-decision'']")}}' depends: write-to-gcs-driver.Succeeded name: write-to-gcs template: system-container-executor @@ -299,69 +217,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - simple-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"simple-two-step-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -379,23 +243,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -411,7 +259,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml index 27f8ea4d398..64fe6f4f5f7 100644 --- a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml +++ b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml @@ -21,6 +21,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -29,80 +30,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - containerized-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"containerized-two-step-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -116,25 +52,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -262,10 +180,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component1-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component1-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component1-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component1-driver''].outputs.result, "$[''cached-decision'']")}}' depends: component1-driver.Succeeded name: component1 template: system-container-executor @@ -287,10 +205,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.component2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''component2-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.component2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''component2-driver''].outputs.result, "$[''cached-decision'']")}}' depends: component2-driver.Succeeded name: component2 template: system-container-executor @@ -300,69 +218,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - containerized-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"containerized-two-step-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -380,23 +244,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -412,7 +260,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index 3edca3d8542..3db2a17e5b4 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -79,6 +79,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -87,80 +88,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"test-data-passing-pipeline-1","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -174,25 +110,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -321,10 +239,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.receive-file-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''receive-file-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.receive-file-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''receive-file-driver''].outputs.result, "$[''cached-decision'']")}}' depends: receive-file-driver.Succeeded name: receive-file template: system-container-executor @@ -345,10 +263,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.send-file-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''send-file-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.send-file-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''send-file-driver''].outputs.result, "$[''cached-decision'']")}}' depends: send-file-driver.Succeeded name: send-file template: system-container-executor @@ -370,10 +288,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.test-uploaded-artifact-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''test-uploaded-artifact-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.test-uploaded-artifact-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''test-uploaded-artifact-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: test-uploaded-artifact-driver.Succeeded name: test-uploaded-artifact template: system-container-executor @@ -383,69 +303,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"test-data-passing-pipeline-1","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -463,23 +329,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -495,7 +345,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml index 4fcdd6e6a6b..8d187fba9e2 100644 --- a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml +++ b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml @@ -283,6 +283,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -291,80 +292,15 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","container":"{{=sprig.b64enc(inputs.parameters[''container''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","kubernetes_config":"{{=sprig.b64enc(inputs.parameters[''kubernetes-config''])}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"xgboost-sample-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"CONTAINER"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -378,25 +314,7 @@ spec: name: kubernetes-config metadata: {} name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -525,10 +443,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.chicago-taxi-trips-dataset-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''chicago-taxi-trips-dataset-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.chicago-taxi-trips-dataset-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''chicago-taxi-trips-dataset-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: chicago-taxi-trips-dataset-driver.Succeeded name: chicago-taxi-trips-dataset template: system-container-executor @@ -550,10 +470,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.convert-csv-to-apache-parquet-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''convert-csv-to-apache-parquet-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.convert-csv-to-apache-parquet-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''convert-csv-to-apache-parquet-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: convert-csv-to-apache-parquet-driver.Succeeded name: convert-csv-to-apache-parquet template: system-container-executor @@ -575,10 +497,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.xgboost-predict-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''xgboost-predict-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.xgboost-predict-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''xgboost-predict-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: xgboost-predict-driver.Succeeded name: xgboost-predict template: system-container-executor @@ -600,10 +524,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.xgboost-predict-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''xgboost-predict-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.xgboost-predict-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''xgboost-predict-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: xgboost-predict-2-driver.Succeeded name: xgboost-predict-2 template: system-container-executor @@ -625,10 +551,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.xgboost-predict-3-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''xgboost-predict-3-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.xgboost-predict-3-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''xgboost-predict-3-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: xgboost-predict-3-driver.Succeeded name: xgboost-predict-3 template: system-container-executor @@ -650,10 +578,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.xgboost-predict-4-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''xgboost-predict-4-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.xgboost-predict-4-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''xgboost-predict-4-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: xgboost-predict-4-driver.Succeeded name: xgboost-predict-4 template: system-container-executor @@ -675,10 +605,10 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.xgboost-train-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''xgboost-train-driver''].outputs.result, "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.xgboost-train-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''xgboost-train-driver''].outputs.result, "$[''cached-decision'']")}}' depends: xgboost-train-driver.Succeeded name: xgboost-train template: system-container-executor @@ -700,10 +630,12 @@ spec: - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.xgboost-train-2-driver.outputs.parameters.pod-spec-patch}}' + value: '{{=jsonpath(tasks[''xgboost-train-2-driver''].outputs.result, + "$[''pod-spec-patch'']")}}' - default: "false" name: cached-decision - value: '{{tasks.xgboost-train-2-driver.outputs.parameters.cached-decision}}' + value: '{{=jsonpath(tasks[''xgboost-train-2-driver''].outputs.result, + "$[''cached-decision'']")}}' depends: xgboost-train-2-driver.Succeeded name: xgboost-train-2 template: system-container-executor @@ -713,69 +645,15 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + - http: + body: '{"cache_disabled":false,"component":"{{=sprig.b64enc(inputs.parameters[''component''])}}","dag_execution_id":"{{inputs.parameters.parent-dag-id}}","http_proxy":"","https_proxy":"","iteration_index":"{{inputs.parameters.iteration-index}}","metadata_tls_enabled":false,"ml_pipeline_server_address":"ml-pipeline.kubeflow.svc.cluster.local","ml_pipeline_server_port":"8887","ml_pipeline_tls_enabled":false,"mlmd_server_address":"metadata-grpc-service.kubeflow.svc.cluster.local","mlmd_server_port":"8080","namespace":"{{workflow.namespace}}","no_proxy":"","pipeline_name":"xgboost-sample-pipeline","run_display_name":"","run_id":"{{workflow.uid}}","run_name":"{{workflow.name}}","runtime_config":"{{=sprig.b64enc(inputs.parameters[''runtime-config''])}}","task":"{{=sprig.b64enc(inputs.parameters[''task''])}}","task_name":"{{inputs.parameters.task-name}}","type":"{{inputs.parameters.driver-type}}"}' + headers: + - name: Content-Type + value: application/json + method: POST + successCondition: response.statusCode == 200 + timeoutSeconds: 600 + url: http://ml-pipeline-driver.kubeflow.svc.cluster.local:8080/api/v1/template.execute inputs: parameters: - name: component @@ -793,23 +671,7 @@ spec: name: driver-type metadata: {} name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + outputs: {} - dag: tasks: - arguments: @@ -825,7 +687,7 @@ spec: - arguments: parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + value: '{{=jsonpath(tasks[''root-driver''].outputs.result, "$[''execution-id'']")}}' - name: condition value: "" depends: root-driver.Succeeded diff --git a/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml b/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml index bdf84eb8391..b27a925bf69 100644 --- a/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml +++ b/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml @@ -44,3 +44,6 @@ rules: - update - patch - delete +- apiGroups: ["argoproj.io"] + resources: ["workflowtasksets/status", "workflowtasksets"] + verbs: ["get","list","watch","update","patch","delete","create"] diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/loop_consume_upstream.py b/test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/loop_consume_upstream.py rename to test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.py diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/loop_consume_upstream.yaml b/test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.yaml similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/loop_consume_upstream.yaml rename to test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.yaml diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_input_child_level.py b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_input_child_level.py rename to test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level.py diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level_compiled.yaml similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_input_child_level_compiled.yaml rename to test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level_compiled.yaml diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_nil.py b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_nil.py rename to test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil.py diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil_compiled.yaml similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_nil_compiled.yaml rename to test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil_compiled.yaml diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_parent_level.py b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_parent_level.py rename to test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level.py diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level_compiled.yaml similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_pipeline_opt_inputs_parent_level_compiled.yaml rename to test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level_compiled.yaml diff --git a/test_data/sdk_compiled_pipelines/valid/essential/dict_input.py b/test_data/sdk_compiled_pipelines/valid/dict_input.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/essential/dict_input.py rename to test_data/sdk_compiled_pipelines/valid/dict_input.py diff --git a/test_data/sdk_compiled_pipelines/valid/dict_input.yaml b/test_data/sdk_compiled_pipelines/valid/dict_input.yaml new file mode 100644 index 00000000000..140b779a2a9 --- /dev/null +++ b/test_data/sdk_compiled_pipelines/valid/dict_input.yaml @@ -0,0 +1,63 @@ +# PIPELINE DEFINITION +# Name: dict-input +# Inputs: +# struct: dict +components: + comp-dict-input: + executorLabel: exec-dict-input + inputDefinitions: + parameters: + struct: + parameterType: STRUCT +deploymentSpec: + executors: + exec-dict-input: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - dict_input + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef dict_input(struct: Dict):\n print(struct)\n\n" + image: python:3.11 +pipelineInfo: + name: dict-input +root: + dag: + tasks: + dict-input: + cachingOptions: + enableCache: true + componentRef: + name: comp-dict-input + inputs: + parameters: + struct: + componentInputParameter: struct + taskInfo: + name: dict-input + inputDefinitions: + parameters: + struct: + parameterType: STRUCT +schemaVersion: 2.1.0 +sdkVersion: kfp-2.14.3 diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline.py b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline.py rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.py diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.yaml similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline.yaml rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.yaml diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_complex.py b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_complex.py rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.py diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_complex.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.yaml similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_complex.yaml rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.yaml diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_loaded_from_yaml.py b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.py similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_loaded_from_yaml.py rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.py diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.yaml similarity index 100% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_in_pipeline_loaded_from_yaml.yaml rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.yaml diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_with_loops.py b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.py similarity index 96% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_with_loops.py rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.py index 909ff0c0a97..d7183d61b46 100644 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_with_loops.py +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.py @@ -35,7 +35,7 @@ def print_struct(struct: Dict): @dsl.pipeline(name='pipeline-with-loops') -def my_pipeline(loop_parameter: List[str] = ["a", "b"]): +def my_pipeline(loop_parameter: List[str]): # Loop argument is from a pipeline input with dsl.ParallelFor(loop_parameter) as item: diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_with_loops.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.yaml similarity index 98% rename from test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_with_loops.yaml rename to test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.yaml index 778d2379e40..8cf4bec3f96 100644 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/pipeline_with_loops.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.yaml @@ -1,7 +1,7 @@ # PIPELINE DEFINITION # Name: pipeline-with-loops # Inputs: -# loop_parameter: list [Default: ['a', 'b']] +# loop_parameter: list components: comp-args-generator-op: executorLabel: exec-args-generator-op @@ -171,7 +171,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -200,7 +200,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -228,7 +228,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -256,7 +256,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -284,7 +284,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -312,7 +312,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -340,7 +340,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -368,7 +368,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.3'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -438,10 +438,6 @@ root: inputDefinitions: parameters: loop_parameter: - defaultValue: - - a - - b - isOptional: true parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.15.2 +sdkVersion: kfp-2.13.0 diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/__init__.py b/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.py b/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.py deleted file mode 100644 index b970d4e4b61..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Pipeline 4: Two components with nested DAG using ParallelFor and use_secret_as_env. - -Tests the most complex scenario: an outer task produces a secret name, -and an inner ParallelFor loop uses that secret name via use_secret_as_env. -This tests cross-DAG taskOutputParameter rewriting. -""" -from kfp import dsl, compiler -from kfp.kubernetes import use_secret_as_env - - -@dsl.component(base_image="python:3.11") -def emit_secret_name() -> str: - """Emits the secret name dynamically.""" - return "test-secret-1" - - -@dsl.component(base_image="python:3.11") -def worker_component(item: str) -> str: - import os - secret_val = os.environ.get("MY_SECRET_KEY", "not-set") - print(f"Item: {item}, Secret set: {secret_val != 'not-set'}") - return secret_val - - -@dsl.pipeline(name="pipeline-4-nested-parallel-for-secret") -def pipeline_nested_parallel_for_secret(): - """Pipeline with outer task output used as secret name inside ParallelFor.""" - secret_name_task = emit_secret_name() - with dsl.ParallelFor(items=["x", "y"], parallelism=1) as item: - task = worker_component(item=item) - use_secret_as_env( - task, - secret_name=secret_name_task.output, - secret_key_to_env={ - "username": "MY_SECRET_KEY", - }, - ) - - -if __name__ == "__main__": - compiler.Compiler().compile( - pipeline_func=pipeline_nested_parallel_for_secret, - package_path="nested_parallel_for_secret.yaml", - ) - print("Compiled nested_parallel_for_secret.yaml") diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.yaml b/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.yaml deleted file mode 100644 index ba6dc73adfd..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/nested_parallel_for_secret.yaml +++ /dev/null @@ -1,148 +0,0 @@ -# PIPELINE DEFINITION -# Name: pipeline-4-nested-parallel-for-secret -# Description: Pipeline with outer task output used as secret name inside ParallelFor. -components: - comp-emit-secret-name: - executorLabel: exec-emit-secret-name - outputDefinitions: - parameters: - Output: - parameterType: STRING - comp-for-loop-2: - dag: - tasks: - worker-component: - cachingOptions: - enableCache: true - componentRef: - name: comp-worker-component - inputs: - parameters: - item: - componentInputParameter: pipelinechannel--loop-item-param-1 - taskInfo: - name: worker-component - inputDefinitions: - parameters: - pipelinechannel--emit-secret-name-Output: - parameterType: STRING - pipelinechannel--loop-item-param-1: - parameterType: STRING - comp-worker-component: - executorLabel: exec-worker-component - inputDefinitions: - parameters: - item: - parameterType: STRING - outputDefinitions: - parameters: - Output: - parameterType: STRING -deploymentSpec: - executors: - exec-emit-secret-name: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - emit_secret_name - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef emit_secret_name() -> str:\n \"\"\"Emits the secret name dynamically.\"\ - \"\"\n return \"test-secret-1\"\n\n" - image: python:3.11 - exec-worker-component: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - worker_component - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef worker_component(item: str) -> str:\n import os\n secret_val\ - \ = os.environ.get(\"MY_SECRET_KEY\", \"not-set\")\n print(f\"Item: {item},\ - \ Secret set: {secret_val != 'not-set'}\")\n return secret_val\n\n" - image: python:3.11 -pipelineInfo: - description: Pipeline with outer task output used as secret name inside ParallelFor. - name: pipeline-4-nested-parallel-for-secret -root: - dag: - tasks: - emit-secret-name: - cachingOptions: - enableCache: true - componentRef: - name: comp-emit-secret-name - taskInfo: - name: emit-secret-name - for-loop-2: - componentRef: - name: comp-for-loop-2 - dependentTasks: - - emit-secret-name - inputs: - parameters: - pipelinechannel--emit-secret-name-Output: - taskOutputParameter: - outputParameterKey: Output - producerTask: emit-secret-name - iteratorPolicy: - parallelismLimit: 1 - parameterIterator: - itemInput: pipelinechannel--loop-item-param-1 - items: - raw: '["x", "y"]' - taskInfo: - name: for-loop-2 -schemaVersion: 2.1.0 -sdkVersion: kfp-2.15.2 ---- -platforms: - kubernetes: - deploymentSpec: - executors: - exec-worker-component: - secretAsEnv: - - keyToEnv: - - envVar: MY_SECRET_KEY - secretKey: username - optional: false - secretNameParameter: - componentInputParameter: pipelinechannel--emit-secret-name-Output diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.py b/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.py deleted file mode 100644 index 3dad0517584..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Pipeline: Single component inside ParallelFor using mount_pvc. - -Regression coverage for pipeline-parameter propagation through ParallelFor -when the parameter is only referenced from Kubernetes platform config. -""" -from kfp import compiler -from kfp import dsl -from kfp import kubernetes - - -@dsl.component(base_image="python:3.11") -def example_component(item: str) -> str: - print(f"Item: {item}") - return item - - -@dsl.pipeline(name="pipeline-parallel-for-mount-pvc") -def pipeline_parallel_for_mount_pvc(my_pvc_name_suffix: str = "test-pvc"): - """Pipeline with 1 component inside ParallelFor using mount_pvc.""" - pvc = kubernetes.CreatePVC( - pvc_name_suffix='-my-pvc', - access_modes=['ReadWriteOnce'], - size='5Mi', - storage_class_name='standard', - ) - - with dsl.ParallelFor(items=["a"]) as item: - task = example_component(item=item) - kubernetes.mount_pvc( - task, - pvc_name=pvc.outputs["name"], - mount_path="/data", - ) - - -if __name__ == "__main__": - compiler.Compiler().compile( - pipeline_func=pipeline_parallel_for_mount_pvc, - package_path="parallel_for_mount_pvc.yaml", - ) - print("Compiled parallel_for_mount_pvc.yaml") diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.yaml b/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.yaml deleted file mode 100644 index cdfbe9a2c6e..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_mount_pvc.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# PIPELINE DEFINITION -# Name: pipeline-parallel-for-mount-pvc -# Description: Pipeline with 1 component inside ParallelFor using mount_pvc. -# Inputs: -# my_pvc_name_suffix: str [Default: 'test-pvc'] -components: - comp-createpvc: - executorLabel: exec-createpvc - inputDefinitions: - parameters: - access_modes: - description: 'AccessModes to request for the provisioned PVC. May - - be one or more of ``''ReadWriteOnce''``, ``''ReadOnlyMany''``, ``''ReadWriteMany''``, - or - - ``''ReadWriteOncePod''``. Corresponds to `PersistentVolumeClaim.spec.accessModes - `_.' - parameterType: LIST - annotations: - description: Annotations for the PVC's metadata. Corresponds to `PersistentVolumeClaim.metadata.annotations - `_. - isOptional: true - parameterType: STRUCT - pvc_name: - description: 'Name of the PVC. Corresponds to `PersistentVolumeClaim.metadata.name - `_. - Only one of ``pvc_name`` and ``pvc_name_suffix`` can - - be provided.' - isOptional: true - parameterType: STRING - pvc_name_suffix: - description: 'Prefix to use for a dynamically generated name, which - - will take the form ``-``. Only one - - of ``pvc_name`` and ``pvc_name_suffix`` can be provided.' - isOptional: true - parameterType: STRING - size: - description: The size of storage requested by the PVC that will be provisioned. - For example, ``'5Gi'``. Corresponds to `PersistentVolumeClaim.spec.resources.requests.storage - `_. - parameterType: STRING - storage_class_name: - defaultValue: '' - description: 'Name of StorageClass from which to provision the PV - - to back the PVC. ``None`` indicates to use the cluster''s default - - storage_class_name. Set to ``''''`` for a statically specified PVC.' - isOptional: true - parameterType: STRING - volume_name: - description: 'Pre-existing PersistentVolume that should back the - - provisioned PersistentVolumeClaim. Used for statically - - specified PV only. Corresponds to `PersistentVolumeClaim.spec.volumeName - `_.' - isOptional: true - parameterType: STRING - outputDefinitions: - parameters: - name: - parameterType: STRING - comp-example-component: - executorLabel: exec-example-component - inputDefinitions: - parameters: - item: - parameterType: STRING - outputDefinitions: - parameters: - Output: - parameterType: STRING - comp-for-loop-2: - dag: - tasks: - example-component: - cachingOptions: - enableCache: true - componentRef: - name: comp-example-component - inputs: - parameters: - item: - componentInputParameter: pipelinechannel--loop-item-param-1 - taskInfo: - name: example-component - inputDefinitions: - parameters: - pipelinechannel--createpvc-name: - parameterType: STRING - pipelinechannel--loop-item-param-1: - parameterType: STRING -deploymentSpec: - executors: - exec-createpvc: - container: - image: argostub/createpvc - exec-example-component: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - example_component - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef example_component(item: str) -> str:\n print(f\"Item: {item}\"\ - )\n return item\n\n" - image: python:3.11 -pipelineInfo: - description: Pipeline with 1 component inside ParallelFor using mount_pvc. - name: pipeline-parallel-for-mount-pvc -root: - dag: - tasks: - createpvc: - cachingOptions: - enableCache: true - componentRef: - name: comp-createpvc - inputs: - parameters: - access_modes: - runtimeValue: - constant: - - ReadWriteOnce - pvc_name_suffix: - runtimeValue: - constant: -my-pvc - size: - runtimeValue: - constant: 5Mi - storage_class_name: - runtimeValue: - constant: standard - taskInfo: - name: createpvc - for-loop-2: - componentRef: - name: comp-for-loop-2 - dependentTasks: - - createpvc - inputs: - parameters: - pipelinechannel--createpvc-name: - taskOutputParameter: - outputParameterKey: name - producerTask: createpvc - parameterIterator: - itemInput: pipelinechannel--loop-item-param-1 - items: - raw: '["a"]' - taskInfo: - name: for-loop-2 - inputDefinitions: - parameters: - my_pvc_name_suffix: - defaultValue: test-pvc - isOptional: true - parameterType: STRING -schemaVersion: 2.1.0 -sdkVersion: kfp-2.15.2 ---- -platforms: - kubernetes: - deploymentSpec: - executors: - exec-example-component: - pvcMount: - - componentInputParameter: pipelinechannel--createpvc-name - mountPath: /data - pvcNameParameter: - componentInputParameter: pipelinechannel--createpvc-name diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.py b/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.py deleted file mode 100644 index 88ff2fb4f51..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Pipeline 2: Single component inside ParallelFor using use_secret_as_env. - -This is the exact bug scenario from RHOAIENG-54342 / kubeflow/pipelines#13078. -The secret name comes from a pipeline parameter and is used inside a -dsl.ParallelFor loop. Without the fix, the driver fails with: - "parent DAG does not have input parameter my_secret_name" -""" -from kfp import dsl, compiler -from kfp.kubernetes import use_secret_as_env - - -@dsl.component(base_image="python:3.11") -def example_component(item: str) -> str: - import os - secret_val = os.environ.get("MY_SECRET_KEY", "not-set") - print(f"Item: {item}, Secret set: {secret_val != 'not-set'}") - return secret_val - - -@dsl.pipeline(name="pipeline-2-parallel-for-secret") -def pipeline_parallel_for_secret(my_secret_name: str = "test-secret-1"): - """Pipeline with 1 component inside ParallelFor using use_secret_as_env.""" - with dsl.ParallelFor(items=["a", "b", "c"], parallelism=1) as item: - task = example_component(item=item) - use_secret_as_env( - task, - secret_name=my_secret_name, - secret_key_to_env={ - "username": "MY_SECRET_KEY", - }, - ) - - -if __name__ == "__main__": - compiler.Compiler().compile( - pipeline_func=pipeline_parallel_for_secret, - package_path="parallel_for_secret.yaml", - ) - print("Compiled parallel_for_secret.yaml") diff --git a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.yaml b/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.yaml deleted file mode 100644 index 000d9e0fcff..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/parallel_and_nested/parallel_for_secret.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# PIPELINE DEFINITION -# Name: pipeline-2-parallel-for-secret -# Description: Pipeline with 1 component inside ParallelFor using use_secret_as_env. -# Inputs: -# my_secret_name: str [Default: 'test-secret-1'] -components: - comp-example-component: - executorLabel: exec-example-component - inputDefinitions: - parameters: - item: - parameterType: STRING - outputDefinitions: - parameters: - Output: - parameterType: STRING - comp-for-loop-2: - dag: - tasks: - example-component: - cachingOptions: - enableCache: true - componentRef: - name: comp-example-component - inputs: - parameters: - item: - componentInputParameter: pipelinechannel--loop-item-param-1 - taskInfo: - name: example-component - inputDefinitions: - parameters: - pipelinechannel--loop-item-param-1: - parameterType: STRING - pipelinechannel--my_secret_name: - parameterType: STRING -deploymentSpec: - executors: - exec-example-component: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - example_component - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef example_component(item: str) -> str:\n import os\n secret_val\ - \ = os.environ.get(\"MY_SECRET_KEY\", \"not-set\")\n print(f\"Item: {item},\ - \ Secret set: {secret_val != 'not-set'}\")\n return secret_val\n\n" - image: python:3.11 -pipelineInfo: - description: Pipeline with 1 component inside ParallelFor using use_secret_as_env. - name: pipeline-2-parallel-for-secret -root: - dag: - tasks: - for-loop-2: - componentRef: - name: comp-for-loop-2 - inputs: - parameters: - pipelinechannel--my_secret_name: - componentInputParameter: my_secret_name - iteratorPolicy: - parallelismLimit: 1 - parameterIterator: - itemInput: pipelinechannel--loop-item-param-1 - items: - raw: '["a", "b", "c"]' - taskInfo: - name: for-loop-2 - inputDefinitions: - parameters: - my_secret_name: - defaultValue: test-secret-1 - isOptional: true - parameterType: STRING -schemaVersion: 2.1.0 -sdkVersion: kfp-2.15.2 ---- -platforms: - kubernetes: - deploymentSpec: - executors: - exec-example-component: - secretAsEnv: - - keyToEnv: - - envVar: MY_SECRET_KEY - secretKey: username - optional: false - secretNameParameter: - componentInputParameter: pipelinechannel--my_secret_name diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.py b/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.py deleted file mode 100644 index 8e2e2a66d44..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2026 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""ParallelFor pipelines that propagate pipeline parameters through sub-DAGs. - -These pipelines verify that pipeline-level parameters are correctly surfaced -through ParallelFor sub-DAG boundaries and accessible to components inside -the loop, alongside the loop item variable. -""" - -from typing import List - -from kfp import compiler -from kfp import dsl -from kfp.dsl import component - - -@component -def print_with_prefix(prefix: str, item: str) -> str: - result = f'{prefix}: {item}' - print(result) - return result - - -@component -def make_prefix() -> str: - return 'dynamic-prefix' - - -@dsl.pipeline(name='pipeline-parallelfor-pipeline-param') -def my_pipeline(prefix: str, loop_parameter: List[str]): - """Pipeline parameter used alongside loop item inside ParallelFor.""" - - # Pipeline parameter propagated into ParallelFor sub-DAG - with dsl.ParallelFor(items=loop_parameter, parallelism=1) as item: - print_with_prefix(prefix=prefix, item=item) - - # Outer task output propagated into ParallelFor sub-DAG - prefix_task = make_prefix() - with dsl.ParallelFor(items=loop_parameter) as item: - print_with_prefix(prefix=prefix_task.output, item=item) - - # Nested ParallelFor with pipeline parameter - static_items = ['x', 'y'] - with dsl.ParallelFor(items=static_items, parallelism=2) as outer_item: - with dsl.ParallelFor(items=loop_parameter) as inner_item: - print_with_prefix(prefix=prefix, item=inner_item) - - -if __name__ == '__main__': - compiler.Compiler().compile( - pipeline_func=my_pipeline, - package_path=__file__.replace('.py', '.yaml')) diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.yaml deleted file mode 100644 index 13f4588a42e..00000000000 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_pipeline_param.yaml +++ /dev/null @@ -1,338 +0,0 @@ -# PIPELINE DEFINITION -# Name: pipeline-parallelfor-pipeline-param -# Description: Pipeline parameter used alongside loop item inside ParallelFor. -# Inputs: -# loop_parameter: list -# prefix: str -components: - comp-for-loop-1: - dag: - tasks: - print-with-prefix: - cachingOptions: - enableCache: true - componentRef: - name: comp-print-with-prefix - inputs: - parameters: - item: - componentInputParameter: pipelinechannel--loop_parameter-loop-item - prefix: - componentInputParameter: pipelinechannel--prefix - taskInfo: - name: print-with-prefix - inputDefinitions: - parameters: - pipelinechannel--loop_parameter: - parameterType: LIST - pipelinechannel--loop_parameter-loop-item: - parameterType: STRING - pipelinechannel--prefix: - parameterType: STRING - comp-for-loop-2: - dag: - tasks: - print-with-prefix-2: - cachingOptions: - enableCache: true - componentRef: - name: comp-print-with-prefix-2 - inputs: - parameters: - item: - componentInputParameter: pipelinechannel--loop_parameter-loop-item - prefix: - componentInputParameter: pipelinechannel--make-prefix-Output - taskInfo: - name: print-with-prefix-2 - inputDefinitions: - parameters: - pipelinechannel--loop_parameter: - parameterType: LIST - pipelinechannel--loop_parameter-loop-item: - parameterType: STRING - pipelinechannel--make-prefix-Output: - parameterType: STRING - comp-for-loop-4: - dag: - tasks: - for-loop-5: - componentRef: - name: comp-for-loop-5 - inputs: - parameters: - pipelinechannel--loop_parameter: - componentInputParameter: pipelinechannel--loop_parameter - pipelinechannel--prefix: - componentInputParameter: pipelinechannel--prefix - parameterIterator: - itemInput: pipelinechannel--loop_parameter-loop-item - items: - inputParameter: pipelinechannel--loop_parameter - taskInfo: - name: for-loop-5 - inputDefinitions: - parameters: - pipelinechannel--loop-item-param-3: - parameterType: STRING - pipelinechannel--loop_parameter: - parameterType: LIST - pipelinechannel--prefix: - parameterType: STRING - comp-for-loop-5: - dag: - tasks: - print-with-prefix-3: - cachingOptions: - enableCache: true - componentRef: - name: comp-print-with-prefix-3 - inputs: - parameters: - item: - componentInputParameter: pipelinechannel--loop_parameter-loop-item - prefix: - componentInputParameter: pipelinechannel--prefix - taskInfo: - name: print-with-prefix-3 - inputDefinitions: - parameters: - pipelinechannel--loop_parameter: - parameterType: LIST - pipelinechannel--loop_parameter-loop-item: - parameterType: STRING - pipelinechannel--prefix: - parameterType: STRING - comp-make-prefix: - executorLabel: exec-make-prefix - outputDefinitions: - parameters: - Output: - parameterType: STRING - comp-print-with-prefix: - executorLabel: exec-print-with-prefix - inputDefinitions: - parameters: - item: - parameterType: STRING - prefix: - parameterType: STRING - outputDefinitions: - parameters: - Output: - parameterType: STRING - comp-print-with-prefix-2: - executorLabel: exec-print-with-prefix-2 - inputDefinitions: - parameters: - item: - parameterType: STRING - prefix: - parameterType: STRING - outputDefinitions: - parameters: - Output: - parameterType: STRING - comp-print-with-prefix-3: - executorLabel: exec-print-with-prefix-3 - inputDefinitions: - parameters: - item: - parameterType: STRING - prefix: - parameterType: STRING - outputDefinitions: - parameters: - Output: - parameterType: STRING -deploymentSpec: - executors: - exec-make-prefix: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - make_prefix - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef make_prefix() -> str:\n return 'dynamic-prefix'\n\n" - image: python:3.11 - exec-print-with-prefix: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - print_with_prefix - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef print_with_prefix(prefix: str, item: str) -> str:\n result\ - \ = f'{prefix}: {item}'\n print(result)\n return result\n\n" - image: python:3.11 - exec-print-with-prefix-2: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - print_with_prefix - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef print_with_prefix(prefix: str, item: str) -> str:\n result\ - \ = f'{prefix}: {item}'\n print(result)\n return result\n\n" - image: python:3.11 - exec-print-with-prefix-3: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - print_with_prefix - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef print_with_prefix(prefix: str, item: str) -> str:\n result\ - \ = f'{prefix}: {item}'\n print(result)\n return result\n\n" - image: python:3.11 -pipelineInfo: - description: Pipeline parameter used alongside loop item inside ParallelFor. - name: pipeline-parallelfor-pipeline-param -root: - dag: - tasks: - for-loop-1: - componentRef: - name: comp-for-loop-1 - inputs: - parameters: - pipelinechannel--loop_parameter: - componentInputParameter: loop_parameter - pipelinechannel--prefix: - componentInputParameter: prefix - iteratorPolicy: - parallelismLimit: 1 - parameterIterator: - itemInput: pipelinechannel--loop_parameter-loop-item - items: - inputParameter: pipelinechannel--loop_parameter - taskInfo: - name: for-loop-1 - for-loop-2: - componentRef: - name: comp-for-loop-2 - dependentTasks: - - make-prefix - inputs: - parameters: - pipelinechannel--loop_parameter: - componentInputParameter: loop_parameter - pipelinechannel--make-prefix-Output: - taskOutputParameter: - outputParameterKey: Output - producerTask: make-prefix - parameterIterator: - itemInput: pipelinechannel--loop_parameter-loop-item - items: - inputParameter: pipelinechannel--loop_parameter - taskInfo: - name: for-loop-2 - for-loop-4: - componentRef: - name: comp-for-loop-4 - inputs: - parameters: - pipelinechannel--loop_parameter: - componentInputParameter: loop_parameter - pipelinechannel--prefix: - componentInputParameter: prefix - iteratorPolicy: - parallelismLimit: 2 - parameterIterator: - itemInput: pipelinechannel--loop-item-param-3 - items: - raw: '["x", "y"]' - taskInfo: - name: for-loop-4 - make-prefix: - cachingOptions: - enableCache: true - componentRef: - name: comp-make-prefix - taskInfo: - name: make-prefix - inputDefinitions: - parameters: - loop_parameter: - parameterType: LIST - prefix: - parameterType: STRING -schemaVersion: 2.1.0 -sdkVersion: kfp-2.15.2