diff --git a/api/generated/crds/config.porch.kpt.dev_functionconfigs.yaml b/api/generated/crds/config.porch.kpt.dev_functionconfigs.yaml index 535ee42ed..1c3916607 100644 --- a/api/generated/crds/config.porch.kpt.dev_functionconfigs.yaml +++ b/api/generated/crds/config.porch.kpt.dev_functionconfigs.yaml @@ -31,9 +31,6 @@ spec: - jsonPath: .status.apiServerObservedGeneration name: Server Applied type: integer - - jsonPath: .status.functionRunnerObservedGeneration - name: FnRunner Applied - type: integer - jsonPath: .status.controllerObservedGeneration name: Controller Applied type: integer @@ -923,12 +920,6 @@ spec: description: Contains an error message if one occurred whilst trying to apply the FunctionConfig type: string - functionRunnerObservedGeneration: - description: FunctionRunnerObservedGeneration indicates which generation - of the config the function-runner has applied to the executable - and pod evaluator - format: int64 - type: integer type: object type: object served: true diff --git a/api/porchconfig/v1alpha1/function_config_types.go b/api/porchconfig/v1alpha1/function_config_types.go index 03c8a2d35..296d38690 100644 --- a/api/porchconfig/v1alpha1/function_config_types.go +++ b/api/porchconfig/v1alpha1/function_config_types.go @@ -23,7 +23,6 @@ import ( // +kubebuilder:subresource:status // +kubebuilder:resource:path=functionconfigs,singular=functionconfig // +kubebuilder:printcolumn:name="Server Applied",type=integer,JSONPath=`.status.apiServerObservedGeneration` -// +kubebuilder:printcolumn:name="FnRunner Applied",type=integer,JSONPath=`.status.functionRunnerObservedGeneration` // +kubebuilder:printcolumn:name="Controller Applied",type=integer,JSONPath=`.status.controllerObservedGeneration` type FunctionConfig struct { metav1.TypeMeta `json:",inline"` @@ -57,8 +56,6 @@ type FunctionConfigStatus struct { // ApiServerObservedGeneration indicates which generation of the config the porch server has applied to the build-in runtime ApiServerObservedGeneration int64 `json:"apiServerObservedGeneration,omitempty"` - // FunctionRunnerObservedGeneration indicates which generation of the config the function-runner has applied to the executable and pod evaluator - FunctionRunnerObservedGeneration int64 `json:"functionRunnerObservedGeneration,omitempty"` // ControllerObservedGeneration indicates which generation of the config the porch controller has applied to its builtin runtime ControllerObservedGeneration int64 `json:"controllerObservedGeneration,omitempty"` // Contains an error message if one occurred whilst trying to apply the FunctionConfig diff --git a/controllers/functionconfigs/functionconfigreconciler.go b/controllers/functionconfigs/functionconfigreconciler.go index 8872a2a35..2f8155589 100644 --- a/controllers/functionconfigs/functionconfigreconciler.go +++ b/controllers/functionconfigs/functionconfigreconciler.go @@ -186,29 +186,29 @@ func (s *FunctionConfigStore) GetBinaryFromCache(image string) (string, bool) { return "", false } -func (s *FunctionConfigStore) GetBinaryFromCacheByConstraint(image, tag string) (string, bool) { +func (s *FunctionConfigStore) GetBinaryFromCacheByConstraint(image, tag string) (string, string, bool) { s.mu.RLock() defer s.mu.RUnlock() parsedImage := imageutil.Parse(image) cacheEntry, ok := s.binaryExecutorCache[parsedImage.BaseName] if !ok { - return "", false + return "", "", false } if !cacheEntry.PrefixRegex.MatchString(parsedImage.Prefix()) { - return "", false + return "", "", false } cacheKeys := slices.Collect(maps.Keys(cacheEntry.Tags)) selectedKey, err := imageutil.FindBestSemverMatch(tag, cacheKeys) if err != nil { - return "", false + return "", "", false } selectedBinary, ok := cacheEntry.Tags[selectedKey] - return selectedBinary, ok + return selectedBinary, selectedKey, ok } func (s *FunctionConfigStore) GetExecCache() map[string]BuiltInCacheEntry { @@ -246,9 +246,8 @@ func (s *FunctionConfigStore) List() []*configapi.FunctionConfig { type ReconcilerFor string const ( - ReconcilerForFunctionRunner ReconcilerFor = "function-runner" - ReconcilerForServer ReconcilerFor = "server" - ReconcilerForController ReconcilerFor = "controller" + ReconcilerForServer ReconcilerFor = "server" + ReconcilerForController ReconcilerFor = "controller" ) type Reconciler struct { @@ -292,15 +291,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl. } defer func() { - patch := client.MergeFrom(obj.DeepCopy()) + patch := client.MergeFromWithOptions(obj.DeepCopy()) if finalErr != nil { obj.Status.Error = finalErr.Error() } else { obj.Status.Error = "" switch r.For { - case ReconcilerForFunctionRunner: - obj.Status.FunctionRunnerObservedGeneration = obj.Generation case ReconcilerForServer: obj.Status.ApiServerObservedGeneration = obj.Generation case ReconcilerForController: @@ -342,8 +339,6 @@ func (r *Reconciler) removeFinalizer(ctx context.Context, obj *configapi.Functio patch := client.MergeFrom(obj.DeepCopy()) switch r.For { - case ReconcilerForFunctionRunner: - controllerutil.RemoveFinalizer(obj, FunctionRunnerFinalizer) case ReconcilerForServer: controllerutil.RemoveFinalizer(obj, ServerFinalizer) case ReconcilerForController: @@ -363,8 +358,6 @@ func (r *Reconciler) addFinalizer(ctx context.Context, obj *configapi.FunctionCo updated := false switch r.For { - case ReconcilerForFunctionRunner: - updated = controllerutil.AddFinalizer(obj, FunctionRunnerFinalizer) case ReconcilerForServer: updated = controllerutil.AddFinalizer(obj, ServerFinalizer) case ReconcilerForController: diff --git a/controllers/functionconfigs/functionconfigreconciler_test.go b/controllers/functionconfigs/functionconfigreconciler_test.go index 55e012645..1a158108a 100644 --- a/controllers/functionconfigs/functionconfigreconciler_test.go +++ b/controllers/functionconfigs/functionconfigreconciler_test.go @@ -229,10 +229,6 @@ func TestFinalizersAdded(t *testing.T) { forValue ReconcilerFor finalizer string }{ - string(ReconcilerForFunctionRunner): { - forValue: ReconcilerForFunctionRunner, - finalizer: FunctionRunnerFinalizer, - }, string(ReconcilerForServer): { forValue: ReconcilerForServer, finalizer: ServerFinalizer, @@ -338,7 +334,7 @@ func TestGetBinaryFromCacheByConstraint(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { - path, found := store.GetBinaryFromCacheByConstraint(tc.image, tc.constraint) + path, _, found := store.GetBinaryFromCacheByConstraint(tc.image, tc.constraint) assert.Equal(t, tc.wantFound, found) if tc.wantFound { assert.Equal(t, tc.wantPath, path) @@ -492,10 +488,6 @@ func TestFinalizersRemoved(t *testing.T) { forValue ReconcilerFor finalizer string }{ - string(ReconcilerForFunctionRunner): { - forValue: ReconcilerForFunctionRunner, - finalizer: FunctionRunnerFinalizer, - }, string(ReconcilerForServer): { forValue: ReconcilerForServer, finalizer: ServerFinalizer, diff --git a/controllers/main.go b/controllers/main.go index 828f3b5e8..288830582 100644 --- a/controllers/main.go +++ b/controllers/main.go @@ -311,7 +311,11 @@ func setupFunctionConfigReconciler(mgr ctrl.Manager) (*functionconfigs.FunctionC if prefix == "" { prefix = runneroptions.GHCRImagePrefix } - functionConfigStore := functionconfigs.NewFunctionConfigStore(prefix, "") + functionCacheDir := os.Getenv("FUNCTION_CACHE_DIR") + if functionCacheDir == "" { + functionCacheDir = "/home/nonroot/functions" + } + functionConfigStore := functionconfigs.NewFunctionConfigStore(prefix, functionCacheDir) rec := &functionconfigs.Reconciler{ Client: mgr.GetClient(), diff --git a/controllers/packagerevisions/pkg/controllers/packagerevision/config.go b/controllers/packagerevisions/pkg/controllers/packagerevision/config.go index 3e328262a..25fe75b77 100644 --- a/controllers/packagerevisions/pkg/controllers/packagerevision/config.go +++ b/controllers/packagerevisions/pkg/controllers/packagerevision/config.go @@ -15,6 +15,7 @@ package packagerevision import ( + "context" "flag" "fmt" "os" @@ -24,8 +25,10 @@ import ( "github.com/kptdev/porch/controllers/packagerevisions/pkg/webhooks" "github.com/kptdev/porch/pkg/cache/contentcache" "github.com/kptdev/porch/pkg/engine" + "github.com/kptdev/porch/pkg/engine/podevaluator" porch "github.com/kptdev/porch/pkg/registry/porch" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) @@ -35,6 +38,7 @@ const ( defaultRenderRequeueDelay = 2 * time.Second defaultRepoOperationRetryAttempts = 3 defaultMaxGRPCMessageSize = 6 * 1024 * 1024 // 6MB + defaultPodNamespace = "porch-fn-system" ) func (r *PackageRevisionReconciler) InitDefaults() { @@ -78,21 +82,60 @@ func (r *PackageRevisionReconciler) Init(mgr ctrl.Manager) error { ) fnRunnerAddr := os.Getenv("FUNCTION_RUNNER_ADDRESS") - functionRuntime, err := engine.NewMultiFunctionRuntime(fnRunnerAddr, r.MaxGRPCMessageSize, r.FunctionConfigStore) - if err != nil { - return fmt.Errorf("failed to create function runtime: %w", err) - } - opts := runneroptions.RunnerOptions{} + wrapperServerImage := os.Getenv("WRAPPER_SERVER_IMAGE") + prefix := os.Getenv("DEFAULT_IMAGE_PREFIX") if prefix == "" { prefix = runneroptions.GHCRImagePrefix } + + var podOpts *podevaluator.PodEvaluatorOptions + var kubeClient client.WithWatch + if wrapperServerImage != "" { + podNamespace := os.Getenv("POD_NAMESPACE") + if podNamespace == "" { + podNamespace = defaultPodNamespace + } + var err error + kubeClient, err = client.NewWithWatch(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()}) + if err != nil { + return fmt.Errorf("failed to create kube client for pod evaluator: %w", err) + } + podOpts = &podevaluator.PodEvaluatorOptions{ + PodNamespace: podNamespace, + WrapperServerImage: wrapperServerImage, + WarmUpPodCacheOnStartup: true, + MaxGrpcMessageSize: r.MaxGRPCMessageSize, + DefaultImagePrefix: prefix, + MaxWaitlistLength: 1, + MaxParallelPodsPerFunction: 2, + } + } + + functionRuntime, err := engine.NewMultiFunctionRuntime(context.Background(), engine.MultiFunctionRuntimeOptions{ + GRPCAddress: fnRunnerAddr, + MaxGrpcMessageSize: r.MaxGRPCMessageSize, + FunctionConfigStore: r.FunctionConfigStore, + PodEvaluator: podOpts, + KubeClient: kubeClient, + DefaultImagePrefix: prefix, + }) + if err != nil { + return fmt.Errorf("failed to create function runtime: %w", err) + } + opts := runneroptions.RunnerOptions{} opts.InitDefaults(prefix) r.Renderer = newKptRenderer(functionRuntime, opts) - if fnRunnerAddr != "" { - ctrl.Log.WithName(r.Name()).Info("function runtime enabled (builtin + fn-runner)", "address", fnRunnerAddr) - } else { - ctrl.Log.WithName(r.Name()).Info("function runtime enabled (builtin only, FUNCTION_RUNNER_ADDRESS not set)") + switch { + case fnRunnerAddr != "" && wrapperServerImage != "": + log.Info("function runtime enabled (builtin + fn-runner + pod evaluator)", + "fnRunner", fnRunnerAddr, "podNamespace", podOpts.PodNamespace) + case fnRunnerAddr != "": + log.Info("function runtime enabled (builtin + fn-runner)", "address", fnRunnerAddr) + case wrapperServerImage != "": + log.Info("function runtime enabled (builtin + pod evaluator)", "podNamespace", podOpts.PodNamespace) + default: + log.Info("function runtime enabled (builtin only)") } // Register PackageRevision validating webhook. diff --git a/deployments/function-pods/README.md b/deployments/function-pods/README.md index 7c78ec539..6630cecc5 100644 --- a/deployments/function-pods/README.md +++ b/deployments/function-pods/README.md @@ -1,35 +1,13 @@ ### Function Pod Template -In order to leverage custom manifests for Pod and frontend Service of Function Pods created by Function Runner, the following additional Kubernetes Resource Manifests (KRM) need to be provisioned in the Porch environment +Function execution pods are created by the **Engine pod evaluator** (porch-server and, when enabled, porch-controllers), not Function Runner. -* A ConfigMap containing 2 data elements: a) a KRM of type Pod under the `template` key and b) a KRM of type Service under `serviceTemplate` key -* A Kubernetes Role providing read access to resource type ConfigMap in the porch-system namespace -* A Kubernetes RoleBinding, binding to the above listed Role to the ServiceAccount (porch-fn-runner) used by Function Runner Pod +Default manifests install Kubernetes `PodTemplate` `base-pod-template` and `ServiceTemplate` `base-service-template` in `porch-fn-system`. See `deployments/porch/22-function-templates.yaml` and [Pod Templates](../../docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/pod-templates.md). -All of the above KRMs are predefined in `deployment.yaml` file present in this folder. +The ConfigMap-based `--function-pod-template` flow documented previously for Function Runner is no longer used. -### How to enable Function Pod Template use by Function Runner +### How to customize -* Apply the [deployment.yaml manifest](deployment.yaml) from this directory +Edit `base-pod-template` in `porch-fn-system` (or the YAML in `deployments/porch/22-function-templates.yaml` before deploy). porch-server and porch-controllers already have RBAC via the `porch-function-executor` Role. -``` -kubectl apply -f deployment.yaml -``` - -* Add an additional argument `--function-pod-template` in command section of function-runner deployment instructing it to use the Function Pod Template ConfigMap, as shown below - -``` -kubectl edit deployment -n porch-system function-runner -``` - -``` - command: - - /server - - --config=/config.yaml - - --functions=/functions - - --pod-namespace=porch-fn-system - - --function-pod-template=kpt-function-eval-pod-template - - --max-request-body-size=6291456 # Keep this in sync with porch-server's corresponding argument -``` - -After the function-runner Pods restart, they will start using the Pod and Service templates from ConfigMap. +This folder's `deployment.yaml` is a historical ConfigMap example and is not wired into current porch-server. diff --git a/deployments/porch/2-function-runner.yaml b/deployments/porch/2-function-runner.yaml index d5eefcaf6..ea130715c 100644 --- a/deployments/porch/2-function-runner.yaml +++ b/deployments/porch/2-function-runner.yaml @@ -43,15 +43,9 @@ spec: runAsGroup: 10001 command: - /home/nonroot/server - - --pod-namespace=porch-fn-system - --max-request-body-size=6291456 # Keep this in sync with porch-server's corresponding argument - - --max-parallel-pods-per-function=2 - - --max-waitlist-length=1 - - --warm-up-pod-cache=true - --functions=/home/nonroot/functions env: - - name: WRAPPER_SERVER_IMAGE - value: ghcr.io/kptdev/porch-wrapper-server:latest - name: OTEL_METRICS_EXPORTER value: "prometheus" - name: OTEL_EXPORTER_PROMETHEUS_HOST diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index 94619a5d0..e8f30f4d3 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -76,6 +76,8 @@ spec: value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER value: none + - name: WRAPPER_SERVER_IMAGE + value: ghcr.io/kptdev/porch-wrapper-server:latest args: - --function-runner=function-runner:9445 - --cache-directory=/cache @@ -86,6 +88,10 @@ spec: - --max-request-body-size=6291456 # Keep this in sync with function-runner's corresponding argument - --cache-type=db - --disable-admission-plugins=MutatingAdmissionPolicy # This can be enabled once kindest/node 1.36.1 is released + - --max-parallel-pods-per-function=2 + - --max-waitlist-length=1 + - --warm-up-pod-cache=true + - --functions=/home/nonroot/functions ports: - containerPort: 9464 name: metrics @@ -119,7 +125,7 @@ spec: periodSeconds: 10 failureThreshold: 6 successThreshold: 1 - timeoutSeconds: 5 + timeoutSeconds: 5 --- apiVersion: v1 kind: Service diff --git a/deployments/porch/6-rbac-bind.yaml b/deployments/porch/6-rbac-bind.yaml index 294bf8536..08351b3a4 100644 --- a/deployments/porch/6-rbac-bind.yaml +++ b/deployments/porch/6-rbac-bind.yaml @@ -48,6 +48,12 @@ roleRef: kind: Role name: porch-function-executor subjects: + - kind: ServiceAccount + name: porch-server + namespace: porch-system + - kind: ServiceAccount + name: porch-controllers + namespace: porch-system - kind: ServiceAccount name: porch-fn-runner namespace: porch-system diff --git a/deployments/porch/9-controllers.yaml b/deployments/porch/9-controllers.yaml index 2859e4faa..35ca72766 100644 --- a/deployments/porch/9-controllers.yaml +++ b/deployments/porch/9-controllers.yaml @@ -70,6 +70,12 @@ spec: value: "true" - name: GIT_CACHE_DIR value: "/cache" + - name: FUNCTION_CACHE_DIR + value: "/home/nonroot/functions" + - name: POD_NAMESPACE + value: "porch-fn-system" + - name: WRAPPER_SERVER_IMAGE + value: ghcr.io/kptdev/porch-wrapper-server:latest - name: OTEL_SERVICE_NAME value: porch-controllers - name: OTEL_METRICS_EXPORTER diff --git a/deployments/porch/README.md b/deployments/porch/README.md index 64b15bd3a..0ee9e0d16 100644 --- a/deployments/porch/README.md +++ b/deployments/porch/README.md @@ -39,7 +39,7 @@ make deployment-config - `IMAGE_REPO`: Set image repository (default: `ghcr.io/kptdev`) - `ENABLED_RECONCILERS`: Comma-separated list of reconcilers (default: `packagevariants,packagevariantsets,repositories`) - `FN_RUNNER_WARM_UP_POD_CACHE`: Enable/disable pod cache warm-up (default: `true`) -- `PORCH_GHCR_PREFIX_URL`: KRM function catalog registry prefix (from `.env` or environment). Applied to porch-server (`--default-image-prefix`), function-runner (`--default-image-prefix`), and porch-controllers (`DEFAULT_IMAGE_PREFIX`) for all `make run-in-kind*` targets that use `deployment-config`. +- `PORCH_GHCR_PREFIX_URL`: KRM function catalog registry prefix (from `.env` or environment). Applied to porch-server (`--default-image-prefix`) and porch-controllers (`DEFAULT_IMAGE_PREFIX`) for all `make run-in-kind*` targets that use `deployment-config`. Examples: diff --git a/docs/content/en/docs/11_glossary/_index.md b/docs/content/en/docs/11_glossary/_index.md index 5e5f4a40a..2118dedc4 100644 --- a/docs/content/en/docs/11_glossary/_index.md +++ b/docs/content/en/docs/11_glossary/_index.md @@ -177,9 +177,9 @@ Package Orchestration Server - "kpt-as-a-service". Porch provides opinionated pa ### Porch Server -The main Porch component implemented as a Kubernetes aggregated API server. It serves PackageRevision, PackageRevisionResources, and Repository APIs, and includes the orchestration engine, package cache, repository adapters, and function runner runtime. +The main Porch component implemented as a Kubernetes aggregated API server. It serves PackageRevision, PackageRevisionResources, and Repository APIs, and includes the orchestration engine, package cache, repository adapters, and the in-process pod evaluator. -*See also*: [Aggregated API Server](#aggregated-api-server), [Function Runner](#function-runner) +*See also*: [Aggregated API Server](#aggregated-api-server), [Function Runner](#function-runner), [Pod Evaluator](#pod-evaluator) ### Aggregated API Server @@ -189,9 +189,15 @@ A Kubernetes extension mechanism that allows adding custom API servers to a clus ### Function Runner -A Porch microservice responsible for evaluating KRM functions. It exposes a gRPC endpoint and maintains a cache of functions to support low-latency execution. Functions can be executed directly (built-in) or in separate pods (on-demand). +A Porch microservice that evaluates **cached** KRM function binaries over gRPC. The Engine supplies `exec_path` from the FunctionConfig store. Container-based (pod) evaluation runs in the Engine (porch-server and the PackageRevision controller), not in Function Runner. -*See also*: [KRM Function](#krm-function), [Rendering](#rendering) +*See also*: [KRM Function](#krm-function), [Rendering](#rendering), [Pod Evaluator](#pod-evaluator) + +### Pod Evaluator + +In-process Engine runtime that executes KRM functions in Kubernetes pods with wrapper-server gRPC, TTL-based pod cache, and image/registry authentication. Required on porch-server (`WRAPPER_SERVER_IMAGE`). Optional on the PackageRevision controller when that env is set. + +*See also*: [Function Runner](#function-runner), [KRM Function](#krm-function) --- @@ -225,7 +231,7 @@ Kubernetes Resource Model - the declarative, intent-based API model and machiner An executable that takes Kubernetes resources as input and produces Kubernetes resources as output. Functions can add, remove, or modify resources. In Porch, functions are declared in a package's Kptfile and executed during rendering. -*See also*: [Function Runner](#function-runner), [Rendering](#rendering) +*See also*: [Function Runner](#function-runner), [Pod Evaluator](#pod-evaluator), [Rendering](#rendering) ### Rendering @@ -233,7 +239,7 @@ The process of executing KRM functions defined in a package's Kptfile pipeline. By default, render failures prevent resources from being persisted. The `porch.kpt.dev/push-on-render-failure` annotation can override this behavior to save work-in-progress packages even when rendering fails. -*See also*: [KRM Function](#krm-function), [Function Runner](#function-runner), [Push on Render Failure](#push-on-render-failure) +*See also*: [KRM Function](#krm-function), [Function Runner](#function-runner), [Pod Evaluator](#pod-evaluator), [Push on Render Failure](#push-on-render-failure) ### Push on Render Failure diff --git a/docs/content/en/docs/1_overview/_index.md b/docs/content/en/docs/1_overview/_index.md index 7fe5034ac..7001d59ec 100644 --- a/docs/content/en/docs/1_overview/_index.md +++ b/docs/content/en/docs/1_overview/_index.md @@ -70,7 +70,7 @@ Porch consists of three main deployable components. The **Porch Server** is a Kubernetes aggregated apiserver that serves the `porch.kpt.dev/v1alpha1` API PackageRevision, PackageRevisionResources, and Package resources. It includes the Engine (orchestration logic), the Cache (repository content), and Repository Adapters that abstract Git backends. As the architecture evolves toward the CRD-based model, the server will remain, and continue to serve PackageRevisionResources (PRR) package file content that can exceed etcd size limits and provides the v1alpha1 API for existing PackageRevisionResources (PRR) clients. -The **Function Runner** is a separate gRPC service that runs KRM functions in containers. It can execute both functions provided by Porch and externally developed function images. +The **Function Runner** is a separate gRPC service that runs **cached** KRM function binaries. The Engine looks up the binary and sends `exec_path`. Container images that are not cached run in the Engine pod evaluator (porch-server / PackageRevision controller). The **Controllers** are a set of Kubernetes controllers that manage package lifecycle and automate operations: diff --git a/docs/content/en/docs/2_concepts/functions.md b/docs/content/en/docs/2_concepts/functions.md index b14dd89bd..e9b8e28ca 100644 --- a/docs/content/en/docs/2_concepts/functions.md +++ b/docs/content/en/docs/2_concepts/functions.md @@ -22,7 +22,7 @@ For details on how to declare and configure functions in the Kptfile pipeline, s ## Function Execution in Porch -Porch executes functions through a **function runner** component that calls kpt to orchestrate containerized function execution. The functions run in isolated containers (Kubernetes pods managed by the `function-runner` microservice). Porch passes the package's resources to kpt, which passes the resources on as a [ResourceList](https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md#resourcelist) to each function in the pipeline in turn. kpt executes the functions sequentially in the order declared in the Kptfile pipeline and passes the function results back to Porch, which stores them in the PackageRevisionResources's `status.renderStatus` field. Execution is triggered automatically following creation or clone of a package revision, update of a package revision, and when a package revision is proposed. +Porch executes functions through the **Engine** function runtime chain: builtin Go functions, the Function Runner (cached binaries via `exec_path`), then the in-process **pod evaluator**. Containerized functions run in Kubernetes pods managed by porch-server (and by the PackageRevision controller when `WRAPPER_SERVER_IMAGE` is set). Porch passes the package's resources to kpt, which passes the resources on as a [ResourceList](https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md#resourcelist) to each function in the pipeline in turn. kpt executes the functions sequentially in the order declared in the Kptfile pipeline and passes the function results back to Porch, which stores them in the PackageRevisionResources's `status.renderStatus` field. Execution is triggered automatically following creation or clone of a package revision, update of a package revision, and when a package revision is proposed. ## When Functions Execute diff --git a/docs/content/en/docs/3_getting_started/installing-porch.md b/docs/content/en/docs/3_getting_started/installing-porch.md index ee99c7970..b852e17a6 100644 --- a/docs/content/en/docs/3_getting_started/installing-porch.md +++ b/docs/content/en/docs/3_getting_started/installing-porch.md @@ -130,18 +130,20 @@ If `kubectl api-resources | grep porch` shows nothing: kubectl get apiservice v1alpha1.porch.kpt.dev -o yaml ``` -### Function runner issues +### Function evaluation issues If function execution fails: -1. Check function-runner logs: +1. Check porch-server logs (pod evaluator) and function-runner logs (exec fast path): ```bash + kubectl logs -n porch-system deployment/porch-server kubectl logs -n porch-system deployment/function-runner ``` -2. Verify function-runner service: +2. Confirm `WRAPPER_SERVER_IMAGE` is set on porch-server, and that the function-runner service is up if you use cached binaries: ```bash kubectl get svc -n porch-system function-runner + kubectl -n porch-system get deploy porch-server -o jsonpath='{.spec.template.spec.containers[0].env}' | grep WRAPPER ``` ## Next Steps diff --git a/docs/content/en/docs/4_tutorials_and_how-tos/working_with_crd_based_packagerevisions/_index.md b/docs/content/en/docs/4_tutorials_and_how-tos/working_with_crd_based_packagerevisions/_index.md index 58c972824..a63e677b2 100644 --- a/docs/content/en/docs/4_tutorials_and_how-tos/working_with_crd_based_packagerevisions/_index.md +++ b/docs/content/en/docs/4_tutorials_and_how-tos/working_with_crd_based_packagerevisions/_index.md @@ -13,7 +13,8 @@ Before enabling the CRD-based architecture, ensure you have: - **Porch** deployed in your cluster (with support for the `Packagerevisions` reconciler) - **kubectl** configured to communicate with the cluster - **Repository Controller** running (this is part of the default Porch deployment) -- **Function runner** deployed and reachable (for KRM function rendering) +- **WRAPPER_SERVER_IMAGE** set on porch-controllers (default manifests already set this) for container-based KRM functions +- **Function runner** optional: set `FUNCTION_RUNNER_ADDRESS` if you also want cached-binary exec ## What is the CRD-based architecture? @@ -106,16 +107,15 @@ You should see log lines indicating the reconciler has started: "Starting workers" controller="packagerevisions" worker count=50 ``` -## Verify Function Runner +## Verify function evaluation -If you plan to use external KRM functions (container-based), confirm the function runner is reachable: +If you plan to use container-based KRM functions, confirm `WRAPPER_SERVER_IMAGE` is set on porch-controllers. Function Runner is optional (exec fast path) and is **not** set on default controller manifests. ```bash +kubectl -n porch-system get deploy porch-controllers -o jsonpath='{.spec.template.spec.containers[0].env}' | grep -o 'WRAPPER_SERVER_IMAGE[^}]*' kubectl -n porch-system get pods -l app=function-runner ``` -The `FUNCTION_RUNNER_ADDRESS` environment variable must be set on the controllers deployment. The default Porch manifests already configure this. - ## Next Steps - [Creating packages]({{% relref "/docs/4_tutorials_and_how-tos/working_with_crd_based_packagerevisions/creating-packages" %}}): create, render, and publish a package using the PR Controller diff --git a/docs/content/en/docs/4_tutorials_and_how-tos/working_with_package_revisions/_index.md b/docs/content/en/docs/4_tutorials_and_how-tos/working_with_package_revisions/_index.md index 28b47dae2..5e7182203 100644 --- a/docs/content/en/docs/4_tutorials_and_how-tos/working_with_package_revisions/_index.md +++ b/docs/content/en/docs/4_tutorials_and_how-tos/working_with_package_revisions/_index.md @@ -87,7 +87,7 @@ PackageRevisions contain structured configuration files that can be modified thr KRM functions defined in the Kptfile automatically transform resources; they run when PackageRevisions are pushed to Porch. Common examples include `set-namespace`, `apply-replacements`, and `search-replace`. {{% alert title="Note" color="primary" %}} -When specifying a function image in the Kptfile pipeline, you can use the shorthand form (e.g. `set-namespace:latest`) without a full container registry path. Porch will automatically resolve it using the default registry prefix configured in the Function Runner. You can also use the full image path (e.g. `ghcr.io/kptdev/krm-functions-catalog/set-namespace:latest`) if you prefer to be explicit. To see which functions are available, run `kubectl get functionconfigs -n porch-fn-system`. +When specifying a function image in the Kptfile pipeline, you can use the shorthand form (e.g. `set-namespace:latest`) without a full container registry path. Porch will automatically resolve it using the default registry prefix configured on porch-server (`--default-image-prefix`). You can also use the full image path (e.g. `ghcr.io/kptdev/krm-functions-catalog/set-namespace:latest`) if you prefer to be explicit. To see which functions are available, run `kubectl get functionconfigs -n porch-fn-system`. {{% /alert %}} **Content Structure:** diff --git a/docs/content/en/docs/5_architecture_and_components/_index.md b/docs/content/en/docs/5_architecture_and_components/_index.md index 400bfa75f..de6b66bdf 100644 --- a/docs/content/en/docs/5_architecture_and_components/_index.md +++ b/docs/content/en/docs/5_architecture_and_components/_index.md @@ -31,8 +31,8 @@ The Kubernetes aggregated API server that serves `porch.kpt.dev/v1alpha1`: ### [Engine]({{% relref "engine" %}}) The Configuration as Data (CaD) Engine: -- In the v1alpha1 path: orchestrates full package lifecycle (creation, tasks, rendering, lifecycle transitions) -- In the v1alpha2 path: provides content read/write for PackageRevisionResources only +- In the v1alpha1 path: orchestrates full package lifecycle (creation, tasks, rendering, lifecycle transitions) and hosts the function runtime chain (builtin, Function Runner exec, pod evaluator) +- In the v1alpha2 path: provides content read/write for PackageRevisionResources only; the PackageRevision controller uses the same Engine runtime chain for renders - Enforces validation rules and business constraints for v1alpha1 operations ### [Package Cache]({{% relref "package-cache" %}}) @@ -45,10 +45,10 @@ The shared caching layer between controllers/Engine and Git repositories: ### [Function Runner]({{% relref "function-runner" %}}) -A standalone gRPC service for executing KRM functions: -- Runs functions in isolated containers or as builtin Go executables -- Manages pod lifecycle with caching and garbage collection -- Used by both the Engine (v1alpha1 renders) and the PR Controller (v1alpha2 renders) +A standalone gRPC service for executing **cached** KRM function binaries: +- The Engine looks up the binary and sends `exec_path` on the gRPC request +- Pod-based evaluation, pod lifecycle, and image/registry management run in the Engine (porch-server and the PackageRevision controller) +- Used as the exec fast path by both the Engine (v1alpha1 renders) and the PR Controller (when `FUNCTION_RUNNER_ADDRESS` is set) ## Data Paths @@ -112,7 +112,7 @@ Both paths coexist in the same cluster and share the same Git repositories ## Design Principles -**Separation of Concerns**: Each component has a well-defined responsibility. The cache owns Git interaction. The function runner owns function execution. Controllers own reconciliation logic. +**Separation of Concerns**: Each component has a well-defined responsibility. The cache owns Git interaction. The Engine owns function evaluation (including the pod evaluator). Function Runner owns cached-binary execution. Controllers own reconciliation logic. **Shared Infrastructure**: Both orchestration paths share the cache, function runner, Git storage format, and lifecycle semantics. A package created via either path looks the same in Git. diff --git a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/_index.md b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/_index.md index 80ccf7155..abc6d5cdb 100644 --- a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/_index.md +++ b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/_index.md @@ -39,8 +39,9 @@ The PR Controller is responsible for: │ ▼ ┌──────────────────────┐ - │ Function Runner │ - │ (gRPC) │ + │ Engine runtimes │ + │ builtin / fn-runner │ + │ exec / pod evaluator │ └──────────────────────┘ ``` @@ -54,7 +55,7 @@ Each reconcile executes four phases in sequence. The reconcile itself is trigger **Source execution** handles one-time package creation. When a user creates a PackageRevision with `spec.source` set (init, clone, copy, or upgrade), the controller executes that source operation to produce the initial package content in the shared cache. Once `status.creationSource` is populated, this phase becomes a no-op on future reconciles. -**Rendering** runs the KRM function pipeline defined in the package's Kptfile. Two events can trigger rendering: a content push via the PRR handler (signalled by the `porch.kpt.dev/render-request` annotation), or the completion of source execution. The controller reads resources from the cache, invokes kpt render through the function runner, and writes the results back. +**Rendering** runs the KRM function pipeline defined in the package's Kptfile. Two events can trigger rendering: a content push via the PRR handler (signalled by the `porch.kpt.dev/render-request` annotation), or the completion of source execution. The controller reads resources from the cache, invokes kpt render through the Engine runtime chain (builtin, optional Function Runner, pod evaluator), and writes the results back. **Lifecycle transition** compares the desired lifecycle in `spec.lifecycle` with the actual lifecycle in the cache. If they differ, the controller transitions the package accordingly. On publish, it assigns a revision number and updates the `latest-revision` label across all revisions of the same package. @@ -74,7 +75,7 @@ The PR Controller is enabled via the `--reconcilers` flag on the controllers dep --reconcilers=repositories,packagerevisions ``` -Make sure the Repository Controller is also running (it populates the shared cache), the `PackageRevision` CRD is installed, and the `FUNCTION_RUNNER_ADDRESS` environment variable is set if external function evaluation is needed. +Make sure the Repository Controller is also running (it populates the shared cache), the `PackageRevision` CRD is installed, and `WRAPPER_SERVER_IMAGE` is set if you need container-based KRM functions. Set `FUNCTION_RUNNER_ADDRESS` only if you also want the Function Runner exec fast path. **Repository Annotation**: For the PR Controller to reconcile packages in a repository, the repository must be annotated with `porch.kpt.dev/v1alpha2-migration: "true"`. Without this annotation, the Repository Controller does not create v1alpha2 PackageRevision CRDs. See the [Working with CRD-Based PackageRevisions tutorial]({{% relref "/docs/4_tutorials_and_how-tos/working_with_crd_based_packagerevisions" %}}) for setup instructions. diff --git a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/rendering.md b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/rendering.md index 015a69da0..769e2de87 100644 --- a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/rendering.md +++ b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/rendering.md @@ -8,7 +8,7 @@ description: | ## Overview -Rendering runs the KRM function pipeline defined in the package's Kptfile. The controller reads resources from the shared cache, invokes kpt render through the function runner (gRPC), and writes the rendered output back to the cache. +Rendering runs the KRM function pipeline defined in the package's Kptfile. The controller reads resources from the shared cache, invokes kpt render through the Engine runtime chain (builtin, optional Function Runner exec, in-process pod evaluator), and writes the rendered output back to the cache. ## Triggers diff --git a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/interactions.md b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/interactions.md index 7af5a37dd..8dc54a953 100644 --- a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/interactions.md +++ b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/interactions.md @@ -60,15 +60,21 @@ The interaction between the API Server and the PR Controller is event-driven thr This handoff means the API Server does not need to know how rendering works; it just signals that new content is available. The PR Controller does not need to know how content was written; it just reads whatever is in the cache. -## Function Runner +## Function evaluation -The PR Controller calls the function runner during the render phase. The function runner is a standalone gRPC service that executes KRM functions, both builtin Go functions compiled into the binary and external functions running in containers. +The PR Controller builds the same Engine multi-runtime used by porch-server: -The controller creates a `kptRenderer` during initialization. This is an internal component that wraps kpt's render library and is configured with the function runner's gRPC address and runner options (image prefix, allowed/disallowed registries, etc.). During render, the controller writes package resources to an in-memory filesystem, invokes the renderer (which calls functions through the gRPC runtime), and reads the results back. +- `FUNCTION_RUNNER_ADDRESS` — optional gRPC address for cached-binary exec. Default controller manifests do **not** set this. +- `WRAPPER_SERVER_IMAGE` — enables the in-process pod evaluator. Default controller manifests set this. +- `POD_NAMESPACE` — function-pod and FunctionConfig namespace (default `porch-fn-system`). +- `FUNCTION_CACHE_DIR` — on-disk FunctionConfig binary cache. +- `DEFAULT_IMAGE_PREFIX` — prefix for short function image names. -Concurrency is bounded by the `max-concurrent-renders` setting. If the function runner is unavailable, renders fail and the Rendered condition is set to False with the error message. The controller does not retry failed renders automatically; it waits for the next trigger (annotation change or manual requeue). +If neither `FUNCTION_RUNNER_ADDRESS` nor `WRAPPER_SERVER_IMAGE` is set, only builtin Go functions are available. External container-based functions will fail. -If `FUNCTION_RUNNER_ADDRESS` is not set, only builtin Go functions are available. External container-based functions will fail. +The controller creates a `kptRenderer` during initialization. During render, it writes package resources to an in-memory filesystem, invokes the renderer (builtin → optional Function Runner with `exec_path` → pod evaluator), and reads the results back. + +Concurrency is bounded by the `max-concurrent-renders` setting. If evaluation fails, the Rendered condition is set to False with the error message. The controller does not retry failed renders automatically; it waits for the next trigger (annotation change or manual requeue). ## PackageVariant and PackageVariantSet Controllers diff --git a/docs/content/en/docs/5_architecture_and_components/engine/_index.md b/docs/content/en/docs/5_architecture_and_components/engine/_index.md index f49ed3fcc..aadcf7c25 100644 --- a/docs/content/en/docs/5_architecture_and_components/engine/_index.md +++ b/docs/content/en/docs/5_architecture_and_components/engine/_index.md @@ -17,6 +17,7 @@ The Engine is responsible for: - **Repository Operations**: Opening repositories from the cache and delegating package operations to the appropriate repository adapter - **Validation and Constraints**: Enforcing business rules like workspace name uniqueness, lifecycle constraints, and package path validation - **Draft Management**: Managing the draft-commit workflow where changes are made to drafts and then closed to create immutable package revisions +- **Function Evaluation**: Running KRM functions through a multi-runtime (builtin, Function Runner exec via `exec_path`, in-process pod evaluator) - **Change Notification**: Notifying watchers of package revision changes for real-time updates ## Role in the Architecture @@ -43,4 +44,41 @@ The Engine sits between the Porch API Server and the lower-level components: 5. **Validation Gateway**: Validates all package operations before execution, including workspace name uniqueness, lifecycle constraints, and task-specific validations -The Engine is instantiated once during Porch API server startup and configured with dependencies (cache, task handler, function runtimes, credential resolvers) through a functional options pattern. +6. **Function evaluation and pod lifecycle**: The Engine hosts the function runtime chain (builtin → Function Runner exec → pod evaluator). The following diagram is the former Function Runner architecture drawing; those boxes now run in-process in porch-server and the PackageRevision controller: + +``` +┌─────────────────────────────────────────────────────────┐ +│ Engine — pod evaluator (porch-server / PR ctrl) │ +│ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ gRPC Server │ │ Evaluators │ │ +│ │ │ ───> │ │ │ +│ │ • FunctionEval │ │ • Pod Evaluator │ │ +│ │ Service │ │ • Exec Evaluator│ │ +│ │ • Health Check │ │ • Multi-Eval │ │ +│ └────────┬─────────┘ └────────┬─────────┘ │ +│ │ │ │ +│ └────────┬────────────────┘ │ +│ ↓ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ Pod Lifecycle │ │ Image & Registry│ │ +│ │ Management │ │ Management │ │ +│ │ │ │ │ │ +│ │ • Pod Cache │ │ • Metadata Cache│ │ +│ │ • Pod Manager │ │ • Auth & TLS │ │ +│ │ • GC & TTL │ │ • Pull Secrets │ │ +│ └────────┬─────────┘ └────────┬─────────┘ │ +│ │ │ │ +│ └────────┬────────────────┘ │ +│ ↓ │ +│ ┌──────────────────────┐ │ +│ │ Kubernetes API │ │ +│ │ & Registries │ │ +│ └──────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ + ↑ + │ + Task Handler / kpt Renderer +``` + +The Engine is instantiated once during Porch API server startup and configured with dependencies (cache, task handler, function runtimes, credential resolvers) through a functional options pattern. The same multi-runtime constructor is used by the PackageRevision controller for v1alpha2 renders. diff --git a/docs/content/en/docs/5_architecture_and_components/engine/design.md b/docs/content/en/docs/5_architecture_and_components/engine/design.md index e3508a61c..88a514062 100644 --- a/docs/content/en/docs/5_architecture_and_components/engine/design.md +++ b/docs/content/en/docs/5_architecture_and_components/engine/design.md @@ -31,7 +31,7 @@ The engine is constructed using a **functional options pattern** that allows fle - Cache implementation (CR-based or DB-based) - Task handler for executing package operations -- Function runtimes (builtin, gRPC, or multi-runtime) +- Function runtimes (builtin, Function Runner exec, in-process pod evaluator) - Credential and reference resolvers - Watcher manager for change notifications - User info provider for audit trails diff --git a/docs/content/en/docs/5_architecture_and_components/engine/functionality/_index.md b/docs/content/en/docs/5_architecture_and_components/engine/functionality/_index.md index 72d88c393..e387b1ccc 100644 --- a/docs/content/en/docs/5_architecture_and_components/engine/functionality/_index.md +++ b/docs/content/en/docs/5_architecture_and_components/engine/functionality/_index.md @@ -6,7 +6,7 @@ description: | Overview of CaDEngine functionality and detailed documentation pages. --- -The Engine provides four core functional areas that work together to manage the complete lifecycle of package revisions: +The Engine provides five core functional areas that work together to manage the complete lifecycle of package revisions: ## Functional Areas @@ -51,11 +51,20 @@ Coordinates task execution by delegating to the Task Handler through: - **ApplyTask**: Execute task during package revision creation (init, clone, edit, upgrade) - **DoPRMutations**: Apply mutations during package revision update - **DoPRResourceMutations**: Apply resource mutations and execute render -- **Function Runtime Integration**: Builtin and gRPC function execution +- **Function Runtime Integration**: Builtin, Function Runner (exec via `exec_path`), and pod evaluator - **Error Handling**: Task errors trigger rollback and cleanup For detailed architecture and process flows, see [Task Coordination]({{% relref "/docs/5_architecture_and_components/engine/functionality/task-coordination.md" %}}). +### Function Evaluation + +Owns the function runtime chain used by both the v1alpha1 Task Handler and the v1alpha2 PackageRevision controller renderer: +- **Multi-runtime**: builtin → Function Runner (cached binaries via `exec_path`) → pod evaluator +- **Pod Evaluator**: In-process pod cache, wrapper-server gRPC, TTL/GC +- **Image and registry**: Digest/entrypoint cache and private registry auth for function pods + +For the runtime chain, see [Function Evaluation]({{% relref "/docs/5_architecture_and_components/engine/functionality/function-evaluation.md" %}}). For pods, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md" %}}). For registries, see [Image and Registry Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/image-registry-management.md" %}}). + ## How They Work Together ``` @@ -93,6 +102,7 @@ For detailed architecture and process flows, see [Task Coordination]({{% relref 2. **Lifecycle Management** enforces state machine rules and constraints 3. **Draft-Commit Orchestration** manages the mutable draft workflow with rollback 4. **Task Coordination** delegates to task handler for package transformations +5. **Function Evaluation** runs the Kptfile pipeline (builtin, Function Runner exec, pod evaluator) Each functional area is documented in detail on its own page with architecture diagrams, process flows, and implementation specifics. @@ -109,4 +119,4 @@ The Engine does **not** implement package/package revision CRUD operations - it These are thin wrappers that open the repository through the cache and delegate to repository adapters. The actual storage operations (Git commits, tags, branches) are handled by repository adapters, not the Engine. -The Engine's real work is **orchestration, validation, lifecycle enforcement, and task coordination** - not storage operations. +The Engine's real work is **orchestration, validation, lifecycle enforcement, task coordination, and function evaluation** - not storage operations. diff --git a/docs/content/en/docs/5_architecture_and_components/engine/functionality/function-evaluation.md b/docs/content/en/docs/5_architecture_and_components/engine/functionality/function-evaluation.md new file mode 100644 index 000000000..03147b48c --- /dev/null +++ b/docs/content/en/docs/5_architecture_and_components/engine/functionality/function-evaluation.md @@ -0,0 +1,467 @@ +--- +title: "Function Evaluation" +type: docs +weight: 5 +description: | + Detailed architecture of function evaluation strategies and execution patterns. +--- + +## Overview + +Function evaluation is hosted by the Engine (porch-server and the PackageRevision controller). This page moved from Function Runner with the pod evaluator. The Engine chains builtin Go functions, the Function Runner executable evaluator (via gRPC `exec_path`), and the in-process pod evaluator. The system uses a strategy pattern where different evaluators handle function execution in different ways (pod-based, executable, or chained), all conforming to a common interface. + +### High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ Function Evaluation System │ +│ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ Evaluator │ │ Execution │ │ +│ │ Interface │ ───> │ Strategies │ │ +│ │ │ │ │ │ +│ │ • Common │ │ • Pod Evaluator │ │ +│ │ Contract │ │ • Exec Evaluator│ │ +│ │ • Pluggable │ │ • Multi Eval │ │ +│ └──────────────────┘ └──────────────────┘ │ +│ │ │ │ +│ └────────┬────────────────┘ │ +│ ↓ │ +│ ┌──────────────────┐ │ +│ │ Wrapper │ │ +│ │ Server │ │ +│ │ │ │ +│ │ • gRPC Frontend │ │ +│ │ • Binary Exec │ │ +│ │ • Result Parse │ │ +│ └──────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +## Evaluator Interface + +All evaluators implement a common interface that defines the contract for function execution. + +### Interface Contract + +**Single operation:** +- **EvaluateFunction**: Accepts context and request, returns response or error + +**Request structure:** +- **Image**: Function container image identifier +- **ResourceList**: Serialized KRM resources as YAML bytes +- **ExecPath**: Absolute path of a cached function binary (gRPC field `exec_path`). Set by the Engine from the FunctionConfig store. If empty, the Function Runner executable evaluator returns NotFoundError so evaluation can fall through to the pod evaluator. + +**Response structure:** +- **ResourceList**: Transformed KRM resources as YAML bytes +- **Log**: Function stderr output as bytes + +**Contract characteristics:** +- **Synchronous**: Blocks until function execution completes +- **Context-aware**: Respects cancellation and deadlines from context +- **Stateless**: No state maintained between calls +- **Error-typed**: Returns NotFoundError for missing functions to enable fallback + +### Evaluator Implementations + +Three evaluator implementations provide different execution strategies: + +**Pod Evaluator:** +- Executes functions in Kubernetes pods +- Uses wrapper server for gRPC interface +- Manages pod cache with TTL-based expiration +- Handles service mesh compatibility via ClusterIP services + +**Executable Evaluator:** +- Runs in the Function Runner gRPC service +- Engine looks up the binary in the FunctionConfig store and sends `exec_path` on the request +- Fast execution without pod overhead +- Empty or missing `exec_path` returns NotFoundError so the Engine can fall through to the pod evaluator + +**Multi-Evaluator:** +- Chains multiple evaluators together +- Tries each evaluator in sequence +- Falls back on NotFoundError +- Returns first successful response + +## Pod Evaluator + +Executes functions in Kubernetes pods with caching and lifecycle management. + +### Channel-Based Communication + +The pod evaluator uses channels for communication with the pod cache manager: + +**Communication pattern:** +- Request sent via channel to pod cache manager +- Blocks waiting for gRPC client from cache manager +- Cache manager handles pod creation/reuse +- Direct gRPC call to wrapper server in pod + +**Channel characteristics:** +- **Buffered channel**: Size 1 to prevent blocking +- **One-way communication**: Request sent, client received +- **Synchronization point**: Blocks until client available +- **Error propagation**: Errors sent through same channel + +**Benefits:** +- Decouples evaluation from pod management +- Single goroutine manages pod cache (no race conditions) +- Natural backpressure when pods unavailable +- Clean separation of concerns + +### Client Connection Management + +The pod cache manager maintains gRPC connections to function pods: + +**Cache structure:** +- Map from image name to pod and gRPC client +- Includes pod object key for validation +- Stores gRPC client connection for reuse + +**Connection validation:** +- Pod still exists (not deleted externally) +- Pod not in Failed state +- Service still exists for pod +- gRPC client target matches service URL +- Pod not being deleted (DeletionTimestamp nil) + +**Failed pod handling:** +- Immediately delete failed pods +- Evict from cache +- Trigger new pod creation +- Prevents reusing broken pods + +### Waitlist Mechanism + +Prevents duplicate pod creation when multiple requests arrive for the same function: + +**Waitlist pattern:** +- Multiple requests for same image queue up +- Single pod creation serves all waiters +- Batch notification when pod ready +- Prevents duplicate pod creation + +**Error handling:** +- Pod creation errors sent to all waiters +- Waitlist cleared on error +- Each waiter receives error independently +- Allows retry on next request + +### Function Execution + +Once gRPC client acquired, function execution proceeds: + +**Execution characteristics:** +- Synchronous gRPC call +- Context passed through for cancellation +- Timeout enforced by context deadline +- Stderr logged even on success +- Detailed logging on errors + +## Executable Evaluator + +Executes pre-cached function binaries locally for fast execution. + +### exec_path resolution + +The Engine resolves cached binaries before calling Function Runner. Function Runner does not map images itself. + +**Resolution:** +- FunctionConfig store lookup by image (and optional tag constraint) +- If a binary exists under `--functions`, Engine sets `exec_path` on `EvaluateFunctionRequest` +- If no binary is cached, Engine does not call Function Runner and returns NotFoundError (fallback to pod evaluator) + +**Function Runner checks:** +- Empty `exec_path` → NotFoundError +- `exec_path` must stay under the `--functions` directory (sandbox) +- Binary is executed with ResourceList on stdin + +### Function Cache Lookup + +**Lookup characteristics:** +- Engine: FunctionConfig store lookup by image name / tag +- Fast path when a binary is already on disk +- NotFoundError triggers fallback in the Engine multi-runtime +- Function Runner performs no image-to-path mapping + +### Local Execution + +Binary execution happens in-process: + +**Execution characteristics:** +- Direct process execution (no container) +- ResourceList passed via stdin +- Output captured from stdout +- Stderr captured for logging +- Context-aware (respects cancellation) + +**Performance benefits:** +- No pod startup latency +- No image pull time +- No Kubernetes API overhead +- Millisecond execution times +- Predictable performance + +## Multi-Evaluator + +Chains multiple evaluators with fallback logic. + +### Evaluator Chaining + +**Chaining characteristics:** +- Sequential evaluation (not parallel) +- First success wins +- Only NotFoundError triggers fallback +- Other errors returned immediately +- Preserves error semantics + +**Typical chain (Engine multi-runtime):** +1. **Builtin runtime** (in-process Go functions) +2. **Function Runner executable evaluator** (gRPC, `exec_path` from FunctionConfig cache) +3. **Pod evaluator** (in-process in porch-server / PR controller) + +### Fallback Strategy + +**Fallback conditions:** +- Only on NotFoundError from evaluator +- Other errors (timeout, execution failure) don't trigger fallback +- Preserves error information from failed evaluator + +**Fallback rationale:** +- NotFoundError indicates function not available in current evaluator +- Other errors indicate actual execution problems +- Fallback only makes sense for missing functions +- Prevents masking real errors + +## Wrapper Server + +Provides gRPC interface for function execution inside pods. + +### Wrapper Server Architecture + +``` +┌─────────────────────────────────────────┐ +│ Function Pod │ +│ │ +│ ┌──────────────────┐ │ +│ │ Init Container │ │ +│ │ │ │ +│ │ • Copy wrapper │ │ +│ │ server binary │ │ +│ │ • To shared vol │ │ +│ └────────┬─────────┘ │ +│ ↓ │ +│ ┌──────────────────┐ │ +│ │ Main Container │ │ +│ │ │ │ +│ │ Entrypoint: │ │ +│ │ wrapper-server │ │ +│ │ │ │ +│ │ Args: │ │ +│ │ • --port 9446 │ │ +│ │ • -- │ │ +│ │ • [function │ │ +│ │ entrypoint] │ │ +│ └──────────────────┘ │ +└─────────────────────────────────────────┘ +``` + +**Wrapper server responsibilities:** +- Accept gRPC EvaluateFunction requests +- Execute function entrypoint with ResourceList +- Capture stdout/stderr from function +- Parse structured results from output +- Return gRPC response with results +- Provide health check endpoint + +### Entrypoint Wrapping + +The wrapper server wraps the original function entrypoint: + +**Entrypoint characteristics:** +- Original function entrypoint preserved +- Passed as arguments after separator +- Executed as subprocess +- Stdin/stdout/stderr captured + +### Resource List Processing + +**Processing characteristics:** +- ResourceList passed as raw bytes to stdin +- Function reads from stdin +- Function writes to stdout +- Standard KRM function protocol +- No modification of ResourceList format + +### Structured Results Handling + +The wrapper server parses structured results from function output: + +**Structured results:** +- ResourceList contains results array +- Each result has severity, message, tags +- Exit code indicates overall success/failure +- Results provide detailed feedback + +**Exit code semantics:** +- **0**: Function succeeded +- **Non-zero**: Function failed +- **Parse failure**: Wrapper server error +- **Execution error**: System error + +## Execution Characteristics + +### Synchronous Execution + +**All evaluators execute synchronously:** +- Block until function completes +- No async callbacks or futures +- Simple request-response pattern +- Caller waits for result + +**Benefits:** +- Simple programming model +- Easy error handling +- Predictable behavior +- No concurrency complexity + +### Context Cancellation + +**Context propagation:** +- Context passed through all layers +- Execution stops on cancellation +- Resources cleaned up +- Error returned to caller + +### Timeout Handling + +**Timeout sources:** +- Task Handler sets context deadline +- Function Runner respects deadline +- Evaluators check context +- Function execution cancelled on timeout + +**Timeout behavior:** +- Execution stops immediately +- Partial results discarded +- Timeout error returned +- Resources cleaned up + +## Error Handling + +The evaluation system handles errors at multiple levels. + +### Error Types + +**NotFoundError:** +- Function not available in evaluator +- Triggers fallback in multi-evaluator +- Distinguishable from execution errors +- Final NotFoundError if all evaluators fail + +**Execution errors:** +- Function failed during execution +- Non-zero exit code +- Invalid output format +- Returned immediately without fallback + +**Timeout errors:** +- Execution exceeded deadline +- Context cancellation triggered +- Partial results discarded +- Resources cleaned up + +**System errors:** +- Infrastructure problems +- Pod creation failures +- Network issues +- Kubernetes API errors + +### Error Propagation + +**Error flow:** +- Function execution error captured +- Wrapper server formats error message +- Pod evaluator adds context +- Multi-evaluator checks error type +- Function runner returns to task handler +- Task handler includes in RenderStatus + +### Error Recovery + +**Retry mechanisms:** +- NotFoundError triggers fallback to next evaluator +- Transient errors may succeed on retry +- Waitlist allows retry on next request + +**No retry scenarios:** +- Execution errors (function failed) +- Timeout errors (deadline exceeded) +- System errors (infrastructure problems) + +## Performance Optimization + +The evaluation system employs several performance strategies. + +### Pod Reuse + +**Reuse benefits:** +- Eliminates pod startup latency +- Avoids image pull time +- Reduces Kubernetes API load +- Improves response time + +**Reuse mechanism:** +- Pod cache with TTL +- TTL extended on each use +- Garbage collection removes expired pods +- Failed pods immediately deleted + +### Cache Warming + +**Warming strategy:** +- Pre-create pods for frequently-used functions +- Configuration file specifies functions and TTLs +- Concurrent pod creation at startup +- Reduces first-request latency + +**Warming benefits:** +- Pods ready before first request +- Predictable performance +- No cold start penalty +- Better user experience + +### Concurrent Requests + +**Concurrency handling:** +- Multiple requests can execute concurrently +- Each request gets own gRPC connection +- Pod cache manager coordinates access +- Round-robin load balancing across equal-load pods + +**Concurrency characteristics:** +- Same function, same pod: Parallel execution supported +- Same function, different pods: Concurrent +- Different functions: Fully concurrent +- No artificial concurrency limits + +**Load balancing:** +- Requests distributed to least-loaded pods +- Round-robin among pods with equal load +- Ensures even work distribution +- Prevents hotspotting on single pod + +### Resource Limits + +**Resource considerations:** +- Function pods have resource limits +- Limits prevent resource exhaustion +- Configurable via pod template +- Affects concurrent execution capacity + +**Performance tuning:** +- Adjust pod TTL for reuse frequency +- Configure cache warming for hot functions +- Use executable evaluator for critical path +- Monitor pod resource usage diff --git a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/image-registry-management.md b/docs/content/en/docs/5_architecture_and_components/engine/functionality/image-registry-management.md similarity index 92% rename from docs/content/en/docs/5_architecture_and_components/function-runner/functionality/image-registry-management.md rename to docs/content/en/docs/5_architecture_and_components/engine/functionality/image-registry-management.md index b7fc03130..c09982a7e 100644 --- a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/image-registry-management.md +++ b/docs/content/en/docs/5_architecture_and_components/engine/functionality/image-registry-management.md @@ -1,14 +1,14 @@ --- title: "Image and Registry Management" type: docs -weight: 3 +weight: 7 description: | Detailed architecture of image metadata caching, registry authentication, and secret management. --- ## Overview -The Function Runner manages container image metadata and registry authentication to optimize pod creation and support private registries. The system caches image metadata (digest and entrypoint) to avoid repeated registry calls, handles authentication for private registries using Docker config format, and supports TLS configuration for secure registry connections. +The Engine pod evaluator manages container image metadata and registry authentication to optimize pod creation and support private registries. The system caches image metadata (digest and entrypoint) to avoid repeated registry calls, handles authentication for private registries using Docker config format, and supports TLS configuration for secure registry connections. ### High-Level Architecture @@ -60,7 +60,7 @@ image → digestAndEntrypoint - **In-memory storage**: sync.Map for thread-safe concurrent access - **Key**: Function image name (full image reference) - **Value**: Digest and entrypoint struct -- **Lifetime**: Persists for Function Runner process lifetime +- **Lifetime**: Persists for Engine pod evaluator process lifetime - **No expiration**: Entries never evicted (immutable image metadata) **Cached metadata:** @@ -102,7 +102,7 @@ Pod Creation Request ## Registry Authentication -The Function Runner supports both default and custom registry authentication. +The Engine pod evaluator supports both default and custom registry authentication. ### Authentication Strategy @@ -148,7 +148,7 @@ Image Reference - Standard Docker config.json structure - Contains auths map with registry credentials - Supports multiple registries -- Mounted as secret into Function Runner pod +- Mounted as secret into the porch-server (or porch-controllers) pod ## Secret Management @@ -175,7 +175,7 @@ Ensure Auth Secret - Secret created on first use - Inspected and updated if Docker config changes - Synchronized with mounted configuration -- Persists across Function Runner restarts +- Persists across Engine pod evaluator restarts ### Image Pull Secret Injection @@ -201,7 +201,7 @@ Pod Creation ## TLS Configuration -The Function Runner supports custom TLS certificates for secure registry connections. +The Engine pod evaluator supports custom TLS certificates for secure registry connections. ### TLS Certificate Management @@ -372,12 +372,12 @@ The image management system handles errors at multiple levels. ## Cache Lifecycle -The image metadata cache follows the Function Runner lifecycle. +The image metadata cache follows the Engine pod evaluator lifecycle. ### Initialization **Startup:** -- Cache created at Function Runner startup +- Cache created at Engine pod evaluator startup - Empty initially (lazy population) - No pre-warming or pre-loading - Memory allocated on demand @@ -393,7 +393,7 @@ The image metadata cache follows the Function Runner lifecycle. ### Shutdown **Cleanup:** -- Cache discarded on Function Runner shutdown +- Cache discarded on Engine pod evaluator shutdown - No persistent storage - No cleanup operations needed - Rebuilt from scratch on restart diff --git a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md b/docs/content/en/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md similarity index 94% rename from docs/content/en/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md rename to docs/content/en/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md index 2313aa64b..3e738a026 100644 --- a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md +++ b/docs/content/en/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md @@ -1,7 +1,7 @@ --- title: "Pod Lifecycle Management" type: docs -weight: 2 +weight: 6 description: | Detailed architecture of pod cache management, lifecycle operations, and garbage collection. --- @@ -21,7 +21,7 @@ Key design characteristics: - **Single-threaded cache management** eliminates race conditions - **Channel-based communication** provides clean separation between components - **Service mesh compatibility** through ClusterIP services fronting each pod -- **Template-based pod creation** supports ConfigMap-based and inline specifications +- **Template-based pod creation** supports the `base-pod-template` PodTemplate CR (and inline defaults if it is missing) - **TTL-based lifecycle** with automatic garbage collection - **Failed pod detection** with immediate deletion and cache eviction - **Pod warming** capability for pre-creating pods at startup @@ -112,19 +112,19 @@ The Pod Manager handles low-level Kubernetes operations for function pods and th **Image Metadata Operations** - Cache image digests and entrypoints, inspect container images to extract configuration, handle private registry authentication and TLS configuration, manage image pull secrets for function pods. -**Template System** - Load pod templates from ConfigMaps or use inline defaults, load service templates from ConfigMaps or use inline defaults, track template versions to detect changes requiring pod replacement, patch templates with function-specific configuration. +**Template System** - Load the `base-pod-template` PodTemplate and `base-service-template` ServiceTemplate CRs (see `deployments/porch/22-function-templates.yaml`), or create them from inline defaults if missing. Track template ResourceVersion to detect changes requiring pod replacement. Patch templates with function-specific configuration and FunctionConfig TemplateOverrides. ### Pod Template System -The pod manager supports two template sources: ConfigMap-based templates for customization and inline templates as fallback defaults. +The pod manager supports two template sources: cluster PodTemplate/ServiceTemplate CRs for customization and inline templates as fallback defaults. -**ConfigMap-Based Templates:** +**PodTemplate CRs:** -When functionPodTemplateName is configured, the pod manager retrieves a ConfigMap containing template and serviceTemplate keys. The ConfigMap's ResourceVersion is used as the template version for tracking changes. When the ConfigMap is updated, existing pods with old template versions are replaced on next use. +The pod manager gets `base-pod-template` (core `PodTemplate`) and `base-service-template` (`ServiceTemplate`) from the function-pod namespace (`porch-fn-system`). The object's ResourceVersion is used as the template version for tracking changes. When the template is updated, existing pods with old template versions are replaced on next use. If the CRs are missing, the manager creates them from the inline defaults. **Inline Templates:** -When no ConfigMap is configured, the pod manager uses hardcoded inline templates with sensible defaults including init container for wrapper-server binary, main container with wrapper-server as entrypoint, EmptyDir volume for tools, readiness probe using grpc-health-probe, and cluster autoscaler safe-to-evict annotation. +When `base-pod-template` is missing, the pod manager uses hardcoded inline templates with sensible defaults including init container for wrapper-server binary, main container with wrapper-server as entrypoint, EmptyDir volume for tools, readiness probe using grpc-health-probe, and cluster autoscaler safe-to-evict annotation. ### Container Configuration diff --git a/docs/content/en/docs/5_architecture_and_components/engine/functionality/task-coordination.md b/docs/content/en/docs/5_architecture_and_components/engine/functionality/task-coordination.md index 3e04deb47..bb0ace858 100644 --- a/docs/content/en/docs/5_architecture_and_components/engine/functionality/task-coordination.md +++ b/docs/content/en/docs/5_architecture_and_components/engine/functionality/task-coordination.md @@ -312,13 +312,13 @@ NewCaDEngine(opts...) **Runtime types:** - **Builtin Runtime**: For built-in functions (set-namespace, etc.) -- **gRPC Runtime**: For external function runner service -- **Multi-Runtime**: Chains multiple runtimes together +- **gRPC Runtime**: For Function Runner cached binaries (`exec_path`) +- **Pod evaluator**: In-process when `WRAPPER_SERVER_IMAGE` is set +- **Multi-Runtime**: Chains builtin → Function Runner → pod evaluator -Runtime selection is configured at Porch server startup. It is passed to task handler during engine initialization, -which uses runtime for function execution. +Runtime selection is configured at porch-server startup (`--function-runner`, `WRAPPER_SERVER_IMAGE`, pod-evaluator flags) and passed to the task handler. The PackageRevision controller builds the same chain from `FUNCTION_RUNNER_ADDRESS` and `WRAPPER_SERVER_IMAGE`. -For details on function runtime implementations, see [Function Runner Design]({{% relref "/docs/5_architecture_and_components/function-runner/design.md" %}}). +For details on function runtime implementations, see [Function Evaluation]({{% relref "/docs/5_architecture_and_components/engine/functionality/function-evaluation.md" %}}) and [Function Runner Design]({{% relref "/docs/5_architecture_and_components/function-runner/design.md" %}}). ## Error Handling diff --git a/docs/content/en/docs/5_architecture_and_components/engine/interactions.md b/docs/content/en/docs/5_architecture_and_components/engine/interactions.md index d0879d88c..70c0534df 100644 --- a/docs/content/en/docs/5_architecture_and_components/engine/interactions.md +++ b/docs/content/en/docs/5_architecture_and_components/engine/interactions.md @@ -154,8 +154,9 @@ the draft. The Task handler has no direct repository access. The task handler uses function runtimes configured in the engine: - **Builtin Runtime**: For built-in functions (set-namespace, etc.) -- **gRPC Runtime**: For external function runner service -- **Multi-Runtime**: Chains multiple runtimes together +- **gRPC Runtime**: For Function Runner cached binaries (`exec_path`) +- **Pod evaluator**: In-process in porch-server / PR controller +- **Multi-Runtime**: Chains builtin → gRPC exec → pod evaluator (NotFound fallback) The engine configures these runtimes during initialization and passes them to the task handler. diff --git a/docs/content/en/docs/5_architecture_and_components/function-runner/_index.md b/docs/content/en/docs/5_architecture_and_components/function-runner/_index.md index 52e779cf6..7ade30519 100644 --- a/docs/content/en/docs/5_architecture_and_components/function-runner/_index.md +++ b/docs/content/en/docs/5_architecture_and_components/function-runner/_index.md @@ -12,15 +12,13 @@ The **Function Runner** is a standalone gRPC service that executes KRM (Kubernet The Function Runner is responsible for: -- **Function Execution**: Running KRM functions in isolated pods or as local executables -- **Pod Lifecycle Management**: Creating, caching, and garbage collecting function execution pods -- **Image Management**: Caching image metadata and handling private registry authentication -- **Service Mesh Compatibility**: Using ClusterIP services as frontends for function pods -- **Resource Isolation**: Ensuring functions execute in separate environments with controlled resources +- **Function Execution**: Running cached KRM function binaries over gRPC using `exec_path` supplied by the Engine. Arbitrary function images are evaluated by the **pod evaluator** in the Engine (porch-server and the PackageRevision controller). +- **Pod Lifecycle Management**, **Image Management**, and **Service Mesh Compatibility**: These still work as shown in the diagram below, but they now run in the Engine. See [Engine Function Evaluation]({{% relref "/docs/5_architecture_and_components/engine/functionality/function-evaluation.md" %}}), [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md" %}}), and [Image and Registry Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/image-registry-management.md" %}}). +- **Resource Isolation**: Cached binaries run in the Function Runner process; other functions run in Engine-managed pods. ## Role in the Architecture -The Function Runner sits as a separate service that the Task Handler communicates with via gRPC: +The Function Runner sits as a separate service that the Task Handler (and the PackageRevision controller, when `FUNCTION_RUNNER_ADDRESS` is set) communicates with via gRPC. The pod evaluator, pod lifecycle, and image/registry boxes in this diagram now run **inside the Engine**; Function Runner itself hosts the gRPC server and the executable evaluator (`exec_path`). ``` ┌─────────────────────────────────────────────────────────┐ @@ -66,10 +64,8 @@ The Function Runner sits as a separate service that the Task Handler communicate **Key architectural responsibilities:** 1. **Separate Service Deployment**: Runs independently from Porch server, enabling independent scaling, isolation, and separate failure domains -2. **Multiple Evaluator Strategies**: Supports pod-based execution (default), executable evaluation (fast path), and multi-evaluator chaining with fallback -3. **Pod-Based Execution Infrastructure**: Creates and manages function execution pods with TTL-based caching, garbage collection, and ClusterIP service frontends -4. **Image and Registry Integration**: Caches image metadata, supports private registries with authentication, and handles TLS certificates -5. **gRPC Communication Protocol**: Exposes FunctionEvaluator service accepting serialized ResourceList and returning transformed resources -6. **Wrapper Server Pattern**: Injects wrapper-server binary into function pods to provide gRPC interface and structured result handling +2. **Executable evaluation (fast path)**: Runs pre-cached function binaries using `exec_path` from the Engine. The Engine falls through to the in-process pod evaluator when no binary is cached +3. **gRPC Communication Protocol**: Exposes FunctionEvaluator service accepting serialized ResourceList (and `exec_path`) and returning transformed resources +4. **Pod-Based Execution Infrastructure**, **Image and Registry Integration**, and **Wrapper Server Pattern**: Now implemented in the Engine (see the Pod Lifecycle box in the diagram and the [Engine]({{% relref "/docs/5_architecture_and_components/engine" %}}) pages) -The Function Runner is instantiated as a separate deployment and configured with evaluator types (pod, executable, or both) through command-line flags. +The Function Runner is instantiated as a separate deployment. Porch-server reaches it with `--function-runner`. The only runtime in this binary is `exec` (`--disable-runtimes` accepts `exec`). diff --git a/docs/content/en/docs/5_architecture_and_components/function-runner/design.md b/docs/content/en/docs/5_architecture_and_components/function-runner/design.md index 27b888867..f53f5ca73 100644 --- a/docs/content/en/docs/5_architecture_and_components/function-runner/design.md +++ b/docs/content/en/docs/5_architecture_and_components/function-runner/design.md @@ -37,9 +37,9 @@ Go Code Runner Runtimes 1. **Builtin Runtime** (in Engine, not Function Runner): Executes specific functions in-process within the Porch server as compiled Go code (apply-replacements, set-namespace, starlark) 2. **gRPC Runtime** (in Engine): Calls the external Function Runner service via gRPC -3. **Multi Runtime** (in Engine): Chains builtin and gRPC runtimes together with fallback logic +3. **Multi Runtime** (in Engine): Chains builtin, gRPC Function Runner (exec), and pod evaluator with NotFound fallback -This documentation focuses on the **Function Runner service** (the gRPC-based external service), not the builtin runtime which is part of the Engine component. +This documentation focuses on the **Function Runner service** (the gRPC-based external service), not the builtin runtime which is part of the Engine component. The pod evaluator now runs in the Engine; Function Runner only hosts the executable evaluator. The Engine looks up a cached binary and sets gRPC `exec_path`; empty `exec_path` is treated as NotFound. ## Evaluator Interface @@ -107,15 +107,17 @@ The multi-evaluator tries evaluators in sequence until one succeeds: The server supports dynamic evaluator selection through command-line flags: **Configuration mechanism:** -1. Start with all available evaluators (exec, pod) -2. Remove evaluators specified in `--disable-runtimes` flag -3. Initialize only enabled evaluators -4. Wrap in MultiEvaluator for unified interface +1. Engine starts with builtin, optional Function Runner exec, and pod evaluator +2. Function Runner `--disable-runtimes` can disable `exec` only +3. Engine hosts the pod evaluator when `WRAPPER_SERVER_IMAGE` is set +4. Engine wraps runtimes in a multi-runtime chain (builtin → exec → pod) **Configuration examples:** -- `--disable-runtimes=exec`: Pod evaluator only -- `--disable-runtimes=pod`: Executable evaluator only -- No flag: Both evaluators with exec as fast path +- Function Runner `--disable-runtimes=exec`: Engine uses builtin + pod evaluator +- No Function Runner address: Engine uses builtin + pod evaluator +- Default porch-server: Function Runner exec fast path, then pod evaluator + +(`--disable-runtimes=pod` was a Function Runner flag; the pod evaluator is not in that binary.) **Pattern benefits:** - Fast path for cached functions (executable evaluator) @@ -189,13 +191,13 @@ Choosing between evaluators depends on deployment requirements and function char ### Default Configuration -Function Runner deploys with **pod evaluator by default** when no evaluators are explicitly disabled. This provides universal function support out of the box while allowing opt-in to executable evaluator for performance optimization. +Porch-server deploys with the **pod evaluator by default** (`WRAPPER_SERVER_IMAGE` is required). Function Runner is the executable fast path. **Configuration method:** -- Use `--disable-runtimes` flag to disable specific evaluators -- Pod evaluator: No additional configuration needed -- Executable evaluator: Requires configuration file mapping images to binaries -- Multi-evaluator: Automatically used when multiple evaluators enabled +- Function Runner `--disable-runtimes` accepts `exec` only +- Pod evaluator: configured on porch-server / PackageRevision controller +- Executable evaluator: Engine looks up `exec_path` in the FunctionConfig store +- Multi-runtime: Engine chains builtin → Function Runner → pod evaluator ### Migration Considerations diff --git a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/_index.md b/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/_index.md index 7e95b8982..ccfbe4ba3 100644 --- a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/_index.md +++ b/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/_index.md @@ -6,7 +6,7 @@ description: | Overview of function runner functionality and detailed documentation pages. --- -The Function Runner provides three core functional areas that work together to execute KRM functions in isolated environments: +The Function Runner provides executable evaluation of cached KRM function binaries (`exec_path`). Pod lifecycle and image/registry management moved to the Engine; the pages below still describe those areas and now live under Engine. ## Functional Areas @@ -15,8 +15,8 @@ The Function Runner provides three core functional areas that work together to e Executes KRM functions through pluggable evaluator strategies: - **Evaluator Interface**: Common contract for all function execution strategies - **Pod Evaluator**: Executes functions in Kubernetes pods with wrapper server integration -- **Executable Evaluator**: Runs pre-cached function binaries locally for fast execution -- **Multi-Evaluator**: Chains evaluators with fallback logic (exec → pod) +- **Executable Evaluator**: Runs pre-cached function binaries using `exec_path` from the Engine +- **Multi-Evaluator / Engine chain**: Engine tries builtin → Function Runner exec → pod evaluator - **Request Channel Pattern**: Channel-based communication for pod cache coordination - **Wrapper Server Integration**: gRPC wrapper injected into function pods for structured execution @@ -33,7 +33,7 @@ Manages function execution pods with caching and garbage collection: - **Garbage Collection**: Periodic cleanup of expired pods and failed pod handling - **Pod Warming**: Pre-creates pods for frequently-used functions -For detailed architecture and process flows, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md" %}}). +For detailed architecture and process flows, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md" %}}). ### Image and Registry Management @@ -45,7 +45,7 @@ Caches image metadata and handles private registry authentication: - **Secret Management**: Creates and attaches image pull secrets to function pods - **Registry Operations**: Handles manifest retrieval, authentication retry, and error handling -For detailed architecture and process flows, see [Image and Registry Management]({{% relref "/docs/5_architecture_and_components/function-runner/functionality/image-registry-management.md" %}}). +For detailed architecture and process flows, see [Image and Registry Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/image-registry-management.md" %}}). ## How They Work Together diff --git a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/function-evaluation.md b/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/function-evaluation.md index 2a66865d5..248498f11 100644 --- a/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/function-evaluation.md +++ b/docs/content/en/docs/5_architecture_and_components/function-runner/functionality/function-evaluation.md @@ -8,7 +8,7 @@ description: | ## Overview -Function evaluation is the core responsibility of the Function Runner - executing KRM (Kubernetes Resource Model) functions through pluggable evaluator strategies. The system uses a strategy pattern where different evaluators handle function execution in different ways (pod-based, executable, or chained), all conforming to a common interface. +Function evaluation is the core responsibility of the Function Runner for **cached binaries**. The pod evaluator sections below now run in the Engine; see [Engine Function Evaluation]({{% relref "/docs/5_architecture_and_components/engine/functionality/function-evaluation.md" %}}). The Engine sends `exec_path` on the gRPC request; Function Runner executes that binary or returns NotFoundError if `exec_path` is empty. ### High-Level Architecture @@ -72,10 +72,10 @@ Three evaluator implementations provide different execution strategies: - Handles service mesh compatibility via ClusterIP services **Executable Evaluator:** -- Executes pre-cached function binaries locally -- Configuration file maps images to binary paths +- Executes pre-cached function binaries locally using `exec_path` from the request +- Engine maps FunctionConfig cache entries to `exec_path`; Function Runner does not read a `--config` file - Fast execution without pod overhead -- Returns NotFoundError for uncached functions +- Empty `exec_path` returns NotFoundError for Engine fallback to the pod evaluator **Multi-Evaluator:** - Chains multiple evaluators together @@ -162,20 +162,14 @@ Once gRPC client acquired, function execution proceeds: Executes pre-cached function binaries locally for fast execution. -### Configuration-Based Caching +### exec_path resolution -The executable evaluator uses a configuration file to map images to binaries: +The Engine resolves cached binaries and sets `exec_path` on the gRPC request. Function Runner does not map images via a configuration file. -**Configuration structure:** -- YAML file with functions array -- Each function has name and images list -- Images map to binary in cache directory - -**Configuration benefits:** -- Explicit control over cached functions -- No automatic caching (predictable behavior) -- Simple file-based configuration -- Easy to update without restart +**Resolution:** +- FunctionConfig store lookup by image (and optional tag) +- `exec_path` must be under Function Runner `--functions` +- Empty `exec_path` → NotFoundError (Engine falls through to the pod evaluator) ### Function Cache Lookup diff --git a/docs/content/en/docs/5_architecture_and_components/function-runner/interactions.md b/docs/content/en/docs/5_architecture_and_components/function-runner/interactions.md index eb2910e38..4e3eeb269 100644 --- a/docs/content/en/docs/5_architecture_and_components/function-runner/interactions.md +++ b/docs/content/en/docs/5_architecture_and_components/function-runner/interactions.md @@ -8,7 +8,7 @@ description: | ## Overview -The Function Runner is a **separate gRPC service** that interacts with multiple systems: the Task Handler (via gRPC), Kubernetes API (for pod management), container registries (for image metadata), and wrapper servers (for function execution). It operates independently from the Porch server, enabling isolated function execution. +The Function Runner is a **separate gRPC service** that interacts with the Task Handler (via gRPC) to run cached function binaries. Kubernetes API, image cache, and function-pod boxes in the diagram now run in the Engine; Function Runner still serves EvaluateFunction for cached binaries using `exec_path`. ### High-Level Architecture @@ -59,12 +59,13 @@ Function Runner Service - Task Handler uses gRPC Runtime to communicate with Function Runner - Single persistent connection shared across all function executions - Connection established at Porch startup, closed on shutdown -- Function Runner address configured via `--function-runner-address` flag +- Function Runner address configured via `--function-runner` on porch-server (`FUNCTION_RUNNER_ADDRESS` on the PackageRevision controller) +- Engine looks up `exec_path` in the FunctionConfig store; Function Runner executes that binary or returns NotFoundError **Request-response flow:** - Task Handler serializes ResourceList as YAML - gRPC Runtime sends EvaluateFunctionRequest -- Function Runner selects appropriate evaluator (exec or pod) +- Function Runner executes the binary at `exec_path` (or returns NotFoundError if it is empty) - Response includes transformed ResourceList and function logs - NotFoundError triggers fallback to next evaluator in multi-evaluator chain @@ -148,7 +149,7 @@ Pod Pod Pod - Wrapper server executes function binary and returns results - Multiple evaluations can execute in parallel on the same pod -**For detailed pod lifecycle, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md" %}}).** +**For detailed pod lifecycle, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md" %}}).** ### Executable-Based Execution @@ -206,7 +207,7 @@ Delete Delete Retrieval Config - Inline templates as fallback defaults - Template version tracking for pod replacement on changes -**For detailed pod management, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md" %}}).** +**For detailed pod management, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md" %}}).** ### Service Mesh Compatibility @@ -256,7 +257,7 @@ Pod Creation - Faster pod creation (no digest resolution delay) - Cache persists for Function Runner lifetime -**For detailed image management, see [Image and Registry Management]({{% relref "/docs/5_architecture_and_components/function-runner/functionality/image-registry-management.md" %}}).** +**For detailed image management, see [Image and Registry Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/image-registry-management.md" %}}).** ### Authentication and TLS @@ -451,4 +452,4 @@ The Function Runner handles concurrent operations safely: - Resource limits enforced by Kubernetes - No shared state between function executions -**For detailed concurrency patterns, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md" %}}).** +**For detailed concurrency patterns, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md" %}}).** diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/_index.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/_index.md index 9f57ce34f..8cacaf13d 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/_index.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/_index.md @@ -14,9 +14,10 @@ Configure individual Porch components: - [Porch Server]({{% relref "components/porch-server-config" %}}) - API server configuration - [Git Authentication]({{% relref "components/porch-server-config/git-authentication" %}}) - Git repository authentication - [Cert Manager Webhooks]({{% relref "components/porch-server-config/cert-manager-webhooks" %}}) - Webhook certificate management + - [Private Registries]({{% relref "components/porch-server-config/private-registries-config" %}}) - Container registry authentication for function pods + - [Pod Templates]({{% relref "components/porch-server-config/pod-templates" %}}) - Function pod specifications - [Porch Controllers]({{% relref "components/porch-controllers-config" %}}) - Repository, PackageRevision, and variant controller settings -- [Function Runner]({{% relref "components/function-runner-config" %}}) - Function execution environment - - [Private Registries]({{% relref "components/function-runner-config/private-registries-config" %}}) - Container registry authentication +- [Function Runner]({{% relref "components/function-runner-config" %}}) - Cached-binary (exec) function execution ### OTEL Metrics & Tracing @@ -37,7 +38,7 @@ Configure Git repository synchronization with ConfigSync or other GitOps tools. ## Configuration Best Practices - Start with default CR cache for simplicity -- Configure private registries only if using private KRM functions in Function Runner +- Configure private registries only if using private KRM functions (porch-server pod evaluator) - Enable tracing in development environments for debugging - Use cert-manager for production TLS certificate management - Set appropriate resource limits for each component diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/_index.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/_index.md index 5f2e07af1..023665b24 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/_index.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/_index.md @@ -13,10 +13,11 @@ Configure each Porch component individually for optimal performance and security The main API server that handles package operations and Git repository interactions: - [Git Authentication]({{% relref "porch-server-config/git-authentication" %}}) - Repository authentication methods - [Cert Manager Webhooks]({{% relref "porch-server-config/cert-manager-webhooks" %}}) - Webhook certificate management +- [Private Registry Access]({{% relref "porch-server-config/private-registries-config" %}}) - Container registry authentication for function pods +- [Pod Templates]({{% relref "porch-server-config/pod-templates" %}}) - Function pod specifications (`base-pod-template`) ### [Porch Controllers]({{% relref "porch-controllers-config" %}}) Manage the lifecycle of Repositories, PackageRevisions, PackageVariants, and PackageVariantSets. ### [Function Runner]({{% relref "function-runner-config" %}}) -Executes KRM functions in isolated containers: -- [Private Registry Access]({{% relref "function-runner-config/private-registries-config" %}}) - Container registry authentication \ No newline at end of file +Executes cached KRM function binaries over gRPC (executable fast path). Function pods are configured under Porch Server. \ No newline at end of file diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/_index.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/_index.md index 89a7c7c0d..1a2423b3e 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/_index.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/_index.md @@ -5,12 +5,12 @@ weight: 3 description: "Configure the Function Runner component" --- +The Function Runner executes cached KRM function binaries over gRPC. Pod runtime flags moved to [Porch Server]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config" %}}) with the pod evaluator. + {{% alert title="Note" color="primary" %}} KPT functions and KRM functions are synonymous terms referring to the same containerized functions. {{% /alert %}} -The Function Runner executes KRM functions in a secure, isolated environment. - ## Configuration Options ### Command Line Arguments @@ -19,7 +19,7 @@ The Function Runner executes KRM functions in a secure, isolated environment. ```bash args: - --port=9445 # Server port (default: 9445) -- --disable-runtimes=exec,pod # Disable specific runtimes (exec, pod) +- --disable-runtimes=exec # Disable the exec runtime (the only runtime in this binary) - --log-level=2 # Log verbosity level 0-5 (default: 2) ``` @@ -27,10 +27,14 @@ args: ```bash args: - --functions=./functions # Path to cached functions (default: ./functions) -- --config=./config.yaml # Path to exec runtime config file (default: ./config.yaml) +- --max-request-body-size=6291456 # Max gRPC message size in bytes (default: 6MB) ``` +The Engine looks up binaries in the FunctionConfig store and sends `exec_path` on the gRPC request. Function Runner does not read a `--config` image-to-binary mapping file. + #### Pod Runtime Arguments + +These flags now belong to **porch-server**. See [Porch Server]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config" %}}). They moved with the pod evaluator: ```bash args: - --pod-cache-config=/pod-cache-config/pod-cache-config.yaml # Pod cache config file path @@ -38,13 +42,14 @@ args: - --pod-namespace=porch-fn-system # Namespace for KRM function pods (default: porch-fn-system) - --pod-ttl=30m # Pod TTL before GC (default: 30m) - --scan-interval=1m # GC scan interval (default: 1m) -- --function-pod-template= # ConfigMap with pod specification - --max-request-body-size=6291456 # Max gRPC message size in bytes (default: 6MB) - --max-waitlist-length # Maximum waitlist length per pod - --max-parallel-pods-per-function # Maximum parallel pods per function ``` #### Private Registry Arguments + +These flags now belong to **porch-server**. See [Private Registries]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config/private-registries-config" %}}). ```bash args: - --enable-private-registries=false # Enable private registry support @@ -56,24 +61,19 @@ args: ### Environment Variables +`WRAPPER_SERVER_IMAGE` is required on **porch-server** (and on porch-controllers when the PackageRevision controller should run the pod evaluator): + ```bash env: - name: WRAPPER_SERVER_IMAGE - value: "" # Required for pod runtime + value: "" # Required for the Engine pod evaluator ``` ## Advanced Configuration ### Pod Templates -Customize function evaluator pod specifications using ConfigMap templates: - -```bash -args: -- --function-pod-template=kpt-function-eval-pod-template # ConfigMap name -``` - -For detailed pod template configuration, see [Pod Templates]({{% relref "pod-templates" %}}) documentation. +Customize function evaluator pod specifications using the `base-pod-template` PodTemplate CR. See [Pod Templates]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config/pod-templates" %}}). ## Runtime Configuration @@ -83,33 +83,26 @@ The exec runtime runs functions as local executables: ```bash args: -- --functions=./functions # Directory containing cached function executables -- --config=./config.yaml # Configuration file for exec runtime +- --functions=/home/nonroot/functions # Directory containing cached function executables ``` +The Engine supplies `exec_path`; Function Runner does not use `--config`. + ### Pod Runtime -The pod runtime runs functions as Kubernetes pods: - -```bash -args: -- --pod-namespace=porch-fn-system # Namespace for function pods -- --pod-ttl=30m # How long pods live before cleanup -- --scan-interval=1m # How often to scan for expired pods -- --warm-up-pod-cache=true # Pre-deploy common function pods -``` +The pod runtime runs in the Engine (porch-server). See [Porch Server]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config" %}}). ### Disabling Runtimes -To disable specific runtimes: +To disable the exec runtime: ```bash args: -- --disable-runtimes=exec # Disable exec runtime only -- --disable-runtimes=pod # Disable pod runtime only -- --disable-runtimes=exec,pod # Disable both runtimes +- --disable-runtimes=exec # Disable exec runtime ``` +`--disable-runtimes=pod` is not valid; the pod evaluator is not in this binary. + ## Resource Limits ```bash @@ -164,14 +157,8 @@ spec: args: - --port=9445 - --log-level=2 - - --pod-namespace=porch-fn-system - - --pod-ttl=30m - - --scan-interval=1m - - --warm-up-pod-cache=true + - --functions=/home/nonroot/functions - --max-request-body-size=6291456 - env: - - name: WRAPPER_SERVER_IMAGE - value: "wrapper-server:latest" ports: - containerPort: 9445 protocol: TCP @@ -196,6 +183,6 @@ spec: {{% alert title="Note" color="primary" %}} For advanced configuration options: -- [Pod Templates]({{% relref "pod-templates" %}}) - Customize function pod specifications -- [Private Registries]({{% relref "private-registries-config" %}}) - Configure private registry access +- [Pod Templates]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config/pod-templates" %}}) - Customize function pod specifications +- [Private Registries]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config/private-registries-config" %}}) - Configure private registry access {{% /alert %}} \ No newline at end of file diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-controllers-config.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-controllers-config.md index 64a1ae376..9a8ab07b8 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-controllers-config.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-controllers-config.md @@ -101,7 +101,11 @@ args: | Variable | Required | Description | |----------|----------|-------------| -| `FUNCTION_RUNNER_ADDRESS` | For external functions | gRPC address of the function runner service. If unset, only builtin Go functions are available. | +| `FUNCTION_RUNNER_ADDRESS` | Optional | gRPC address of Function Runner (exec fast path). Default manifests omit this. | +| `WRAPPER_SERVER_IMAGE` | For container functions | Wrapper-server image. When set, the controller runs the Engine pod evaluator in-process. Default manifests set this. | +| `POD_NAMESPACE` | No | Function-pod / FunctionConfig namespace (default `porch-fn-system`). | +| `FUNCTION_CACHE_DIR` | No | On-disk FunctionConfig binary cache (default `/home/nonroot/functions`). | +| `DEFAULT_IMAGE_PREFIX` | No | Prefix for short function image names. | **Prerequisites:** @@ -109,7 +113,8 @@ The PR Controller requires: - The Repository Controller to be running (provides the shared cache) - The `PackageRevision` CRD (`porch.kpt.dev/v1alpha2`) to be installed in the cluster -- The function runner service to be reachable (if external KRM functions are used) +- `WRAPPER_SERVER_IMAGE` for external (container) KRM functions +- `FUNCTION_RUNNER_ADDRESS` only if you also want cached-binary exec via Function Runner **Tuning Guidance:** diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/_index.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/_index.md index 921232eba..cba3887f1 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/_index.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/_index.md @@ -42,9 +42,38 @@ args: #### Function Runtime Arguments ```bash args: +- --function-runner=function-runner:9445 # Function Runner gRPC address (exec fast path) - --default-image-prefix=ghcr.io/kptdev/krm-functions-catalog # Default function image prefix +- --functions=/home/nonroot/functions # On-disk FunctionConfig binary cache +- --pod-namespace=porch-fn-system # Namespace for function pods and FunctionConfigs ``` +#### Pod Evaluator Arguments + +Porch-server **requires** `WRAPPER_SERVER_IMAGE` and will not start without it. These flags configure the in-process pod evaluator: + +```bash +args: +- --warm-up-pod-cache=true # Pre-create pods from the warm-up config (default: true) +- --pod-ttl=30m # Pod TTL before GC (default: 30m) +- --scan-interval=1m # GC scan interval (default: 1m) +- --max-waitlist-length=1 # Max waiters per pod (deployment default; flag default is 2) +- --max-parallel-pods-per-function=2 # Max parallel pods per function image +- --enable-private-registries=false +- --registry-auth-secret-path=/var/tmp/config-secret/.dockerconfigjson +- --registry-auth-secret-name=auth-secret +- --enable-private-registries-tls=false +- --tls-secret-path=/var/tmp/tls-secret/ +``` + +```bash +env: +- name: WRAPPER_SERVER_IMAGE + value: "ghcr.io/kptdev/porch-wrapper-server:latest" # Required +``` + +For function pod specs see [Pod Templates]({{% relref "pod-templates" %}}). For registry auth see [Private Registries]({{% relref "private-registries-config" %}}). + ### Environment Variables #### Database Configuration (when using DB cache) diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/pod-templates.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/pod-templates.md similarity index 65% rename from docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/pod-templates.md rename to docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/pod-templates.md index 5937b4733..a223fa2e4 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/pod-templates.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/pod-templates.md @@ -2,14 +2,14 @@ title: "Pod Templates" type: docs weight: 2 -description: "Customize function evaluator pod specifications using ConfigMap templates" +description: "Customize function evaluator pod specifications using PodTemplate CRs" --- -The Function Runner supports customizing the pod specifications used for KRM function evaluation through ConfigMap-based templates. This allows you to configure resource limits, security contexts, node selectors, tolerations, and other pod-level settings for function execution pods. +The Engine pod evaluator (porch-server and the PackageRevision controller) customizes KRM function pods through Kubernetes `PodTemplate` and `ServiceTemplate` objects named `base-pod-template` and `base-service-template` in the function-pod namespace (`porch-fn-system`). This page moved from Function Runner with the pod evaluator. ## Overview -By default, the Function Runner uses an inline pod template with sensible defaults. For advanced use cases requiring customization, you can provide a ConfigMap containing custom pod and service templates. The Function Runner will use these templates when creating function evaluator pods. +By default, the Engine uses an inline pod template and creates `base-pod-template` / `base-service-template` in `porch-fn-system` if they are missing. Default manifests ship these objects in `deployments/porch/22-function-templates.yaml`. Edit those CRs to customize resource limits, security contexts, node selectors, tolerations, and other pod-level settings. The pod template system provides: - **Resource customization** - Configure CPU/memory limits for function pods @@ -18,7 +18,7 @@ The pod template system provides: - **Network policies** - Customize service specifications for service mesh integration - **Volume management** - Add additional volumes and volume mounts -For architectural details on how pod templates are used in the pod lifecycle, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/function-runner/functionality/pod-lifecycle-management.md" %}}). +For architectural details on how pod templates are used in the pod lifecycle, see [Pod Lifecycle Management]({{% relref "/docs/5_architecture_and_components/engine/functionality/pod-lifecycle-management.md" %}}). ## Template Contract @@ -29,84 +29,21 @@ Any custom pod template must fulfill the following requirements: 3. **Image replacement** - The `function` container's image can be set to any KRM function image without breaking the wrapper server entrypoint 4. **Entrypoint arguments** - The `function` container's args can be appended with entries from the function image's Dockerfile ENTRYPOINT -The Function Runner automatically patches the template with function-specific configuration before creating pods. +The Engine automatically patches the template with function-specific configuration (image, entrypoint, pull secrets, FunctionConfig TemplateOverrides) before creating pods. ## Enabling Pod Templates -### Step 1: Configure RBAC +Default Porch manifests already apply `base-pod-template` and `base-service-template` in `porch-fn-system` (`deployments/porch/22-function-templates.yaml`). porch-server and porch-controllers are bound to the `porch-function-executor` Role, which can get/create those objects. There is no `--function-pod-template` flag. -The Function Runner requires read access to the pod template ConfigMap. Create a Role and RoleBinding in the Function Runner's namespace: - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: porch-fn-runner-configmap-reader - namespace: porch-system -rules: - - apiGroups: [""] - resources: ["configmaps"] - resourceNames: ["kpt-function-eval-pod-template"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: porch-fn-runner-configmap-reader - namespace: porch-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: porch-fn-runner-configmap-reader -subjects: - - kind: ServiceAccount - name: porch-fn-runner - namespace: porch-system -``` - -### Step 2: Configure Function Runner - -Add the `--function-pod-template` argument to the Function Runner deployment, specifying the ConfigMap name: - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: function-runner - namespace: porch-system -spec: - template: - spec: - serviceAccountName: porch-fn-runner - containers: - - name: function-runner - image: ghcr.io/kptdev/porch-function-runner:latest - args: - - --port=9445 - - --pod-namespace=porch-fn-system - - --function-pod-template=kpt-function-eval-pod-template - env: - - name: WRAPPER_SERVER_IMAGE - value: ghcr.io/kptdev/porch-wrapper-server:latest -``` - -### Step 3: Create the ConfigMap - -Create a ConfigMap in the same namespace where the Function Runner is deployed (typically `porch-system`). The ConfigMap must contain two keys: - -- `template` - Pod specification in YAML format -- `serviceTemplate` - Service specification in YAML format +To customize, edit the `PodTemplate` in `porch-fn-system` (or replace the shipped YAML before deploy). Example: ```yaml apiVersion: v1 -kind: ConfigMap +kind: PodTemplate metadata: - name: kpt-function-eval-pod-template - namespace: porch-system -data: - template: | - apiVersion: v1 - kind: Pod + name: base-pod-template + namespace: porch-fn-system +template: metadata: annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" @@ -133,19 +70,12 @@ data: volumes: - name: wrapper-server-tools emptyDir: {} - serviceTemplate: | - apiVersion: v1 - kind: Service - spec: - type: ClusterIP - ports: - - port: 9446 - protocol: TCP - targetPort: 9446 - selector: - fn.kpt.dev/image: to-be-replaced ``` +The matching service frontend is `ServiceTemplate` `base-service-template` in the same namespace (see `deployments/porch/22-function-templates.yaml`). + +The snippets below are pod spec fragments to merge into `base-pod-template`. + ## Template Customization Examples ### Resource Limits @@ -265,18 +195,18 @@ data: ## Template Versioning -The Function Runner tracks the ConfigMap's `ResourceVersion` to detect template changes. When the ConfigMap is updated: +The Engine tracks the PodTemplate's `ResourceVersion` to detect template changes. When the template is updated: -1. The Function Runner detects the new version on the next pod creation +1. The Engine detects the new version on the next pod creation 2. Existing pods with the old template version continue running -3. When an old pod is reused, the Function Runner detects the version mismatch +3. When an old pod is reused, the Engine detects the version mismatch 4. The old pod is deleted and a new pod is created with the updated template This ensures zero-downtime template updates while maintaining cache efficiency. ## Default Template -When no ConfigMap is specified, the Function Runner uses this inline default template: +When `base-pod-template` is missing, the Engine creates it from this inline default template: ```yaml apiVersion: v1 @@ -321,26 +251,37 @@ spec: ### Template Validation Errors -If the Function Runner fails to parse the template: +If the Engine fails to parse the template: + +```bash +kubectl logs -n porch-system deployment/porch-server | grep "unable to decode" +``` + +Common issues: +- Invalid YAML syntax in the PodTemplate +- Missing required fields (function container) +- Incorrect indentation + +If the Engine cannot read the template: ```bash -kubectl logs -n porch-system deployment/function-runner | grep "unable to decode" +kubectl logs -n porch-system deployment/porch-server | grep "PodTemplate" ``` Common issues: -- Invalid YAML syntax in the ConfigMap +- Invalid YAML syntax in the PodTemplate - Missing required `function` container - Incorrect indentation ### RBAC Permission Errors -If the Function Runner cannot read the ConfigMap: +If porch-server cannot read `base-pod-template`: ```bash -kubectl logs -n porch-system deployment/function-runner | grep "Could not get Configmap" +kubectl logs -n porch-system deployment/porch-server | grep "PodTemplate" ``` -Verify the Role and RoleBinding are correctly configured and the ServiceAccount name matches. +Verify the `porch-function-executor` RoleBinding includes the `porch-server` (and `porch-controllers`) ServiceAccount. ### Pod Creation Failures diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/private-registries-config.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/private-registries-config.md similarity index 84% rename from docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/private-registries-config.md rename to docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/private-registries-config.md index 8c2dddf1e..7805c883d 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/private-registries-config.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/components/porch-server-config/private-registries-config.md @@ -2,14 +2,14 @@ title: "Private Registries" type: docs weight: 4 -description: "Configure Function Runner access to private container registries" +description: "Configure porch-server access to private container registries" --- {{% alert title="Note" color="primary" %}} KPT functions and KRM functions are synonymous terms referring to the same containerized functions. {{% /alert %}} -Configure the Function Runner to access private container registries for KRM functions. +Configure **porch-server** to access private container registries for KRM functions. These flags moved from Function Runner with the pod evaluator. ## Use Cases @@ -20,20 +20,20 @@ Private registries are commonly used for: ## Default Public Registries -By default, Function Runner uses public registries: +By default, porch-server uses public registries: - `ghcr.io/kptdev/krm-functions-catalog` - GitHub Container Registry for KRM functions - Other public registries as configured ## Private Registry Authentication -To use private container registries for KRM functions, configure authentication in the Function Runner. +To use private container registries for KRM functions, configure authentication in the porch-server. ### 1. Create Docker Configuration Secret Create a secret using Docker configuration format: {{% alert title="Note" color="primary" %}} -The secret must be in the same namespace as the function runner deployment. By default, this is the *porch-system* namespace. +The secret must be in the same namespace as the **porch-server** deployment. By default, this is the *porch-system* namespace. {{% /alert %}} ```bash @@ -60,21 +60,21 @@ Example `config.json` format: The `auth` value is base64 encoded `username:password`. -### 2. Mount Secret in Function Runner +### 2. Mount Secret in porch-server -Update the Function Runner deployment: +Update the porch-server deployment: ```yaml apiVersion: apps/v1 kind: Deployment metadata: - name: function-runner + name: porch-server namespace: porch-system spec: template: spec: containers: - - name: function-runner + - name: porch-server args: - --enable-private-registries=true - --registry-auth-secret-path=/var/tmp/auth-secret/.dockerconfigjson @@ -91,7 +91,7 @@ spec: ### 3. Configuration Arguments -Required Function Runner arguments: +Required porch-server arguments: - `--enable-private-registries=true` - Enable private registry functionality - `--registry-auth-secret-path` - Path to mounted secret (default: `/var/tmp/auth-secret/.dockerconfigjson`) - `--registry-auth-secret-name` - Name of the secret (default: `auth-secret`) @@ -102,7 +102,7 @@ Use dedicated subdirectories for mount paths to avoid overwriting directory perm ## How It Works -When configured, the Function Runner: +When configured, the porch-server: 1. Replicates the registry secret to the `porch-fn-system` namespace 2. Uses it as an `imagePullSecret` for KRM function pods 3. Enables function pods to pull images from private registries @@ -140,7 +140,7 @@ spec: template: spec: containers: - - name: function-runner + - name: porch-server args: - --enable-private-registries-tls=true - --tls-secret-path=/var/tmp/tls-secret/ @@ -162,14 +162,14 @@ Additional arguments for TLS: ## TLS Connection Logic -When TLS is enabled, Function Runner attempts connection in this order: +When TLS is enabled, porch-server attempts connection in this order: 1. Using the mounted TLS certificate 2. Using system intermediate certificates (for well-known CAs) 3. Without TLS as fallback 4. Returns error if all attempts fail {{% alert title="Warning" color="warning" %}} -Ensure Kubernetes nodes are configured with the same TLS certificate information. The Function Runner can pull images, but KRM function pods need node-level certificate configuration to run successfully. +Ensure Kubernetes nodes are configured with the same TLS certificate information. The porch-server can pull images, but KRM function pods need node-level certificate configuration to run successfully. {{% /alert %}} ## Complete Example @@ -180,13 +180,13 @@ Combining both authentication and TLS: apiVersion: apps/v1 kind: Deployment metadata: - name: function-runner + name: porch-server namespace: porch-system spec: template: spec: containers: - - name: function-runner + - name: porch-server args: - --enable-private-registries=true - --registry-auth-secret-path=/var/tmp/auth-secret/.dockerconfigjson diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md index 5e6a32b7d..fac123dc0 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md @@ -385,9 +385,9 @@ spec: ### Wrapper Server Configuration via Pod Templating -The wrapper-server component can be configured with OpenTelemetry settings through the pod templating mechanism used by the function runner. This is done by creating a ConfigMap with a pod template that includes the necessary environment variables. +The wrapper-server component can be configured with OpenTelemetry settings through `base-pod-template` used by the Engine pod evaluator. Put the environment variables on the `function` container. -#### ConfigMap Pod Template with OpenTelemetry Configuration +#### PodTemplate fragment with OpenTelemetry configuration ```yaml apiVersion: v1 @@ -447,16 +447,7 @@ data: emptyDir: {} ``` -The function runner must be configured to use this template by specifying the `--function-pod-template` argument: - -```yaml -command: - - /server - - --config=/config.yaml - - --functions=/functions - - --pod-namespace=porch-fn-system - - --function-pod-template=kpt-function-eval-pod-template -``` +Apply this OpenTelemetry configuration by editing `base-pod-template` in `porch-fn-system` (see [Pod Templates]({{% relref "/docs/6_configuration_and_deployments/configurations/components/porch-server-config/pod-templates" %}})). There is no `--function-pod-template` flag. ## Context Propagation diff --git a/docs/content/en/docs/6_configuration_and_deployments/deployments/catalog-deployment.md b/docs/content/en/docs/6_configuration_and_deployments/deployments/catalog-deployment.md index 002ea1ffa..820778150 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/deployments/catalog-deployment.md +++ b/docs/content/en/docs/6_configuration_and_deployments/deployments/catalog-deployment.md @@ -30,9 +30,7 @@ These **optional** features must be configured **before** deployment if you need - [Cert-Manager Webhooks]({{% relref "../configurations/components/porch-server-config/cert-manager-webhooks" %}}) - Enable cert-manager webhook integration (requires deployment env vars) - [OpenTelemetry]({{% relref "../configurations/opentelemetry" %}}) - Enable distributed tracing and metrics (requires deployment env vars) - [Git Custom TLS]({{% relref "../configurations/components/porch-server-config/git-authentication#3-httpstls-configuration" %}}) - Enable custom TLS certificates for Git repositories (requires `--use-git-cabundle=true` arg) - -#### Function Runner -- [Private Registries]({{% relref "../configurations/components/function-runner-config/private-registries-config" %}}) - Configure private container registries (requires deployment args and volume mounts) +- [Private Registries]({{% relref "../configurations/components/porch-server-config/private-registries-config" %}}) - Configure private container registries for function pods (requires deployment args and volume mounts) ### Post-deployment Configuration diff --git a/func/evaluator/evaluator.pb.go b/func/evaluator/evaluator.pb.go index 6d122f5ea..7cd21600d 100644 --- a/func/evaluator/evaluator.pb.go +++ b/func/evaluator/evaluator.pb.go @@ -21,13 +21,12 @@ package evaluator import ( - reflect "reflect" - sync "sync" - unsafe "unsafe" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" _ "google.golang.org/protobuf/types/known/structpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" ) const ( @@ -44,7 +43,10 @@ type EvaluateFunctionRequest struct { // kpt image identifying the function to evaluate Image string `protobuf:"bytes,2,opt,name=image,proto3" json:"image,omitempty"` // optional field for function description. - Tag string `protobuf:"bytes,3,opt,name=Tag,proto3" json:"Tag,omitempty"` + Tag string `protobuf:"bytes,3,opt,name=Tag,proto3" json:"Tag,omitempty"` + // exec_path is the path of the executable to run when an executable runner + // is available for this function. If empty, the executable runner is skipped. + ExecPath string `protobuf:"bytes,4,opt,name=exec_path,json=execPath,proto3" json:"exec_path,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -100,6 +102,13 @@ func (x *EvaluateFunctionRequest) GetTag() string { return "" } +func (x *EvaluateFunctionRequest) GetExecPath() string { + if x != nil { + return x.ExecPath + } + return "" +} + // ConfigMap wraps a map for use in oneof clause. type ConfigMap struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -204,11 +213,12 @@ var File_evaluator_proto protoreflect.FileDescriptor const file_evaluator_proto_rawDesc = "" + "\n" + - "\x0fevaluator.proto\x12\tevaluator\x1a\fstruct.proto\"f\n" + + "\x0fevaluator.proto\x12\tevaluator\x1a\fstruct.proto\"\x83\x01\n" + "\x17EvaluateFunctionRequest\x12#\n" + "\rresource_list\x18\x01 \x01(\fR\fresourceList\x12\x14\n" + "\x05image\x18\x02 \x01(\tR\x05image\x12\x10\n" + - "\x03Tag\x18\x03 \x01(\tR\x03Tag\"x\n" + + "\x03Tag\x18\x03 \x01(\tR\x03Tag\x12\x1b\n" + + "\texec_path\x18\x04 \x01(\tR\bexecPath\"x\n" + "\tConfigMap\x122\n" + "\x04data\x18\x01 \x03(\v2\x1e.evaluator.ConfigMap.DataEntryR\x04data\x1a7\n" + "\tDataEntry\x12\x10\n" + @@ -218,7 +228,7 @@ const file_evaluator_proto_rawDesc = "" + "\rresource_list\x18\x01 \x01(\fR\fresourceList\x12\x10\n" + "\x03log\x18\x02 \x01(\fR\x03log2r\n" + "\x11FunctionEvaluator\x12]\n" + - "\x10EvaluateFunction\x12\".evaluator.EvaluateFunctionRequest\x1a#.evaluator.EvaluateFunctionResponse\"\x00B0Z.github.com/kptdev/porch/func/evaluatorb\x06proto3" + "\x10EvaluateFunction\x12\".evaluator.EvaluateFunctionRequest\x1a#.evaluator.EvaluateFunctionResponse\"\x00B(Z&github.com/kptdev/porch/func/evaluatorb\x06proto3" var ( file_evaluator_proto_rawDescOnce sync.Once diff --git a/func/evaluator/evaluator.proto b/func/evaluator/evaluator.proto index 229fb7b83..e4b390e7b 100644 --- a/func/evaluator/evaluator.proto +++ b/func/evaluator/evaluator.proto @@ -36,6 +36,10 @@ message EvaluateFunctionRequest { // optional field for function description. string Tag = 3; + + // exec_path is the path of the executable to run when an executable runner + // is available for this function. If empty, the executable runner is skipped. + string exec_path = 4; } // ConfigMap wraps a map for use in oneof clause. diff --git a/func/internal/executableevaluator.go b/func/internal/executableevaluator.go index 2a16457ed..c5995fdd9 100644 --- a/func/internal/executableevaluator.go +++ b/func/internal/executableevaluator.go @@ -18,67 +18,45 @@ import ( "bytes" "context" "fmt" + "os" "os/exec" + "path/filepath" + "strings" kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" "github.com/kptdev/kpt/pkg/fn" - "github.com/kptdev/porch/controllers/functionconfigs" pb "github.com/kptdev/porch/func/evaluator" - regclientref "github.com/regclient/regclient/types/ref" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog/v2" ) -type ExecutableEvaluatorOptions struct { - FunctionCacheDir string // Path to cached functions -} - type executableEvaluator struct { - // Fast-path function cache - FunctionConfigStore *functionconfigs.FunctionConfigStore + functionCacheDir string } var _ Evaluator = &executableEvaluator{} -func NewExecutableEvaluator(FunctionConfigStore *functionconfigs.FunctionConfigStore) (Evaluator, error) { - return &executableEvaluator{ - FunctionConfigStore: FunctionConfigStore, - }, nil +func NewExecutableEvaluator(functionCacheDir string) (Evaluator, error) { + return &executableEvaluator{functionCacheDir: functionCacheDir}, nil } func (e *executableEvaluator) EvaluateFunction(ctx context.Context, req *pb.EvaluateFunctionRequest) (*pb.EvaluateFunctionResponse, error) { - var selectedBinary string - if req.Tag != "" { - ref, err := regclientref.New(req.Image) - if err != nil { - return nil, fmt.Errorf("failed to parse image %q as reference: %w", req.Image, err) + if req.ExecPath == "" { + return nil, &fn.NotFoundError{ + Function: kptfilev1.Function{Image: req.Image}, } - ref.Tag = "" - ref.Digest = "" - req.Image = ref.CommonName() + } - binary, exists := e.FunctionConfigStore.GetBinaryFromCacheByConstraint(req.Image, req.Tag) - if !exists { - return nil, &fn.NotFoundError{ - Function: kptfilev1.Function{Image: req.Image}, - } - } - selectedBinary = binary - } else { - klog.V(2).Infof("Image tag is empty, using the image with explicit tag: %q", req.Image) - binary, exists := e.FunctionConfigStore.GetBinaryFromCache(req.Image) - if !exists { - return nil, &fn.NotFoundError{ - Function: kptfilev1.Function{Image: req.Image}, - } - } - selectedBinary = binary + execPath := filepath.Clean(req.ExecPath) + base := filepath.Clean(e.functionCacheDir) + string(os.PathSeparator) + if !strings.HasPrefix(execPath, base) { + return nil, fmt.Errorf("exec_path %q is outside functions dir", req.ExecPath) } klog.Infof("Evaluating %q in executable mode", req.Image) var stdout, stderr bytes.Buffer - cmd := exec.CommandContext(ctx, selectedBinary) // #nosec G204 -- variables controlled internally + cmd := exec.CommandContext(ctx, execPath) // #nosec G204 -- variables controlled internally cmd.Stdin = bytes.NewReader(req.ResourceList) cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -98,3 +76,7 @@ func (e *executableEvaluator) EvaluateFunction(ctx context.Context, req *pb.Eval Log: stderr.Bytes(), }, nil } + +func (e *executableEvaluator) Name() string { + return "exec" +} diff --git a/func/internal/executableevaluator_test.go b/func/internal/executableevaluator_test.go index 3647d5f20..8001527e6 100644 --- a/func/internal/executableevaluator_test.go +++ b/func/internal/executableevaluator_test.go @@ -15,7 +15,6 @@ package internal import ( - "bytes" "flag" "fmt" "os" @@ -24,8 +23,6 @@ import ( kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" "github.com/kptdev/kpt/pkg/fn" - configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" - "github.com/kptdev/porch/controllers/functionconfigs" pb "github.com/kptdev/porch/func/evaluator" imageutil "github.com/kptdev/porch/pkg/util/image" "github.com/stretchr/testify/assert" @@ -40,51 +37,10 @@ const ( testImageName = "test-image" ) -func getFunctionConfigStore(binaryDir string) *functionconfigs.FunctionConfigStore { - starlarkConfig := &configapi.FunctionConfig{ - Spec: configapi.FunctionConfigSpec{ - Image: starlarkFunction, - Prefixes: []string{ - "", - }, - BinaryExecutor: &configapi.BinaryExecutorConfig{ - Tags: []string{ - "v0.5.2", - "v0.5.3", - }, - Path: starlarkFunction, - }, - }, - } - setImageConfig := &configapi.FunctionConfig{ - Spec: configapi.FunctionConfigSpec{ - Image: setImageFunction, - Prefixes: []string{ - "", - }, - BinaryExecutor: &configapi.BinaryExecutorConfig{ - Tags: []string{ - "v0.1.2", - "v0.1.3", - }, - Path: setImageFunction, - }, - }, - } - fstore := functionconfigs.NewFunctionConfigStore(defaultKRMImagePrefix, binaryDir) - fstore.UpdateBinaryCache(starlarkFunction, starlarkConfig) - fstore.UpdateBinaryCache(setImageFunction, setImageConfig) - return fstore -} - func TestNewExecutableEvaluator(t *testing.T) { const tempCacheDir = "/tmp/func_cache" t.Run("no errors", func(t *testing.T) { - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tempCacheDir, - } - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - _, err := NewExecutableEvaluator(fStore) + _, err := NewExecutableEvaluator("/tmp/function-cache") assert.NoError(t, err) }) } @@ -95,11 +51,8 @@ func TestEvaluateExecutableFunction(t *testing.T) { _ = flagSet.Parse([]string{"--v", "3"}) const tempCacheDir = "/tmp/func_cache" - t.Run("invalid semver constraint will cause function not found error", func(t *testing.T) { + t.Run("no exec_path returns function not found error", func(t *testing.T) { ctx := t.Context() - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tempCacheDir, - } req := &pb.EvaluateFunctionRequest{ ResourceList: []byte("req-rl"), @@ -107,8 +60,7 @@ func TestEvaluateExecutableFunction(t *testing.T) { Tag: ">> 0.1.3 < 0.2.0", // Invalid semver constraint, '>>' is not a valid operator } - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - evaluator, _ := NewExecutableEvaluator(fStore) + evaluator, _ := NewExecutableEvaluator("/tmp/function-cache") resp, err := evaluator.EvaluateFunction(ctx, req) assert.Nil(t, resp) @@ -116,60 +68,20 @@ func TestEvaluateExecutableFunction(t *testing.T) { Function: kptfilev1.Function{Image: req.Image}, }, err) }) - t.Run("function is not included in the config", func(t *testing.T) { - ctx := t.Context() - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tempCacheDir, - } - - req := &pb.EvaluateFunctionRequest{ - ResourceList: []byte("req-rl"), - // This image is not included in the config.yaml -> function not found - Image: imageutil.Join(defaultKRMImagePrefix, testImageName), - Tag: "> 0.1.3 < 0.2.0", // This is a valid semver constraint syntax - } - - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - evaluator, _ := NewExecutableEvaluator(fStore) - _, err := evaluator.EvaluateFunction(ctx, req) - assert.Equal(t, fmt.Sprintf("function \"%s\" not found", req.Image), err.Error()) - }) - t.Run("function does not match the semantic version constraints", func(t *testing.T) { - ctx := t.Context() - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tempCacheDir, - } - - req := &pb.EvaluateFunctionRequest{ - ResourceList: []byte("req-rl"), - Image: imageutil.Join(defaultKRMImagePrefix, setImageFunction), - Tag: "> 0.1.3 < 0.2.0", - } - - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - evaluator, _ := NewExecutableEvaluator(fStore) - _, err := evaluator.EvaluateFunction(ctx, req) - assert.ErrorContains(t, err, fmt.Sprintf("function \"%s\" not found", req.Image), err.Error()) - }) t.Run("failed to execute function", func(t *testing.T) { ctx := t.Context() - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tempCacheDir, // function cache dir is not exist - } req := &pb.EvaluateFunctionRequest{ ResourceList: []byte("req-rl"), Image: imageutil.Join(defaultKRMImagePrefix, setImageFunction), - Tag: ">= 0.1.2 < 0.2.0", + ExecPath: "/tmp/function-cache/nonexistent-binary", } - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - - evaluator, _ := NewExecutableEvaluator(fStore) + evaluator, _ := NewExecutableEvaluator("/tmp/function-cache") _, err := evaluator.EvaluateFunction(ctx, req) assert.ErrorContains(t, err, fmt.Sprintf("Failed to execute function \"%s\":", req.Image)) }) - t.Run("successful function execution with semantic versioning", func(t *testing.T) { + t.Run("successful function execution", func(t *testing.T) { ctx := t.Context() // Create a temporary directory for the function cache @@ -183,132 +95,8 @@ cat exit 0 ` err := os.WriteFile(testBinary, []byte(testScript), 0755) - assert.NoError(t, err) - - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tmpDir, - } - - // Use a valid KRM ResourceList format - const resourceList = `apiVersion: config.kubernetes.io/v1 -kind: ResourceList -items: [] -` - - // This constraint matches both v0.1.2 and v0.1.3 from config.yaml - // We expect v0.1.3 to be selected as it's the greatest version - req := &pb.EvaluateFunctionRequest{ - ResourceList: []byte(resourceList), - Image: imageutil.Join(defaultKRMImagePrefix, setImageFunction), - Tag: ">= 0.1.2 < 0.2.0", - } - - // Capture klog output by redirecting stderr - oldStderr := os.Stderr - r, w, _ := os.Pipe() - os.Stderr = w - - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - evaluator, err := NewExecutableEvaluator(fStore) require.NoError(t, err) - resp, err := evaluator.EvaluateFunction(ctx, req) - - // Flush klog and restore stderr - klog.Flush() - w.Close() - os.Stderr = oldStderr - - // Read captured output - var logBuffer bytes.Buffer - logBuffer.ReadFrom(r) - logOutput := logBuffer.String() - - assert.NoError(t, err) - assert.NotNil(t, resp) - - // Verify the klog message contains the expected version selection - assert.Contains(t, logOutput, `Selected tag "v0.1.3"`) - }) - t.Run("successful function execution with explicit tagging", func(t *testing.T) { - ctx := t.Context() - - // Create a temporary directory for the function cache - tmpDir := t.TempDir() - - // Create a simple test executable that echoes input as a valid KRM function - testBinary := filepath.Join(tmpDir, setImageFunction) - const testScript = `#!/bin/sh -# Emulating the KRM function execution by running this shell script -cat -exit 0 -` - err := os.WriteFile(testBinary, []byte(testScript), 0755) - require.NoError(t, err) - - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tmpDir, - } - - // Use a valid KRM ResourceList format - const resourceList = `apiVersion: config.kubernetes.io/v1 -kind: ResourceList -items: [] -` - - // Explicit tagging - req := &pb.EvaluateFunctionRequest{ - ResourceList: []byte(resourceList), - Image: imageutil.Join(defaultKRMImagePrefix, setImageFunction) + ":v0.1.3", - } - - // Capture klog output by redirecting stderr - oldStderr := os.Stderr - r, w, _ := os.Pipe() - os.Stderr = w - - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - evaluator, err := NewExecutableEvaluator(fStore) - require.NoError(t, err) - - resp, err := evaluator.EvaluateFunction(ctx, req) - - // Flush klog and restore stderr - klog.Flush() - w.Close() - os.Stderr = oldStderr - - // Read captured output - var logBuffer bytes.Buffer - logBuffer.ReadFrom(r) - logOutput := logBuffer.String() - - require.NoError(t, err) - assert.NotNil(t, resp) - - // Verify the klog message contains the expected version selection - assert.Contains(t, logOutput, `Image tag is empty, using the image with explicit tag: "ghcr.io/kptdev/krm-functions-catalog/set-image:v0.1.3"`) - }) - t.Run("successful execution with explicit tagging + Tag field set", func(t *testing.T) { - ctx := t.Context() - - // Create a temporary directory for the function cache - tmpDir := t.TempDir() - - // Create a simple test executable that echoes input as a valid KRM function - testBinary := filepath.Join(tmpDir, setImageFunction) - const testScript = `#!/bin/sh -# Emulating the KRM function execution by running this shell script -cat -exit 0 -` - err := os.WriteFile(testBinary, []byte(testScript), 0755) - require.NoError(t, err) - - executableEvaluatorOptions := ExecutableEvaluatorOptions{ - FunctionCacheDir: tmpDir, - } - // Use a valid KRM ResourceList format const resourceList = `apiVersion: config.kubernetes.io/v1 kind: ResourceList @@ -317,35 +105,17 @@ items: [] req := &pb.EvaluateFunctionRequest{ ResourceList: []byte(resourceList), - Image: imageutil.Join(defaultKRMImagePrefix, setImageFunction) + ":v0.0.1", - Tag: ">= 0.1.2 < 0.2.0", + Image: imageutil.Join(defaultKRMImagePrefix, setImageFunction), + ExecPath: testBinary, } - // Capture klog output by redirecting stderr - oldStderr := os.Stderr - r, w, _ := os.Pipe() - os.Stderr = w - - fStore := getFunctionConfigStore(executableEvaluatorOptions.FunctionCacheDir) - evaluator, err := NewExecutableEvaluator(fStore) + evaluator, err := NewExecutableEvaluator(tmpDir) require.NoError(t, err) resp, err := evaluator.EvaluateFunction(ctx, req) - // Flush klog and restore stderr - klog.Flush() - w.Close() - os.Stderr = oldStderr - - // Read captured output - var logBuffer bytes.Buffer - logBuffer.ReadFrom(r) - logOutput := logBuffer.String() - require.NoError(t, err) assert.NotNil(t, resp) - - // Verify the klog message contains the expected version selection - assert.Contains(t, logOutput, `Selected tag "v0.1.3"`) + assert.Equal(t, []byte(resourceList), resp.ResourceList) }) } diff --git a/func/server/server.go b/func/server/server.go index fd9da7fb1..9b6662722 100644 --- a/func/server/server.go +++ b/func/server/server.go @@ -15,46 +15,29 @@ package main import ( - "context" "flag" "fmt" "net" - "net/http" "os" "strconv" "strings" "time" - "github.com/kptdev/kpt/pkg/lib/runneroptions" - configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" - "github.com/kptdev/porch/controllers/functionconfigs" pb "github.com/kptdev/porch/func/evaluator" "github.com/kptdev/porch/func/healthchecker" "github.com/kptdev/porch/func/internal" "github.com/kptdev/porch/internal/telemetry" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" contextsignal "k8s.io/apiserver/pkg/server" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/klog/v2/textlogger" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client/config" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/predicate" ) const ( execRuntime = "exec" - podRuntime = "pod" - - wrapperServerImageEnv = "WRAPPER_SERVER_IMAGE" ) type options struct { @@ -65,37 +48,22 @@ type options struct { // The verbosity level of the logs (0-5) logLevel int - defaultImagePrefix string + MaxGrpcMessageSize int // Parameters of ExecEvaluator - exec internal.ExecutableEvaluatorOptions - // Parameters of PodEvaluator - pod internal.PodEvaluatorOptions + FunctionCacheDir string } func main() { o := &options{} // generic flags flag.IntVar(&o.port, "port", 9445, "The server port") - flag.StringVar(&o.disableRuntimes, "disable-runtimes", "", fmt.Sprintf("The runtime(s) to disable. Multiple runtimes should separated by `,`. Available runtimes: `%v`, `%v`.", execRuntime, podRuntime)) + flag.StringVar(&o.disableRuntimes, "disable-runtimes", "", fmt.Sprintf("The runtime(s) to disable. Multiple runtimes should separated by `,`. Available runtimes: `%v`.", execRuntime)) flag.IntVar(&o.logLevel, "log-level", 2, "The verbosity level of the logs (0-5)") - flag.StringVar(&o.defaultImagePrefix, "default-image-prefix", runneroptions.GHCRImagePrefix, "Default prefix for unqualified function names") // flags for the exec runtime - flag.StringVar(&o.exec.FunctionCacheDir, "functions", "./functions", "Path to cached functions.") + flag.StringVar(&o.FunctionCacheDir, "functions", "./functions", "Path to cached functions.") // flags for the pod runtime - flag.BoolVar(&o.pod.WarmUpPodCacheOnStartup, "warm-up-pod-cache", true, "if true, pod-cache-config image pods will be deployed at startup") - flag.StringVar(&o.pod.PodNamespace, "pod-namespace", "porch-fn-system", "Namespace to run KRM functions pods.") - flag.DurationVar(&o.pod.PodTTL, "pod-ttl", 30*time.Minute, "TTL for pods before GC.") - flag.DurationVar(&o.pod.GcScanInterval, "scan-interval", time.Minute, "The interval of GC between scans.") - flag.BoolVar(&o.pod.EnablePrivateRegistries, "enable-private-registries", false, "if true enables the use of private registries and their authentication") - flag.StringVar(&o.pod.RegistryAuthSecretPath, "registry-auth-secret-path", "/var/tmp/config-secret/.dockerconfigjson", "The path of the secret used for authenticating to custom registries") - flag.StringVar(&o.pod.RegistryAuthSecretName, "registry-auth-secret-name", "auth-secret", "The name of the secret used for authenticating to custom registries") - flag.BoolVar(&o.pod.EnablePrivateRegistriesTls, "enable-private-registries-tls", false, "if enabled, will prioritize use of user provided TLS secret when accessing registries") - flag.StringVar(&o.pod.TlsSecretPath, "tls-secret-path", "/var/tmp/tls-secret/", "The path of the secret used in tls configuration") - flag.IntVar(&o.pod.MaxGrpcMessageSize, "max-request-body-size", 6*1024*1024, "Maximum size of grpc messages in bytes. Keep this in sync with porch-server's corresponding argument.") - flag.IntVar(&o.pod.MaxWaitlistLength, "max-waitlist-length", 2, "Maximum waitlist length per pod") - flag.IntVar(&o.pod.MaxParallelPodsPerFunction, "max-parallel-pods-per-function", 1, "Maximum parallel pods per function") - flag.IntVar(&o.pod.MaxGrpcRetries, "max-grpc-retries", 2, "Maximum number of retries on gRPC Unavailable errors") + flag.IntVar(&o.MaxGrpcMessageSize, "max-request-body-size", 6*1024*1024, "Maximum size of grpc messages in bytes. Keep this in sync with porch-server's corresponding argument.") flag.Parse() @@ -138,7 +106,6 @@ func run(o *options) error { availableRuntimes := map[string]struct{}{ execRuntime: {}, - podRuntime: {}, } if o.disableRuntimes != "" { runtimesFromFlag := strings.SplitSeq(o.disableRuntimes, ",") @@ -147,35 +114,15 @@ func run(o *options) error { } } - scheme, err := buildScheme() - if err != nil { - return err - } - fnConfigReconciler, err := buildFnConfigReconciler(o, scheme) - if err != nil { - return err - } - runtimes := []internal.Evaluator{} for rt := range availableRuntimes { switch rt { case execRuntime: - execEval, err := internal.NewExecutableEvaluator(fnConfigReconciler.FunctionConfigStore) + execEval, err := internal.NewExecutableEvaluator(o.FunctionCacheDir) if err != nil { return fmt.Errorf("failed to initialize executable evaluator: %w", err) } runtimes = append(runtimes, execEval) - case podRuntime: - o.pod.WrapperServerImage = os.Getenv(wrapperServerImageEnv) - if o.pod.WrapperServerImage == "" { - return fmt.Errorf("environment variable %v must be set to use pod function evaluator runtime", wrapperServerImageEnv) - } - o.pod.DefaultImagePrefix = o.defaultImagePrefix - podEval, err := internal.NewPodEvaluator(ctx, o.pod, fnConfigReconciler.Client, fnConfigReconciler.FunctionConfigStore) - if err != nil { - return fmt.Errorf("failed to initialize pod evaluator: %w", err) - } - runtimes = append(runtimes, podEval) } } if len(runtimes) == 0 { @@ -187,8 +134,8 @@ func run(o *options) error { // Start the gRPC server server := grpc.NewServer( - grpc.MaxRecvMsgSize(o.pod.MaxGrpcMessageSize), - grpc.MaxSendMsgSize(o.pod.MaxGrpcMessageSize), + grpc.MaxRecvMsgSize(o.MaxGrpcMessageSize), + grpc.MaxSendMsgSize(o.MaxGrpcMessageSize), grpc.StatsHandler(otelgrpc.NewServerHandler()), ) go func() { @@ -203,91 +150,3 @@ func run(o *options) error { } return nil } - -func getRestConfig() (*rest.Config, error) { - restCfg, err := config.GetConfig() - if err != nil { - return nil, err - } - - // Give it a slightly higher QPS to prevent unnecessary client-side throttling. - if restCfg.QPS < 30 { - restCfg.QPS = 30.0 - restCfg.Burst = 45 - } - - restCfg.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { - return otelhttp.NewTransport(rt) - } - - return restCfg, nil -} - -func buildScheme() (*runtime.Scheme, error) { - scheme := runtime.NewScheme() - if err := configapi.AddToScheme(scheme); err != nil { - return nil, err - } - if err := corev1.AddToScheme(scheme); err != nil { - return nil, err - } - - return scheme, nil -} - -func buildFnConfigReconciler(o *options, scheme *runtime.Scheme) (*functionconfigs.Reconciler, error) { - restCfg, err := getRestConfig() - if err != nil { - return nil, err - } - - var cacheOpts cache.Options - - cacheOpts.Scheme = scheme - cacheOpts.DefaultNamespaces = map[string]cache.Config{ - o.pod.PodNamespace: {}, - } - - mgr, err := ctrl.NewManager(restCfg, ctrl.Options{ - Scheme: scheme, - Cache: cacheOpts, - // Disable controller-runtime's default :8080 /metrics listener. Port 8080 is reserved for - // optional pprof (PORCH_PPROF_PORT); controller-runtime metrics are exposed on :9464 via OpenTelemetry. - Metrics: metricsserver.Options{ - BindAddress: "0", - }, - }) - if err != nil { - return nil, err - } - - functionConfigStore := functionconfigs.NewFunctionConfigStore(o.defaultImagePrefix, o.exec.FunctionCacheDir) - - rec := &functionconfigs.Reconciler{ - Client: mgr.GetClient(), - FunctionConfigStore: functionConfigStore, - For: functionconfigs.ReconcilerForFunctionRunner, - } - - if err := ctrl.NewControllerManagedBy(mgr). - For(&configapi.FunctionConfig{}). - WithEventFilter(predicate.GenerationChangedPredicate{}). - Complete(rec); err != nil { - panic(err) - } - - ctx, cancel := context.WithCancel(context.Background()) - _ = cancel - - go func() { - if err := mgr.Start(ctx); err != nil { - klog.Infof("manager stopped: %v", err) - } - }() - - if ok := mgr.GetCache().WaitForCacheSync(ctx); !ok { - return nil, fmt.Errorf("cache didn't sync: %w", err) - } - - return rec, nil -} diff --git a/pkg/apiserver/config.go b/pkg/apiserver/config.go index 7c9580758..c93c6b6ce 100644 --- a/pkg/apiserver/config.go +++ b/pkg/apiserver/config.go @@ -31,6 +31,7 @@ import ( "github.com/kptdev/porch/pkg/cache" cachetypes "github.com/kptdev/porch/pkg/cache/types" "github.com/kptdev/porch/pkg/engine" + "github.com/kptdev/porch/pkg/engine/podevaluator" "github.com/kptdev/porch/pkg/registry/porch" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "google.golang.org/api/option" @@ -102,7 +103,10 @@ type ExtraConfig struct { GRPCRuntimeOptions engine.GRPCRuntimeOptions CacheOptions cachetypes.CacheOptions - HAOptions HAConfig + PodEvaluatorOptions podevaluator.PodEvaluatorOptions + + ExecEvaluatorOptions engine.ExecutableEvaluatorOptions + HAOptions HAConfig PodNameSpace string FunctionStore *functionconfigs.FunctionConfigStore @@ -269,6 +273,11 @@ func (c *completedConfig) buildManager(restConfig *rest.Config, scheme *runtime. probePort = fmt.Sprintf(":%d", c.ExtraConfig.ProbePort) } + byObject := map[client.Object]ctrlcache.ByObject{ + // The informer should pre-cache all the repositories at startup + &configapi.Repository{}: {}, + } + mgr, err := c.deps.newManager(restConfig, ctrl.Options{ Scheme: scheme, LeaderElection: c.ExtraConfig.HAOptions.LeaderElection, @@ -277,11 +286,8 @@ func (c *completedConfig) buildManager(restConfig *rest.Config, scheme *runtime. RenewDeadline: zeroToNil(c.ExtraConfig.HAOptions.RenewDeadline), RetryPeriod: zeroToNil(c.ExtraConfig.HAOptions.RetryPeriod), Cache: ctrlcache.Options{ - Scheme: scheme, - ByObject: map[client.Object]ctrlcache.ByObject{ - // The informer should pre-cache all the repositories at startup - &configapi.Repository{}: {}, - }, + Scheme: scheme, + ByObject: byObject, }, HealthProbeBindAddress: probePort, // Disable controller-runtime's default :8080 /metrics listener. Port 8080 is reserved for @@ -347,7 +353,10 @@ func (c *completedConfig) registerFunctionConfigController(mgr manager.Manager) return c.deps.registerFCController(mgr) } - functionConfigStore := functionconfigs.NewFunctionConfigStore(c.ExtraConfig.GRPCRuntimeOptions.DefaultImagePrefix, "") + functionConfigStore := functionconfigs.NewFunctionConfigStore( + c.ExtraConfig.GRPCRuntimeOptions.DefaultImagePrefix, + c.ExtraConfig.ExecEvaluatorOptions.FunctionCacheDir, + ) controller := &functionconfigs.Reconciler{ Client: mgr.GetClient(), @@ -482,10 +491,19 @@ func (c *completedConfig) New(ctx context.Context) (manager.Manager, *PorchServe return runnerOptions } + coreClientWithoutCache, err := client.NewWithWatch(restConfig, client.Options{ + Scheme: scheme, + }) + + if err != nil { + return nil, nil, fmt.Errorf("error building watching client: %w", err) + } + cad, err := c.deps.newEngine( engine.WithCache(cacheImpl), engine.WithBuiltinFunctionRuntime(c.ExtraConfig.FunctionStore), - engine.WithGRPCFunctionRuntime(c.ExtraConfig.GRPCRuntimeOptions), + engine.WithGRPCFunctionRuntime(c.ExtraConfig.GRPCRuntimeOptions, c.ExtraConfig.FunctionStore), + engine.WithPodEvaluatorRuntime(ctx, c.ExtraConfig.PodEvaluatorOptions, coreClientWithoutCache, c.ExtraConfig.FunctionStore), engine.WithCredentialResolver(credentialResolver), engine.WithRunnerOptionsResolver(runnerOptionsResolver), engine.WithReferenceResolver(referenceResolver), diff --git a/pkg/apiserver/config_test.go b/pkg/apiserver/config_test.go index 053395d0a..fcad43d10 100644 --- a/pkg/apiserver/config_test.go +++ b/pkg/apiserver/config_test.go @@ -23,6 +23,8 @@ import ( sampleopenapi "github.com/kptdev/porch/api/generated/openapi" configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" + "github.com/kptdev/porch/controllers/functionconfigs" + "github.com/kptdev/porch/pkg/engine" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -145,6 +147,36 @@ func TestRegisterFunctionConfigController(t *testing.T) { assert.NotNil(t, completed.ExtraConfig.FunctionStore) } +func TestFunctionConfigStoreUsesFunctionCacheDir(t *testing.T) { + completed := completedConfigForTest(t, ExtraConfig{ + ExecEvaluatorOptions: engine.ExecutableEvaluatorOptions{ + FunctionCacheDir: "/home/nonroot/functions", + }, + }) + + store := functionconfigs.NewFunctionConfigStore( + completed.ExtraConfig.GRPCRuntimeOptions.DefaultImagePrefix, + completed.ExtraConfig.ExecEvaluatorOptions.FunctionCacheDir, + ) + + obj := &configapi.FunctionConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "set-annotations"}, + Spec: configapi.FunctionConfigSpec{ + Image: "set-annotations", + Prefixes: []string{""}, + BinaryExecutor: &configapi.BinaryExecutorConfig{ + Tags: []string{"v0.1.5"}, + Path: "set-annotations", + }, + }, + } + store.UpdateBinaryCache(obj.Name, obj) + + path, found := store.GetBinaryFromCache("set-annotations:v0.1.5") + require.True(t, found) + assert.Equal(t, "/home/nonroot/functions/set-annotations", path) +} + func TestLeaderElectionID(t *testing.T) { assert.Equal(t, "porch-server", LeaderElectionID) } diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index b55da9b3f..b0fb5c519 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -36,6 +36,7 @@ import ( "github.com/kptdev/porch/pkg/apiserver" cachetypes "github.com/kptdev/porch/pkg/cache/types" "github.com/kptdev/porch/pkg/engine" + "github.com/kptdev/porch/pkg/engine/podevaluator" "github.com/kptdev/porch/pkg/externalrepo/git" externalrepotypes "github.com/kptdev/porch/pkg/externalrepo/types" "k8s.io/apimachinery/pkg/runtime/schema" @@ -54,6 +55,8 @@ const ( defaultEtcdPathPrefix = "/registry/porch.kpt.dev" OpenAPITitle = "Porch" OpenAPIVersion = "0.1" + + wrapperServerImageEnv = "WRAPPER_SERVER_IMAGE" ) // PorchServerOptions contains state for master/api server @@ -93,6 +96,9 @@ type PorchServerOptions struct { PodNamespace string + PodEvaluatorOptions podevaluator.PodEvaluatorOptions + + Exec engine.ExecutableEvaluatorOptions ProbePort int HAOptions apiserver.HAConfig @@ -303,6 +309,11 @@ func (o *PorchServerOptions) Config() (*apiserver.Config, error) { return nil, fmt.Errorf("error creating self-signed certificates: %w", err) } + o.PodEvaluatorOptions.WrapperServerImage = os.Getenv(wrapperServerImageEnv) + if o.PodEvaluatorOptions.WrapperServerImage == "" { + return nil, fmt.Errorf("required environment variable %s is not set; porch-server cannot start", wrapperServerImageEnv) + } + o.RecommendedOptions.ExtraAdmissionInitializers = func(c *genericapiserver.RecommendedConfig) ([]admission.PluginInitializer, error) { client, err := clientset.NewForConfig(c.LoopbackClientConfig) if err != nil { @@ -365,8 +376,26 @@ func (o *PorchServerOptions) buildExtraConfig() apiserver.ExtraConfig { DbPushDraftsToGit: o.DbPushDrafsToGit, }, PodNameSpace: o.PodNamespace, - ProbePort: o.ProbePort, - HAOptions: o.HAOptions, + PodEvaluatorOptions: podevaluator.PodEvaluatorOptions{ + WrapperServerImage: o.PodEvaluatorOptions.WrapperServerImage, + GcScanInterval: o.PodEvaluatorOptions.GcScanInterval, + PodTTL: o.PodEvaluatorOptions.PodTTL, + WarmUpPodCacheOnStartup: o.PodEvaluatorOptions.WarmUpPodCacheOnStartup, + EnablePrivateRegistries: o.PodEvaluatorOptions.EnablePrivateRegistries, + RegistryAuthSecretPath: o.PodEvaluatorOptions.RegistryAuthSecretPath, + RegistryAuthSecretName: o.PodEvaluatorOptions.RegistryAuthSecretName, + EnablePrivateRegistriesTls: o.PodEvaluatorOptions.EnablePrivateRegistriesTls, + TlsSecretPath: o.PodEvaluatorOptions.TlsSecretPath, + MaxWaitlistLength: o.PodEvaluatorOptions.MaxWaitlistLength, + MaxParallelPodsPerFunction: o.PodEvaluatorOptions.MaxParallelPodsPerFunction, + PodNamespace: o.PodNamespace, + MaxGrpcMessageSize: o.MaxRequestBodySize, + }, + ExecEvaluatorOptions: engine.ExecutableEvaluatorOptions{ + FunctionCacheDir: o.Exec.FunctionCacheDir, + }, + ProbePort: o.ProbePort, + HAOptions: o.HAOptions, } } @@ -498,6 +527,20 @@ func (o *PorchServerOptions) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&o.ListTimeoutPerRepository, "list-timeout-per-repo", 20*time.Second, "Maximum amount of time to wait for a repository list request.") fs.IntVar(&o.MaxConcurrentLists, "max-parallel-repo-lists", 10, "Maximum number of repositories to list in parallel.") + // Pod evaluator related flags + fs.DurationVar(&o.PodEvaluatorOptions.GcScanInterval, "scan-interval", time.Minute, "The interval of GC between scans.") + fs.DurationVar(&o.PodEvaluatorOptions.PodTTL, "pod-ttl", 30*time.Minute, "TTL for pods before GC.") + fs.BoolVar(&o.PodEvaluatorOptions.WarmUpPodCacheOnStartup, "warm-up-pod-cache", true, "if true, pod-cache-config image pods will be deployed at startup") + fs.BoolVar(&o.PodEvaluatorOptions.EnablePrivateRegistries, "enable-private-registries", false, "if true enables the use of private registries and their authentication") + fs.StringVar(&o.PodEvaluatorOptions.RegistryAuthSecretPath, "registry-auth-secret-path", "/var/tmp/config-secret/.dockerconfigjson", "The path of the secret used for authenticating to custom registries") + fs.StringVar(&o.PodEvaluatorOptions.RegistryAuthSecretName, "registry-auth-secret-name", "auth-secret", "The name of the secret used for authenticating to custom registries") + fs.BoolVar(&o.PodEvaluatorOptions.EnablePrivateRegistriesTls, "enable-private-registries-tls", false, "if enabled, will prioritize use of user provided TLS secret when accessing registries") + fs.StringVar(&o.PodEvaluatorOptions.TlsSecretPath, "tls-secret-path", "/var/tmp/tls-secret/", "The path of the secret used in tls configuration") + fs.IntVar(&o.PodEvaluatorOptions.MaxWaitlistLength, "max-waitlist-length", 2, "Maximum waitlist length per pod") + fs.IntVar(&o.PodEvaluatorOptions.MaxParallelPodsPerFunction, "max-parallel-pods-per-function", 1, "Maximum parallel pods per function") + + // executable evaluator flags + fs.StringVar(&o.Exec.FunctionCacheDir, "functions", "./functions", "Path to cached functions.") fs.IntVar(&o.ProbePort, "probe-port", 0, "If > 0, start serving controller-runtime /healthz and /readyz on this port (in addition to the API server's built-in probes at `--secure-port`); a liveness-style check is available as /healthz/livez") fs.BoolVar(&o.HAOptions.LeaderElection, "leader-elect", false, "If true, the porch-server will attempt to acquire leader election lock") diff --git a/pkg/cmd/server/start_test.go b/pkg/cmd/server/start_test.go index 928737b77..59fb4d536 100644 --- a/pkg/cmd/server/start_test.go +++ b/pkg/cmd/server/start_test.go @@ -345,6 +345,8 @@ func TestBuildExtraConfig(t *testing.T) { } func TestConfigMapsExtraConfig(t *testing.T) { + t.Setenv(wrapperServerImageEnv, "test-wrapper-server") + ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { ln.Close() }) diff --git a/pkg/engine/grpcruntime.go b/pkg/engine/grpcruntime.go index 0404d4cfe..7dba33a57 100644 --- a/pkg/engine/grpcruntime.go +++ b/pkg/engine/grpcruntime.go @@ -22,14 +22,21 @@ import ( kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" "github.com/kptdev/kpt/pkg/fn" "github.com/kptdev/kpt/pkg/lib/kptops" + "github.com/kptdev/kpt/pkg/lib/runneroptions" "github.com/kptdev/porch/controllers/functionconfigs" "github.com/kptdev/porch/func/evaluator" + "github.com/kptdev/porch/pkg/engine/podevaluator" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" ) +type ExecutableEvaluatorOptions struct { + FunctionCacheDir string +} + type GRPCRuntimeOptions struct { FunctionRunnerAddress string MaxGrpcMessageSize int @@ -37,11 +44,21 @@ type GRPCRuntimeOptions struct { } type grpcRuntime struct { - cc *grpc.ClientConn - client evaluator.FunctionEvaluatorClient + cc *grpc.ClientConn + client evaluator.FunctionEvaluatorClient + functionConfigStore *functionconfigs.FunctionConfigStore +} + +func (gr *grpcRuntime) getExecutablePath(fn *kptfilev1.Function) (string, bool) { + if fn.Tag != "" { + execPath, _, exists := gr.functionConfigStore.GetBinaryFromCacheByConstraint(fn.Image, fn.Tag) + return execPath, exists + } + klog.V(2).Infof("Image tag is empty, using the image with explicit tag: %q", fn.Image) + return gr.functionConfigStore.GetBinaryFromCache(fn.Image) } -func newGRPCFunctionRuntime(options GRPCRuntimeOptions) (*grpcRuntime, error) { +func newGRPCFunctionRuntime(options GRPCRuntimeOptions, functionConfigStore *functionconfigs.FunctionConfigStore) (*grpcRuntime, error) { if options.FunctionRunnerAddress == "" { return nil, fmt.Errorf("address is required to instantiate gRPC function runtime") } @@ -61,21 +78,29 @@ func newGRPCFunctionRuntime(options GRPCRuntimeOptions) (*grpcRuntime, error) { } return &grpcRuntime{ - cc: cc, - client: evaluator.NewFunctionEvaluatorClient(cc), + cc: cc, + client: evaluator.NewFunctionEvaluatorClient(cc), + functionConfigStore: functionConfigStore, }, err } var _ kptops.FunctionRuntime = &grpcRuntime{} -func (gr *grpcRuntime) GetRunner(ctx context.Context, fn *kptfilev1.Function) (fn.FunctionRunner, error) { - // TODO: Check if the function is actually available? - return &grpcRunner{ - ctx: ctx, - client: gr.client, - image: fn.Image, - tag: fn.Tag, - }, nil +func (gr *grpcRuntime) GetRunner(ctx context.Context, function *kptfilev1.Function) (fn.FunctionRunner, error) { + klog.Infof("[grpcRuntime::GetRunner] Current state of client connection: %s", gr.cc.GetState().String()) + + if execPath, exists := gr.getExecutablePath(function); exists { + return &grpcRunner{ + ctx: ctx, + client: gr.client, + image: function.Image, + tag: function.Tag, + execPath: execPath, + }, nil + } + return nil, &fn.NotFoundError{ + Function: *function, + } } func (gr *grpcRuntime) Close() error { @@ -90,10 +115,11 @@ func (gr *grpcRuntime) Close() error { } type grpcRunner struct { - ctx context.Context - client evaluator.FunctionEvaluatorClient - image string - tag string + ctx context.Context + client evaluator.FunctionEvaluatorClient + image string + tag string + execPath string } var _ fn.FunctionRunner = &grpcRunner{} @@ -108,6 +134,7 @@ func (gr *grpcRunner) Run(r io.Reader, w io.Writer) error { ResourceList: in, Image: gr.image, Tag: gr.tag, + ExecPath: gr.execPath, }) if err != nil { return fmt.Errorf("func eval %q failed: %w", gr.image, err) @@ -118,22 +145,52 @@ func (gr *grpcRunner) Run(r io.Reader, w io.Writer) error { return nil } -// NewMultiFunctionRuntime creates a FunctionRuntime that tries builtin functions -// first, then falls back to the gRPC fn-runner. -func NewMultiFunctionRuntime(grpcAddress string, maxGrpcMessageSize int, functionConfigStore *functionconfigs.FunctionConfigStore) (fn.FunctionRuntime, error) { - builtin := newBuiltinRuntime(functionConfigStore) +// MultiFunctionRuntimeOptions configures the function runtime chain: builtin, +// optional gRPC fn-runner (exec), and optional pod evaluator. +type MultiFunctionRuntimeOptions struct { + GRPCAddress string + MaxGrpcMessageSize int + FunctionConfigStore *functionconfigs.FunctionConfigStore + PodEvaluator *podevaluator.PodEvaluatorOptions + KubeClient client.WithWatch + DefaultImagePrefix string +} - if grpcAddress == "" { - return builtin, nil +// NewMultiFunctionRuntime creates a FunctionRuntime that tries builtin functions +// first, then gRPC fn-runner (exec), then pod evaluator when configured. +func NewMultiFunctionRuntime(ctx context.Context, opts MultiFunctionRuntimeOptions) (fn.FunctionRuntime, error) { + runtimes := []fn.FunctionRuntime{newBuiltinRuntime(opts.FunctionConfigStore)} + + if opts.GRPCAddress != "" { + grpc, err := newGRPCFunctionRuntime(GRPCRuntimeOptions{ + FunctionRunnerAddress: opts.GRPCAddress, + MaxGrpcMessageSize: opts.MaxGrpcMessageSize, + }, opts.FunctionConfigStore) + if err != nil { + return nil, err + } + runtimes = append(runtimes, grpc) } - grpc, err := newGRPCFunctionRuntime(GRPCRuntimeOptions{ - FunctionRunnerAddress: grpcAddress, - MaxGrpcMessageSize: maxGrpcMessageSize, - }) - if err != nil { - return nil, err + if opts.PodEvaluator != nil && opts.PodEvaluator.WrapperServerImage != "" { + if opts.KubeClient == nil { + return nil, fmt.Errorf("kube client is required for pod evaluator runtime") + } + podOpts := *opts.PodEvaluator + if podOpts.DefaultImagePrefix == "" { + podOpts.DefaultImagePrefix = opts.DefaultImagePrefix + if podOpts.DefaultImagePrefix == "" { + podOpts.DefaultImagePrefix = runneroptions.GHCRImagePrefix + } + } + if podOpts.MaxGrpcMessageSize == 0 { + podOpts.MaxGrpcMessageSize = opts.MaxGrpcMessageSize + } + runtimes = append(runtimes, podevaluator.NewPodEvaluatorRuntime(ctx, podOpts, opts.KubeClient, opts.FunctionConfigStore)) } - return fn.NewMultiRuntime([]fn.FunctionRuntime{builtin, grpc}), nil + if len(runtimes) == 1 { + return runtimes[0], nil + } + return fn.NewMultiRuntime(runtimes), nil } diff --git a/pkg/engine/grpcruntime_test.go b/pkg/engine/grpcruntime_test.go index e8eb34aa0..003627904 100644 --- a/pkg/engine/grpcruntime_test.go +++ b/pkg/engine/grpcruntime_test.go @@ -23,7 +23,10 @@ import ( "strings" "testing" + kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" v1 "github.com/kptdev/kpt/api/kptfile/v1" + "github.com/kptdev/kpt/pkg/lib/runneroptions" + configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" "github.com/kptdev/porch/controllers/functionconfigs" "github.com/kptdev/porch/func/evaluator" "github.com/stretchr/testify/assert" @@ -31,24 +34,30 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( - testImage = "test-image" - testTag = "latest" + testImage = "test-image" + testTag = "latest" + functionCacheDir = "/functions" + defaultImagePrefix = "ghcr.io/kptdev/krm-functions-catalog/" + testNamespace = "porch-fn-system" ) func TestNewGRPCFunctionRuntimeSuccess(t *testing.T) { addr, stop := startMockServer(t) defer stop() + functionConfigStore := functionconfigs.NewFunctionConfigStore(runneroptions.GHCRImagePrefix, "/functions") + options := GRPCRuntimeOptions{ FunctionRunnerAddress: addr, MaxGrpcMessageSize: 1024, DefaultImagePrefix: "gcr.io/", } - runtime, err := newGRPCFunctionRuntime(options) + runtime, err := newGRPCFunctionRuntime(options, functionConfigStore) require.NoError(t, err) require.NotNil(t, runtime) assert.NotNil(t, runtime.cc) @@ -60,7 +69,8 @@ func TestNewGRPCFunctionRuntimeEmptyAddress(t *testing.T) { options := GRPCRuntimeOptions{ MaxGrpcMessageSize: 1024, } - runtime, err := newGRPCFunctionRuntime(options) + functionConfigStore := functionconfigs.NewFunctionConfigStore(runneroptions.GHCRImagePrefix, "/functions") + runtime, err := newGRPCFunctionRuntime(options, functionConfigStore) require.Error(t, err) assert.Contains(t, err.Error(), "address is required") if runtime != nil { @@ -77,13 +87,33 @@ func TestGRPCRuntimeGetRunner(t *testing.T) { MaxGrpcMessageSize: 1024, } - runtime, err := newGRPCFunctionRuntime(options) + sampleFunctionConfig := &configapi.FunctionConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-image", + Namespace: testNamespace, + }, + Spec: configapi.FunctionConfigSpec{ + Image: "test-image", + Prefixes: []string{ + "ghcr.io/kptdev/krm-functions-catalog", + }, + BinaryExecutor: &configapi.BinaryExecutorConfig{ + Tags: []string{ + "latest", + }, + Path: "test-image", + }, + }, + } + functionConfigStore := functionconfigs.NewFunctionConfigStore(defaultImagePrefix, functionCacheDir) + functionConfigStore.UpdateBinaryCache("test-image", sampleFunctionConfig) + + runtime, err := newGRPCFunctionRuntime(options, functionConfigStore) require.NoError(t, err) defer runtime.Close() - fn := &v1.Function{ - Image: testImage, - Tag: testTag, + fn := &kptfilev1.Function{ + Image: "ghcr.io/kptdev/krm-functions-catalog/test-image:latest", } runner, err := runtime.GetRunner(t.Context(), fn) @@ -95,6 +125,7 @@ func TestGRPCRuntimeGetRunner(t *testing.T) { assert.Equal(t, fn.Image, grpcRunner.image) assert.NotNil(t, grpcRunner.ctx) assert.NotNil(t, grpcRunner.client) + assert.NotEmpty(t, grpcRunner.execPath) } func TestGRPCRuntimeCloseWithConnection(t *testing.T) { @@ -106,7 +137,8 @@ func TestGRPCRuntimeCloseWithConnection(t *testing.T) { MaxGrpcMessageSize: 1024, } - runtime, err := newGRPCFunctionRuntime(options) + functionConfigStore := functionconfigs.NewFunctionConfigStore(runneroptions.GHCRImagePrefix, "/functions") + runtime, err := newGRPCFunctionRuntime(options, functionConfigStore) require.NoError(t, err) err = runtime.Close() @@ -123,7 +155,8 @@ func TestGRPCRuntimeCloseMultipleCalls(t *testing.T) { MaxGrpcMessageSize: 1024, } - runtime, err := newGRPCFunctionRuntime(options) + functionConfigStore := functionconfigs.NewFunctionConfigStore(runneroptions.GHCRImagePrefix, "/functions") + runtime, err := newGRPCFunctionRuntime(options, functionConfigStore) require.NoError(t, err) err = runtime.Close() @@ -153,10 +186,10 @@ items: }, } runner := &grpcRunner{ - ctx: t.Context(), - client: client, - image: testImage, - tag: testTag, + ctx: t.Context(), + client: client, + image: testImage, + execPath: "/path/to/binary", } reader := strings.NewReader(`apiVersion: config.kubernetes.io/v1alpha1 @@ -190,10 +223,10 @@ func TestGRPCRunnerRunEvaluationError(t *testing.T) { } runner := &grpcRunner{ - ctx: t.Context(), - client: client, - image: testImage, - tag: testTag, + ctx: t.Context(), + client: client, + image: testImage, + execPath: "/path/to/binary", } reader := strings.NewReader(`apiVersion: config.kubernetes.io/v1alpha1 @@ -317,7 +350,10 @@ func (e *errorWriter) Write(p []byte) (n int, err error) { func TestNewMultiFunctionRuntime_BuiltinOnly(t *testing.T) { store := newTestFunctionConfigStore() - runtime, err := NewMultiFunctionRuntime("", 1024, store) + runtime, err := NewMultiFunctionRuntime(t.Context(), MultiFunctionRuntimeOptions{ + MaxGrpcMessageSize: 1024, + FunctionConfigStore: store, + }) require.NoError(t, err) require.NotNil(t, runtime) @@ -331,7 +367,11 @@ func TestNewMultiFunctionRuntime_WithGRPC(t *testing.T) { defer stop() store := newTestFunctionConfigStore() - runtime, err := NewMultiFunctionRuntime(addr, 1024, store) + runtime, err := NewMultiFunctionRuntime(t.Context(), MultiFunctionRuntimeOptions{ + GRPCAddress: addr, + MaxGrpcMessageSize: 1024, + FunctionConfigStore: store, + }) require.NoError(t, err) require.NotNil(t, runtime) @@ -341,7 +381,10 @@ func TestNewMultiFunctionRuntime_WithGRPC(t *testing.T) { } func TestNewMultiFunctionRuntime_NilStorePanics(t *testing.T) { - runtime, err := NewMultiFunctionRuntime("", 1024, nil) + runtime, err := NewMultiFunctionRuntime(t.Context(), MultiFunctionRuntimeOptions{ + MaxGrpcMessageSize: 1024, + FunctionConfigStore: nil, + }) require.NoError(t, err) // Panic occurs on lookup, not construction diff --git a/pkg/engine/options.go b/pkg/engine/options.go index 536c0978f..d08a80f50 100644 --- a/pkg/engine/options.go +++ b/pkg/engine/options.go @@ -15,13 +15,16 @@ package engine import ( + "context" "fmt" "github.com/kptdev/kpt/pkg/fn" "github.com/kptdev/kpt/pkg/lib/runneroptions" "github.com/kptdev/porch/controllers/functionconfigs" cachetypes "github.com/kptdev/porch/pkg/cache/types" + "github.com/kptdev/porch/pkg/engine/podevaluator" "github.com/kptdev/porch/pkg/repository" + "sigs.k8s.io/controller-runtime/pkg/client" ) type EngineOption interface { @@ -58,9 +61,9 @@ func WithBuiltinFunctionRuntime(functionConfigStore *functionconfigs.FunctionCon }) } -func WithGRPCFunctionRuntime(options GRPCRuntimeOptions) EngineOption { +func WithGRPCFunctionRuntime(options GRPCRuntimeOptions, functionConfigStore *functionconfigs.FunctionConfigStore) EngineOption { return EngineOptionFunc(func(engine *cadEngine) error { - runtime, err := newGRPCFunctionRuntime(options) + runtime, err := newGRPCFunctionRuntime(options, functionConfigStore) if err != nil { return fmt.Errorf("failed to create function runtime: %w", err) } @@ -75,6 +78,20 @@ func WithGRPCFunctionRuntime(options GRPCRuntimeOptions) EngineOption { }) } +func WithPodEvaluatorRuntime(ctx context.Context, podEvaluatorOptions podevaluator.PodEvaluatorOptions, kubeClient client.WithWatch, functionConfigStore *functionconfigs.FunctionConfigStore) EngineOption { + return EngineOptionFunc(func(engine *cadEngine) error { + runtime := podevaluator.NewPodEvaluatorRuntime(ctx, podEvaluatorOptions, kubeClient, functionConfigStore) + if engine.taskHandler.GetRuntime() == nil { + engine.taskHandler.SetRuntime(runtime) + } else if mr, ok := engine.taskHandler.GetRuntime().(*fn.MultiRuntime); ok { + mr.Add(runtime) + } else { + engine.taskHandler.SetRuntime(fn.NewMultiRuntime([]fn.FunctionRuntime{engine.taskHandler.GetRuntime(), runtime})) + } + return nil + }) +} + func WithRunnerOptionsResolver(fn func(namespace string) runneroptions.RunnerOptions) EngineOption { return EngineOptionFunc(func(engine *cadEngine) error { engine.taskHandler.SetRunnerOptionsResolver(fn) diff --git a/func/internal/podcachemanager.go b/pkg/engine/podevaluator/podcachemanager.go similarity index 99% rename from func/internal/podcachemanager.go rename to pkg/engine/podevaluator/podcachemanager.go index 1feb54205..d93405564 100644 --- a/func/internal/podcachemanager.go +++ b/pkg/engine/podevaluator/podcachemanager.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "context" diff --git a/func/internal/podcachemanager_eventloop_test.go b/pkg/engine/podevaluator/podcachemanager_eventloop_test.go similarity index 99% rename from func/internal/podcachemanager_eventloop_test.go rename to pkg/engine/podevaluator/podcachemanager_eventloop_test.go index d24b7a8af..1f30c934d 100644 --- a/func/internal/podcachemanager_eventloop_test.go +++ b/pkg/engine/podevaluator/podcachemanager_eventloop_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "context" diff --git a/func/internal/podcachemanager_unit_test.go b/pkg/engine/podevaluator/podcachemanager_unit_test.go similarity index 99% rename from func/internal/podcachemanager_unit_test.go rename to pkg/engine/podevaluator/podcachemanager_unit_test.go index 806bcd681..a53792a1d 100644 --- a/func/internal/podcachemanager_unit_test.go +++ b/pkg/engine/podevaluator/podcachemanager_unit_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "fmt" diff --git a/func/internal/podevaluator.go b/pkg/engine/podevaluator/podevaluator.go similarity index 85% rename from func/internal/podevaluator.go rename to pkg/engine/podevaluator/podevaluator.go index 7771430ba..7fa63d92d 100644 --- a/func/internal/podevaluator.go +++ b/pkg/engine/podevaluator/podevaluator.go @@ -12,14 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "context" "fmt" + "io" "sync/atomic" "time" + kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" + "github.com/kptdev/kpt/pkg/fn" "github.com/kptdev/kpt/pkg/fn/runtime" fnconf "github.com/kptdev/porch/controllers/functionconfigs" "github.com/kptdev/porch/func/evaluator" @@ -58,6 +61,10 @@ type podEvaluator struct { maxGrpcRetries int } +type podEvaluatorRuntime struct { + pe *podEvaluator +} + type PodEvaluatorOptions struct { PodNamespace string // Namespace to run KRM functions pods in WrapperServerImage string // Container image name of the wrapper server @@ -76,8 +83,6 @@ type PodEvaluatorOptions struct { MaxGrpcRetries int // Maximum number of retries on gRPC Unavailable errors } -var _ Evaluator = &podEvaluator{} - type podData struct { // the OCI image name of the KRM function image string @@ -110,7 +115,66 @@ type podReadyResponse struct { err error } -func NewPodEvaluator(ctx context.Context, o PodEvaluatorOptions, cl client.Client, functionConfigStore *fnconf.FunctionConfigStore) (Evaluator, error) { +func (pe *podEvaluator) Close() error { + return nil +} + +func NewPodEvaluatorRuntime(ctx context.Context, options PodEvaluatorOptions, cl client.WithWatch, functionConfigStore *fnconf.FunctionConfigStore) *podEvaluatorRuntime { + pe, err := NewPodEvaluator(ctx, options, cl, functionConfigStore) + if err != nil { + klog.Errorf("failed to create pod evaluator: %v", err) + return &podEvaluatorRuntime{} + } + return &podEvaluatorRuntime{ + pe: pe, + } +} + +func (pe *podEvaluatorRuntime) GetRunner(ctx context.Context, funct *kptfilev1.Function) (fn.FunctionRunner, error) { + if pe.pe == nil { + return nil, fmt.Errorf("pod evaluator runtime is not properly initialized") + } + return &podevalRunner{ + ctx: ctx, + pe: *pe.pe, + image: funct.Image, + tag: funct.Tag, + }, nil +} + +type podevalRunner struct { + ctx context.Context + pe podEvaluator + image string + tag string +} + +var _ fn.FunctionRunner = &podevalRunner{} + +func (runner *podevalRunner) Run(r io.Reader, w io.Writer) error { + in, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("failed to read function runner input: %w", err) + } + + var res *evaluator.EvaluateFunctionResponse + res, err = runner.pe.EvaluateFunction(context.Background(), &evaluator.EvaluateFunctionRequest{ + ResourceList: in, + Image: runner.image, + Tag: runner.tag, + }) + + if err != nil { + return fmt.Errorf("func eval %q failed: %w", runner.image, err) + } + + if _, err := w.Write(res.ResourceList); err != nil { + return fmt.Errorf("failed to write function runner output: %w", err) + } + return nil +} + +func NewPodEvaluator(ctx context.Context, o PodEvaluatorOptions, cl client.WithWatch, functionConfigStore *fnconf.FunctionConfigStore) (*podEvaluator, error) { maxWaitlist := o.MaxWaitlistLength if maxWaitlist <= 0 { maxWaitlist = defaultMaxWaitlistLength diff --git a/func/internal/podevaluator_podcachemanager_test.go b/pkg/engine/podevaluator/podevaluator_podcachemanager_test.go similarity index 99% rename from func/internal/podevaluator_podcachemanager_test.go rename to pkg/engine/podevaluator/podevaluator_podcachemanager_test.go index 2e9b94ce2..d3d2c0bea 100644 --- a/func/internal/podevaluator_podcachemanager_test.go +++ b/pkg/engine/podevaluator/podevaluator_podcachemanager_test.go @@ -14,7 +14,7 @@ limitations under the License. */ -package internal +package podevaluator import ( "context" diff --git a/func/internal/podevaluator_podmanager_test.go b/pkg/engine/podevaluator/podevaluator_podmanager_test.go similarity index 99% rename from func/internal/podevaluator_podmanager_test.go rename to pkg/engine/podevaluator/podevaluator_podmanager_test.go index 92dfd3897..5f3dc2104 100644 --- a/func/internal/podevaluator_podmanager_test.go +++ b/pkg/engine/podevaluator/podevaluator_podmanager_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "bytes" diff --git a/func/internal/podevaluator_porch_parallel_execution_test.go b/pkg/engine/podevaluator/podevaluator_porch_parallel_execution_test.go similarity index 99% rename from func/internal/podevaluator_porch_parallel_execution_test.go rename to pkg/engine/podevaluator/podevaluator_porch_parallel_execution_test.go index 6ad0846ba..0f1ddccf4 100644 --- a/func/internal/podevaluator_porch_parallel_execution_test.go +++ b/pkg/engine/podevaluator/podevaluator_porch_parallel_execution_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "context" diff --git a/func/internal/podevaluator_tag_resolution_test.go b/pkg/engine/podevaluator/podevaluator_tag_resolution_test.go similarity index 96% rename from func/internal/podevaluator_tag_resolution_test.go rename to pkg/engine/podevaluator/podevaluator_tag_resolution_test.go index 31e198e8d..0028e26fe 100644 --- a/func/internal/podevaluator_tag_resolution_test.go +++ b/pkg/engine/podevaluator/podevaluator_tag_resolution_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "context" @@ -31,6 +31,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + testImageName = "test-image" + defaultKRMImagePrefix = "ghcr.io/kptdev/krm-functions-catalog" +) + type fakeLister struct { tags map[string][]string err string diff --git a/func/internal/podevaluator_unit_test.go b/pkg/engine/podevaluator/podevaluator_unit_test.go similarity index 57% rename from func/internal/podevaluator_unit_test.go rename to pkg/engine/podevaluator/podevaluator_unit_test.go index 5fd924363..434dec50a 100644 --- a/func/internal/podevaluator_unit_test.go +++ b/pkg/engine/podevaluator/podevaluator_unit_test.go @@ -12,19 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( + "bytes" "context" "fmt" "net" + "strings" "sync/atomic" "testing" "time" - "github.com/kptdev/kpt/pkg/fn/runtime" - "github.com/kptdev/kpt/pkg/lib/runneroptions" - fnconf "github.com/kptdev/porch/controllers/functionconfigs" + kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" + kptfnruntime "github.com/kptdev/kpt/pkg/fn/runtime" + "github.com/kptdev/porch/controllers/functionconfigs" pb "github.com/kptdev/porch/func/evaluator" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -32,8 +34,11 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) // startFakeEvalServer starts a gRPC function evaluator server on a dynamic port. @@ -53,35 +58,12 @@ func startFakeEvalServer(t *testing.T, evalFunc func(ctx context.Context, req *p } } -func TestNewPodEvaluator(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - kubeClient := fake.NewClientBuilder().Build() - store := fnconf.NewFunctionConfigStore(runneroptions.GHCRImagePrefix, "/functions") - - eval, err := NewPodEvaluator(ctx, PodEvaluatorOptions{ - PodNamespace: "test-ns", - WrapperServerImage: "ghcr.io/kptdev/wrapper-server:latest", - GcScanInterval: time.Minute, - PodTTL: time.Minute, - }, kubeClient, store) - require.NoError(t, err) - require.NotNil(t, eval) - - pe, ok := eval.(*podEvaluator) - require.True(t, ok) - assert.Equal(t, defaultMaxWaitlistLength, pe.podCacheManager.maxWaitlistLength) - assert.Equal(t, defaultMaxParallelPodsPerFunction, pe.podCacheManager.maxParallelPodsPerFunction) - assert.Equal(t, defaultMaxGrpcRetries, pe.maxGrpcRetries) -} - func TestEvaluateFunction_ErrorInResponse(t *testing.T) { reqCh := make(chan *connectionRequest, 1) pe := &podEvaluator{requestCh: reqCh, podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -106,7 +88,7 @@ func TestEvaluateFunction_NilGrpcConnection(t *testing.T) { pe := &podEvaluator{requestCh: reqCh, podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -142,7 +124,7 @@ func TestEvaluateFunction_GrpcCallFails(t *testing.T) { pe := &podEvaluator{requestCh: reqCh, podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -182,7 +164,7 @@ func TestEvaluateFunction_SuccessWithStderr(t *testing.T) { pe := &podEvaluator{requestCh: reqCh, podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -222,7 +204,7 @@ func TestEvaluateFunction_SuccessClean(t *testing.T) { pe := &podEvaluator{requestCh: reqCh, podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -260,7 +242,7 @@ func TestEvaluateFunction_CounterDecrement(t *testing.T) { pe := &podEvaluator{requestCh: reqCh, podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -313,7 +295,7 @@ func TestEvaluateFunction_Unavailable_EvictsAndRetries(t *testing.T) { maxGrpcRetries: 2, podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -389,7 +371,7 @@ func TestEvaluateFunction_ExhaustsRetries(t *testing.T) { maxGrpcRetries: 1, // only 1 retry allowed podCacheManager: &podCacheManager{ podManager: &podManager{ - tagResolver: runtime.TagResolver{}, + tagResolver: kptfnruntime.TagResolver{}, }, }, } @@ -431,3 +413,262 @@ func TestEvaluateFunction_ExhaustsRetries(t *testing.T) { assert.Nil(t, resp) assert.Contains(t, err.Error(), "after retries") } + +func newTestKubeClient(t *testing.T, funcs interceptor.Funcs) client.WithWatch { + t.Helper() + scheme := k8sruntime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + return fake.NewClientBuilder(). + WithScheme(scheme). + WithInterceptorFuncs(funcs). + Build() +} + +func newTestPodEvaluatorOptions() PodEvaluatorOptions { + return PodEvaluatorOptions{ + PodNamespace: defaultNamespace, + WrapperServerImage: defaultWrapperServerImage, + } +} + +func TestNewPodEvaluator(t *testing.T) { + t.Run("success applies defaults", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + kubeClient := newTestKubeClient(t, interceptor.Funcs{}) + functionConfigStore := functionconfigs.NewFunctionConfigStore(defaultRegistry, "/functions") + + pe, err := NewPodEvaluator(ctx, newTestPodEvaluatorOptions(), kubeClient, functionConfigStore) + require.NoError(t, err) + require.NotNil(t, pe) + assert.NotNil(t, pe.requestCh) + assert.NotNil(t, pe.evictionCh) + assert.NotNil(t, pe.podCacheManager) + assert.Equal(t, defaultMaxGrpcRetries, pe.maxGrpcRetries) + assert.Equal(t, defaultMaxWaitlistLength, pe.podCacheManager.maxWaitlistLength) + assert.Equal(t, defaultMaxParallelPodsPerFunction, pe.podCacheManager.maxParallelPodsPerFunction) + assert.Equal(t, defaultManagerNamespace, pe.podCacheManager.podManager.managerNamespace) + assert.Equal(t, defaultNamespace, pe.podCacheManager.podManager.namespace) + assert.Equal(t, defaultWrapperServerImage, pe.podCacheManager.podManager.wrapperServerImage) + }) + + t.Run("success applies custom options", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + kubeClient := newTestKubeClient(t, interceptor.Funcs{}) + functionConfigStore := functionconfigs.NewFunctionConfigStore(defaultRegistry, "/functions") + options := newTestPodEvaluatorOptions() + options.MaxWaitlistLength = 5 + options.MaxParallelPodsPerFunction = 3 + options.MaxGrpcRetries = 7 + + pe, err := NewPodEvaluator(ctx, options, kubeClient, functionConfigStore) + require.NoError(t, err) + assert.Equal(t, 7, pe.maxGrpcRetries) + assert.Equal(t, 5, pe.podCacheManager.maxWaitlistLength) + assert.Equal(t, 3, pe.podCacheManager.maxParallelPodsPerFunction) + }) + + t.Run("retrieveFunctionPods failure", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + kubeClient := newTestKubeClient(t, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return fmt.Errorf("forced get error") + }, + }) + functionConfigStore := functionconfigs.NewFunctionConfigStore(defaultRegistry, "/functions") + + pe, err := NewPodEvaluator(ctx, newTestPodEvaluatorOptions(), kubeClient, functionConfigStore) + require.Error(t, err) + assert.Nil(t, pe) + assert.Contains(t, err.Error(), "failed to retrieve existing pods") + }) +} + +func TestNewPodEvaluatorRuntime(t *testing.T) { + t.Run("success", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + kubeClient := newTestKubeClient(t, interceptor.Funcs{}) + functionConfigStore := functionconfigs.NewFunctionConfigStore(defaultRegistry, "/functions") + + runtime := NewPodEvaluatorRuntime(ctx, newTestPodEvaluatorOptions(), kubeClient, functionConfigStore) + require.NotNil(t, runtime) + require.NotNil(t, runtime.pe) + }) + + t.Run("returns uninitialized runtime on constructor failure", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + kubeClient := newTestKubeClient(t, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return fmt.Errorf("forced get error") + }, + }) + functionConfigStore := functionconfigs.NewFunctionConfigStore(defaultRegistry, "/functions") + + runtime := NewPodEvaluatorRuntime(ctx, newTestPodEvaluatorOptions(), kubeClient, functionConfigStore) + require.NotNil(t, runtime) + assert.Nil(t, runtime.pe) + }) +} + +func TestPodEvaluatorRuntimeGetRunner(t *testing.T) { + t.Run("success", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + kubeClient := newTestKubeClient(t, interceptor.Funcs{}) + functionConfigStore := functionconfigs.NewFunctionConfigStore(defaultRegistry, "/functions") + runtime := NewPodEvaluatorRuntime(ctx, newTestPodEvaluatorOptions(), kubeClient, functionConfigStore) + + funct := &kptfilev1.Function{ + Image: "ghcr.io/kptdev/krm-functions-catalog/set-image:latest", + Tag: ">= 1.0.0", + } + + runner, err := runtime.GetRunner(ctx, funct) + require.NoError(t, err) + require.NotNil(t, runner) + + podRunner, ok := runner.(*podevalRunner) + require.True(t, ok) + assert.Equal(t, ctx, podRunner.ctx) + assert.Equal(t, funct.Image, podRunner.image) + assert.Equal(t, funct.Tag, podRunner.tag) + assert.NotNil(t, podRunner.pe.requestCh) + }) + + t.Run("uninitialized runtime", func(t *testing.T) { + runtime := &podEvaluatorRuntime{} + + runner, err := runtime.GetRunner(t.Context(), &kptfilev1.Function{Image: "test-image"}) + require.Error(t, err) + assert.Nil(t, runner) + assert.Contains(t, err.Error(), "not properly initialized") + }) +} + +func TestPodevalRunnerRun(t *testing.T) { + setupRunner := func(t *testing.T, evalFunc func(ctx context.Context, req *pb.EvaluateFunctionRequest) (*pb.EvaluateFunctionResponse, error)) *podevalRunner { + t.Helper() + + addr, cleanup := startFakeEvalServer(t, evalFunc) + t.Cleanup(cleanup) + + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + + counter := &atomic.Int32{} + counter.Store(1) + + reqCh := make(chan *connectionRequest, 1) + pe := &podEvaluator{ + requestCh: reqCh, + maxGrpcRetries: defaultMaxGrpcRetries, + podCacheManager: &podCacheManager{ + podManager: &podManager{ + tagResolver: kptfnruntime.TagResolver{}, + }, + }, + } + + go func() { + req := <-reqCh + req.responseCh <- &connectionResponse{ + podData: podData{image: "test-image", grpcConnection: conn}, + concurrentEvaluations: counter, + } + }() + + return &podevalRunner{ + ctx: t.Context(), + pe: *pe, + image: "test-image", + tag: "latest", + } + } + + t.Run("success", func(t *testing.T) { + const input = `apiVersion: config.kubernetes.io/v1 +kind: ResourceList +items: [] +` + const output = `apiVersion: config.kubernetes.io/v1 +kind: ResourceList +items: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: transformed +` + + runner := setupRunner(t, func(_ context.Context, req *pb.EvaluateFunctionRequest) (*pb.EvaluateFunctionResponse, error) { + assert.Equal(t, []byte(input), req.ResourceList) + assert.Equal(t, "docker.io/library/test-image:latest", req.Image) + assert.Equal(t, "latest", req.Tag) + return &pb.EvaluateFunctionResponse{ResourceList: []byte(output)}, nil + }) + + var writer bytes.Buffer + err := runner.Run(strings.NewReader(input), &writer) + require.NoError(t, err) + assert.Equal(t, output, writer.String()) + }) + + t.Run("read error", func(t *testing.T) { + runner := &podevalRunner{ + ctx: t.Context(), + image: "test-image", + } + + var writer bytes.Buffer + err := runner.Run(ioErrReader{err: fmt.Errorf("read failed")}, &writer) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to read function runner input") + }) + + t.Run("evaluation error", func(t *testing.T) { + runner := setupRunner(t, func(_ context.Context, _ *pb.EvaluateFunctionRequest) (*pb.EvaluateFunctionResponse, error) { + return nil, status.Error(codes.Internal, "evaluation failed") + }) + + var writer bytes.Buffer + err := runner.Run(strings.NewReader("input"), &writer) + require.Error(t, err) + assert.Contains(t, err.Error(), `func eval "test-image" failed`) + }) + + t.Run("write error", func(t *testing.T) { + runner := setupRunner(t, func(_ context.Context, _ *pb.EvaluateFunctionRequest) (*pb.EvaluateFunctionResponse, error) { + return &pb.EvaluateFunctionResponse{ResourceList: []byte("output")}, nil + }) + + err := runner.Run(strings.NewReader("input"), ioErrWriter{err: fmt.Errorf("write failed")}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to write function runner output") + }) +} + +type ioErrReader struct { + err error +} + +func (r ioErrReader) Read([]byte) (int, error) { + return 0, r.err +} + +type ioErrWriter struct { + err error +} + +func (w ioErrWriter) Write([]byte) (int, error) { + return 0, w.err +} diff --git a/func/internal/podmanager.go b/pkg/engine/podevaluator/podmanager.go similarity index 99% rename from func/internal/podmanager.go rename to pkg/engine/podevaluator/podmanager.go index ece25fb7c..56da91d5b 100644 --- a/func/internal/podmanager.go +++ b/pkg/engine/podevaluator/podmanager.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "context" diff --git a/func/internal/podmanager_tls_test.go b/pkg/engine/podevaluator/podmanager_tls_test.go similarity index 99% rename from func/internal/podmanager_tls_test.go rename to pkg/engine/podevaluator/podmanager_tls_test.go index b82b76d92..37e64155e 100644 --- a/func/internal/podmanager_tls_test.go +++ b/pkg/engine/podevaluator/podmanager_tls_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "crypto/ecdsa" diff --git a/func/internal/podmanager_unit_test.go b/pkg/engine/podevaluator/podmanager_unit_test.go similarity index 99% rename from func/internal/podmanager_unit_test.go rename to pkg/engine/podevaluator/podmanager_unit_test.go index 01c7f0029..f0fd3b3d7 100644 --- a/func/internal/podmanager_unit_test.go +++ b/pkg/engine/podevaluator/podmanager_unit_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package podevaluator import ( "os" diff --git a/func/internal/testdata/config.yaml b/pkg/engine/podevaluator/testdata/config.yaml similarity index 100% rename from func/internal/testdata/config.yaml rename to pkg/engine/podevaluator/testdata/config.yaml diff --git a/func/internal/testdata/config_bad_format.yaml b/pkg/engine/podevaluator/testdata/config_bad_format.yaml similarity index 100% rename from func/internal/testdata/config_bad_format.yaml rename to pkg/engine/podevaluator/testdata/config_bad_format.yaml diff --git a/scripts/create-deployment-blueprint.sh b/scripts/create-deployment-blueprint.sh index 149fc634c..5c1cc3d48 100755 --- a/scripts/create-deployment-blueprint.sh +++ b/scripts/create-deployment-blueprint.sh @@ -40,7 +40,7 @@ Supported Flags: --enabled-reconcilers RECONCILERS ... comma-separated list of reconcilers that should be enabled in porch controller --ghcr-image-prefix PREFIX ... GHCR image url prefix for running porch behind a proxy - --fn-runner-warm-up-pod-cache BOOL ... disable warm-up-pod-cache in function runner + --fn-runner-warm-up-pod-cache BOOL ... disable warm-up-pod-cache on porch-server --porch-cache-type TYPE ... porch cache type (CR or DB) --db-push-drafts-to-git BOOL ... enable db-push-drafts-to-git flag for porch-server --create-v1alpha2-rpkg BOOL ... enable v1alpha2 PackageRevision CRD creation by repo controller @@ -200,18 +200,6 @@ for resource in ctx.resource_list['items']: container['args'].append('--default-image-prefix=${GHCR_IMAGE_PREFIX}') " - kpt fn eval ${DESTINATION} \ - --image ${STARLARK_IMG} \ - --match-kind Deployment \ - --match-name function-runner \ - --match-namespace porch-system \ - -- "source= -for resource in ctx.resource_list['items']: - containers = resource['spec']['template']['spec']['containers'] - for container in containers: - container['command'].append('--default-image-prefix=${GHCR_IMAGE_PREFIX}') -" - kpt fn eval ${DESTINATION} \ --image ${STARLARK_IMG} \ --match-kind Deployment \ @@ -234,7 +222,7 @@ function disable_fn_runner_warm_up_pod_cache() { kpt fn eval ${DESTINATION} \ --image ${SEARCH_REPLACE_IMG} \ --match-kind Deployment \ - --match-name function-runner \ + --match-name porch-server \ --match-namespace porch-system \ -- by-value="--warm-up-pod-cache=true" put-value="--warm-up-pod-cache=false" } diff --git a/test/e2e/api/fn_runner_test.go b/test/e2e/api/fn_runner_test.go index bab819e77..6f49264dd 100644 --- a/test/e2e/api/fn_runner_test.go +++ b/test/e2e/api/fn_runner_test.go @@ -352,18 +352,18 @@ func (t *PorchSuite) TestPodEvaluatorParallelExecution() { sleepDuration = 3 * time.Second singleFunctionTime = sleepDuration - // Defaults from func/server/server.go flag definitions + // Defaults from pkg/cmd/server/start.go flag definitions defaultMaxWaitlistLength = 2 defaultMaxParallelPodsPerFunction = 1 ) - // Read the actual function-runner args to determine scaling parameters - maxWaitList, maxParallelPods := t.getFunctionRunnerScalingParams(defaultMaxWaitlistLength, defaultMaxParallelPodsPerFunction) + // Read porch-server args where the pod evaluator is configured. + maxWaitList, maxParallelPods := t.getPodEvaluatorScalingParams(defaultMaxWaitlistLength, defaultMaxParallelPodsPerFunction) expectedPodCount := min((parallelRequestCount+maxWaitList-1)/maxWaitList, maxParallelPods) expectedSequentialTime := parallelRequestCount * sleepDuration * 5 / 2 // add some buffer to account for overhead and slow ci pollTimeout := expectedSequentialTime * 5 / 4 // +0.25 headroom - t.Logf("Function-runner scaling params: maxWaitList=%d, maxParallelPods=%d, expectedPodCount=%d", maxWaitList, maxParallelPods, expectedPodCount) + t.Logf("Pod evaluator scaling params: maxWaitList=%d, maxParallelPods=%d, expectedPodCount=%d", maxWaitList, maxParallelPods, expectedPodCount) t.RegisterGitRepositoryF(t.GetPorchTestRepoURL(), repoName, "", suiteutils.GiteaUser, suiteutils.GiteaPassword) @@ -442,14 +442,14 @@ func (t *PorchSuite) TestPodEvaluatorParallelExecution() { t.Log("All parallel evaluations completed, and duration check passed.") } -// getFunctionRunnerScalingParams reads the deployed function-runner container args +// getPodEvaluatorScalingParams reads the deployed porch-server container args // and returns the max-waitlist-length and max-parallel-pods-per-function values. // Falls back to the provided defaults if the args are not found. -func (t *PorchSuite) getFunctionRunnerScalingParams(defaultMaxWaitlist, defaultMaxParallelPods int) (int, int) { +func (t *PorchSuite) getPodEvaluatorScalingParams(defaultMaxWaitlist, defaultMaxParallelPods int) (int, int) { porchSvcKey := t.PorchServerServiceKey() - container := t.FindFirstContainerByImageName(porchSvcKey.Namespace, "porch-function-runner", "porch-fnrunner") + container := t.FindFirstContainerByImageName(porchSvcKey.Namespace, "porch-server") if container == nil { - t.Logf("Could not find function-runner container, using defaults: maxWaitlist=%d, maxParallelPods=%d", defaultMaxWaitlist, defaultMaxParallelPods) + t.Logf("Could not find porch-server container, using defaults: maxWaitlist=%d, maxParallelPods=%d", defaultMaxWaitlist, defaultMaxParallelPods) return defaultMaxWaitlist, defaultMaxParallelPods } diff --git a/test/e2e/cli/testdata/rpkg-push/config.yaml b/test/e2e/cli/testdata/rpkg-push/config.yaml index 3708d5796..001465745 100644 --- a/test/e2e/cli/testdata/rpkg-push/config.yaml +++ b/test/e2e/cli/testdata/rpkg-push/config.yaml @@ -266,10 +266,10 @@ commands: - git.test-package.push - /tmp/porch-e2e/testing-invalid-render - args: - - sh - - -c - - | - echo "pipeline:\n mutators:\n - image: ghcr.io/kptdev/krm-functions-catalog/set-namespace:v0.4.5\n configMap:\n namespace: example-ns\n - image: ghcr.io/kptdev/krm-functions-catalog/set-annotations:v0.1.7\n" >> /tmp/porch-e2e/testing-invalid-render/Kptfile + - sh + - -c + - | + echo "pipeline:\n mutators:\n - image: ghcr.io/kptdev/krm-functions-catalog/set-namespace:v0.4.5\n configMap:\n namespace: example-ns\n - image: ghcr.io/kptdev/krm-functions-catalog/set-annotations:v0.1.7\n" >> /tmp/porch-e2e/testing-invalid-render/Kptfile - args: - porchctl - rpkg diff --git a/test/e2e/crd/podevaluator_test.go b/test/e2e/crd/podevaluator_test.go new file mode 100644 index 000000000..4baaa37b3 --- /dev/null +++ b/test/e2e/crd/podevaluator_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crd + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const podEvaluatorMutatorImage = "ghcr.io/kptdev/krm-functions-catalog/set-annotations:v0.1.4" + +var _ = Describe("PodEvaluator", Ordered, Label("content"), func() { + var env *testEnv + + BeforeAll(func() { + env = sharedEnv() + }) + + It("should render a non-builtin function via the controller pod evaluator", func() { + if !allInCluster { + Skip("pod evaluator test requires in-cluster porch-controllers") + } + + By("verifying porch-controllers is configured with WRAPPER_SERVER_IMAGE") + deploy := &appsv1.Deployment{} + Expect(k8sClient.Get(env.Ctx, client.ObjectKey{ + Namespace: "porch-system", + Name: "porch-controllers", + }, deploy)).To(Succeed()) + Expect(deploymentEnv(deploy, "WRAPPER_SERVER_IMAGE")).NotTo(BeEmpty(), + "porch-controllers must set WRAPPER_SERVER_IMAGE for the in-process pod evaluator") + + By("creating a draft package") + pr := newPackageRevision(env.Namespace, env.RepoName, "podeval-pkg", "v1", withInit("pod evaluator test")) + Expect(k8sClient.Create(env.Ctx, pr)).To(Succeed()) + waitForReady(env.Ctx, pr) + waitForPRRVisible(env.Ctx, env.Namespace, pr.Name) + + By("pushing a pipeline that requires the pod evaluator") + updatePRRResources(env.Ctx, env.Namespace, pr.Name, map[string]string{ + "Kptfile": "apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: podeval-pkg\npipeline:\n mutators:\n - image: " + podEvaluatorMutatorImage + "\n configMap:\n foo: bar\n", + "cm.yaml": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: podeval-cm\ndata:\n key: value\n", + }) + + By("waiting for Rendered=True") + waitForRendered(env.Ctx, pr) + waitForReady(env.Ctx, pr) + + By("verifying the mutator annotated the ConfigMap") + Eventually(func(g Gomega) { + resources := getPRRResources(env.Ctx, env.Namespace, pr.Name) + g.Expect(resources).To(HaveKey("cm.yaml")) + g.Expect(resources["cm.yaml"]).To(ContainSubstring("foo: bar")) + }).WithTimeout(defaultTimeout).WithPolling(defaultInterval).Should(Succeed()) + + By("verifying a function pod was created in porch-fn-system") + Eventually(func(g Gomega) { + podList := &corev1.PodList{} + g.Expect(k8sClient.List(env.Ctx, podList, client.InNamespace(fnNamespace))).To(Succeed()) + g.Expect(podsMatchingFunctionImage(podList, podEvaluatorMutatorImage)).NotTo(BeEmpty(), + "expected a function pod whose image is %s", podEvaluatorMutatorImage) + }).WithTimeout(defaultTimeout).WithPolling(defaultInterval).Should(Succeed()) + }) +}) + +func deploymentEnv(deploy *appsv1.Deployment, name string) string { + for i := range deploy.Spec.Template.Spec.Containers { + for _, envVar := range deploy.Spec.Template.Spec.Containers[i].Env { + if envVar.Name == name { + return envVar.Value + } + } + } + return "" +} + +func podsMatchingFunctionImage(podList *corev1.PodList, image string) []corev1.Pod { + var matches []corev1.Pod + for i := range podList.Items { + pod := podList.Items[i] + for _, c := range pod.Spec.Containers { + if c.Image == image || strings.Contains(c.Image, image) { + matches = append(matches, pod) + break + } + } + } + return matches +} diff --git a/test/e2e/suiteutils/suite_utils.go b/test/e2e/suiteutils/suite_utils.go index 14f139bfe..58bc3e34c 100644 --- a/test/e2e/suiteutils/suite_utils.go +++ b/test/e2e/suiteutils/suite_utils.go @@ -833,7 +833,7 @@ func (t *TestSuite) FnNamespaceName() string { porchSvcKey := t.PorchServerServiceKey() - container := t.FindFirstContainerByImageName(porchSvcKey.Namespace, "porch-function-runner", "porch-fnrunner") + container := t.FindFirstContainerByImageName(porchSvcKey.Namespace, "porch-server") if container != nil { for _, arg := range container.Args { if strings.Contains(arg, "pod-namespace") { diff --git a/test/performance/README.md b/test/performance/README.md index 1d8b2b9c5..2a927a6df 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -235,7 +235,7 @@ Tests handle `SIGINT`/`SIGTERM` gracefully: in-flight work stops and results col | `-gitea-username` | `porch` | Gitea username | | `-gitea-password` | `secret` | Gitea password | -The KRM function registry URL is configured via `PORCH_GHCR_PREFIX_URL` in the repo root `.env` file. It is applied at deploy time to porch-server, function-runner, and porch-controllers (`make run-in-kind`, `make run-in-kind-db-cache`, and `make run-in-kind-v1alpha2` all read `.env` automatically via `make deployment-config`). Package `Kptfile` images use short names (for example `set-namespace:v0.4.1`); porch-server and function-runner resolve them with `--default-image-prefix`, and controllers use the `DEFAULT_IMAGE_PREFIX` environment variable. The `CHANGE_NAMESPACE` placeholder in Kptfiles is substituted at test runtime. +The KRM function registry URL is configured via `PORCH_GHCR_PREFIX_URL` in the repo root `.env` file. It is applied at deploy time to porch-server and porch-controllers (`make run-in-kind`, `make run-in-kind-db-cache`, and `make run-in-kind-v1alpha2` all read `.env` automatically via `make deployment-config`). Package `Kptfile` images use short names (for example `set-namespace:v0.4.1`); porch-server resolves them with `--default-image-prefix`, and controllers use the `DEFAULT_IMAGE_PREFIX` environment variable. The `CHANGE_NAMESPACE` placeholder in Kptfiles is substituted at test runtime. ## 6. Output Files