Add middleware update e2e test

.github/workflows/test-e2e-bundle.yml

@@ -4,6 +4,10 @@ on:
   push:
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test-bundle-e2e:
     name: Bundle E2E Tests

.github/workflows/test-e2e.yml

@@ -4,6 +4,10 @@ on:
   push:
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test-e2e:
     name: E2E Tests

Makefile

@@ -120,7 +120,7 @@ test: manifests generate fmt vet setup-envtest ## Run tests.
 
 .PHONY: test-e2e ## Run e2e tests.
 test-e2e:
-	go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=1h -ginkgo.label-filter="!bundle"
+	go test -timeout 1h ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=1h -ginkgo.label-filter="!bundle"
 
 .PHONY: test-e2e-bundle ## Run bundle e2e tests.
 test-e2e-bundle: operator-sdk docker-build docker-push bundle bundle-build bundle-push install-olm-in-cluster

hack/create-kind-cluster.sh

@@ -177,7 +177,12 @@ function install_gitea() {
     --set gitea.admin.username=giteaadmin \
     --set gitea.admin.password=giteapass \
     --set gitea.admin.email=admin@gitea.local \
-    --set persistence.enabled=false
+    --set persistence.enabled=false \
+    --set postgresql-ha.enabled=false \
+    --set postgresql.enabled=true \
+    --set postgresql.persistence.enabled=false \
+    --set redis-cluster.enabled=false \
+    --set redis.enabled=false
 
   header_text "Waiting for Gitea to become ready"
   kubectl wait deployment --all --timeout=-1s --for=condition=Available --namespace gitea

test/e2e/e2e_test.go

@@ -217,3 +217,30 @@ func getMetricsOutput() string {
 	Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK"))
 	return metricsOutput
 }
+
+// logFailedTestDetails logs function resource and controller logs on test failure
+func logFailedTestDetails(functionName, functionNamespace string) {
+	specReport := CurrentSpecReport()
+	if !specReport.Failed() {
+		return
+	}
+
+	if functionName != "" {
+		cmd := exec.Command("kubectl", "get", "function", functionName, "-n", functionNamespace, "-o", "yaml")
+		function, err := utils.Run(cmd)
+		if err == nil {
+			_, _ = fmt.Fprintf(GinkgoWriter, "Function:\n %s", function)
+		} else {
+			_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get function: %s", err)
+		}
+	}
+
+	By("Fetching controller manager pod logs")
+	cmd := exec.Command("kubectl", "logs", "-l", "control-plane=controller-manager", "-n", namespace)
+	controllerLogs, err := utils.Run(cmd)
+	if err == nil {
+		_, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs)
+	} else {
+		_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err)
+	}
+}

test/e2e/func_deploy_test.go

@@ -63,19 +63,17 @@ var _ = Describe("Operator", func() {
 			DeferCleanup(cleanupNamespaces, functionNamespace)
 
 			// Deploy function using func CLI
-			cmd := exec.Command("func", "deploy",
+			out, err := utils.RunFunc("deploy",
 				"--namespace", functionNamespace,
 				"--path", repoDir,
 				"--registry", registry,
 				"--registry-insecure", strconv.FormatBool(registryInsecure))
-			out, err := utils.Run(cmd)
 			Expect(err).NotTo(HaveOccurred())
 			_, _ = fmt.Fprint(GinkgoWriter, out)
 
 			// Cleanup func deployment
 			DeferCleanup(func() {
-				cmd := exec.Command("func", "delete", "--path", repoDir, "--namespace", functionNamespace)
-				_, _ = utils.Run(cmd)
+				_, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace)
 			})
 
 			// Commit func.yaml changes
@@ -84,27 +82,7 @@ var _ = Describe("Operator", func() {
 		})
 
 		AfterEach(func() {
-			specReport := CurrentSpecReport()
-			if specReport.Failed() {
-				if functionName != "" {
-					cmd := exec.Command("kubectl", "get", "function", functionName, "-n", functionNamespace, "-o", "yaml")
-					function, err := utils.Run(cmd)
-					if err == nil {
-						_, _ = fmt.Fprintf(GinkgoWriter, "Function:\n %s", function)
-					} else {
-						_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get function: %s", err)
-					}
-				}
-
-				By("Fetching controller manager pod logs")
-				cmd := exec.Command("kubectl", "logs", "-l", "control-plane=controller-manager", "-n", namespace)
-				controllerLogs, err := utils.Run(cmd)
-				if err == nil {
-					_, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs)
-				} else {
-					_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err)
-				}
-			}
+			logFailedTestDetails(functionName, functionNamespace)
 
 			// Cleanup function resource
 			if functionName != "" {

test/e2e/func_middleware_update_test.go

@@ -0,0 +1,272 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	functionsdevv1alpha1 "github.com/functions-dev/func-operator/api/v1alpha1"
+	"github.com/functions-dev/func-operator/internal/function"
+	"github.com/functions-dev/func-operator/test/utils"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	funcfn "knative.dev/func/pkg/functions"
+)
+
+var _ = Describe("Middleware Update", func() {
+
+	SetDefaultEventuallyTimeout(2 * time.Minute)
+	SetDefaultEventuallyPollingInterval(time.Second)
+
+	Context("with a function deployed using old func CLI", func() {
+		var repoURL string
+		var repoDir string
+		var functionName, functionNamespace string
+
+		BeforeEach(func() {
+			var err error
+
+			// Create repository provider resources with automatic cleanup
+			username, password, _, cleanup, err := repoProvider.CreateRandomUser()
+			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(cleanup)
+
+			_, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false)
+			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(cleanup)
+
+			functionNamespace, err = utils.GetTestNamespace()
+			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(cleanupNamespaces, functionNamespace)
+
+			// Initialize repository with function code using OLD func CLI version
+			// v1.20.1 has no middleware-version label and uses instance-compatible templates
+			oldFuncVersion := "v1.20.1"
+			repoDir, err = utils.InitializeRepoWithFunctionVersion(repoURL, username, password, "go", oldFuncVersion)
+			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(os.RemoveAll, repoDir)
+
+			// Deploy function using the same OLD func CLI version
+			out, err := utils.RunFuncWithVersion(oldFuncVersion, "deploy",
+				"--namespace", functionNamespace,
+				"--path", repoDir,
+				"--registry", registry,
+				"--registry-insecure", strconv.FormatBool(registryInsecure))
+			Expect(err).NotTo(HaveOccurred())
+			_, _ = fmt.Fprint(GinkgoWriter, out)
+
+			// Cleanup func deployment
+			DeferCleanup(func() {
+				_, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace)
+			})
+
+			// Commit func.yaml changes
+			err = utils.CommitAndPush(repoDir, "Update func.yaml after deploy", "func.yaml")
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			logFailedTestDetails(functionName, functionNamespace)
+
+			// Cleanup function resource
+			if functionName != "" {
+				cmd := exec.Command("kubectl", "delete", "function", functionName, "-n", functionNamespace, "--ignore-not-found")
+				_, err := utils.Run(cmd)
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		It("should update the middleware and mark the function as ready", func() {
+			// Get function metadata to retrieve the deployed function name
+			funcMetadata, err := function.Metadata(repoDir)
+			Expect(err).NotTo(HaveOccurred())
+			deployedFunctionName := funcMetadata.Name
+
+			// NOTE: We use skopeo to verify the middleware version because func describe
+			// cannot access the image registry in our test environment.
+			//
+			// func describe calls MiddlewareVersion() which uses go-containerregistry's
+			// remote.Get() to fetch the image manifest and read the middleware-version label.
+			// However, the image reference is kind-registry:5000/..., where "kind-registry"
+			// is a Docker container name, not a real hostname.
+			//
+			// Docker CLI can resolve container names on Docker networks (which is why
+			// func deploy --registry kind-registry:5000 works for pushing images), but
+			// go-containerregistry is not Docker-aware and tries regular DNS resolution,
+			// which fails. The error is silently ignored in describers.
+			//
+			// We use skopeo with localhost:5001 (port-forward to the registry) to
+			// directly inspect the OCI image labels and verify the middleware was updated.
+
+			// Get initial image digest from func describe (deployed with v1.20.1)
+			out, err := utils.RunFunc("describe", deployedFunctionName, "-n", functionNamespace, "-o", "yaml")
+			Expect(err).NotTo(HaveOccurred())
+
+			var initialInstance funcfn.Instance
+			err = yaml.Unmarshal([]byte(out), &initialInstance)
+			Expect(err).NotTo(HaveOccurred())
+
+			initialImage := initialInstance.Image
+			Expect(initialImage).NotTo(BeEmpty(), "Initial image should be available from func describe")
+			_, _ = fmt.Fprintf(GinkgoWriter, "Initial image (deployed with v1.20.1): %s\n", initialImage)
+
+			// Verify initial image has no middleware-version label (v1.20.1 doesn't set it)
+			initialImageLocal := strings.Replace(initialImage, "kind-registry:5000", "localhost:5001", 1)
+			// Remove tag if both tag and digest are present (skopeo doesn't support this format)
+			if strings.Contains(initialImageLocal, "@") {
+				atIndex := strings.Index(initialImageLocal, "@")
+				slashIndex := strings.LastIndex(initialImageLocal[:atIndex], "/")
+				if slashIndex != -1 {
+					betweenSlashAndAt := initialImageLocal[slashIndex+1 : atIndex]
+					if strings.Contains(betweenSlashAndAt, ":") {
+						colonIndex := strings.Index(betweenSlashAndAt, ":")
+						initialImageLocal = initialImageLocal[:slashIndex+1+colonIndex] + initialImageLocal[atIndex:]
+					}
+				}
+			}
+			cmd := exec.Command("skopeo",
+				"inspect",
+				"--tls-verify=false",
+				"--no-tags",
+				"docker://"+initialImageLocal)
+			skopeoOutput, err := utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+
+			var initialImageLabels struct {
+				Labels map[string]string `json:"Labels"`
+			}
+			err = json.Unmarshal([]byte(skopeoOutput), &initialImageLabels)
+			Expect(err).NotTo(HaveOccurred())
+
+			initialMiddlewareVersion := initialImageLabels.Labels["middleware-version"]
+			_, _ = fmt.Fprintf(GinkgoWriter, "Initial middleware-version label: '%s' (expected empty for v1.20.1)\n",
+				initialMiddlewareVersion)
+
+			// Create a Function resource
+			fn := &functionsdevv1alpha1.Function{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "my-function-",
+					Namespace:    functionNamespace,
+				},
+				Spec: functionsdevv1alpha1.FunctionSpec{
+					Source: functionsdevv1alpha1.FunctionSpecSource{
+						RepositoryURL: repoURL,
+					},
+					Registry: functionsdevv1alpha1.FunctionSpecRegistry{
+						Path:     registry,
+						Insecure: registryInsecure,
+					},
+				},
+			}
+
+			err = k8sClient.Create(ctx, fn)
+			Expect(err).NotTo(HaveOccurred())
+
+			functionName = fn.Name
+
+			funcBecomeReady := func(g Gomega) {
+				f := &functionsdevv1alpha1.Function{}
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: fn.Name, Namespace: fn.Namespace}, f)
+				g.Expect(err).NotTo(HaveOccurred())
+
+				for _, cond := range f.Status.Conditions {
+					if cond.Type == functionsdevv1alpha1.TypeReady {
+						g.Expect(cond.Status).To(Equal(metav1.ConditionTrue))
+						return
+					}
+				}
+				g.Expect(false).To(BeTrue(), "Ready condition not found")
+			}
+
+			// Middleware update could take a bit longer therefore give more time
+			Eventually(funcBecomeReady, 6*time.Minute).Should(Succeed())
+
+			// Verify middleware was actually updated by inspecting the new image
+			out, err = utils.RunFunc("describe", deployedFunctionName, "-n", functionNamespace, "-o", "yaml")
+			Expect(err).NotTo(HaveOccurred())
+
+			var updatedInstance funcfn.Instance
+			err = yaml.Unmarshal([]byte(out), &updatedInstance)
+			Expect(err).NotTo(HaveOccurred())
+
+			updatedImage := updatedInstance.Image
+			Expect(updatedImage).NotTo(BeEmpty(), "Updated image should be available from func describe")
+			_, _ = fmt.Fprintf(GinkgoWriter, "Updated image (redeployed by operator): %s\n", updatedImage)
+
+			// Verify the image actually changed
+			Expect(updatedImage).NotTo(Equal(initialImage), "Image should have changed after operator redeploy")
+
+			// Verify updated image has middleware-version label set
+			updatedImageLocal := strings.Replace(updatedImage, "kind-registry:5000", "localhost:5001", 1)
+			// Remove tag if both tag and digest are present (skopeo doesn't support this format)
+			// Format: registry/name:tag@digest -> registry/name@digest
+			if strings.Contains(updatedImageLocal, "@") {
+				atIndex := strings.Index(updatedImageLocal, "@")
+				slashIndex := strings.LastIndex(updatedImageLocal[:atIndex], "/")
+				if slashIndex != -1 {
+					// Check if there's a colon between last slash and @
+					betweenSlashAndAt := updatedImageLocal[slashIndex+1 : atIndex]
+					if strings.Contains(betweenSlashAndAt, ":") {
+						// Remove the :tag part
+						colonIndex := strings.Index(betweenSlashAndAt, ":")
+						updatedImageLocal = updatedImageLocal[:slashIndex+1+colonIndex] + updatedImageLocal[atIndex:]
+					}
+				}
+			}
+			cmd = exec.Command("skopeo", "inspect", "--tls-verify=false", "--no-tags", "docker://"+updatedImageLocal)
+			skopeoOutput, err = utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+
+			var updatedImageLabels struct {
+				Labels map[string]string `json:"Labels"`
+			}
+			err = json.Unmarshal([]byte(skopeoOutput), &updatedImageLabels)
+			Expect(err).NotTo(HaveOccurred())
+
+			updatedMiddlewareVersion := updatedImageLabels.Labels["middleware-version"]
+			_, _ = fmt.Fprintf(GinkgoWriter, "Updated middleware-version label: '%s'\n", updatedMiddlewareVersion)
+
+			// The operator should have set a middleware version
+			Expect(updatedMiddlewareVersion).NotTo(BeEmpty(), "Operator should have deployed with middleware-version label set")
+
+			// Verify MiddlewareUpToDate condition is True
+			updatedFunction := &functionsdevv1alpha1.Function{}
+			err = k8sClient.Get(ctx, types.NamespacedName{Name: fn.Name, Namespace: fn.Namespace}, updatedFunction)
+			Expect(err).NotTo(HaveOccurred())
+
+			var middlewareUpToDate bool
+			for _, cond := range updatedFunction.Status.Conditions {
+				if cond.Type == functionsdevv1alpha1.TypeMiddlewareUpToDate {
+					middlewareUpToDate = cond.Status == metav1.ConditionTrue
+					_, _ = fmt.Fprintf(GinkgoWriter, "MiddlewareUpToDate condition: status=%s, reason=%s\n",
+						cond.Status, cond.Reason)
+					break
+				}
+			}
+			Expect(middlewareUpToDate).To(BeTrue(), "MiddlewareUpToDate condition should be True")
+		})
+	})
+})