Add end-to-end verification of middleware update via skopeo

- Use skopeo to inspect OCI image labels and verify middleware-version
- Initial image (v1.20.0 deploy) has no middleware-version label
- Updated image (operator redeploy) has middleware-version set
- Handle image references with both tag and digest (remove tag for skopeo)
- Test now fully verifies the core operator feature end-to-end

Why skopeo instead of func describe:
- func describe uses go-containerregistry to read middleware-version from image
- kind-registry is a Docker container name, not a real hostname
- Docker CLI can resolve it (func deploy works), but go-containerregistry cannot
- func describe silently fails to populate middleware.version field
- skopeo with localhost:5001 successfully inspects the image labels

Author: creydr

test/e2e/func_middleware_update_test.go

@@ -17,18 +17,23 @@ limitations under the License.
 package e2e
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/exec"
 	"strconv"
+	"strings"
 	"time"
 
 	functionsdevv1alpha1 "github.com/functions-dev/func-operator/api/v1alpha1"
+	"github.com/functions-dev/func-operator/internal/function"
 	"github.com/functions-dev/func-operator/test/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	funcfn "knative.dev/func/pkg/functions"
 )
 
 var _ = Describe("Middleware Update", func() {
@@ -113,10 +118,70 @@ var _ = Describe("Middleware Update", func() {
 		})
 
 		It("should update the middleware and mark the function as ready", func() {
-			// Note: We can't reliably check middleware version via func describe in e2e tests
-			// because func describe requires inspecting OCI image labels, which doesn't work
-			// with insecure registries (kind-registry:5000).
-			// Instead, we'll verify the middleware update by checking the CR conditions.
+			// Get function metadata to retrieve the deployed function name
+			funcMetadata, err := function.Metadata(repoDir)
+			Expect(err).NotTo(HaveOccurred())
+			deployedFunctionName := funcMetadata.Name
+
+			// NOTE: We use skopeo to verify the middleware version because func describe
+			// cannot access the image registry in our test environment.
+			//
+			// func describe calls MiddlewareVersion() which uses go-containerregistry's
+			// remote.Get() to fetch the image manifest and read the middleware-version label.
+			// However, the image reference is kind-registry:5000/..., where "kind-registry"
+			// is a Docker container name, not a real hostname.
+			//
+			// Docker CLI can resolve container names on Docker networks (which is why
+			// func deploy --registry kind-registry:5000 works for pushing images), but
+			// go-containerregistry is not Docker-aware and tries regular DNS resolution,
+			// which fails. The error is silently ignored in describers.
+			//
+			// We use skopeo with localhost:5001 (port-forward to the registry) to
+			// directly inspect the OCI image labels and verify the middleware was updated.
+
+			// Get initial image digest from func describe (deployed with old v1.20.0)
+			out, err := utils.RunFunc("describe", deployedFunctionName, "-n", functionNamespace, "-o", "yaml")
+			Expect(err).NotTo(HaveOccurred())
+
+			var initialInstance funcfn.Instance
+			err = yaml.Unmarshal([]byte(out), &initialInstance)
+			Expect(err).NotTo(HaveOccurred())
+
+			initialImage := initialInstance.Image
+			Expect(initialImage).NotTo(BeEmpty(), "Initial image should be available from func describe")
+			_, _ = fmt.Fprintf(GinkgoWriter, "Initial image (deployed with v1.20.0): %s\n", initialImage)
+
+			// Verify initial image has no middleware-version label (old func CLI)
+			initialImageLocal := strings.Replace(initialImage, "kind-registry:5000", "localhost:5001", 1)
+			// Remove tag if both tag and digest are present (skopeo doesn't support this format)
+			if strings.Contains(initialImageLocal, "@") {
+				atIndex := strings.Index(initialImageLocal, "@")
+				slashIndex := strings.LastIndex(initialImageLocal[:atIndex], "/")
+				if slashIndex != -1 {
+					betweenSlashAndAt := initialImageLocal[slashIndex+1 : atIndex]
+					if strings.Contains(betweenSlashAndAt, ":") {
+						colonIndex := strings.Index(betweenSlashAndAt, ":")
+						initialImageLocal = initialImageLocal[:slashIndex+1+colonIndex] + initialImageLocal[atIndex:]
+					}
+				}
+			}
+			cmd := exec.Command("skopeo",
+				"inspect",
+				"--tls-verify=false",
+				"--no-tags",
+				"docker://"+initialImageLocal)
+			skopeoOutput, err := utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+
+			var initialImageLabels struct {
+				Labels map[string]string `json:"Labels"`
+			}
+			err = json.Unmarshal([]byte(skopeoOutput), &initialImageLabels)
+			Expect(err).NotTo(HaveOccurred())
+
+			initialMiddlewareVersion := initialImageLabels.Labels["middleware-version"]
+			_, _ = fmt.Fprintf(GinkgoWriter, "Initial middleware-version label: '%s' (expected empty for v1.20.0)\n",
+				initialMiddlewareVersion)
 
 			// Create a Function resource
 			fn := &functionsdevv1alpha1.Function{
@@ -135,7 +200,7 @@ var _ = Describe("Middleware Update", func() {
 				},
 			}
 
-			err := k8sClient.Create(ctx, fn)
+			err = k8sClient.Create(ctx, fn)
 			Expect(err).NotTo(HaveOccurred())
 
 			functionName = fn.Name
@@ -157,26 +222,69 @@ var _ = Describe("Middleware Update", func() {
 			// Middleware update could take a bit longer therefore give more time
 			Eventually(funcBecomeReady, 6*time.Minute).Should(Succeed())
 
-			// Verify middleware was updated by checking the MiddlewareUpToDate condition
+			// Verify middleware was actually updated by inspecting the new image
+			out, err = utils.RunFunc("describe", deployedFunctionName, "-n", functionNamespace, "-o", "yaml")
+			Expect(err).NotTo(HaveOccurred())
+
+			var updatedInstance funcfn.Instance
+			err = yaml.Unmarshal([]byte(out), &updatedInstance)
+			Expect(err).NotTo(HaveOccurred())
+
+			updatedImage := updatedInstance.Image
+			Expect(updatedImage).NotTo(BeEmpty(), "Updated image should be available from func describe")
+			_, _ = fmt.Fprintf(GinkgoWriter, "Updated image (redeployed by operator): %s\n", updatedImage)
+
+			// Verify the image actually changed
+			Expect(updatedImage).NotTo(Equal(initialImage), "Image should have changed after operator redeploy")
+
+			// Verify updated image has middleware-version label set
+			updatedImageLocal := strings.Replace(updatedImage, "kind-registry:5000", "localhost:5001", 1)
+			// Remove tag if both tag and digest are present (skopeo doesn't support this format)
+			// Format: registry/name:tag@digest -> registry/name@digest
+			if strings.Contains(updatedImageLocal, "@") {
+				atIndex := strings.Index(updatedImageLocal, "@")
+				slashIndex := strings.LastIndex(updatedImageLocal[:atIndex], "/")
+				if slashIndex != -1 {
+					// Check if there's a colon between last slash and @
+					betweenSlashAndAt := updatedImageLocal[slashIndex+1 : atIndex]
+					if strings.Contains(betweenSlashAndAt, ":") {
+						// Remove the :tag part
+						colonIndex := strings.Index(betweenSlashAndAt, ":")
+						updatedImageLocal = updatedImageLocal[:slashIndex+1+colonIndex] + updatedImageLocal[atIndex:]
+					}
+				}
+			}
+			cmd = exec.Command("skopeo", "inspect", "--tls-verify=false", "--no-tags", "docker://"+updatedImageLocal)
+			skopeoOutput, err = utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+
+			var updatedImageLabels struct {
+				Labels map[string]string `json:"Labels"`
+			}
+			err = json.Unmarshal([]byte(skopeoOutput), &updatedImageLabels)
+			Expect(err).NotTo(HaveOccurred())
+
+			updatedMiddlewareVersion := updatedImageLabels.Labels["middleware-version"]
+			_, _ = fmt.Fprintf(GinkgoWriter, "Updated middleware-version label: '%s'\n", updatedMiddlewareVersion)
+
+			// The operator should have set a middleware version
+			Expect(updatedMiddlewareVersion).NotTo(BeEmpty(), "Operator should have deployed with middleware-version label set")
+
+			// Verify MiddlewareUpToDate condition is True
 			updatedFunction := &functionsdevv1alpha1.Function{}
 			err = k8sClient.Get(ctx, types.NamespacedName{Name: fn.Name, Namespace: fn.Namespace}, updatedFunction)
 			Expect(err).NotTo(HaveOccurred())
 
-			// Check that MiddlewareUpToDate condition is now True
 			var middlewareUpToDate bool
-			var middlewareCondition metav1.Condition
 			for _, cond := range updatedFunction.Status.Conditions {
 				if cond.Type == functionsdevv1alpha1.TypeMiddlewareUpToDate {
 					middlewareUpToDate = cond.Status == metav1.ConditionTrue
-					middlewareCondition = cond
+					_, _ = fmt.Fprintf(GinkgoWriter, "MiddlewareUpToDate condition: status=%s, reason=%s\n",
+						cond.Status, cond.Reason)
 					break
 				}
 			}
-			_, _ = fmt.Fprintf(GinkgoWriter, "MiddlewareUpToDate condition: status=%s, reason=%s\n",
-				middlewareCondition.Status, middlewareCondition.Reason)
-
-			Expect(middlewareUpToDate).To(BeTrue(),
-				"MiddlewareUpToDate condition should be True after operator redeploys")
+			Expect(middlewareUpToDate).To(BeTrue(), "MiddlewareUpToDate condition should be True")
 		})
 	})
 })