diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 6c434ed..9d897b0 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -29,6 +29,10 @@ jobs: - name: Install helm uses: azure/setup-helm@v4.3.0 + # func CLI is needed in some e2e tests ATM + - name: Install func cli + uses: functions-dev/action@main + - name: Setup KinD cluster run: make create-kind-cluster @@ -36,4 +40,7 @@ jobs: run: make docker-build docker-push deploy - name: Running Test e2e + env: + REGISTRY_INSECURE: true + REGISTRY: kind-registry:5000 run: make test-e2e diff --git a/.gitignore b/.gitignore index b3828d1..70e1559 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,7 @@ go.work.sum .idea/ .vscode/ -bin/ \ No newline at end of file +bin/ + +# Local registry certificates +hack/registry-certs/ \ No newline at end of file diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index 77c8ec6..2038b86 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -41,8 +41,31 @@ function setup_local_registry() { fi if [ "$(docker inspect -f '{{.State.Running}}' "${REGISTRY_NAME}" 2>/dev/null || true)" != 'true' ]; then - header_text "create registry container for port ${REGISTRY_PORT}" - docker run -d --restart=always -p "127.0.0.1:${REGISTRY_PORT}:5000" --name "${REGISTRY_NAME}" docker.io/registry:2 + header_text "create registry container for port ${REGISTRY_PORT} with HTTPS" + + # Create persistent directory for registry certs in project + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REGISTRY_CERTS_DIR="${SCRIPT_DIR}/registry-certs" + mkdir -p "${REGISTRY_CERTS_DIR}" + + # Generate self-signed certificate if it doesn't exist + if [ ! -f "${REGISTRY_CERTS_DIR}/registry.crt" ]; then + header_text "Generating self-signed certificate for registry" + openssl req -newkey rsa:4096 -nodes -sha256 \ + -keyout "${REGISTRY_CERTS_DIR}/registry.key" \ + -x509 -days 365 -out "${REGISTRY_CERTS_DIR}/registry.crt" \ + -subj "/CN=${REGISTRY_NAME}" \ + -addext "subjectAltName=DNS:kind-registry,DNS:localhost,IP:127.0.0.1" + fi + + # Run registry with HTTPS + docker run -d --restart=always \ + -p "127.0.0.1:${REGISTRY_PORT}:5000" \ + -v "${REGISTRY_CERTS_DIR}:/certs" \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/registry.key \ + --name "${REGISTRY_NAME}" \ + docker.io/registry:2 fi } @@ -61,7 +84,7 @@ nodes: containerdConfigPatches: - |- [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:$REGISTRY_PORT"] - endpoint = ["http://$REGISTRY_NAME:5000"] + endpoint = ["https://$REGISTRY_NAME:5000"] EOF } @@ -84,6 +107,12 @@ data: help: "https://kind.sigs.k8s.io/docs/user/local-registry/" EOF + # Install registry certificate on all nodes + header_text "Installing registry certificate on cluster nodes" + for node in $(kind get nodes --name "$CLUSTER_NAME"); do + docker exec "$REGISTRY_NAME" cat /certs/registry.crt | docker exec -i "$node" bash -c 'mkdir -p /usr/local/share/ca-certificates && cat > /usr/local/share/ca-certificates/kind-registry.crt && update-ca-certificates' + docker exec "$node" systemctl restart containerd + done } function install_tekton() { diff --git a/internal/funccli/manager.go b/internal/funccli/manager.go index c12574c..107e486 100644 --- a/internal/funccli/manager.go +++ b/internal/funccli/manager.go @@ -219,11 +219,14 @@ func (m *managerImpl) Deploy(ctx context.Context, repoPath string, namespace str "--registry", opts.Registry, "--git-url", opts.GitUrl, "--builder", opts.Builder, - "--registry-authfile", opts.RegistryAuthFile, + } + + if opts.RegistryAuthFile != "" { + deployArgs = append(deployArgs, "--registry-authfile", opts.RegistryAuthFile) } if opts.InsecureRegistry { - deployArgs = append(deployArgs, "--insecure-registry") + deployArgs = append(deployArgs, "--registry-insecure") } out, err := m.Run(ctx, repoPath, deployArgs...) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b47f1d1..1604222 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -17,11 +17,27 @@ limitations under the License. package e2e import ( + "context" "fmt" + "os" + "strings" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + functionsdevv1alpha1 "github.com/functions-dev/func-operator/api/v1alpha1" +) + +var ( + k8sClient client.Client + ctx context.Context + + registry string + registryInsecure bool ) // TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, @@ -33,3 +49,34 @@ func TestE2E(t *testing.T) { _, _ = fmt.Fprintf(GinkgoWriter, "Starting func-operator integration test suite\n") RunSpecs(t, "e2e suite") } + +var _ = BeforeSuite(func() { + ctx = context.Background() + + // Register the Function API scheme + err := functionsdevv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // Load kubeconfig and create client + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + cfg, err := kubeConfig.ClientConfig() + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // Setup vars from env + registry = os.Getenv("REGISTRY") + if registry == "" { + registry = "kind-registry:5000" + } + + registryInsecure = false + if sec := os.Getenv("REGISTRY_INSECURE"); strings.ToLower(sec) == "true" { + registryInsecure = true + } +}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7462ad5..b500437 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -121,37 +121,46 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyControllerUp).Should(Succeed()) }) - It("should ensure the metrics endpoint is serving metrics", func() { - By("validating that the metrics service is available") - cmd := exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") - - By("waiting for the metrics endpoint to be ready") - verifyMetricsEndpointReady := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring(metricsPort), "Metrics endpoint is not ready") - } - Eventually(verifyMetricsEndpointReady).Should(Succeed()) - - By("verifying that the controller manager is serving the metrics server") - verifyMetricsServerStarted := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), - "Metrics server not yet started") - } - Eventually(verifyMetricsServerStarted).Should(Succeed()) - - By("creating the curl-metrics pod to access the metrics endpoint") - cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", - "--namespace", namespace, - "--image=curlimages/curl:latest", - "--overrides", - fmt.Sprintf(`{ + Context("with curl-metrics-pod", func() { + curlMetricPodName := "curl-metrics" + + AfterEach(func() { + cmd := exec.Command("kubectl", "delete", "pod", curlMetricPodName, "-n", namespace, "--ignore-not-found") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("validating that the metrics service is available") + cmd := exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring(metricsPort), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", curlMetricPodName, "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ "spec": { "containers": [{ "name": "curl", @@ -173,37 +182,29 @@ var _ = Describe("Manager", Ordered, func() { "serviceAccount": "%s" } }`, metricsServiceName, namespace, metricsPort, serviceAccountName)) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") - - By("waiting for the curl-metrics pod to complete.") - verifyCurlUp := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", - "-o", "jsonpath={.status.phase}", - "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") - } - Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) - - By("getting the metrics by checking curl-metrics logs") - metricsOutput := getMetricsOutput() - Expect(metricsOutput).To(ContainSubstring( - "controller_runtime_reconcile_total", - )) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", curlMetricPodName, + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) }) // +kubebuilder:scaffold:e2e-webhooks-checks - - // TODO: Customize the e2e test suite with scenarios specific to your project. - // Consider applying sample/CR(s) and check their status and/or verifying - // the reconciliation by using the metrics, i.e.: - // metricsOutput := getMetricsOutput() - // Expect(metricsOutput).To(ContainSubstring( - // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, - // strings.ToLower(), - // )) }) }) diff --git a/test/e2e/func_deploy_test.go b/test/e2e/func_deploy_test.go new file mode 100644 index 0000000..699f259 --- /dev/null +++ b/test/e2e/func_deploy_test.go @@ -0,0 +1,191 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "time" + + functionsdevv1alpha1 "github.com/functions-dev/func-operator/api/v1alpha1" + "github.com/functions-dev/func-operator/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" +) + +var _ = Describe("Operator", Ordered, func() { + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("with a deployed function", func() { + var tempDir string + var functionName, functionNamespace string + + BeforeEach(func() { + var err error + // deploy function + tempDir = fmt.Sprintf("%s/func-operator-e2e-%s", os.TempDir(), rand.String(10)) + Expect(err).NotTo(HaveOccurred()) + + cmd := exec.Command("git", "clone", "https://github.com/creydr/func-go-hello-world", tempDir) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + cmd = exec.Command("func", "deploy", + "--path", tempDir, + "--registry", registry, + "--registry-insecure", strconv.FormatBool(registryInsecure)) + out, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + _, _ = fmt.Fprint(GinkgoWriter, out) + }) + + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + if functionName != "" { + cmd := exec.Command("kubectl", "get", "function", functionName, "-n", functionNamespace, "-o", "yaml") + function, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Function:\n %s", function) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get function: %s", err) + } + } + + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", "-l", "control-plane=controller-manager", "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + } + + if tempDir != "" { + cmd := exec.Command("func", "delete", "--path", tempDir) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + } + + cmd := exec.Command("kubectl", "delete", "function", functionName, "-n", functionNamespace, "--ignore-not-found") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should mark the function as ready", func() { + // Create a Function resource + function := &functionsdevv1alpha1.Function{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "my-function-", + Namespace: "default", + }, + Spec: functionsdevv1alpha1.FunctionSpec{ + Source: functionsdevv1alpha1.FunctionSpecSource{ + RepositoryURL: "https://github.com/creydr/func-go-hello-world", + }, + Registry: functionsdevv1alpha1.FunctionSpecRegistry{ + Path: registry, + Insecure: registryInsecure, + }, + }, + } + + err := k8sClient.Create(ctx, function) + Expect(err).NotTo(HaveOccurred()) + + functionName = function.Name + functionNamespace = function.Namespace + + funcBecomeReady := func(g Gomega) { + fn := &functionsdevv1alpha1.Function{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: function.Name, Namespace: function.Namespace}, fn) + g.Expect(err).NotTo(HaveOccurred()) + + for _, cond := range fn.Status.Conditions { + if cond.Type == functionsdevv1alpha1.TypeReady { + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + return + } + } + g.Expect(false).To(BeTrue(), "Ready condition not found") + } + + // redeploy could take a bit longer therefore give a bit more time + Eventually(funcBecomeReady, 6*time.Minute).Should(Succeed()) + }) + }) + Context("with a not yet deployed function", func() { + var functionName, functionNamespace string + + AfterEach(func() { + cmd := exec.Command("kubectl", "delete", "function", functionName, "-n", functionNamespace, "--ignore-not-found") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should mark the function as not ready", func() { + // Create a Function resource + function := &functionsdevv1alpha1.Function{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "my-undeployed-function-", + Namespace: "default", + }, + Spec: functionsdevv1alpha1.FunctionSpec{ + Source: functionsdevv1alpha1.FunctionSpecSource{ + RepositoryURL: "https://github.com/creydr/func-go-hello-world", + }, + Registry: functionsdevv1alpha1.FunctionSpecRegistry{ + Path: registry, + Insecure: registryInsecure, + }, + }, + } + + err := k8sClient.Create(ctx, function) + Expect(err).NotTo(HaveOccurred()) + + functionName = function.Name + functionNamespace = function.Namespace + + funcBecomeReady := func(g Gomega) { + fn := &functionsdevv1alpha1.Function{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: function.Name, Namespace: function.Namespace}, fn) + g.Expect(err).NotTo(HaveOccurred()) + + for _, cond := range fn.Status.Conditions { + if cond.Type == functionsdevv1alpha1.TypeDeployed { + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal("NotDeployed")) + g.Expect(cond.Message).To(ContainSubstring("Function not deployed yet")) + return + } + } + g.Expect(false).To(BeTrue(), "Deployed condition not found") + } + + Eventually(funcBecomeReady, 2*time.Minute).Should(Succeed()) + }) + }) +})