diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7273240..149a66f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -398,6 +398,23 @@ jobs: E2E_SHARD_TOTAL: "4" run: make test-e2e + - name: Collect failure diagnostics + if: ${{ failure() && !cancelled() }} + run: | + mkdir -p failure-diagnostics + kind export logs --name kind failure-diagnostics/kind || true + kubectl get pods,jobs -A -o wide > failure-diagnostics/pods-jobs.txt 2>&1 || true + kubectl get events -A --sort-by=.metadata.creationTimestamp > failure-diagnostics/events.txt 2>&1 || true + + - name: Upload failure diagnostics + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: ${{ failure() && !cancelled() }} + with: + name: e2e-kind-logs-shard-${{ matrix.shard }} + path: failure-diagnostics + if-no-files-found: warn + retention-days: 7 + - name: Upload test report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: ${{ !cancelled() }} diff --git a/Makefile b/Makefile index 01142f1..137f3f3 100644 --- a/Makefile +++ b/Makefile @@ -154,7 +154,7 @@ fuzz: fuzz-keeper fuzz-clickhouse ## Run all fuzz tests # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: ## Run all e2e tests. Honors E2E_SHARD_INDEX / E2E_SHARD_TOTAL for CI sharding. - go test ./test/e2e/ -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml + go test ./test/e2e/ -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml --ginkgo.json-report=report/ginkgo-report.json .PHONY: test-keeper-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-keeper-e2e: ## Run keeper e2e tests. diff --git a/internal/controller/versionprobe.go b/internal/controller/versionprobe.go index b92c089..b3d18fe 100644 --- a/internal/controller/versionprobe.go +++ b/internal/controller/versionprobe.go @@ -267,11 +267,16 @@ func (rm *ResourceManager) buildVersionProbeJob(cfg VersionProbeConfig, revision Image: cfg.ContainerTemplate.Image.String(), ImagePullPolicy: cfg.ContainerTemplate.ImagePullPolicy, SecurityContext: DefaultContainerSecurityContext(), - TerminationMessagePolicy: corev1.TerminationMessageReadFile, + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, TerminationMessagePath: corev1.TerminationMessagePathDefault, Command: []string{versionProbeBinary}, - Args: []string{"local", "--query", versionProbeQuery}, - Env: []corev1.EnvVar{{Name: "MALLOC_CONF", Value: "narenas:2,dirty_decay_ms:0,muzzy_decay_ms:0,thp:never"}}, + Args: []string{ + "local", + "--logger.console=1", + "--logger.level=debug", + "--query", versionProbeQuery, + }, + Env: []corev1.EnvVar{{Name: "MALLOC_CONF", Value: "narenas:2,dirty_decay_ms:0,muzzy_decay_ms:0,thp:never"}}, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(DefaultProbeCPURequest), diff --git a/internal/controller/versionprobe_test.go b/internal/controller/versionprobe_test.go index 5bb057a..5a987e6 100644 --- a/internal/controller/versionprobe_test.go +++ b/internal/controller/versionprobe_test.go @@ -387,7 +387,13 @@ var _ = Describe("VersionProbe caching", func() { Expect(jobs.Items[0].Spec.Template.Spec.ActiveDeadlineSeconds).NotTo(BeNil()) Expect(*jobs.Items[0].Spec.Template.Spec.ActiveDeadlineSeconds).To(Equal(versionProbeDeadline)) Expect(container.Command).To(Equal([]string{"/usr/bin/clickhouse"})) - Expect(container.Args).To(Equal([]string{"local", "--query", "INSERT INTO FUNCTION file('/dev/termination-log', 'RawBLOB', 'version String') SELECT version()"})) + Expect(container.Args).To(Equal([]string{ + "local", + "--logger.console=1", + "--logger.level=debug", + "--query", "INSERT INTO FUNCTION file('/dev/termination-log', 'RawBLOB', 'version String') SELECT version()", + })) + Expect(container.TerminationMessagePolicy).To(Equal(corev1.TerminationMessageFallbackToLogsOnError)) }) It("should miss cache when image changes", func(ctx context.Context) { diff --git a/test/testutil/utils.go b/test/testutil/utils.go index 2dee4e5..3562bd0 100644 --- a/test/testutil/utils.go +++ b/test/testutil/utils.go @@ -227,7 +227,16 @@ func DumpNamespaceDiagnostics(ctx context.Context, config *rest.Config, cli clie } if strings.TrimSpace(events) != "" { + path, writeErr := (&DumpResult{Full: events}).WriteFull(dir, fmt.Sprintf("events-%s.log", report.FullText())) + if writeErr != nil { + GinkgoWriter.Printf("failed to write events dump: %v\n", writeErr) + } + GinkgoWriter.Printf("\n=== Namespace Events (since test start) ===\n%s\n", events) + + if path != "" { + GinkgoWriter.Printf("Full events dump: %s\n", path) + } } }