test: retain e2e failure diagnostics and reports

.github/workflows/ci.yaml

@@ -398,6 +398,23 @@ jobs:
           E2E_SHARD_TOTAL: "4"
         run:  make test-e2e
 
+      - name: Collect failure diagnostics
+        if: ${{ failure() && !cancelled() }}
+        run: |
+          mkdir -p failure-diagnostics
+          kind export logs --name kind failure-diagnostics/kind || true
+          kubectl get pods,jobs -A -o wide > failure-diagnostics/pods-jobs.txt 2>&1 || true
+          kubectl get events -A --sort-by=.metadata.creationTimestamp > failure-diagnostics/events.txt 2>&1 || true
+
+      - name: Upload failure diagnostics
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: ${{ failure() && !cancelled() }}
+        with:
+          name: e2e-kind-logs-shard-${{ matrix.shard }}
+          path: failure-diagnostics
+          if-no-files-found: warn
+          retention-days: 7
+
       - name: Upload test report
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         if: ${{ !cancelled() }}

Makefile

@@ -154,7 +154,7 @@ fuzz: fuzz-keeper fuzz-clickhouse ## Run all fuzz tests
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e: ## Run all e2e tests. Honors E2E_SHARD_INDEX / E2E_SHARD_TOTAL for CI sharding.
-	go test ./test/e2e/ -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
+	go test ./test/e2e/ -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml --ginkgo.json-report=report/ginkgo-report.json
 
 .PHONY: test-keeper-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-keeper-e2e: ## Run keeper e2e tests.

internal/controller/versionprobe.go

@@ -267,11 +267,16 @@ func (rm *ResourceManager) buildVersionProbeJob(cfg VersionProbeConfig, revision
 							Image:                    cfg.ContainerTemplate.Image.String(),
 							ImagePullPolicy:          cfg.ContainerTemplate.ImagePullPolicy,
 							SecurityContext:          DefaultContainerSecurityContext(),
-							TerminationMessagePolicy: corev1.TerminationMessageReadFile,
+							TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
 							TerminationMessagePath:   corev1.TerminationMessagePathDefault,
 							Command:                  []string{versionProbeBinary},
-							Args:                     []string{"local", "--query", versionProbeQuery},
-							Env:                      []corev1.EnvVar{{Name: "MALLOC_CONF", Value: "narenas:2,dirty_decay_ms:0,muzzy_decay_ms:0,thp:never"}},
+							Args: []string{
+								"local",
+								"--logger.console=1",
+								"--logger.level=debug",
+								"--query", versionProbeQuery,
+							},
+							Env: []corev1.EnvVar{{Name: "MALLOC_CONF", Value: "narenas:2,dirty_decay_ms:0,muzzy_decay_ms:0,thp:never"}},
 							Resources: corev1.ResourceRequirements{
 								Requests: corev1.ResourceList{
 									corev1.ResourceCPU:    resource.MustParse(DefaultProbeCPURequest),

internal/controller/versionprobe_test.go

@@ -387,7 +387,13 @@ var _ = Describe("VersionProbe caching", func() {
 		Expect(jobs.Items[0].Spec.Template.Spec.ActiveDeadlineSeconds).NotTo(BeNil())
 		Expect(*jobs.Items[0].Spec.Template.Spec.ActiveDeadlineSeconds).To(Equal(versionProbeDeadline))
 		Expect(container.Command).To(Equal([]string{"/usr/bin/clickhouse"}))
-		Expect(container.Args).To(Equal([]string{"local", "--query", "INSERT INTO FUNCTION file('/dev/termination-log', 'RawBLOB', 'version String') SELECT version()"}))
+		Expect(container.Args).To(Equal([]string{
+			"local",
+			"--logger.console=1",
+			"--logger.level=debug",
+			"--query", "INSERT INTO FUNCTION file('/dev/termination-log', 'RawBLOB', 'version String') SELECT version()",
+		}))
+		Expect(container.TerminationMessagePolicy).To(Equal(corev1.TerminationMessageFallbackToLogsOnError))
 	})
 
 	It("should miss cache when image changes", func(ctx context.Context) {

test/testutil/utils.go

@@ -227,7 +227,16 @@ func DumpNamespaceDiagnostics(ctx context.Context, config *rest.Config, cli clie
 	}
 
 	if strings.TrimSpace(events) != "" {
+		path, writeErr := (&DumpResult{Full: events}).WriteFull(dir, fmt.Sprintf("events-%s.log", report.FullText()))
+		if writeErr != nil {
+			GinkgoWriter.Printf("failed to write events dump: %v\n", writeErr)
+		}
+
 		GinkgoWriter.Printf("\n=== Namespace Events (since test start) ===\n%s\n", events)
+
+		if path != "" {
+			GinkgoWriter.Printf("Full events dump: %s\n", path)
+		}
 	}
 }