[QA] Add Kubernetes Connections report; split mesheryctl results dirs

.github/workflows/publish-allure-report.yml

@@ -10,6 +10,8 @@ on:
       - "meshery-results/**"
       - "meshery-server-results/**"
       - "mesheryctl-results/**"
+      - "mesheryctl-bats-results/**"
+      - "mesheryctl-unit-results/**"
       - "remote-provider-results/**"
       - "allurerc.mjs"
   workflow_dispatch:

Makefile

@@ -19,33 +19,48 @@ include build/Makefile.show-help.mk
 # Helpers
 # --------------------------------------------------
 
+# Replace the destination ONLY after the source is validated AND the copy has
+# fully succeeded. The copy lands in a temporary sibling dir first; the
+# destination is removed and swapped in only once that copy completes, so a
+# misconfigured source var, a mid-copy I/O error, or a disappearing source can
+# never leave the committed results half-deleted. An unset/missing source skips
+# entirely and leaves the destination intact. `cp -a "$src/."` copies directory
+# contents (and succeeds on an empty source) without relying on glob expansion.
 define results-sync
-	@rm -rf $(2)
-	@mkdir -p $(2)
 	@if [ -n "$($(1))" ] && [ -d "$($(1))" ]; then \
 		echo "Copying results from $($(1)) → $(2)"; \
-		cp -r "$($(1))"/* $(2)/ ; \
+		tmp="$(2).tmp.$$$$"; \
+		rm -rf "$$tmp"; mkdir -p "$$tmp"; \
+		if cp -a "$($(1))/." "$$tmp/"; then \
+			rm -rf "$(2)"; mv "$$tmp" "$(2)"; \
+		else \
+			echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \
+		fi; \
 	else \
-		echo "$(1) not set or directory does not exist, skipping"; \
+		echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \
 	fi
 endef
 
 
 define results-sync-path
-	@rm -rf $(2)
-	@mkdir -p $(2)
 	@if [ -n "$(1)" ] && [ -d "$(1)" ]; then \
 		echo "Copying results from $(1) → $(2)"; \
-		cp -r "$(1)"/* $(2)/ ; \
+		tmp="$(2).tmp.$$$$"; \
+		rm -rf "$$tmp"; mkdir -p "$$tmp"; \
+		if cp -a "$(1)/." "$$tmp/"; then \
+			rm -rf "$(2)"; mv "$$tmp" "$(2)"; \
+		else \
+			echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \
+		fi; \
 	else \
-		echo "$(1) not set or directory does not exist, skipping"; \
+		echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \
 	fi
 endef
 
 # --------------------------------------------------
 # Targets
 # --------------------------------------------------
-.PHONY: report-generate meshery-results-sync meshery-server-results-sync mesheryctl-results-sync report-open report
+.PHONY: report-generate meshery-results-sync meshery-server-results-sync mesheryctl-results-sync mesheryctl-bats-results-sync mesheryctl-unit-results-sync report-open report
 
 ## Sync Meshery Test Results
 meshery-results-sync: 
@@ -57,11 +72,23 @@ meshery-server-results-sync:
 	@echo "Syncing Server Go Unit Test Results..."
 	$(call results-sync,MESHERY_SERVER_RESULTS_PATH,meshery-server-results)
 
-## Sync mesheryctl Test Results
+## Sync mesheryctl Test Results (DEPRECATED - shared by BATS e2e and go unit
+## feeders, which clobbered each other since results-sync wipes its target.
+## Kept as a back-compat alias; new feeders use the split targets below.)
 mesheryctl-results-sync:
 	@echo "Syncing mesheryctl Test Results..."
 	$(call results-sync,MESHERYCTL_RESULTS_PATH,mesheryctl-results)
 
+## Sync mesheryctl BATS e2e Test Results
+mesheryctl-bats-results-sync:
+	@echo "Syncing mesheryctl BATS e2e Test Results..."
+	$(call results-sync,MESHERYCTL_BATS_RESULTS_PATH,mesheryctl-bats-results)
+
+## Sync mesheryctl Go unit Test Results
+mesheryctl-unit-results-sync:
+	@echo "Syncing mesheryctl Go unit Test Results..."
+	$(call results-sync,MESHERYCTL_UNIT_RESULTS_PATH,mesheryctl-unit-results)
+
 ## Generic sync - make project-results-sync RESULTS_DIR=path/to/results PROJECT=myproject
 project-results-sync:
 	@if [ -z "$(RESULTS_DIR)" ] || [ -z "$(PROJECT)" ]; then \
@@ -91,7 +118,12 @@ report-build:
 	cp kanvas-results/* allure-results/ || true
 	cp meshery-results/* allure-results/ || true
 	cp meshery-server-results/* allure-results/ || true
-	cp mesheryctl-results/* allure-results/ || true
+	# NOTE: the legacy mesheryctl-results/ dir is intentionally NOT copied. Its
+	# results are frozen (both feeders now write to the split dirs below), so
+	# including it would merge stale pre-split results with current ones. The
+	# split dirs repopulate on the next CI run of each feeder.
+	cp mesheryctl-bats-results/* allure-results/ || true
+	cp mesheryctl-unit-results/* allure-results/ || true
 	cp remote-provider-results/* allure-results/ || true
 	npm run report:generate
 

README.md

@@ -79,7 +79,30 @@ A self-service engineering platform, <a href="https://meshery.io">Meshery</a>, i
 
 Visit https://qa.meshery.io for the Meshery project quality assurance dashboard.
 
+### Published reports
 
+The dashboard is generated by `make report-build` (config in `allurerc.mjs`) and
+deployed to GitHub Pages. Each report is a filtered view over one shared pool of
+Allure results, selected by test labels:
+
+| Report | Selects results where |
+| --- | --- |
+| Dashboard | all results (aggregate) |
+| Meshery | `project == "Meshery"` |
+| Mesheryctl | `project == "mesheryctl"` |
+| **Kubernetes Connections** | `epic == "Kubernetes Connections"`; results with no `epic` label also match via the Kubernetes `componentUnderTest` fallback (UI + CLI connection tests, grouped by `client`) |
+| Extension: Remote Provider Layer5 Cloud | `project == "Layer5Cloud"` |
+| Extension: Kanvas | `project == "Kanvas"` |
+
+The Kubernetes Connections report is a cross-client behavior lens: connection
+tests are tagged at their source (UI Playwright specs, CLI converters) with
+`epic="Kubernetes Connections"`, `componentUnderTest`, `testId` (`TC-<n>`), and
+`client` (`UI`|`CLI`), sourced from the Meshery Test Plan. Tagged tests still
+appear in their `project` report; the Connections report is an additional view.
+As a fallback, a result that carries no `epic` label at all is included when its
+`componentUnderTest` matches Kubernetes, so results predating the `epic`
+convention still appear; a result with a *different* `epic` value is not pulled
+in by the component fallback.
 
 <p style="clear:both;">&nbsp;</p>
 
@@ -98,7 +121,9 @@ The following directories hold generated test data and are safe to exclude (they
 | --- | --- |
 | `meshery-results/` | Raw Allure results for Meshery integration tests |
 | `meshery-server-results/` | Raw Allure results for Meshery Server Go unit tests |
-| `mesheryctl-results/` | Raw Allure results for `mesheryctl` tests |
+| `mesheryctl-results/` | Raw Allure results for `mesheryctl` tests (deprecated - split into the two dirs below) |
+| `mesheryctl-bats-results/` | Raw Allure results for `mesheryctl` BATS e2e tests |
+| `mesheryctl-unit-results/` | Raw Allure results for `mesheryctl` Go unit tests |
 | `kanvas-results/` | Raw Allure results for Kanvas (extension) |
 | `remote-provider-results/` | Raw Allure results for the Remote Provider (extension) |
 | `pr-reports/` | Generated per-PR HTML reports (the largest contributor to repo size) |
@@ -116,6 +141,8 @@ git sparse-checkout set --no-cone \
   '!/meshery-results/' \
   '!/meshery-server-results/' \
   '!/mesheryctl-results/' \
+  '!/mesheryctl-bats-results/' \
+  '!/mesheryctl-unit-results/' \
   '!/kanvas-results/' \
   '!/remote-provider-results/' \
   '!/pr-reports/'
@@ -135,6 +162,7 @@ cd qa
 git sparse-checkout set --no-cone \
   '/*' \
   '!/meshery-results/' '!/meshery-server-results/' '!/mesheryctl-results/' \
+  '!/mesheryctl-bats-results/' '!/mesheryctl-unit-results/' \
   '!/kanvas-results/' '!/remote-provider-results/' '!/pr-reports/'
 ```
 

allurerc.mjs

@@ -13,6 +13,33 @@ const PROJECTS = {
 const isProject = (labels, projectName) =>
   labels.find(({ name, value }) => name === "project" && value === projectName);
 
+// Behavior-scoped report keys. A Connections result is tagged at its source
+// (UI Playwright specs and CLI converters) with epic="Kubernetes Connections",
+// componentUnderTest (Test Plan col C), testId=TC-<n> (Test Plan col A), and
+// client (UI|CLI). See https://qa.meshery.io and the meshery test-tagging docs.
+const CONNECTIONS_EPIC = "Kubernetes Connections";
+
+// Matches componentUnderTest values that denote Kubernetes connection behavior.
+// Used as a fallback selector when a result predates the epic label.
+const CONNECTION_COMPONENT_RE = /kubernetes/i;
+
+// Select a result into the Connections report: prefer the explicit epic label.
+// The componentUnderTest fallback applies ONLY to results that carry no epic
+// label at all (tagged before the epic convention) - a result with a different
+// epic value must not be pulled in just because its component is Kubernetes.
+const isConnectionBehavior = (labels) => {
+  const hasEpic = labels.some(({ name }) => name === "epic");
+  if (hasEpic) {
+    return labels.some(
+      ({ name, value }) => name === "epic" && value === CONNECTIONS_EPIC,
+    );
+  }
+  return labels.some(
+    ({ name, value }) =>
+      name === "componentUnderTest" && CONNECTION_COMPONENT_RE.test(value),
+  );
+};
+
 export default defineConfig({
   name: "Meshery Quality Dashboard",
   output: "./allure-report",
@@ -96,6 +123,26 @@ export default defineConfig({
         groupBy: ["parentSuite", "suite", "subSuite"],
       },
     },
+    // Cross-client behavior report: aggregates Kubernetes Connection tests from
+    // BOTH the UI (project=Meshery) and CLI (project=mesheryctl) pools. It keys
+    // on the epic label, NOT project, so it is an additional lens - connection
+    // tests still appear in the Meshery and Mesheryctl reports above.
+    connections: {
+      import: "@allurereport/plugin-awesome",
+      options: {
+        reportName: "Kubernetes Connections",
+        singleFile: false,
+        reportLanguage: "en",
+        open: false,
+        logo: "https://raw.githubusercontent.com/meshery-extensions/qa/refs/heads/master/.github/assets/images/meshery/icon-only/meshery-light-icon.svg",
+        filter: ({ labels }) => isConnectionBehavior(labels),
+        // Group by client (UI vs CLI) first, then the suite hierarchy. The
+        // awesome plugin (preciseTreeLabels) keeps only label names present on
+        // at least one result, so results missing "client" fall back to the
+        // suite/subSuite grouping automatically.
+        groupBy: ["client", "suite", "subSuite"],
+      },
+    },
 
     log: {
       options: {