From 8f66ff65d17243d323065f7eeee95ae44bf4fbdd Mon Sep 17 00:00:00 2001 From: marblom007 <158522975+marblom007@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:13:52 -0500 Subject: [PATCH 1/5] [QA] Add Kubernetes Connections report; split mesheryctl results dirs Add a dedicated 'Kubernetes Connections' Allure report as a filtered view in allurerc.mjs: an @allurereport/plugin-awesome block whose filter selects results labelled epic="Kubernetes Connections" (falling back to componentUnderTest matching Kubernetes for results tagged before the epic convention), grouped by client (UI vs CLI) then suite/subSuite. It aggregates connection tests from both the Meshery (UI) and mesheryctl (CLI) pools without removing them from those reports. Fix the mesheryctl-results clobber (qa side): the BATS e2e and Go unit feeders both synced into mesheryctl-results/ via results-sync, which wipes its target, so whichever committed last erased the other. Add dedicated mesheryctl-bats-results- sync and mesheryctl-unit-results-sync targets writing to distinct dirs, merged at report-build; keep the legacy mesheryctl-results-sync as a back-compat alias so the change is order-independent with the meshery-side workflow change. Update the Pages workflow paths trigger and the README (report list + results-dir table + sparse-clone examples) for the new dirs. Signed-off-by: marblom007 <158522975+marblom007@users.noreply.github.com> --- .github/workflows/publish-allure-report.yml | 2 ++ Makefile | 18 ++++++++-- README.md | 26 +++++++++++++- allurerc.mjs | 39 +++++++++++++++++++++ 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-allure-report.yml b/.github/workflows/publish-allure-report.yml index fe42fd206f5..f4767d3f373 100644 --- a/.github/workflows/publish-allure-report.yml +++ b/.github/workflows/publish-allure-report.yml @@ -10,6 +10,8 @@ on: - "meshery-results/**" - "meshery-server-results/**" - "mesheryctl-results/**" + - "mesheryctl-bats-results/**" + - "mesheryctl-unit-results/**" - "remote-provider-results/**" - "allurerc.mjs" workflow_dispatch: diff --git a/Makefile b/Makefile index c5ea0ff23dd..057607986d2 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ endef # -------------------------------------------------- # Targets # -------------------------------------------------- -.PHONY: report-generate meshery-results-sync meshery-server-results-sync mesheryctl-results-sync report-open report +.PHONY: report-generate meshery-results-sync meshery-server-results-sync mesheryctl-results-sync mesheryctl-bats-results-sync mesheryctl-unit-results-sync report-open report ## Sync Meshery Test Results meshery-results-sync: @@ -57,11 +57,23 @@ meshery-server-results-sync: @echo "Syncing Server Go Unit Test Results..." $(call results-sync,MESHERY_SERVER_RESULTS_PATH,meshery-server-results) -## Sync mesheryctl Test Results +## Sync mesheryctl Test Results (DEPRECATED - shared by BATS e2e and go unit +## feeders, which clobbered each other since results-sync wipes its target. +## Kept as a back-compat alias; new feeders use the split targets below.) mesheryctl-results-sync: @echo "Syncing mesheryctl Test Results..." $(call results-sync,MESHERYCTL_RESULTS_PATH,mesheryctl-results) +## Sync mesheryctl BATS e2e Test Results +mesheryctl-bats-results-sync: + @echo "Syncing mesheryctl BATS e2e Test Results..." + $(call results-sync,MESHERYCTL_BATS_RESULTS_PATH,mesheryctl-bats-results) + +## Sync mesheryctl Go unit Test Results +mesheryctl-unit-results-sync: + @echo "Syncing mesheryctl Go unit Test Results..." + $(call results-sync,MESHERYCTL_UNIT_RESULTS_PATH,mesheryctl-unit-results) + ## Generic sync - make project-results-sync RESULTS_DIR=path/to/results PROJECT=myproject project-results-sync: @if [ -z "$(RESULTS_DIR)" ] || [ -z "$(PROJECT)" ]; then \ @@ -92,6 +104,8 @@ report-build: cp meshery-results/* allure-results/ || true cp meshery-server-results/* allure-results/ || true cp mesheryctl-results/* allure-results/ || true + cp mesheryctl-bats-results/* allure-results/ || true + cp mesheryctl-unit-results/* allure-results/ || true cp remote-provider-results/* allure-results/ || true npm run report:generate diff --git a/README.md b/README.md index 621375e931b..57391b58c42 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,26 @@ A self-service engineering platform, Meshery, i Visit https://qa.meshery.io for the Meshery project quality assurance dashboard. +### Published reports +The dashboard is generated by `make report-build` (config in `allurerc.mjs`) and +deployed to GitHub Pages. Each report is a filtered view over one shared pool of +Allure results, selected by test labels: + +| Report | Selects results where | +| --- | --- | +| Dashboard | all results (aggregate) | +| Meshery | `project == "Meshery"` | +| Mesheryctl | `project == "mesheryctl"` | +| **Kubernetes Connections** | `epic == "Kubernetes Connections"` (UI + CLI connection tests, grouped by `client`) | +| Extension: Remote Provider Layer5 Cloud | `project == "Layer5Cloud"` | +| Extension: Kanvas | `project == "Kanvas"` | + +The Kubernetes Connections report is a cross-client behavior lens: connection +tests are tagged at their source (UI Playwright specs, CLI converters) with +`epic="Kubernetes Connections"`, `componentUnderTest`, `testId` (`TC-`), and +`client` (`UI`|`CLI`), sourced from the Meshery Test Plan. Tagged tests still +appear in their `project` report; the Connections report is an additional view.

 

@@ -98,7 +117,9 @@ The following directories hold generated test data and are safe to exclude (they | --- | --- | | `meshery-results/` | Raw Allure results for Meshery integration tests | | `meshery-server-results/` | Raw Allure results for Meshery Server Go unit tests | -| `mesheryctl-results/` | Raw Allure results for `mesheryctl` tests | +| `mesheryctl-results/` | Raw Allure results for `mesheryctl` tests (deprecated - split into the two dirs below) | +| `mesheryctl-bats-results/` | Raw Allure results for `mesheryctl` BATS e2e tests | +| `mesheryctl-unit-results/` | Raw Allure results for `mesheryctl` Go unit tests | | `kanvas-results/` | Raw Allure results for Kanvas (extension) | | `remote-provider-results/` | Raw Allure results for the Remote Provider (extension) | | `pr-reports/` | Generated per-PR HTML reports (the largest contributor to repo size) | @@ -116,6 +137,8 @@ git sparse-checkout set --no-cone \ '!/meshery-results/' \ '!/meshery-server-results/' \ '!/mesheryctl-results/' \ + '!/mesheryctl-bats-results/' \ + '!/mesheryctl-unit-results/' \ '!/kanvas-results/' \ '!/remote-provider-results/' \ '!/pr-reports/' @@ -135,6 +158,7 @@ cd qa git sparse-checkout set --no-cone \ '/*' \ '!/meshery-results/' '!/meshery-server-results/' '!/mesheryctl-results/' \ + '!/mesheryctl-bats-results/' '!/mesheryctl-unit-results/' \ '!/kanvas-results/' '!/remote-provider-results/' '!/pr-reports/' ``` diff --git a/allurerc.mjs b/allurerc.mjs index 8b5c222691e..2c89290ac1e 100644 --- a/allurerc.mjs +++ b/allurerc.mjs @@ -13,6 +13,25 @@ const PROJECTS = { const isProject = (labels, projectName) => labels.find(({ name, value }) => name === "project" && value === projectName); +// Behavior-scoped report keys. A Connections result is tagged at its source +// (UI Playwright specs and CLI converters) with epic="Kubernetes Connections", +// componentUnderTest (Test Plan col C), testId=TC- (Test Plan col A), and +// client (UI|CLI). See https://qa.meshery.io and the meshery test-tagging docs. +const CONNECTIONS_EPIC = "Kubernetes Connections"; + +// componentUnderTest values that denote Kubernetes connection behavior. Used as +// a fallback selector when a result predates the epic label. +const CONNECTION_COMPONENTS = /kubernetes/i; + +// Select a result into the Connections report: prefer the explicit epic label, +// fall back to componentUnderTest for results tagged before the epic convention. +const isConnectionBehavior = (labels) => + labels.some(({ name, value }) => name === "epic" && value === CONNECTIONS_EPIC) || + labels.some( + ({ name, value }) => + name === "componentUnderTest" && CONNECTION_COMPONENTS.test(value), + ); + export default defineConfig({ name: "Meshery Quality Dashboard", output: "./allure-report", @@ -96,6 +115,26 @@ export default defineConfig({ groupBy: ["parentSuite", "suite", "subSuite"], }, }, + // Cross-client behavior report: aggregates Kubernetes Connection tests from + // BOTH the UI (project=Meshery) and CLI (project=mesheryctl) pools. It keys + // on the epic label, NOT project, so it is an additional lens - connection + // tests still appear in the Meshery and Mesheryctl reports above. + connections: { + import: "@allurereport/plugin-awesome", + options: { + reportName: "Kubernetes Connections", + singleFile: false, + reportLanguage: "en", + open: false, + logo: "https://raw.githubusercontent.com/meshery-extensions/qa/refs/heads/master/.github/assets/images/meshery/icon-only/meshery-light-icon.svg", + filter: ({ labels }) => isConnectionBehavior(labels), + // Group by client (UI vs CLI) first, then the suite hierarchy. The + // awesome plugin (preciseTreeLabels) keeps only label names present on + // at least one result, so results missing "client" fall back to the + // suite/subSuite grouping automatically. + groupBy: ["client", "suite", "subSuite"], + }, + }, log: { options: { From e7f3e1a16cfc4d9372d9f410c7c5b037ecbab9dd Mon Sep 17 00:00:00 2001 From: marblom007 <158522975+marblom007@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:44:37 -0500 Subject: [PATCH 2/5] [QA] Address review: guard epic fallback, fix results-sync data loss, drop stale legacy dir - allurerc.mjs: the Connections filter's componentUnderTest fallback now applies ONLY when a result has no epic label at all. Previously a result with a different epic (e.g. epic="Other") plus componentUnderTest=Kubernetes was wrongly pulled into the report. Verified: matching epic, legacy no-epic fallback, and different-epic-excluded cases all behave correctly. - Makefile results-sync/results-sync-path: only rm -rf the destination after the source var is confirmed set and present. Wiping first meant a misconfigured or empty source var destroyed the committed destination results before the guard. - Makefile report-build: stop copying the frozen legacy mesheryctl-results/ dir; merging it with the split dirs would mix stale pre-split results with current ones. The split dirs repopulate on the next CI run of each feeder. - README: document the no-epic componentUnderTest fallback (and that a different epic is not pulled in by the component fallback). Signed-off-by: marblom007 <158522975+marblom007@users.noreply.github.com> --- Makefile | 20 +++++++++++++------- README.md | 6 +++++- allurerc.mjs | 18 +++++++++++++----- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 057607986d2..a59b4f52a5c 100644 --- a/Makefile +++ b/Makefile @@ -19,26 +19,29 @@ include build/Makefile.show-help.mk # Helpers # -------------------------------------------------- +# Wipe-and-replace the destination ONLY after the source is confirmed set and +# present. Wiping first (unconditionally) meant a misconfigured/empty source var +# still destroyed the committed destination results before the guard ran. define results-sync - @rm -rf $(2) - @mkdir -p $(2) @if [ -n "$($(1))" ] && [ -d "$($(1))" ]; then \ echo "Copying results from $($(1)) → $(2)"; \ + rm -rf $(2); \ + mkdir -p $(2); \ cp -r "$($(1))"/* $(2)/ ; \ else \ - echo "$(1) not set or directory does not exist, skipping"; \ + echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \ fi endef define results-sync-path - @rm -rf $(2) - @mkdir -p $(2) @if [ -n "$(1)" ] && [ -d "$(1)" ]; then \ echo "Copying results from $(1) → $(2)"; \ + rm -rf $(2); \ + mkdir -p $(2); \ cp -r "$(1)"/* $(2)/ ; \ else \ - echo "$(1) not set or directory does not exist, skipping"; \ + echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \ fi endef @@ -103,7 +106,10 @@ report-build: cp kanvas-results/* allure-results/ || true cp meshery-results/* allure-results/ || true cp meshery-server-results/* allure-results/ || true - cp mesheryctl-results/* allure-results/ || true + # NOTE: the legacy mesheryctl-results/ dir is intentionally NOT copied. Its + # results are frozen (both feeders now write to the split dirs below), so + # including it would merge stale pre-split results with current ones. The + # split dirs repopulate on the next CI run of each feeder. cp mesheryctl-bats-results/* allure-results/ || true cp mesheryctl-unit-results/* allure-results/ || true cp remote-provider-results/* allure-results/ || true diff --git a/README.md b/README.md index 57391b58c42..50c202019a3 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Allure results, selected by test labels: | Dashboard | all results (aggregate) | | Meshery | `project == "Meshery"` | | Mesheryctl | `project == "mesheryctl"` | -| **Kubernetes Connections** | `epic == "Kubernetes Connections"` (UI + CLI connection tests, grouped by `client`) | +| **Kubernetes Connections** | `epic == "Kubernetes Connections"`; results with no `epic` label also match via the Kubernetes `componentUnderTest` fallback (UI + CLI connection tests, grouped by `client`) | | Extension: Remote Provider Layer5 Cloud | `project == "Layer5Cloud"` | | Extension: Kanvas | `project == "Kanvas"` | @@ -99,6 +99,10 @@ tests are tagged at their source (UI Playwright specs, CLI converters) with `epic="Kubernetes Connections"`, `componentUnderTest`, `testId` (`TC-`), and `client` (`UI`|`CLI`), sourced from the Meshery Test Plan. Tagged tests still appear in their `project` report; the Connections report is an additional view. +As a fallback, a result that carries no `epic` label at all is included when its +`componentUnderTest` matches Kubernetes, so results predating the `epic` +convention still appear; a result with a *different* `epic` value is not pulled +in by the component fallback.

 

diff --git a/allurerc.mjs b/allurerc.mjs index 2c89290ac1e..68576cb80ed 100644 --- a/allurerc.mjs +++ b/allurerc.mjs @@ -23,14 +23,22 @@ const CONNECTIONS_EPIC = "Kubernetes Connections"; // a fallback selector when a result predates the epic label. const CONNECTION_COMPONENTS = /kubernetes/i; -// Select a result into the Connections report: prefer the explicit epic label, -// fall back to componentUnderTest for results tagged before the epic convention. -const isConnectionBehavior = (labels) => - labels.some(({ name, value }) => name === "epic" && value === CONNECTIONS_EPIC) || - labels.some( +// Select a result into the Connections report: prefer the explicit epic label. +// The componentUnderTest fallback applies ONLY to results that carry no epic +// label at all (tagged before the epic convention) - a result with a different +// epic value must not be pulled in just because its component is Kubernetes. +const isConnectionBehavior = (labels) => { + const hasEpic = labels.some(({ name }) => name === "epic"); + if (hasEpic) { + return labels.some( + ({ name, value }) => name === "epic" && value === CONNECTIONS_EPIC, + ); + } + return labels.some( ({ name, value }) => name === "componentUnderTest" && CONNECTION_COMPONENTS.test(value), ); +}; export default defineConfig({ name: "Meshery Quality Dashboard", From 9d7e413391cdcf5c9229354978540cd681e0edd1 Mon Sep 17 00:00:00 2001 From: marblom007 <158522975+marblom007@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:51:46 -0500 Subject: [PATCH 3/5] [QA] Rename CONNECTION_COMPONENTS to CONNECTION_COMPONENT_RE for clarity The constant is a RegExp; the plural name read like a collection. Rename it so the type is obvious at the use site. No behavior change. Signed-off-by: marblom007 <158522975+marblom007@users.noreply.github.com> --- allurerc.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/allurerc.mjs b/allurerc.mjs index 68576cb80ed..6fb14e42944 100644 --- a/allurerc.mjs +++ b/allurerc.mjs @@ -19,9 +19,9 @@ const isProject = (labels, projectName) => // client (UI|CLI). See https://qa.meshery.io and the meshery test-tagging docs. const CONNECTIONS_EPIC = "Kubernetes Connections"; -// componentUnderTest values that denote Kubernetes connection behavior. Used as -// a fallback selector when a result predates the epic label. -const CONNECTION_COMPONENTS = /kubernetes/i; +// Matches componentUnderTest values that denote Kubernetes connection behavior. +// Used as a fallback selector when a result predates the epic label. +const CONNECTION_COMPONENT_RE = /kubernetes/i; // Select a result into the Connections report: prefer the explicit epic label. // The componentUnderTest fallback applies ONLY to results that carry no epic @@ -36,7 +36,7 @@ const isConnectionBehavior = (labels) => { } return labels.some( ({ name, value }) => - name === "componentUnderTest" && CONNECTION_COMPONENTS.test(value), + name === "componentUnderTest" && CONNECTION_COMPONENT_RE.test(value), ); }; From db7d81cbef370780be21849378b478ba6c9346a4 Mon Sep 17 00:00:00 2001 From: marblom007 <158522975+marblom007@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:58:43 -0500 Subject: [PATCH 4/5] [QA] Make results-sync atomic: copy to temp sibling, swap only on success Previously the destination was rm -rf'd before cp completed, so a mid-copy I/O error, a disappearing source, or an empty glob could delete valid committed results and leave the destination empty. Copy the validated source into a temporary sibling dir first (cp -a src/. so it also succeeds on an empty source without glob expansion), then remove and swap in the destination only after the copy succeeds; on copy failure the destination is left intact and the target fails loudly. Applied to both results-sync and results-sync-path. Signed-off-by: marblom007 <158522975+marblom007@users.noreply.github.com> --- Makefile | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index a59b4f52a5c..08ae513f8b8 100644 --- a/Makefile +++ b/Makefile @@ -19,15 +19,23 @@ include build/Makefile.show-help.mk # Helpers # -------------------------------------------------- -# Wipe-and-replace the destination ONLY after the source is confirmed set and -# present. Wiping first (unconditionally) meant a misconfigured/empty source var -# still destroyed the committed destination results before the guard ran. +# Replace the destination ONLY after the source is validated AND the copy has +# fully succeeded. The copy lands in a temporary sibling dir first; the +# destination is removed and swapped in only once that copy completes, so a +# misconfigured source var, a mid-copy I/O error, or a disappearing source can +# never leave the committed results half-deleted. An unset/missing source skips +# entirely and leaves the destination intact. `cp -a "$src/."` copies directory +# contents (and succeeds on an empty source) without relying on glob expansion. define results-sync @if [ -n "$($(1))" ] && [ -d "$($(1))" ]; then \ echo "Copying results from $($(1)) → $(2)"; \ - rm -rf $(2); \ - mkdir -p $(2); \ - cp -r "$($(1))"/* $(2)/ ; \ + tmp="$(2).tmp.$$$$"; \ + rm -rf "$$tmp"; mkdir -p "$$tmp"; \ + if cp -a "$($(1))/." "$$tmp/"; then \ + rm -rf $(2); mv "$$tmp" $(2); \ + else \ + echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \ + fi; \ else \ echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \ fi @@ -37,9 +45,13 @@ endef define results-sync-path @if [ -n "$(1)" ] && [ -d "$(1)" ]; then \ echo "Copying results from $(1) → $(2)"; \ - rm -rf $(2); \ - mkdir -p $(2); \ - cp -r "$(1)"/* $(2)/ ; \ + tmp="$(2).tmp.$$$$"; \ + rm -rf "$$tmp"; mkdir -p "$$tmp"; \ + if cp -a "$(1)/." "$$tmp/"; then \ + rm -rf $(2); mv "$$tmp" $(2); \ + else \ + echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \ + fi; \ else \ echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \ fi From 471e287333671c63ebfc122dddf82e9111a77ddf Mon Sep 17 00:00:00 2001 From: marblom007 <158522975+marblom007@users.noreply.github.com> Date: Sun, 2 Aug 2026 12:04:06 -0500 Subject: [PATCH 5/5] [QA] Quote destination path in results-sync swap for whitespace safety project-results-sync feeds $(PROJECT)-results into $(2); an unquoted rm -rf/mv would split a destination containing whitespace into multiple paths. Quote both uses in results-sync and results-sync-path. Verified with a PROJECT containing a space: a single correctly-named dir is created, no stray paths. Signed-off-by: marblom007 <158522975+marblom007@users.noreply.github.com> --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 08ae513f8b8..5c503d9eeeb 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ define results-sync tmp="$(2).tmp.$$$$"; \ rm -rf "$$tmp"; mkdir -p "$$tmp"; \ if cp -a "$($(1))/." "$$tmp/"; then \ - rm -rf $(2); mv "$$tmp" $(2); \ + rm -rf "$(2)"; mv "$$tmp" "$(2)"; \ else \ echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \ fi; \ @@ -48,7 +48,7 @@ define results-sync-path tmp="$(2).tmp.$$$$"; \ rm -rf "$$tmp"; mkdir -p "$$tmp"; \ if cp -a "$(1)/." "$$tmp/"; then \ - rm -rf $(2); mv "$$tmp" $(2); \ + rm -rf "$(2)"; mv "$$tmp" "$(2)"; \ else \ echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \ fi; \