diff --git a/.github/workflows/publish-allure-report.yml b/.github/workflows/publish-allure-report.yml index fe42fd206f5..f4767d3f373 100644 --- a/.github/workflows/publish-allure-report.yml +++ b/.github/workflows/publish-allure-report.yml @@ -10,6 +10,8 @@ on: - "meshery-results/**" - "meshery-server-results/**" - "mesheryctl-results/**" + - "mesheryctl-bats-results/**" + - "mesheryctl-unit-results/**" - "remote-provider-results/**" - "allurerc.mjs" workflow_dispatch: diff --git a/Makefile b/Makefile index c5ea0ff23dd..5c503d9eeeb 100644 --- a/Makefile +++ b/Makefile @@ -19,33 +19,48 @@ include build/Makefile.show-help.mk # Helpers # -------------------------------------------------- +# Replace the destination ONLY after the source is validated AND the copy has +# fully succeeded. The copy lands in a temporary sibling dir first; the +# destination is removed and swapped in only once that copy completes, so a +# misconfigured source var, a mid-copy I/O error, or a disappearing source can +# never leave the committed results half-deleted. An unset/missing source skips +# entirely and leaves the destination intact. `cp -a "$src/."` copies directory +# contents (and succeeds on an empty source) without relying on glob expansion. define results-sync - @rm -rf $(2) - @mkdir -p $(2) @if [ -n "$($(1))" ] && [ -d "$($(1))" ]; then \ echo "Copying results from $($(1)) → $(2)"; \ - cp -r "$($(1))"/* $(2)/ ; \ + tmp="$(2).tmp.$$$$"; \ + rm -rf "$$tmp"; mkdir -p "$$tmp"; \ + if cp -a "$($(1))/." "$$tmp/"; then \ + rm -rf "$(2)"; mv "$$tmp" "$(2)"; \ + else \ + echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \ + fi; \ else \ - echo "$(1) not set or directory does not exist, skipping"; \ + echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \ fi endef define results-sync-path - @rm -rf $(2) - @mkdir -p $(2) @if [ -n "$(1)" ] && [ -d "$(1)" ]; then \ echo "Copying results from $(1) → $(2)"; \ - cp -r "$(1)"/* $(2)/ ; \ + tmp="$(2).tmp.$$$$"; \ + rm -rf "$$tmp"; mkdir -p "$$tmp"; \ + if cp -a "$(1)/." "$$tmp/"; then \ + rm -rf "$(2)"; mv "$$tmp" "$(2)"; \ + else \ + echo "Copy failed; destination $(2) left intact"; rm -rf "$$tmp"; exit 1; \ + fi; \ else \ - echo "$(1) not set or directory does not exist, skipping"; \ + echo "$(1) not set or directory does not exist, skipping (destination $(2) left intact)"; \ fi endef # -------------------------------------------------- # Targets # -------------------------------------------------- -.PHONY: report-generate meshery-results-sync meshery-server-results-sync mesheryctl-results-sync report-open report +.PHONY: report-generate meshery-results-sync meshery-server-results-sync mesheryctl-results-sync mesheryctl-bats-results-sync mesheryctl-unit-results-sync report-open report ## Sync Meshery Test Results meshery-results-sync: @@ -57,11 +72,23 @@ meshery-server-results-sync: @echo "Syncing Server Go Unit Test Results..." $(call results-sync,MESHERY_SERVER_RESULTS_PATH,meshery-server-results) -## Sync mesheryctl Test Results +## Sync mesheryctl Test Results (DEPRECATED - shared by BATS e2e and go unit +## feeders, which clobbered each other since results-sync wipes its target. +## Kept as a back-compat alias; new feeders use the split targets below.) mesheryctl-results-sync: @echo "Syncing mesheryctl Test Results..." $(call results-sync,MESHERYCTL_RESULTS_PATH,mesheryctl-results) +## Sync mesheryctl BATS e2e Test Results +mesheryctl-bats-results-sync: + @echo "Syncing mesheryctl BATS e2e Test Results..." + $(call results-sync,MESHERYCTL_BATS_RESULTS_PATH,mesheryctl-bats-results) + +## Sync mesheryctl Go unit Test Results +mesheryctl-unit-results-sync: + @echo "Syncing mesheryctl Go unit Test Results..." + $(call results-sync,MESHERYCTL_UNIT_RESULTS_PATH,mesheryctl-unit-results) + ## Generic sync - make project-results-sync RESULTS_DIR=path/to/results PROJECT=myproject project-results-sync: @if [ -z "$(RESULTS_DIR)" ] || [ -z "$(PROJECT)" ]; then \ @@ -91,7 +118,12 @@ report-build: cp kanvas-results/* allure-results/ || true cp meshery-results/* allure-results/ || true cp meshery-server-results/* allure-results/ || true - cp mesheryctl-results/* allure-results/ || true + # NOTE: the legacy mesheryctl-results/ dir is intentionally NOT copied. Its + # results are frozen (both feeders now write to the split dirs below), so + # including it would merge stale pre-split results with current ones. The + # split dirs repopulate on the next CI run of each feeder. + cp mesheryctl-bats-results/* allure-results/ || true + cp mesheryctl-unit-results/* allure-results/ || true cp remote-provider-results/* allure-results/ || true npm run report:generate diff --git a/README.md b/README.md index 621375e931b..50c202019a3 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,30 @@ A self-service engineering platform, Meshery, i Visit https://qa.meshery.io for the Meshery project quality assurance dashboard. +### Published reports +The dashboard is generated by `make report-build` (config in `allurerc.mjs`) and +deployed to GitHub Pages. Each report is a filtered view over one shared pool of +Allure results, selected by test labels: + +| Report | Selects results where | +| --- | --- | +| Dashboard | all results (aggregate) | +| Meshery | `project == "Meshery"` | +| Mesheryctl | `project == "mesheryctl"` | +| **Kubernetes Connections** | `epic == "Kubernetes Connections"`; results with no `epic` label also match via the Kubernetes `componentUnderTest` fallback (UI + CLI connection tests, grouped by `client`) | +| Extension: Remote Provider Layer5 Cloud | `project == "Layer5Cloud"` | +| Extension: Kanvas | `project == "Kanvas"` | + +The Kubernetes Connections report is a cross-client behavior lens: connection +tests are tagged at their source (UI Playwright specs, CLI converters) with +`epic="Kubernetes Connections"`, `componentUnderTest`, `testId` (`TC-`), and +`client` (`UI`|`CLI`), sourced from the Meshery Test Plan. Tagged tests still +appear in their `project` report; the Connections report is an additional view. +As a fallback, a result that carries no `epic` label at all is included when its +`componentUnderTest` matches Kubernetes, so results predating the `epic` +convention still appear; a result with a *different* `epic` value is not pulled +in by the component fallback.

 

@@ -98,7 +121,9 @@ The following directories hold generated test data and are safe to exclude (they | --- | --- | | `meshery-results/` | Raw Allure results for Meshery integration tests | | `meshery-server-results/` | Raw Allure results for Meshery Server Go unit tests | -| `mesheryctl-results/` | Raw Allure results for `mesheryctl` tests | +| `mesheryctl-results/` | Raw Allure results for `mesheryctl` tests (deprecated - split into the two dirs below) | +| `mesheryctl-bats-results/` | Raw Allure results for `mesheryctl` BATS e2e tests | +| `mesheryctl-unit-results/` | Raw Allure results for `mesheryctl` Go unit tests | | `kanvas-results/` | Raw Allure results for Kanvas (extension) | | `remote-provider-results/` | Raw Allure results for the Remote Provider (extension) | | `pr-reports/` | Generated per-PR HTML reports (the largest contributor to repo size) | @@ -116,6 +141,8 @@ git sparse-checkout set --no-cone \ '!/meshery-results/' \ '!/meshery-server-results/' \ '!/mesheryctl-results/' \ + '!/mesheryctl-bats-results/' \ + '!/mesheryctl-unit-results/' \ '!/kanvas-results/' \ '!/remote-provider-results/' \ '!/pr-reports/' @@ -135,6 +162,7 @@ cd qa git sparse-checkout set --no-cone \ '/*' \ '!/meshery-results/' '!/meshery-server-results/' '!/mesheryctl-results/' \ + '!/mesheryctl-bats-results/' '!/mesheryctl-unit-results/' \ '!/kanvas-results/' '!/remote-provider-results/' '!/pr-reports/' ``` diff --git a/allurerc.mjs b/allurerc.mjs index 8b5c222691e..6fb14e42944 100644 --- a/allurerc.mjs +++ b/allurerc.mjs @@ -13,6 +13,33 @@ const PROJECTS = { const isProject = (labels, projectName) => labels.find(({ name, value }) => name === "project" && value === projectName); +// Behavior-scoped report keys. A Connections result is tagged at its source +// (UI Playwright specs and CLI converters) with epic="Kubernetes Connections", +// componentUnderTest (Test Plan col C), testId=TC- (Test Plan col A), and +// client (UI|CLI). See https://qa.meshery.io and the meshery test-tagging docs. +const CONNECTIONS_EPIC = "Kubernetes Connections"; + +// Matches componentUnderTest values that denote Kubernetes connection behavior. +// Used as a fallback selector when a result predates the epic label. +const CONNECTION_COMPONENT_RE = /kubernetes/i; + +// Select a result into the Connections report: prefer the explicit epic label. +// The componentUnderTest fallback applies ONLY to results that carry no epic +// label at all (tagged before the epic convention) - a result with a different +// epic value must not be pulled in just because its component is Kubernetes. +const isConnectionBehavior = (labels) => { + const hasEpic = labels.some(({ name }) => name === "epic"); + if (hasEpic) { + return labels.some( + ({ name, value }) => name === "epic" && value === CONNECTIONS_EPIC, + ); + } + return labels.some( + ({ name, value }) => + name === "componentUnderTest" && CONNECTION_COMPONENT_RE.test(value), + ); +}; + export default defineConfig({ name: "Meshery Quality Dashboard", output: "./allure-report", @@ -96,6 +123,26 @@ export default defineConfig({ groupBy: ["parentSuite", "suite", "subSuite"], }, }, + // Cross-client behavior report: aggregates Kubernetes Connection tests from + // BOTH the UI (project=Meshery) and CLI (project=mesheryctl) pools. It keys + // on the epic label, NOT project, so it is an additional lens - connection + // tests still appear in the Meshery and Mesheryctl reports above. + connections: { + import: "@allurereport/plugin-awesome", + options: { + reportName: "Kubernetes Connections", + singleFile: false, + reportLanguage: "en", + open: false, + logo: "https://raw.githubusercontent.com/meshery-extensions/qa/refs/heads/master/.github/assets/images/meshery/icon-only/meshery-light-icon.svg", + filter: ({ labels }) => isConnectionBehavior(labels), + // Group by client (UI vs CLI) first, then the suite hierarchy. The + // awesome plugin (preciseTreeLabels) keeps only label names present on + // at least one result, so results missing "client" fall back to the + // suite/subSuite grouping automatically. + groupBy: ["client", "suite", "subSuite"], + }, + }, log: { options: {