[Improve] Make mutation findings advisory (#1610)

* ci: make mutation findings advisory

* fix(ci): surface mutation advisories

---------

Co-authored-by: Roomote <roomote@roomote.dev>

Author: zoomote[bot]

.github/workflows/mutation-testing.yml

@@ -22,8 +22,10 @@ jobs:
             - name: Record merge-queue enforcement
               if: github.event_name == 'merge_group'
               run: |
-                  echo "## Changed-code mutation testing" >> "$GITHUB_STEP_SUMMARY"
-                  echo "Mutation testing was enforced on each pull request before it entered the merge queue." >> "$GITHUB_STEP_SUMMARY"
+                  {
+                      echo "## Changed-code mutation testing"
+                      echo "Mutation testing was enforced on each pull request before it entered the merge queue."
+                  } >> "$GITHUB_STEP_SUMMARY" || echo "::warning title=Mutation test advisory::Could not write the job summary"
 
             - name: Checkout pull request merge result
               if: github.event_name == 'pull_request'
@@ -50,7 +52,7 @@ jobs:
               if: github.event_name == 'pull_request'
               run: pnpm test:mutation-ci
 
-            - name: Mutate changed executable lines
+            - name: Enforce executable-line scope and run advisory mutation testing
               if: github.event_name == 'pull_request'
               env:
                   BASE_SHA: ${{ github.event.pull_request.base.sha }}
@@ -60,6 +62,7 @@ jobs:
             - name: Upload mutation reports
               id: mutation_report
               if: always() && github.event_name == 'pull_request'
+              continue-on-error: true
               uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
               with:
                   name: changed-code-mutation-report
@@ -76,4 +79,4 @@ jobs:
                       echo ""
                       echo "### Download mutation reports"
                       echo "[Open the changed-code-mutation-report artifact]($ARTIFACT_URL), then open the package's mutation.html file."
-                  } >> "$GITHUB_STEP_SUMMARY"
+                  } >> "$GITHUB_STEP_SUMMARY" || echo "::warning title=Mutation test advisory::Could not write the job summary"

scripts/stryker-diff.mjs

@@ -198,6 +198,7 @@ export function packageForPath(filePath) {
 
 export function buildManifest(entries, readSource, diffForPath) {
 	const packages = new Map()
+	const advisories = []
 
 	for (const entry of entries) {
 		if (!new Set(["A", "M", "R"]).has(entry.status)) continue
@@ -211,7 +212,11 @@ export function buildManifest(entries, readSource, diffForPath) {
 				? new Set(Array.from({ length: sourceLineCount }, (_, index) => index + 1))
 				: parseChangedLines(diffForPath(entry.path))
 
-		validateDisableDirectives(source, new Set(source.split(/\r?\n/).map((_, index) => index + 1)), entry.path)
+		try {
+			validateDisableDirectives(source, new Set(source.split(/\r?\n/).map((_, index) => index + 1)), entry.path)
+		} catch (error) {
+			advisories.push(error.message)
+		}
 		const executableLines = executableChangedLines(source, changedLines, entry.path)
 		if (executableLines.size === 0) continue
 
@@ -243,7 +248,7 @@ export function buildManifest(entries, readSource, diffForPath) {
 		}
 	}
 
-	return { packages: [...packages.values()] }
+	return { packages: [...packages.values()], advisories }
 }
 
 function validateSha(value, name) {
@@ -446,7 +451,11 @@ function escapeWorkflowProperty(value) {
 }
 
 export function formatAnnotationCommand(annotation) {
-	return `::error file=${escapeWorkflowProperty(annotation.file)},line=${annotation.line},title=Mutation test gap::${escapeWorkflowData(annotation.message)}`
+	return `::warning file=${escapeWorkflowProperty(annotation.file)},line=${annotation.line},title=Mutation test advisory::${escapeWorkflowData(annotation.message)}`
+}
+
+export function formatAdvisoryCommand(advisory) {
+	return `::warning title=Mutation test advisory::${escapeWorkflowData(advisory)}`
 }
 
 export function formatAnnotations(blockingMutants, packageRoot, state = { total: 0, perFile: new Map() }) {
@@ -523,7 +532,7 @@ export function formatBlockingMutants(blockingMutants, packageRoot) {
 	return lines
 }
 
-export function formatSummary(rows, failures, manifest = {}) {
+export function formatSummary(rows, advisories, manifest = {}) {
 	const lines = [
 		"## Changed-code mutation testing",
 		"",
@@ -557,7 +566,7 @@ export function formatSummary(rows, failures, manifest = {}) {
 			"",
 			"### All surviving and uncovered mutants",
 			"",
-			"Annotations highlight up to 20 unique locations (maximum 7 per file). This summary lists every blocking mutant.",
+			"Warning annotations highlight up to 20 unique locations (maximum 7 per file). This summary lists every advisory mutant.",
 			"",
 		)
 		for (const row of blockingRows) {
@@ -573,7 +582,7 @@ export function formatSummary(rows, failures, manifest = {}) {
 			"const result = condition ? value : fallback",
 			"```",
 			"",
-			"Broad `all` exclusions and exclusions without a concrete reason are rejected by the gate.",
+			"Broad `all` exclusions and exclusions without a concrete reason are reported as advisories.",
 		)
 	}
 
@@ -600,14 +609,14 @@ export function formatSummary(rows, failures, manifest = {}) {
 		)
 	}
 
-	if (failures.length > 0) {
+	if (advisories.length > 0) {
 		lines.push(
 			"",
-			"### Failures",
+			"### Advisory findings",
 			"",
-			...failures.map((failure) => {
+			...advisories.map((advisory) => {
 				const detail =
-					failure.length > 4_000 ? `${failure.slice(0, 4_000)}\n[truncated; see the step log]` : failure
+					advisory.length > 4_000 ? `${advisory.slice(0, 4_000)}\n[truncated; see the step log]` : advisory
 				return `- ${detail.replaceAll("\n", "\n  ")}`
 			}),
 		)
@@ -616,37 +625,45 @@ export function formatSummary(rows, failures, manifest = {}) {
 	return `${lines.join("\n")}\n`
 }
 
-function appendSummary(rows, failures, manifest) {
+export function appendSummary(rows, advisories, manifest) {
+	for (const advisory of advisories) console.warn(formatAdvisoryCommand(advisory))
 	if (!process.env.GITHUB_STEP_SUMMARY) return
-	fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(rows, failures, manifest))
+	try {
+		fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(rows, advisories, manifest))
+	} catch (error) {
+		console.warn(
+			`::warning title=Mutation test advisory::Could not write the job summary: ${escapeWorkflowData(error.message)}`,
+		)
+	}
 }
 
 export function evaluateReport(report, packageEntry) {
 	const counts = mutantCounts(report)
+	const advisories = []
 	if (counts.valid > MAX_MUTANTS) {
-		throw new Error(
+		advisories.push(
 			`${packageEntry.id} generated ${counts.valid} valid mutants (limit ${MAX_MUTANTS}). ` +
 				"Split the PR or obtain a maintainer-reviewed narrow exclusion.",
 		)
 	}
 	if (counts.timeout > 10 || (counts.valid > 0 && counts.timeout / counts.valid > 0.15)) {
-		throw new Error(
+		advisories.push(
 			`${packageEntry.id} timed out ${counts.timeout} of ${counts.valid} valid mutants. ` +
-				"The result is inconclusive; fix flaky or slow tests, or reduce the changed scope before merge.",
+				"The result is inconclusive; consider fixing flaky or slow tests, or reducing the changed scope.",
 		)
 	}
 	if (counts.blocking.length > 0) {
-		throw new Error(
+		advisories.push(
 			`${packageEntry.id} has ${counts.survived} surviving and ${counts.noCoverage} uncovered changed-code mutants. ` +
-				"Add or strengthen focused tests before merge.",
+				"Consider adding or strengthening focused tests.",
 		)
 	}
-	return counts
+	return { ...counts, advisories }
 }
 
 export function runManifest(repoRoot, manifest, reportRoot) {
 	const rows = []
-	const failures = []
+	const advisories = [...(manifest.advisories ?? [])]
 	const annotationState = { total: 0, perFile: new Map() }
 
 	for (const packageEntry of manifest.packages) {
@@ -694,15 +711,15 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 			const jsonReportPath = path.join(reportRoot, packageEntry.id, "mutation.json")
 			const report = JSON.parse(fs.readFileSync(jsonReportPath, "utf8"))
 			packageEntry.testFiles = testsFromMutationReport(report, packageEntry.testFiles)
-			counts = mutantCounts(report)
+			counts = evaluateReport(report, packageEntry)
 			for (const annotation of formatAnnotations(
 				counts.blocking,
 				packageEntry.runRoot ?? packageEntry.root,
 				annotationState,
 			)) {
 				console.log(formatAnnotationCommand(annotation))
 			}
-			evaluateReport(report, packageEntry)
+			advisories.push(...counts.advisories)
 			rows.push({
 				id: packageEntry.id,
 				root: packageEntry.root,
@@ -712,10 +729,10 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 				reportPath,
 				changedLines: packageEntry.changedExecutableLines,
 				...counts,
-				result: "Passed",
+				result: counts.advisories.length > 0 ? "Advisory findings" : "Passed",
 			})
 		} catch (error) {
-			failures.push(error.message)
+			advisories.push(error.message)
 			rows.push({
 				id: packageEntry.id,
 				root: packageEntry.root,
@@ -730,13 +747,12 @@ export function runManifest(repoRoot, manifest, reportRoot) {
 				survived: counts?.survived ?? 0,
 				noCoverage: counts?.noCoverage ?? 0,
 				blocking: counts?.blocking ?? [],
-				result: "Failed",
+				result: "Advisory incomplete",
 			})
 		}
 	}
 
-	appendSummary(rows, failures, manifest)
-	if (failures.length > 0) throw new Error(failures.join("\n"))
+	appendSummary(rows, advisories, manifest)
 	return rows
 }
 
@@ -758,7 +774,7 @@ function main() {
 	const reportRoot = path.resolve(repoRoot, argument("--reports") ?? "reports/mutation")
 	const manifest = selectFromGit(repoRoot, baseSha, headSha)
 	if (manifest.packages.length === 0) {
-		appendSummary([], [], manifest)
+		appendSummary([], manifest.advisories, manifest)
 		console.log("No changed executable lines in mutation-tested packages; mutation testing is not applicable.")
 		return
 	}