From dd6aa669d329f448d15095ec65be02fe5d4a2c3b Mon Sep 17 00:00:00 2001 From: Pushpak Jaiswal <149317014+PUSHPAK-JAISWAL@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:07:06 +0530 Subject: [PATCH 1/3] feat: feature added failure intelligence --- CHANGELOG.md | 2 + ROADMAP.md | 13 +- docs-site/docs/changelog.md | 10 +- docs-site/docs/guides/failure-intelligence.md | 113 ++++++++++++ docs-site/docs/reporting/html-report.md | 31 ++-- .../seleniumboot/ai/AiFailureAnalyzer.java | 5 + .../seleniumboot/cucumber/CucumberHooks.java | 2 +- .../seleniumboot/failure/FailureCategory.java | 32 ++++ .../failure/FailureClassifier.java | 170 +++++++++++++----- .../seleniumboot/failure/FailureDetail.java | 67 ++++++- .../com/seleniumboot/failure/FailureType.java | 26 +++ .../metrics/ExecutionMetrics.java | 4 +- .../reporting/HtmlReportGenerator.java | 23 ++- .../reporting/JUnitXmlReporter.java | 6 +- src/main/resources/report-template.html | 9 +- .../ai/AiFailureAnalyzerPromptTest.java | 3 + .../unit/FailureClassifierTest.java | 75 +++++++- .../unit/FailureDetailExportTest.java | 4 + .../unit/FailureDetailReportTest.java | 25 +++ .../seleniumboot/unit/FailureDetailTest.java | 2 + 20 files changed, 543 insertions(+), 79 deletions(-) create mode 100644 docs-site/docs/guides/failure-intelligence.md create mode 100644 src/main/java/com/seleniumboot/failure/FailureType.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 452732c..f6da132 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Unreleased +- **Failure Intelligence: know why a test failed, with no API key** — bridges the gap from "a test failed" to "I know why and what to do", working out of the box on the first run with zero configuration and no AI or API key required. Every failure is classified into a high-level `FailureType` (`Product bug`, `Test bug`, `Environment`, `Flaky`, `Unknown`) with deterministic, context-aware remediation guidance ("Suggested Action") based on locator details, console errors, HTTP 5xx responses, and retry/history context. The HTML report surfaces color-coded type badges and prescriptive action banners; metrics JSON exports `failureType` and `suggestedAction`; and JUnit XML includes the classification tags and actions. Opt-in AI analysis (`ai.failureAnalysis: true`) is enriched with this baseline intelligence and prompts for structured root cause and fixes. Includes full user guide and documentation. (#138) + - **AI provider is configurable** — `ai.provider` selects the backend for failure analysis and the run summary: `anthropic` (default, unchanged), `gemini`, or `openai` (any OpenAI-compatible `/chat/completions` endpoint — OpenAI, Azure, Groq, local Ollama via `ai.baseUrl`). `ai.model` now defaults per provider. Existing configs keep working with no change. - **AI run summary (opt-in)** — `ai.runSummary: true` adds one short paragraph to the HTML dashboard summarising the run's failures: which same-cause groups account for most of them, which look like product bugs versus test or environment problems, and what to check first. One API call per run, independent of `ai.failureAnalysis`; only grouped error messages and test names are sent (see the data note in the HTML report docs). Absent when disabled, when nothing failed, or on any error; the suite result is never affected. The text is exported as `aiRunSummary` in the metrics JSON. diff --git a/ROADMAP.md b/ROADMAP.md index 8d4311c..f30aaae 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -40,12 +40,13 @@ Every item below is a real, scoped issue with acceptance criteria. Comment to cl - [#136](https://github.com/seleniumboot/selenium-boot/issues/136) Assertions: `isFocused`, `isEmpty`, regex overloads `help wanted` - [#137](https://github.com/seleniumboot/selenium-boot/issues/137) Failed tests and trace links in the GitHub Actions job summary `help wanted` -**Failure Intelligence** (know why a test failed, with no API key; design in #138, build in this order) +**Failure Intelligence** (know why a test failed, with no API key; design in #138, fully implemented) -- [#139](https://github.com/seleniumboot/selenium-boot/issues/139) Rule-based failure classification `help wanted` -- [#140](https://github.com/seleniumboot/selenium-boot/issues/140) Group failures by root cause in the report `help wanted` -- [#141](https://github.com/seleniumboot/selenium-boot/issues/141) Retry- and history-aware failure status `help wanted` -- [#142](https://github.com/seleniumboot/selenium-boot/issues/142) Richer context for AI failure analysis `help wanted` +- [#138](https://github.com/seleniumboot/selenium-boot/issues/138) Failure Intelligence: high-level classification & prescriptive remediation `completed` +- [#139](https://github.com/seleniumboot/selenium-boot/issues/139) Rule-based failure classification `completed` +- [#140](https://github.com/seleniumboot/selenium-boot/issues/140) Group failures by root cause in the report `completed` +- [#141](https://github.com/seleniumboot/selenium-boot/issues/141) Retry- and history-aware failure status `completed` +- [#142](https://github.com/seleniumboot/selenium-boot/issues/142) Richer context for AI failure analysis `completed` **Design discussion** @@ -53,7 +54,7 @@ Every item below is a real, scoped issue with acceptance criteria. Comment to cl ### Already done (don't duplicate) -Per-page SEO descriptions · Why pages (Selenium Boot / plain Selenium / Playwright) · six recipes (upload, download PDF, iframes, Shadow DOM, drag & drop, new windows) · migration guides (Selenium+TestNG, WebDriverManager, Playwright) · homepage before/after · Edge and Safari local browsers. +Failure Intelligence (design in #138: rule-based `FailureType` classification, actionable remediation, root cause grouping #140, retry & history status #141, bounded failure detail records #142) · Per-page SEO descriptions · Why pages (Selenium Boot / plain Selenium / Playwright) · six recipes (upload, download PDF, iframes, Shadow DOM, drag & drop, new windows) · migration guides (Selenium+TestNG, WebDriverManager, Playwright) · homepage before/after · Edge and Safari local browsers. ### Ongoing diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index a28340d..63a7c07 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -11,6 +11,8 @@ All notable changes to Selenium Boot are documented here. ## [Unreleased] +- **Failure Intelligence: know why a test failed, with no API key** — bridges the gap from "a test failed" to "I know why and what to do", working out of the box on the first run with zero configuration and no AI or API key required. Every failure is classified into a high-level `FailureType` (`Product bug`, `Test bug`, `Environment`, `Flaky`, `Unknown`) with deterministic, context-aware remediation guidance ("Suggested Action") based on locator details, console errors, HTTP 5xx responses, and retry/history context. The HTML report surfaces color-coded type badges and prescriptive action banners; metrics JSON exports `failureType` and `suggestedAction`; and JUnit XML includes the classification tags and actions. Opt-in AI analysis (`ai.failureAnalysis: true`) is enriched with this baseline intelligence and prompts for structured root cause and fixes. Includes full user guide and documentation. (#138) + - **AI provider is configurable** — `ai.provider` selects the backend for failure analysis and the run summary: `anthropic` (default, unchanged), `gemini`, or `openai` (any OpenAI-compatible `/chat/completions` endpoint — OpenAI, Azure, Groq, local Ollama via `ai.baseUrl`). `ai.model` now defaults per provider. Existing configs keep working with no change. - **AI run summary (opt-in)** — `ai.runSummary: true` adds one short paragraph to the HTML dashboard summarising the run's failures: which same-cause groups account for most of them, which look like product bugs versus test or environment problems, and what to check first. One API call per run, independent of `ai.failureAnalysis`; only grouped error messages and test names are sent (see the data note in the HTML report docs). Absent when disabled, when nothing failed, or on any error; the suite result is never affected. The text is exported as `aiRunSummary` in the metrics JSON. @@ -35,14 +37,10 @@ All notable changes to Selenium Boot are documented here. - **Failure grouping** — the report's Failures tab now opens with a "N tests, same cause" panel: failed tests are grouped by a signature of category, exception type, normalised message (ids, numbers and quoted values stripped) and top application stack frame. The flat per-class list is unchanged below it. With `ai.failureAnalysis` on, one Claude call is made per group; the other tests in the group reuse its analysis. The metrics JSON gains `failureSignature`. (#140) -- **`Locator` form actions: `clear()`, `check()`, `uncheck()`, `isChecked()`, `selectOption(text)`, `selectOptionByValue(value)`, `inputValue()`** — form controls can now be manipulated and inspected directly on `Locator` without falling back to `BasePage` helpers or raw `WebElement`s. `check()` and `uncheck()` are idempotent (no click if already in the target state), `isChecked()` queries checked state without waiting, and terminal actions (`clear`, `check`, `uncheck`, `selectOption`, `selectOptionByValue`, `inputValue`) auto-wait, re-resolving from scratch on every poll to survive stale or late-appearing elements. (#133) - ---- - -## [Unreleased] - - **Failure classification** — every failed test is tagged with a category (`LOCATOR_NOT_FOUND`, `ELEMENT_NOT_INTERACTABLE`, `STALE_ELEMENT`, `TIMEOUT`, `ASSERTION_MISMATCH`, `NAVIGATION_ERROR`, `BROWSER_CRASH`, or `UNKNOWN`) and the evidence for it, from the exception class chain and message alone. No AI, no API key, no network. Shown as a badge in the HTML report's failure panel, as `failureCategory` / `failureEvidence` in the metrics JSON, and as the `type` of `` in the JUnit XML. The raw error and stack trace are unchanged. (#139) +- **`Locator` form actions: `clear()`, `check()`, `uncheck()`, `isChecked()`, `selectOption(text)`, `selectOptionByValue(value)`, `inputValue()`** — form controls can now be manipulated and inspected directly on `Locator` without falling back to `BasePage` helpers or raw `WebElement`s. `check()` and `uncheck()` are idempotent (no click if already in the target state), `isChecked()` queries checked state without waiting, and terminal actions (`clear`, `check`, `uncheck`, `selectOption`, `selectOptionByValue`, `inputValue`) auto-wait, re-resolving from scratch on every poll to survive stale or late-appearing elements. (#133) + --- ## [3.8.0] — 2026-10-11 diff --git a/docs-site/docs/guides/failure-intelligence.md b/docs-site/docs/guides/failure-intelligence.md new file mode 100644 index 0000000..d045920 --- /dev/null +++ b/docs-site/docs/guides/failure-intelligence.md @@ -0,0 +1,113 @@ +--- +description: "Failure Intelligence in Selenium Boot: know why a test failed and what to do, with zero setup and no API key." +id: failure-intelligence +title: Failure Intelligence +sidebar_position: 12 +--- + +# Failure Intelligence + +**Failure Intelligence** takes you from *"a test failed"* to *"I know why and what to do"*, working on the first run with zero configuration and no API key. + +--- + +## Why Failure Intelligence? + +When automated test suites grow, diagnosing failures quickly becomes the biggest bottleneck: +- **Alert fatigue**: 40 tests failing due to a single backend outage read as 40 independent bugs. +- **Ambiguity**: Error messages like `TimeoutException` or `NoSuchElementException` don't distinguish between an application defect, a broken locator, or an environment timeout. +- **Trial-and-error triage**: Engineers often have to reproduce the failure locally or search through massive stack traces to figure out what to fix. +- **AI lock-in**: Most triage tools require commercial LLM API keys and send test code or stack traces to external servers. + +Selenium Boot's Failure Intelligence solves this by diagnosing failures **locally**, using rule-based classification, history awareness, and root-cause grouping. + +--- + +## 1. High-level triage classification (`FailureType`) + +Every failed test is automatically classified into a high-level triage type: + +| Classification | Meaning | Typical Trigger | +|---|---|---| +| **`Product bug`** | The application behavior diverged from expectations (potential regression). | Assertion failure (`AssertionError`, `ComparisonFailure`), especially when newly failing. | +| **`Test bug`** | An issue in test code, selectors, or test sequencing. | `LocatorException`, `NoSuchElementException`, `ElementNotInteractableException`, `StaleElementReferenceException`. | +| **`Environment`** | Infrastructure, network, browser, or backend server outage. | `NoSuchSessionException`, `UnreachableBrowserException`, `net::ERR_*`, DNS failures, HTTP 5xx errors. | +| **`Flaky`** | Non-deterministic test outcome due to timing or shared state. | Passed on retry, or marked `INTERMITTENT` based on run history. | +| **`Unknown`** | Unrecognised or unexpected exception. Raw stack trace is preserved. | Any unmapped exception. The classifier never guesses. | + +Shown as a color-coded badge (`[Product bug]`, `[Test bug]`, `[Environment]`, `[Flaky]`) in the HTML report and exported as `failureType` in `selenium-boot-metrics.json`. + +--- + +## 2. Granular category and evidence (`FailureCategory`) + +Below the high-level classification, each failure is tagged with a fine-grained category and the exact evidence extracted from the exception cause chain: + +| Category | Triggered by | +|---|---| +| `LOCATOR_NOT_FOUND` | `LocatorException`, `NoSuchElementException`, `NoSuchFrameException`, `NoSuchShadowRootException` anywhere in the cause chain. | +| `ELEMENT_NOT_INTERACTABLE` | `ElementNotInteractableException`, `ElementClickInterceptedException`, `InvalidElementStateException`. | +| `STALE_ELEMENT` | `StaleElementReferenceException` anywhere in the cause chain. | +| `TIMEOUT` | A `TimeoutException` with no locator or navigation cause. | +| `ASSERTION_MISMATCH` | Outermost error is an `AssertionError` (or TestNG/JUnit/`DbAssertException` equivalent), or soft assertions failed. | +| `NAVIGATION_ERROR` | `net::ERR_*`, DNS failures, renderer timeouts, page load timeouts, or HTTP 5xx status codes in the message. | +| `BROWSER_CRASH` | `NoSuchSessionException`, `UnreachableBrowserException`, "invalid session id", or browser process crash. | +| `UNKNOWN` | None of the above. | + +--- + +## 3. Actionable remediation ("What to do", No API Key) + +Failure Intelligence does not just report *why* a test failed — it provides prescriptive, context-aware remediation guidance: + +- **Locator not found**: Names the exact failing locator when available and recommends checking if the selector changed, or if the element is inside an iframe, shadow DOM, or delayed by async loading. +- **Element not interactable**: Identifies whether the element is covered by an overlay/modal, disabled, or outside the viewport, recommending scroll or modal dismissal. +- **Stale element**: Recommends using dynamic `$(...)` locators which auto-wait and re-resolve instead of holding cached `WebElement` references. +- **Assertion mismatch**: Highlights expected vs. actual values and advises checking whether product requirements changed or the test assertion is outdated. +- **Navigation error (HTTP 5xx)**: Instructs checking backend service health, database connectivity, and `execution.baseUrl`. +- **Browser crash**: Suggests reviewing driver binary compatibility, memory limits, and container resource limits. +- **Flaky**: Recommends replacing arbitrary thread sleeps with explicit `Locator` condition waits and isolating test state. + +The remediation guidance appears in an **Action** callout box in the HTML report and is exported as `suggestedAction` in metrics. + +--- + +## 4. Root cause grouping ("N tests, same cause") + +When multiple tests fail for the same reason (for example, a shared login locator change or a backend service outage), Failure Intelligence groups them by a signature: +- Failure category +- Outermost exception class +- Normalised error message (UUIDs, numbers, timestamps, and quoted strings replaced with placeholders) +- Top application stack frame (page object or test method) + +The HTML report's **Failures** tab opens with a *"Same cause (N tests)"* section showing the largest failure groups first. One outage no longer reads as forty disconnected failures. + +--- + +## 5. Retry and history status (`TestStatus`) + +Failures are compared against the test's recent history in `target/metrics-history/` (or `-Dseleniumboot.history.dir`): + +- **Newly failing**: Passed in all earlier runs in the window, but failed in this run. These are sorted to the top of the Failures tab as the most likely regressions. +- **Consistently failing**: Failed in this run and in earlier runs without passing. +- **Intermittent**: Alternates between passes and failures across recent runs. +- **Passed on retry**: Failed on initial attempt, but recovered upon retry. + +--- + +## 6. Optional AI enhancement (`ai.failureAnalysis`) + +When you configure an LLM API key (`ai.apiKey`), Selenium Boot passes the structured Failure Detail record (including rule-based classification, failing locator, console errors, self-healing events, and same-cause group context) to the model. + +Crucially: +- **Tests in the same-cause group share one AI call**, reducing API costs by 90%+. +- **AI output adheres to the framework's classification taxonomy**: + ```markdown + **Classification:** (Product Bug | Test Bug | Environment | Flaky) + **Root Cause:** (1-2 sentences explaining what likely went wrong) + **Suggested Fix:** + - (bullet 1) + - (bullet 2, if needed) + ``` +- **Privacy by default**: If no API key is provided, zero data leaves your machine. Classification and remediation work 100% locally. + diff --git a/docs-site/docs/reporting/html-report.md b/docs-site/docs/reporting/html-report.md index 0950c52..9be5112 100644 --- a/docs-site/docs/reporting/html-report.md +++ b/docs-site/docs/reporting/html-report.md @@ -59,22 +59,25 @@ Same detail panels as Test Cases tab, but only for failed tests. Detail panels a --- -## Failure classification +## Failure classification & intelligence -Each failed test is tagged with a category and the evidence for it, worked out from the exception class chain and message the framework already records. It needs no AI, no API key and no network. The badge and evidence appear at the top of the failure's detail panel; the raw error and stack trace stay right below. +Each failed test is tagged with a high-level triage classification (`Product bug`, `Test bug`, `Environment`, `Flaky`), a fine-grained category, and the evidence for it, worked out from the exception class chain and message the framework already records. It also provides actionable remediation advice ("Action: ...") right out of the box with no AI, no API key, and no network. The triage badge, category, and action callout appear at the top of the failure's detail panel; the raw error and stack trace stay right below. -| Category | Triggered by | -|---|---| -| `LOCATOR_NOT_FOUND` | `LocatorException`, `NoSuchElementException`, `NoSuchFrameException` or `NoSuchShadowRootException` anywhere in the cause chain, including a `LocatorException` caused by a `TimeoutException` | -| `ELEMENT_NOT_INTERACTABLE` | `ElementNotInteractableException`, `ElementClickInterceptedException`, `InvalidElementStateException` | -| `STALE_ELEMENT` | `StaleElementReferenceException` anywhere in the cause chain | -| `TIMEOUT` | a `TimeoutException` with no locator or navigation cause | -| `ASSERTION_MISMATCH` | the outermost error is an `AssertionError` (or TestNG/JUnit/`DbAssertException` equivalent), or soft assertions failed | -| `NAVIGATION_ERROR` | `net::ERR_*`, DNS failures, "Timed out receiving message from renderer", page load timeouts, HTTP 5xx in the message | -| `BROWSER_CRASH` | `NoSuchSessionException`, `UnreachableBrowserException`, "invalid session id", page crash | -| `UNKNOWN` | none of the above. The classifier never guesses | - -The category is also written to the metrics JSON as `failureCategory` and `failureEvidence`, and to the JUnit XML as the `type` of ``. +| Classification | Category | Triggered by | +|---|---|---| +| `Product bug` | `ASSERTION_MISMATCH` | the outermost error is an `AssertionError` (or TestNG/JUnit/`DbAssertException` equivalent), or soft assertions failed | +| `Test bug` | `LOCATOR_NOT_FOUND` | `LocatorException`, `NoSuchElementException`, `NoSuchFrameException` or `NoSuchShadowRootException` anywhere in the cause chain, including a `LocatorException` caused by a `TimeoutException` | +| `Test bug` | `ELEMENT_NOT_INTERACTABLE` | `ElementNotInteractableException`, `ElementClickInterceptedException`, `InvalidElementStateException` | +| `Test bug` | `STALE_ELEMENT` | `StaleElementReferenceException` anywhere in the cause chain | +| `Environment` | `TIMEOUT` | a `TimeoutException` with no locator or navigation cause | +| `Environment` | `NAVIGATION_ERROR` | `net::ERR_*`, DNS failures, "Timed out receiving message from renderer", page load timeouts, HTTP 5xx in the message | +| `Environment` | `BROWSER_CRASH` | `NoSuchSessionException`, `UnreachableBrowserException`, "invalid session id", page crash | +| `Flaky` | Any | When retried and passed (`Passed on retry`) or history indicates `Intermittent` execution | +| `Unknown` | `UNKNOWN` | none of the above. The classifier never guesses | + +The classification and action are also written to the metrics JSON as `failureType`, `failureCategory`, `failureEvidence`, and `suggestedAction`, and to the JUnit XML. + +For full architecture and design details, see [Failure Intelligence](../guides/failure-intelligence.md). --- diff --git a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java index eec7da4..3ae99d1 100644 --- a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java +++ b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java @@ -143,8 +143,12 @@ static String buildPrompt( com.seleniumboot.failure.FailureClassifier.Result classification = com.seleniumboot.failure.FailureClassifier.classify(timing); sb.append("\n## Failure Classification\n"); + sb.append("- Type: ").append(classification.type().label()).append(" (").append(classification.type().name()).append(")\n"); sb.append("- Category: ").append(classification.category().name()).append("\n"); sb.append("- Evidence: ").append(classification.evidence()).append("\n"); + if (classification.suggestedAction() != null && !classification.suggestedAction().isBlank()) { + sb.append("- Baseline recommendation: ").append(classification.suggestedAction()).append("\n"); + } detail.appendPrompt(sb); @@ -173,6 +177,7 @@ static String buildPrompt( sb.append("\n## Your Task\n"); sb.append("Provide a concise failure analysis in this exact format:\n\n"); + sb.append("**Classification:** (Product Bug | Test Bug | Environment | Flaky)\n\n"); sb.append("**Root Cause:** (1-2 sentences explaining what likely went wrong)\n\n"); sb.append("**Suggested Fix:**\n- (bullet 1)\n- (bullet 2, if needed)\n\n"); sb.append("Be specific and actionable. Do not repeat the error message verbatim."); diff --git a/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java b/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java index bd4f00a..0b488b6 100644 --- a/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java +++ b/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java @@ -219,7 +219,7 @@ private static void applyRetryTag(Scenario scenario) { static void checkCucumberQuarantine(Scenario scenario) { try { com.seleniumboot.config.SeleniumBootConfig.Quarantine cfg = - SeleniumBootContext.getConfig().getQuarantine(); + SeleniumBootContext.isInitialized() ? SeleniumBootContext.getConfig().getQuarantine() : null; if (cfg != null && !cfg.isEnabled()) return; // ── 1. In-file tag check (user adds @quarantine to the scenario) ── diff --git a/src/main/java/com/seleniumboot/failure/FailureCategory.java b/src/main/java/com/seleniumboot/failure/FailureCategory.java index cae5c3c..f46ebff 100644 --- a/src/main/java/com/seleniumboot/failure/FailureCategory.java +++ b/src/main/java/com/seleniumboot/failure/FailureCategory.java @@ -20,4 +20,36 @@ public enum FailureCategory { public String label() { return label; } + + /** The default high-level {@link FailureType} associated with this category. */ + public FailureType defaultType() { + return switch (this) { + case ASSERTION_MISMATCH -> FailureType.PRODUCT_BUG; + case LOCATOR_NOT_FOUND, ELEMENT_NOT_INTERACTABLE, STALE_ELEMENT -> FailureType.TEST_BUG; + case NAVIGATION_ERROR, BROWSER_CRASH, TIMEOUT -> FailureType.ENVIRONMENT; + case UNKNOWN -> FailureType.UNKNOWN; + }; + } + + /** Actionable remediation advice for this failure category without an API key. */ + public String defaultAction() { + return switch (this) { + case LOCATOR_NOT_FOUND -> + "Check if the element selector has changed in the application, or if it renders inside an iframe, shadow DOM, or after an asynchronous animation."; + case ELEMENT_NOT_INTERACTABLE -> + "Element is present in the DOM but not clickable or visible. Check if obscured by an overlay/modal, disabled, or needs scrolling into view."; + case STALE_ELEMENT -> + "Element detached from DOM after retrieval. Use dynamic Locator actions $(...) which auto-wait and re-resolve instead of caching WebElement references."; + case TIMEOUT -> + "Operation timed out waiting for condition. Check if page response or network is unusually slow, increase timeouts.explicit if expected, or adjust wait condition."; + case ASSERTION_MISMATCH -> + "Assertion failed. Compare expected vs actual values; check if product functionality changed or test expectation is outdated."; + case NAVIGATION_ERROR -> + "Page navigation or network request failed. Verify target URL, server status (HTTP 5xx), DNS resolution, and execution.baseUrl configuration."; + case BROWSER_CRASH -> + "Browser process or driver session closed unexpectedly. Check driver binary compatibility, available container memory, and headless mode options."; + case UNKNOWN -> + "Unclassified failure. Review the root exception and application stack trace below for details."; + }; + } } diff --git a/src/main/java/com/seleniumboot/failure/FailureClassifier.java b/src/main/java/com/seleniumboot/failure/FailureClassifier.java index 076b2ab..8125a64 100644 --- a/src/main/java/com/seleniumboot/failure/FailureClassifier.java +++ b/src/main/java/com/seleniumboot/failure/FailureClassifier.java @@ -1,5 +1,6 @@ package com.seleniumboot.failure; +import com.seleniumboot.flakiness.TestStatus; import com.seleniumboot.metrics.TestTiming; import java.util.ArrayList; @@ -9,8 +10,9 @@ import java.util.regex.Pattern; /** - * Tags a failed test with a {@link FailureCategory} and the evidence for it, using only the - * error message and stack trace the framework already records. No AI, no network. + * Tags a failed test with a {@link FailureCategory}, high-level {@link FailureType}, evidence, + * and actionable remediation advice, using only the error message, stack trace, and test metrics + * the framework already records. No AI, no network. * *

The exception class chain is read from the printed stack trace (the outer throwable on the * first line, each cause on a {@code Caused by:} line), so a {@code LocatorException} whose cause @@ -19,8 +21,17 @@ */ public final class FailureClassifier { - /** Category plus the one-line reason it was chosen. */ - public record Result(FailureCategory category, String evidence) { } + /** Category, high-level triage type, the one-line reason, and actionable remediation advice. */ + public record Result(FailureCategory category, FailureType type, String evidence, String suggestedAction) { + public Result(FailureCategory category, String evidence) { + this(category, category != null ? category.defaultType() : FailureType.UNKNOWN, + evidence, category != null ? category.defaultAction() : ""); + } + + public Result(FailureCategory category, FailureType type, String evidence) { + this(category, type, evidence, category != null ? category.defaultAction() : ""); + } + } private static final Pattern THROWABLE_LINE = Pattern.compile( "^(?:Caused by: )?((?:[A-Za-z_$][\\w$]*\\.)+[A-Za-z_$][\\w$]*|[A-Za-z_$][\\w$]*(?:Exception|Error))" @@ -31,11 +42,28 @@ public record Result(FailureCategory category, String evidence) { } private FailureClassifier() { } public static Result classify(TestTiming timing) { + return classify(timing, null); + } + + public static Result classify(TestTiming timing, TestStatus status) { if (timing == null) return unknown("no failure data"); - return classify(timing.getErrorMessage(), timing.getStackTrace()); + Result base = classify(timing.getErrorMessage(), timing.getStackTrace(), status); + if (base.category() == FailureCategory.LOCATOR_NOT_FOUND) { + String locator = timing.getFailingLocator(); + if (locator != null && !locator.isBlank()) { + String enrichedAction = "Check if locator '" + locator + + "' has changed or if the element is rendered in an iframe/shadow DOM or delayed by async loading."; + return new Result(base.category(), base.type(), base.evidence(), enrichedAction); + } + } + return base; } public static Result classify(String errorMessage, String stackTrace) { + return classify(errorMessage, stackTrace, null); + } + + public static Result classify(String errorMessage, String stackTrace, TestStatus status) { List chain = causeChain(stackTrace); String outer = chain.isEmpty() ? null : chain.get(0); String haystack = ((errorMessage != null ? errorMessage : "") + "\n" @@ -45,50 +73,101 @@ public static Result classify(String errorMessage, String stackTrace) { return unknown("no error message or stack trace recorded"); } + FailureCategory category = null; + String evidence = null; + // A failed assertion is the test doing its job, whatever text its message happens to contain. if (outer != null && isAssertion(outer)) { - return new Result(FailureCategory.ASSERTION_MISMATCH, "assertion failed (" + simple(outer) + ")"); + category = FailureCategory.ASSERTION_MISMATCH; + evidence = "assertion failed (" + simple(outer) + ")"; + } else { + String hit = firstMatch(chain, "NoSuchSessionException", "UnreachableBrowserException", + "SessionNotCreatedException"); + if (hit != null) { + category = FailureCategory.BROWSER_CRASH; + evidence = simple(hit) + " in cause chain"; + } else { + String crash = containsAny(haystack, "invalid session id", "session deleted because of page crash", + "chrome not reachable", "not connected to devtools", "target frame detached", + "browser has closed the connection", "tab crashed"); + if (crash != null) { + category = FailureCategory.BROWSER_CRASH; + evidence = "message contains \"" + crash + "\""; + } else { + String nav = containsAny(haystack, "net::err_", "err_name_not_resolved", "dnsnotfound", + "reached error page", "unknownhostexception", "timed out receiving message from renderer", + "page load timeout", "pageloadstrategy"); + if (nav != null) { + category = FailureCategory.NAVIGATION_ERROR; + evidence = "message contains \"" + nav + "\""; + } else { + Matcher http = Pattern.compile("(?:http|status)[ :=]+(5\\d\\d)\\b").matcher(haystack); + if (http.find()) { + category = FailureCategory.NAVIGATION_ERROR; + evidence = "HTTP " + http.group(1) + " in message"; + } else { + hit = firstMatch(chain, "StaleElementReferenceException"); + if (hit != null) { + category = FailureCategory.STALE_ELEMENT; + evidence = simple(hit) + " in cause chain"; + } else { + hit = firstMatch(chain, "ElementNotInteractableException", "ElementClickInterceptedException", + "InvalidElementStateException", "ElementNotSelectableException"); + if (hit != null) { + category = FailureCategory.ELEMENT_NOT_INTERACTABLE; + evidence = simple(hit) + " in cause chain"; + } else { + hit = firstMatch(chain, "LocatorException", "NoSuchElementException", "NoSuchFrameException", + "NoSuchShadowRootException"); + if (hit != null) { + category = FailureCategory.LOCATOR_NOT_FOUND; + evidence = simple(hit) + " in cause chain"; + } else { + hit = firstMatch(chain, "TimeoutException"); + if (hit != null) { + category = FailureCategory.TIMEOUT; + evidence = simple(hit) + " with no locator or navigation cause"; + } else { + for (String c : chain) { + if (isAssertion(c)) { + category = FailureCategory.ASSERTION_MISMATCH; + evidence = simple(c) + " in cause chain"; + break; + } + } + if (category == null && haystack.contains("the following asserts failed")) { + category = FailureCategory.ASSERTION_MISMATCH; + evidence = "soft assertions failed"; + } + } + } + } + } + } + } + } + } } - String hit = firstMatch(chain, "NoSuchSessionException", "UnreachableBrowserException", - "SessionNotCreatedException"); - if (hit != null) return result(FailureCategory.BROWSER_CRASH, simple(hit) + " in cause chain"); - String crash = containsAny(haystack, "invalid session id", "session deleted because of page crash", - "chrome not reachable", "not connected to devtools", "target frame detached", - "browser has closed the connection", "tab crashed"); - if (crash != null) return result(FailureCategory.BROWSER_CRASH, "message contains \"" + crash + "\""); - - String nav = containsAny(haystack, "net::err_", "err_name_not_resolved", "dnsnotfound", - "reached error page", "unknownhostexception", "timed out receiving message from renderer", - "page load timeout", "pageloadstrategy"); - if (nav != null) return result(FailureCategory.NAVIGATION_ERROR, "message contains \"" + nav + "\""); - Matcher http = Pattern.compile("(?:http|status)[ :=]+(5\\d\\d)\\b").matcher(haystack); - if (http.find()) return result(FailureCategory.NAVIGATION_ERROR, "HTTP " + http.group(1) + " in message"); - - hit = firstMatch(chain, "StaleElementReferenceException"); - if (hit != null) return result(FailureCategory.STALE_ELEMENT, simple(hit) + " in cause chain"); - - hit = firstMatch(chain, "ElementNotInteractableException", "ElementClickInterceptedException", - "InvalidElementStateException", "ElementNotSelectableException"); - if (hit != null) return result(FailureCategory.ELEMENT_NOT_INTERACTABLE, simple(hit) + " in cause chain"); - - hit = firstMatch(chain, "LocatorException", "NoSuchElementException", "NoSuchFrameException", - "NoSuchShadowRootException"); - if (hit != null) return result(FailureCategory.LOCATOR_NOT_FOUND, simple(hit) + " in cause chain"); - - hit = firstMatch(chain, "TimeoutException"); - if (hit != null) return result(FailureCategory.TIMEOUT, simple(hit) + " with no locator or navigation cause"); - - for (String c : chain) { - if (isAssertion(c)) { - return result(FailureCategory.ASSERTION_MISMATCH, simple(c) + " in cause chain"); - } + if (category == null) { + return unknown(outer != null ? "unrecognised " + simple(outer) : "unrecognised error"); } - if (haystack.contains("the following asserts failed")) { - return result(FailureCategory.ASSERTION_MISMATCH, "soft assertions failed"); + + FailureType type = category.defaultType(); + String action = category.defaultAction(); + + if (status == TestStatus.PASSED_ON_RETRY || status == TestStatus.INTERMITTENT) { + type = FailureType.FLAKY; + action = "Test is non-deterministic (" + status.label().toLowerCase() + + "). Replace arbitrary sleeps with explicit Locator waits and ensure tests do not share state."; + } else if (status == TestStatus.NEWLY_FAILING && category == FailureCategory.ASSERTION_MISMATCH) { + type = FailureType.PRODUCT_BUG; + action = "Newly failing assertion (passed in previous runs). Check recent application changes or deployments for a regression."; + } else if (category == FailureCategory.NAVIGATION_ERROR && evidence != null && evidence.startsWith("HTTP 5")) { + action = "Server returned an HTTP 5xx error. Inspect backend service logs and verify application server health."; } - return unknown(outer != null ? "unrecognised " + simple(outer) : "unrecognised error"); + return result(category, type, evidence, action); } static List causeChainOf(String stackTrace) { @@ -143,11 +222,12 @@ private static String simple(String className) { return i >= 0 ? className.substring(i + 1) : className; } - private static Result result(FailureCategory c, String evidence) { - return new Result(c, evidence.length() > EVIDENCE_MAX ? evidence.substring(0, EVIDENCE_MAX) + "…" : evidence); + private static Result result(FailureCategory c, FailureType type, String evidence, String action) { + String boundedEvidence = evidence.length() > EVIDENCE_MAX ? evidence.substring(0, EVIDENCE_MAX) + "…" : evidence; + return new Result(c, type, boundedEvidence, action); } private static Result unknown(String evidence) { - return new Result(FailureCategory.UNKNOWN, evidence); + return new Result(FailureCategory.UNKNOWN, FailureType.UNKNOWN, evidence, FailureCategory.UNKNOWN.defaultAction()); } } diff --git a/src/main/java/com/seleniumboot/failure/FailureDetail.java b/src/main/java/com/seleniumboot/failure/FailureDetail.java index 85e5c48..5d081de 100644 --- a/src/main/java/com/seleniumboot/failure/FailureDetail.java +++ b/src/main/java/com/seleniumboot/failure/FailureDetail.java @@ -24,6 +24,8 @@ public final class FailureDetail { public static final int MAX_URL_LENGTH = 2_048; public static final int MAX_ITEMS = 10; + private final String failureType; + private final String suggestedAction; private final String failingLocator; private final String applicationFrame; private final String lastSuccessfulStep; @@ -35,6 +37,8 @@ public final class FailureDetail { private final Map groupContext; private FailureDetail( + String failureType, + String suggestedAction, String failingLocator, String applicationFrame, String lastSuccessfulStep, @@ -44,6 +48,8 @@ private FailureDetail( List consoleErrors, String testStatus, Map groupContext) { + this.failureType = failureType; + this.suggestedAction = suggestedAction; this.failingLocator = failingLocator; this.applicationFrame = applicationFrame; this.lastSuccessfulStep = lastSuccessfulStep; @@ -62,10 +68,19 @@ public static FailureDetail from( int groupSize, TestTiming representative) { if (timing == null) { - return new FailureDetail(null, null, null, null, null, + return new FailureDetail(null, null, null, null, null, null, null, List.of(), List.of(), null, Map.of()); } + FailureClassifier.Result classification = + (timing.getErrorMessage() != null || timing.getStackTrace() != null) + ? FailureClassifier.classify(timing, status) + : null; + String failureType = classification != null && classification.type() != FailureType.UNKNOWN + ? classification.type().name() : null; + String suggestedAction = classification != null && classification.category() != FailureCategory.UNKNOWN + ? classification.suggestedAction() : null; + List> heals = new ArrayList<>(); for (HealEvent event : limited(HealLog.getForTest(timing.getTestId()))) { Map item = new LinkedHashMap<>(); @@ -90,6 +105,8 @@ public static FailureDetail from( } return new FailureDetail( + bounded(failureType, MAX_TEXT_LENGTH), + bounded(suggestedAction, MAX_TEXT_LENGTH), bounded(failingLocator(timing), MAX_TEXT_LENGTH), bounded(applicationFrame(timing.getStackTrace()), MAX_TEXT_LENGTH), bounded(lastSuccessfulStep(timing.getSteps()), MAX_TEXT_LENGTH), @@ -101,6 +118,50 @@ public static FailureDetail from( Collections.unmodifiableMap(group)); } + public String getFailureType() { + return failureType; + } + + public String getSuggestedAction() { + return suggestedAction; + } + + public String getFailingLocator() { + return failingLocator; + } + + public String getApplicationFrame() { + return applicationFrame; + } + + public String getLastSuccessfulStep() { + return lastSuccessfulStep; + } + + public String getPageUrl() { + return pageUrl; + } + + public String getPageTitle() { + return pageTitle; + } + + public List> getHealEvents() { + return healEvents; + } + + public List getConsoleErrors() { + return consoleErrors; + } + + public String getTestStatus() { + return testStatus; + } + + public Map getGroupContext() { + return groupContext; + } + private static String failingLocator(TestTiming timing) { String finalErrorLocator = locatorFrom(timing.getErrorMessage()); if (finalErrorLocator != null && !finalErrorLocator.isBlank()) { @@ -123,6 +184,8 @@ private static boolean hasLocatorWaitFrame(String stackTrace) { /** JSON-ready representation. Fields without data are deliberately absent. */ public Map toMap() { Map values = new LinkedHashMap<>(); + put(values, "failureType", failureType, MAX_TEXT_LENGTH); + put(values, "suggestedAction", suggestedAction, MAX_TEXT_LENGTH); put(values, "failingLocator", failingLocator, MAX_TEXT_LENGTH); put(values, "applicationFrame", applicationFrame, MAX_TEXT_LENGTH); put(values, "lastSuccessfulStep", lastSuccessfulStep, MAX_TEXT_LENGTH); @@ -139,6 +202,8 @@ public Map toMap() { public void appendPrompt(StringBuilder prompt) { if (toMap().isEmpty()) return; prompt.append("\n## Failure Detail\n"); + line(prompt, "Failure type", failureType); + line(prompt, "Suggested action", suggestedAction); line(prompt, "Failing locator", failingLocator); line(prompt, "Application frame", applicationFrame); line(prompt, "Last successful step", lastSuccessfulStep); diff --git a/src/main/java/com/seleniumboot/failure/FailureType.java b/src/main/java/com/seleniumboot/failure/FailureType.java new file mode 100644 index 0000000..35094b7 --- /dev/null +++ b/src/main/java/com/seleniumboot/failure/FailureType.java @@ -0,0 +1,26 @@ +package com.seleniumboot.failure; + +/** + * High-level triage classification for a failed test. + * + *

Answers whether the failure represents a defect in the application under test, + * an issue in test code or locators, an infrastructure/environment outage, or flakiness. + */ +public enum FailureType { + PRODUCT_BUG("Product bug"), + TEST_BUG("Test bug"), + ENVIRONMENT("Environment"), + FLAKY("Flaky"), + UNKNOWN("Unknown"); + + private final String label; + + FailureType(String label) { + this.label = label; + } + + public String label() { + return label; + } +} + diff --git a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java index 46d9c98..2d62449 100644 --- a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java +++ b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java @@ -431,9 +431,11 @@ public static void exportToJson() { } if ("FAILED".equals(timing.getStatus())) { com.seleniumboot.failure.FailureClassifier.Result failure = - com.seleniumboot.failure.FailureClassifier.classify(timing); + com.seleniumboot.failure.FailureClassifier.classify(timing, testStatus); testEntry.put("failureCategory", failure.category().name()); testEntry.put("failureEvidence", failure.evidence()); + testEntry.put("failureType", failure.type().name()); + testEntry.put("suggestedAction", failure.suggestedAction()); String signature = com.seleniumboot.failure.FailureSignature.of(timing); testEntry.put("failureSignature", signature); com.seleniumboot.failure.FailureDetail detail = diff --git a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java index c9aa361..88732f4 100644 --- a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java +++ b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java @@ -436,14 +436,33 @@ private static String buildFailureWhy(JsonNode test) { label = cat; } String evidence = test.has("failureEvidence") ? test.get("failureEvidence").asText() : ""; - return "

" + escapeHtml(label) + "" - + escapeHtml(evidence) + "
"; + String typeBadge = ""; + if (test.has("failureType")) { + String type = test.get("failureType").asText(); + String typeLabel; + try { + typeLabel = com.seleniumboot.failure.FailureType.valueOf(type).label(); + } catch (IllegalArgumentException e) { + typeLabel = type; + } + String typeClass = "failure-type-" + type.toLowerCase(java.util.Locale.ROOT).replace('_', '-'); + typeBadge = "" + escapeHtml(typeLabel) + ""; + } + String actionHtml = ""; + if (test.has("suggestedAction") && !test.get("suggestedAction").asText().isBlank()) { + actionHtml = "
Action: " + + escapeHtml(test.get("suggestedAction").asText()) + "
"; + } + return "
" + typeBadge + "" + escapeHtml(label) + "" + + escapeHtml(evidence) + "
" + actionHtml; } private static String buildFailureDetailPanel(JsonNode detail) { if (detail == null || detail.isEmpty()) return ""; StringBuilder sb = new StringBuilder("
" + "
Failure detail
    "); + detailItem(sb, "Failure type", detail, "failureType"); + detailItem(sb, "Suggested action", detail, "suggestedAction"); detailItem(sb, "Failing locator", detail, "failingLocator"); detailItem(sb, "Application frame", detail, "applicationFrame"); detailItem(sb, "Last successful step", detail, "lastSuccessfulStep"); diff --git a/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java b/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java index 9002501..0c39bd3 100644 --- a/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java +++ b/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java @@ -102,9 +102,13 @@ private static void writeXml(Collection timings, long totalDurationM xml.append(">\n"); com.seleniumboot.failure.FailureClassifier.Result why = com.seleniumboot.failure.FailureClassifier.classify(t); + String header = "[" + why.type().name() + " | " + why.category().name() + "] " + why.evidence(); + if (why.suggestedAction() != null && !why.suggestedAction().isBlank()) { + header += "\nSuggested Action: " + why.suggestedAction(); + } xml.append(" ") - .append(escapeXml("[" + why.category().name() + "] " + why.evidence() + "\n" + trace)) + .append(escapeXml(header + "\n\n" + trace)) .append("\n"); xml.append(" \n"); break; diff --git a/src/main/resources/report-template.html b/src/main/resources/report-template.html index a0bf3fd..27c0505 100644 --- a/src/main/resources/report-template.html +++ b/src/main/resources/report-template.html @@ -237,7 +237,6 @@ .ai-analysis-section { margin-bottom: 14px; background: rgba(88,166,255,.06); border: 1px solid rgba(88,166,255,.2); border-radius: 6px; overflow: hidden; } .ai-analysis-header { padding: 8px 14px; font-size: 11px; font-weight: 700; color: #58a6ff; text-transform: uppercase; letter-spacing: .5px; border-bottom: 1px solid rgba(88,166,255,.15); } .ai-analysis-body { padding: 12px 14px; font-size: 13px; color: var(--text-primary); white-space: pre-wrap; line-height: 1.6; } - .flakiness-section { } .flakiness-item { display: flex; align-items: center; gap: 10px; padding: 9px 0; border-bottom: 1px solid var(--border); } .flakiness-item:last-child { border-bottom: none; } .flakiness-name { flex: 1; font-size: 13px; color: var(--text-primary); word-break: break-all; } @@ -261,6 +260,14 @@ .failure-group-tests { margin: 0; padding-left: 20px; font-size: 12px; color: var(--text-secondary); } .failure-why { font-size: 12px; color: var(--muted, #6b7280); margin-bottom: 8px; } .failure-badge { display: inline-block; padding: 1px 8px; margin-right: 6px; border-radius: 10px; font-size: 11px; font-weight: 700; letter-spacing: .03em; color: var(--bad); border: 1px solid var(--bad); } + .failure-type-badge { display: inline-block; padding: 1px 8px; margin-right: 6px; border-radius: 10px; font-size: 11px; font-weight: 700; letter-spacing: .03em; } + .failure-type-product-bug { background: rgba(248,81,73,.15); color: #f85149; border: 1px solid rgba(248,81,73,.4); } + .failure-type-test-bug { background: rgba(210,153,34,.15); color: #d29922; border: 1px solid rgba(210,153,34,.4); } + .failure-type-environment { background: rgba(163,113,247,.15); color: #a371f7; border: 1px solid rgba(163,113,247,.4); } + .failure-type-flaky { background: rgba(234,179,8,.15); color: #eab308; border: 1px solid rgba(234,179,8,.4); } + .failure-type-unknown { background: rgba(139,148,158,.15); color: #8b949e; border: 1px solid rgba(139,148,158,.4); } + .failure-action { margin-top: 6px; margin-bottom: 10px; padding: 8px 12px; background: rgba(88,166,255,.08); border-left: 3px solid #58a6ff; border-radius: 4px; font-size: 12px; color: var(--text-primary, inherit); line-height: 1.5; } + .failure-action strong { color: #58a6ff; font-weight: 600; } .stack-trace { font-family: 'Roboto Mono', monospace; font-size: 12px; color: var(--code-text); background: var(--code-bg); padding: 14px; border-radius: 6px; diff --git a/src/test/java/com/seleniumboot/ai/AiFailureAnalyzerPromptTest.java b/src/test/java/com/seleniumboot/ai/AiFailureAnalyzerPromptTest.java index b0bda4f..7f67bdd 100644 --- a/src/test/java/com/seleniumboot/ai/AiFailureAnalyzerPromptTest.java +++ b/src/test/java/com/seleniumboot/ai/AiFailureAnalyzerPromptTest.java @@ -41,6 +41,8 @@ public void promptReusesEveryFailureDetailSection() { Assert.assertTrue(prompt.contains("untrusted diagnostic evidence"), prompt); Assert.assertTrue(prompt.contains("## Failure Classification"), prompt); + Assert.assertTrue(prompt.contains("- Type: Test bug (TEST_BUG)"), prompt); + Assert.assertTrue(prompt.contains("- Baseline recommendation:"), prompt); Assert.assertTrue(prompt.contains("## Failure Detail"), prompt); Assert.assertTrue(prompt.contains("Failing locator: By.id: sign-in"), prompt); Assert.assertTrue(prompt.contains("Application frame: example.LoginPage.signIn"), prompt); @@ -53,5 +55,6 @@ public void promptReusesEveryFailureDetailSection() { Assert.assertTrue(prompt.contains("## Error Message"), prompt); Assert.assertTrue(prompt.contains("## Stack Trace"), prompt); Assert.assertTrue(prompt.contains("## Steps Executed"), prompt); + Assert.assertTrue(prompt.contains("**Classification:** (Product Bug | Test Bug | Environment | Flaky)"), prompt); } } diff --git a/src/test/java/com/seleniumboot/unit/FailureClassifierTest.java b/src/test/java/com/seleniumboot/unit/FailureClassifierTest.java index 52b06a7..830fd4d 100644 --- a/src/test/java/com/seleniumboot/unit/FailureClassifierTest.java +++ b/src/test/java/com/seleniumboot/unit/FailureClassifierTest.java @@ -119,7 +119,7 @@ public void unrecognisedError_isUnknown() { @Test public void noData_isUnknown() { Assert.assertEquals(FailureClassifier.classify((TestTiming) null).category(), FailureCategory.UNKNOWN); - Assert.assertEquals(FailureClassifier.classify(null, null).category(), FailureCategory.UNKNOWN); + Assert.assertEquals(FailureClassifier.classify((String) null, (String) null).category(), FailureCategory.UNKNOWN); } @Test @@ -134,4 +134,77 @@ public void multilineSeleniumMessage_isNotMistakenForCause() { "boom\nCommand: [abc, get {url=x}]\nBuild info: version: '4'"); Assert.assertEquals(classify(t), FailureCategory.UNKNOWN); } + + @Test + public void failureType_mapsCorrectlyAcrossCategories() { + Assert.assertEquals(result(new AssertionError("expected a but got b")).type(), + com.seleniumboot.failure.FailureType.PRODUCT_BUG); + Assert.assertEquals(result(new org.openqa.selenium.NoSuchElementException("missing")).type(), + com.seleniumboot.failure.FailureType.TEST_BUG); + Assert.assertEquals(result(new org.openqa.selenium.ElementNotInteractableException("hidden")).type(), + com.seleniumboot.failure.FailureType.TEST_BUG); + Assert.assertEquals(result(new org.openqa.selenium.StaleElementReferenceException("stale")).type(), + com.seleniumboot.failure.FailureType.TEST_BUG); + Assert.assertEquals(result(new org.openqa.selenium.NoSuchSessionException("crash")).type(), + com.seleniumboot.failure.FailureType.ENVIRONMENT); + Assert.assertEquals(result(new RuntimeException("net::ERR_NAME_NOT_RESOLVED")).type(), + com.seleniumboot.failure.FailureType.ENVIRONMENT); + Assert.assertEquals(result(new org.openqa.selenium.TimeoutException("timeout")).type(), + com.seleniumboot.failure.FailureType.ENVIRONMENT); + Assert.assertEquals(result(new RuntimeException("weird unrecognised error")).type(), + com.seleniumboot.failure.FailureType.UNKNOWN); + } + + @Test + public void failureType_retryOrIntermittentStatus_isFlaky() { + FailureClassifier.Result r1 = FailureClassifier.classify("assertion failed", "AssertionError: boom", + com.seleniumboot.flakiness.TestStatus.PASSED_ON_RETRY); + Assert.assertEquals(r1.type(), com.seleniumboot.failure.FailureType.FLAKY); + Assert.assertTrue(r1.suggestedAction().contains("non-deterministic")); + + FailureClassifier.Result r2 = FailureClassifier.classify("timeout", "TimeoutException: boom", + com.seleniumboot.flakiness.TestStatus.INTERMITTENT); + Assert.assertEquals(r2.type(), com.seleniumboot.failure.FailureType.FLAKY); + } + + @Test + public void failureType_newlyFailingAssertion_isProductBugWithRegressionNote() { + FailureClassifier.Result r = FailureClassifier.classify("expected 10 but found 5", + "java.lang.AssertionError: expected 10 but found 5", + com.seleniumboot.flakiness.TestStatus.NEWLY_FAILING); + Assert.assertEquals(r.type(), com.seleniumboot.failure.FailureType.PRODUCT_BUG); + Assert.assertTrue(r.suggestedAction().contains("regression"), r.suggestedAction()); + } + + @Test + public void suggestedAction_enrichedWithFailingLocator() { + TestTiming timing = new TestTiming("com.example.CartTest.checkout", "main"); + timing.setFailingLocator("By.cssSelector: #checkout-button"); + timing.setErrorMessage("Element not found By.cssSelector: #checkout-button"); + timing.setStackTrace("org.openqa.selenium.NoSuchElementException: not found\n\tat example.Cart.pay(Cart.java:55)"); + + FailureClassifier.Result result = FailureClassifier.classify(timing); + Assert.assertEquals(result.category(), FailureCategory.LOCATOR_NOT_FOUND); + Assert.assertEquals(result.type(), com.seleniumboot.failure.FailureType.TEST_BUG); + Assert.assertTrue(result.suggestedAction().contains("#checkout-button"), result.suggestedAction()); + } + + @Test + public void suggestedAction_http5xx_givesBackendGuidance() { + FailureClassifier.Result r = FailureClassifier.classify("HTTP 503 Service Unavailable", "status: 503"); + Assert.assertEquals(r.category(), FailureCategory.NAVIGATION_ERROR); + Assert.assertTrue(r.suggestedAction().contains("HTTP 5xx"), r.suggestedAction()); + } + + @Test + public void result_backwardCompatibleConstructors() { + FailureClassifier.Result r2 = new FailureClassifier.Result(FailureCategory.LOCATOR_NOT_FOUND, "evidence"); + Assert.assertEquals(r2.category(), FailureCategory.LOCATOR_NOT_FOUND); + Assert.assertEquals(r2.type(), com.seleniumboot.failure.FailureType.TEST_BUG); + Assert.assertNotNull(r2.suggestedAction()); + + FailureClassifier.Result r3 = new FailureClassifier.Result(FailureCategory.LOCATOR_NOT_FOUND, + com.seleniumboot.failure.FailureType.FLAKY, "evidence"); + Assert.assertEquals(r3.type(), com.seleniumboot.failure.FailureType.FLAKY); + } } diff --git a/src/test/java/com/seleniumboot/unit/FailureDetailExportTest.java b/src/test/java/com/seleniumboot/unit/FailureDetailExportTest.java index 265cfc5..6859c01 100644 --- a/src/test/java/com/seleniumboot/unit/FailureDetailExportTest.java +++ b/src/test/java/com/seleniumboot/unit/FailureDetailExportTest.java @@ -31,7 +31,11 @@ public void failedTestsExportBoundedRecordAndSharedRepresentative() throws Excep Assert.assertEquals(tests.size(), 2); for (JsonNode test : tests) { + Assert.assertEquals(test.path("failureType").asText(), "TEST_BUG"); + Assert.assertTrue(test.has("suggestedAction")); JsonNode detail = test.path("failureDetail"); + Assert.assertEquals(detail.path("failureType").asText(), "TEST_BUG"); + Assert.assertTrue(detail.has("suggestedAction")); Assert.assertEquals(detail.path("failingLocator").asText(), "By.id: submit"); Assert.assertEquals(detail.path("pageUrl").asText(), "https://app.example/checkout"); Assert.assertEquals(detail.path("consoleErrors").get(0).asText(), "ReferenceError: submit is missing"); diff --git a/src/test/java/com/seleniumboot/unit/FailureDetailReportTest.java b/src/test/java/com/seleniumboot/unit/FailureDetailReportTest.java index 2ba1255..e445773 100644 --- a/src/test/java/com/seleniumboot/unit/FailureDetailReportTest.java +++ b/src/test/java/com/seleniumboot/unit/FailureDetailReportTest.java @@ -38,4 +38,29 @@ public void reportShowsRecordAndEscapesCapturedValues() throws Exception { Assert.assertTrue(html.contains("a=1&b=2"), html); Assert.assertFalse(html.contains("