diff --git a/CHANGELOG.md b/CHANGELOG.md index 290336a..73b0ffe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **Trace shows the locators each step used** — every terminal `Locator` action (click, type, getText, …) is now recorded with its time offset and listed in the trace file under "Locators Used", grouped beneath the step it ran in. A lookup that timed out is marked "not found". Collected automatically, no config. (#130) +- **Trace: console errors and failed requests under the step they happened in** — console errors (Chrome) now carry their timestamp and are listed beneath the step they occurred in. New opt-in `tracing.captureNetwork: true` records requests that returned 4xx/5xx or failed outright (CDP on Chrome/Edge, WebDriver BiDi on Firefox) and lists them the same way, so a trace shows the `500` behind a failed step. Off by default; successful requests are never recorded and at most 200 are kept per test. (#130) + --- ### v3.9.0 — 2026-10-21 diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index 575a1b4..1a5c809 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -17,6 +17,8 @@ All notable changes to Selenium Boot are documented here. - **Trace shows the locators each step used** — every terminal `Locator` action (click, type, getText, …) is now recorded with its time offset and listed in the trace file under "Locators Used", grouped beneath the step it ran in. A lookup that timed out is marked "not found". Collected automatically, no config. (#130) +- **Trace: console errors and failed requests under the step they happened in** — console errors (Chrome) now carry their timestamp and are listed beneath the step they occurred in. New opt-in `tracing.captureNetwork: true` records requests that returned 4xx/5xx or failed outright (CDP on Chrome/Edge, WebDriver BiDi on Firefox) and lists them the same way, so a trace shows the `500` behind a failed step. Off by default; successful requests are never recorded and at most 200 are kept per test. (#130) + --- ## [3.9.0] — 2026-10-21 diff --git a/docs-site/docs/guides/trace-viewer.md b/docs-site/docs/guides/trace-viewer.md index b3fff16..367c8bf 100644 --- a/docs-site/docs/guides/trace-viewer.md +++ b/docs-site/docs/guides/trace-viewer.md @@ -20,6 +20,7 @@ Trace recording is configured in `selenium-boot.yml`. When `tracing.enabled` is tracing: enabled: true captureOnPass: false # set true to generate traces for passing tests too + captureNetwork: false # set true to list failed requests (4xx/5xx, network errors) in the trace ``` When enabled, a self-contained HTML file is generated for each failed test at: @@ -39,8 +40,11 @@ Each trace file is an independent, single-file HTML document featuring: 1. **Metadata header** — test method name, class, browser name, duration in milliseconds, status badge, and timestamp. 2. **Interactive step timeline** — every named step recorded via `StepLogger.step(...)` appears in the timeline with its offset. Steps that captured screenshots display a camera icon (📷) and open the interactive screenshot panel on click. 3. **Final-state screenshot** — full-browser screenshot captured automatically at the exact moment of test failure. -4. **Failure details** — highlighted error message and full scrollable stack trace. -5. **Zero CDN dependencies** — CSS styles and interactive JavaScript are completely inlined so traces can be viewed offline, archived, or shared without broken assets. +4. **Locators used** — each `Locator` action (click, type, getText, …) listed under the step it ran in; a lookup that timed out is marked "not found". +5. **Console errors** — with `browser.captureConsoleErrors: true`, JS errors are listed under the step they happened in (Chrome; other browsers list them without a step). +6. **Failed requests** — with `tracing.captureNetwork: true`, requests that returned 4xx/5xx or failed outright (method, URL, status or error) are listed under their step. Uses CDP on Chrome/Edge and WebDriver BiDi on Firefox; off by default because it keeps a listener open for the whole test. Successful requests are not recorded, and at most 200 are kept per test. +7. **Failure details** — highlighted error message and full scrollable stack trace. +8. **Zero CDN dependencies** — CSS styles and interactive JavaScript are completely inlined so traces can be viewed offline, archived, or shared without broken assets. --- diff --git a/src/main/java/com/seleniumboot/browser/ConsoleErrorCollector.java b/src/main/java/com/seleniumboot/browser/ConsoleErrorCollector.java index b6eae97..86d1518 100644 --- a/src/main/java/com/seleniumboot/browser/ConsoleErrorCollector.java +++ b/src/main/java/com/seleniumboot/browser/ConsoleErrorCollector.java @@ -60,8 +60,10 @@ public static List collect() { try { driver.manage().logs().get(LogType.BROWSER).getAll().stream() .filter(e -> e.getLevel().intValue() >= Level.SEVERE.intValue()) - .map(LogEntry::getMessage) - .forEach(errors::add); + .forEach(e -> { + errors.add(e.getMessage()); + recordForTrace(e); + }); } catch (Exception ignored) { // Firefox and some drivers don't support browser logs } @@ -80,6 +82,16 @@ public static List collect() { return Collections.unmodifiableList(errors); } + /** Keeps the entry's timestamp so the trace can show it under the step it happened in. */ + private static void recordForTrace(LogEntry e) { + try { + com.seleniumboot.metrics.ExecutionMetrics.recordConsoleEntry( + SeleniumBootContext.getCurrentTestId(), e.getMessage(), e.getTimestamp()); + } catch (RuntimeException ignored) { + // trace detail only — never let it break console collection + } + } + /** * Injects a JS console error shim into the current page. * Call this after page load if using Firefox or browsers without WebDriver log support. diff --git a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java index 45a69bb..0130d65 100644 --- a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java +++ b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java @@ -523,6 +523,7 @@ public static final class Tracing { /** null = not set in selenium-boot.yml; resolved by {@link #isEffectivelyEnabled(boolean)}. */ private Boolean enabled = null; private boolean captureOnPass = false; + private boolean captureNetwork = false; /** The value set in selenium-boot.yml; false when unset. Prefer {@link #isEffectivelyEnabled()}. */ public boolean isEnabled() { return Boolean.TRUE.equals(enabled); } @@ -534,6 +535,10 @@ public static final class Tracing { public boolean isCaptureOnPass() { return captureOnPass; } public void setCaptureOnPass(boolean v) { this.captureOnPass = v; } + + /** Opt-in: listen for failed requests (4xx/5xx, network errors) for the whole test and list them in the trace. */ + public boolean isCaptureNetwork() { return captureNetwork; } + public void setCaptureNetwork(boolean v) { this.captureNetwork = v; } } public static final class Visual { diff --git a/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java b/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java index d2bbefa..c78dcae 100644 --- a/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java +++ b/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java @@ -73,6 +73,7 @@ public void beforeScenario(Scenario scenario) { } ExecutionMetrics.clearSteps(testId); ExecutionMetrics.markStart(testId); + com.seleniumboot.tracing.NetworkFailureRecorder.startIfEnabled(testId); ExecutionMetrics.recordTestClass(testId, featureTitle(scenario.getUri())); ExecutionMetrics.recordDescription(testId, scenario.getName()); java.util.Collection scenarioTags = scenario.getSourceTagNames(); diff --git a/src/main/java/com/seleniumboot/junit5/SeleniumBootExtension.java b/src/main/java/com/seleniumboot/junit5/SeleniumBootExtension.java index 68b4e57..76d1cee 100644 --- a/src/main/java/com/seleniumboot/junit5/SeleniumBootExtension.java +++ b/src/main/java/com/seleniumboot/junit5/SeleniumBootExtension.java @@ -93,6 +93,7 @@ public void beforeEach(ExtensionContext context) { SeleniumBootContext.setCurrentTestId(testId); ExecutionMetrics.clearSteps(testId); ExecutionMetrics.markStart(testId); + com.seleniumboot.tracing.NetworkFailureRecorder.startIfEnabled(testId); ExecutionMetrics.recordTestClass(testId, context.getRequiredTestClass().getSimpleName()); ExecutionMetrics.recordDescription(testId, context.getDisplayName()); java.util.Set tags = context.getTags(); diff --git a/src/main/java/com/seleniumboot/listeners/TestExecutionListener.java b/src/main/java/com/seleniumboot/listeners/TestExecutionListener.java index 023420b..e3beaa1 100644 --- a/src/main/java/com/seleniumboot/listeners/TestExecutionListener.java +++ b/src/main/java/com/seleniumboot/listeners/TestExecutionListener.java @@ -72,6 +72,7 @@ public void onTestStart(ITestResult result) { SeleniumBootContext.setCurrentTestId(testId); ExecutionMetrics.clearSteps(testId); // discard stale steps from prior retry attempt ExecutionMetrics.markStart(testId); + com.seleniumboot.tracing.NetworkFailureRecorder.startIfEnabled(testId); ExecutionMetrics.recordTestClass(testId, result.getTestClass().getRealClass().getSimpleName()); ExecutionMetrics.recordDescription(testId, result.getMethod().getDescription()); String[] groups = result.getMethod().getGroups(); diff --git a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java index ebe859a..c756668 100644 --- a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java +++ b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java @@ -62,6 +62,26 @@ public static void recordLocatorUse(String testId, String locator, boolean found .addLocatorUse(new com.seleniumboot.steps.LocatorUse(locator, offsetMs, found)); } + /** Stores a console error with its offset from test start; {@code timestampMs} is epoch millis. */ + public static void recordConsoleEntry(String testId, String message, long timestampMs) { + if (testId == null || message == null) return; + long start = getTestStartTime(testId); + long offsetMs = start > 0 ? Math.max(0L, timestampMs - start) : 0L; + TIMINGS.computeIfAbsent(testId, + id -> new TestTiming(id, Thread.currentThread().getName())) + .addConsoleEntry(new com.seleniumboot.steps.ConsoleEntry(message, offsetMs)); + } + + /** Stores a failed request ({@code status} 0 plus {@code error} when no response arrived). */ + public static void recordNetworkFailure(String testId, String method, String url, int status, String error) { + if (testId == null || url == null) return; + long start = getTestStartTime(testId); + long offsetMs = start > 0 ? System.currentTimeMillis() - start : 0L; + TIMINGS.computeIfAbsent(testId, + id -> new TestTiming(id, Thread.currentThread().getName())) + .addNetworkFailure(new com.seleniumboot.steps.NetworkFailure(method, url, status, error, offsetMs)); + } + public static void clearSteps(String testId) { TestTiming t = TIMINGS.get(testId); if (t != null) t.clearSteps(); @@ -241,6 +261,7 @@ public static void recordStatus(String testId, String status) { // ========================================================== public static void markEnd(String testId) { + com.seleniumboot.tracing.NetworkFailureRecorder.stop(testId); Long start = START_TIMES.remove(testId); if (start == null) return; diff --git a/src/main/java/com/seleniumboot/metrics/TestTiming.java b/src/main/java/com/seleniumboot/metrics/TestTiming.java index 0c6e9ae..bcdd80d 100644 --- a/src/main/java/com/seleniumboot/metrics/TestTiming.java +++ b/src/main/java/com/seleniumboot/metrics/TestTiming.java @@ -141,7 +141,7 @@ public java.util.List getConsoleErrors() { new java.util.concurrent.CopyOnWriteArrayList<>(); public void addStep(com.seleniumboot.steps.StepRecord step) { steps.add(step); } - public void clearSteps() { steps.clear(); locatorUses.clear(); } + public void clearSteps() { steps.clear(); locatorUses.clear(); consoleEntries.clear(); networkFailures.clear(); } private static final int MAX_LOCATOR_USES = 1000; private final java.util.List locatorUses = @@ -149,6 +149,26 @@ public java.util.List getConsoleErrors() { public void addLocatorUse(com.seleniumboot.steps.LocatorUse use) { if (locatorUses.size() < MAX_LOCATOR_USES) locatorUses.add(use); } + + private static final int MAX_CONSOLE_ENTRIES = 500; + private static final int MAX_NETWORK_FAILURES = 200; + private final java.util.List consoleEntries = + new java.util.concurrent.CopyOnWriteArrayList<>(); + private final java.util.List networkFailures = + new java.util.concurrent.CopyOnWriteArrayList<>(); + public void addConsoleEntry(com.seleniumboot.steps.ConsoleEntry e) { + if (consoleEntries.size() < MAX_CONSOLE_ENTRIES) consoleEntries.add(e); + } + public java.util.List getConsoleEntries() { + return java.util.Collections.unmodifiableList(consoleEntries); + } + public void addNetworkFailure(com.seleniumboot.steps.NetworkFailure f) { + if (networkFailures.size() < MAX_NETWORK_FAILURES) networkFailures.add(f); + } + public java.util.List getNetworkFailures() { + return java.util.Collections.unmodifiableList(networkFailures); + } + public java.util.List getLocatorUses() { return java.util.Collections.unmodifiableList(locatorUses); } diff --git a/src/main/java/com/seleniumboot/steps/ConsoleEntry.java b/src/main/java/com/seleniumboot/steps/ConsoleEntry.java new file mode 100644 index 0000000..b4be33d --- /dev/null +++ b/src/main/java/com/seleniumboot/steps/ConsoleEntry.java @@ -0,0 +1,16 @@ +package com.seleniumboot.steps; + +/** A browser console error with the time it was logged, so the trace can show it under its step. */ +public class ConsoleEntry { + + private final String message; + private final long offsetMs; + + public ConsoleEntry(String message, long offsetMs) { + this.message = message; + this.offsetMs = offsetMs; + } + + public String getMessage() { return message; } + public long getOffsetMs() { return offsetMs; } +} diff --git a/src/main/java/com/seleniumboot/steps/NetworkFailure.java b/src/main/java/com/seleniumboot/steps/NetworkFailure.java new file mode 100644 index 0000000..ddb2549 --- /dev/null +++ b/src/main/java/com/seleniumboot/steps/NetworkFailure.java @@ -0,0 +1,26 @@ +package com.seleniumboot.steps; + +/** A request that came back 4xx/5xx or failed outright, kept so the trace can show it under its step. */ +public class NetworkFailure { + + private final String method; + private final String url; + private final int status; + private final String error; + private final long offsetMs; + + /** @param status HTTP status, or 0 when the request failed before any response; @param error failure text, else null */ + public NetworkFailure(String method, String url, int status, String error, long offsetMs) { + this.method = method; + this.url = url; + this.status = status; + this.error = error; + this.offsetMs = offsetMs; + } + + public String getMethod() { return method; } + public String getUrl() { return url; } + public int getStatus() { return status; } + public String getError() { return error; } + public long getOffsetMs() { return offsetMs; } +} diff --git a/src/main/java/com/seleniumboot/tracing/NetworkFailureRecorder.java b/src/main/java/com/seleniumboot/tracing/NetworkFailureRecorder.java new file mode 100644 index 0000000..8b19e69 --- /dev/null +++ b/src/main/java/com/seleniumboot/tracing/NetworkFailureRecorder.java @@ -0,0 +1,118 @@ +package com.seleniumboot.tracing; + +import com.seleniumboot.driver.DriverManager; +import com.seleniumboot.internal.SeleniumBootContext; +import com.seleniumboot.metrics.ExecutionMetrics; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.bidi.HasBiDi; +import org.openqa.selenium.bidi.module.Network; +import org.openqa.selenium.chromium.ChromiumDriver; +import org.openqa.selenium.devtools.DevTools; +import org.openqa.selenium.devtools.v153.network.model.LoadingFailed; +import org.openqa.selenium.devtools.v153.network.model.RequestWillBeSent; +import org.openqa.selenium.devtools.v153.network.model.ResponseReceived; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Listens for failed requests (HTTP 4xx/5xx and network errors) for the length of a test so the + * trace can list them under the step they happened in. Opt-in via {@code tracing.captureNetwork}. + * + *

CDP on Chromium, WebDriver BiDi on anything else that supports it; a driver with neither is + * skipped silently. Never fails a test: every problem is logged and swallowed. + * + *

Started when a test starts, stopped from {@link ExecutionMetrics#markEnd(String)}, which every + * runner (TestNG, JUnit 5, Cucumber) calls when a test finishes. + */ +public final class NetworkFailureRecorder { + + private static final Logger LOG = Logger.getLogger(NetworkFailureRecorder.class.getName()); + + private static final Map ACTIVE = new ConcurrentHashMap<>(); + + private NetworkFailureRecorder() {} + + /** Starts listening for the current test if {@code tracing.captureNetwork} (and tracing) is on. */ + public static void startIfEnabled(String testId) { + try { + var tracing = SeleniumBootContext.getConfig().getTracing(); + if (tracing == null || !tracing.isEffectivelyEnabled() || !tracing.isCaptureNetwork()) return; + start(testId, DriverManager.getDriver()); + } catch (RuntimeException e) { + LOG.log(Level.FINE, "[NetworkFailureRecorder] not started: " + e.getMessage()); + } + } + + static void start(String testId, WebDriver driver) { + if (testId == null || driver == null) return; + stop(testId); + try { + AutoCloseable listener = null; + if (driver instanceof ChromiumDriver chromium) { + listener = listenCdp(testId, chromium); + } else if (driver instanceof HasBiDi) { + listener = listenBiDi(testId, driver); + } + if (listener != null) ACTIVE.put(testId, listener); + } catch (Exception | LinkageError e) { + LOG.warning("[NetworkFailureRecorder] could not start for " + testId + ": " + e.getMessage()); + } + } + + /** Stops listening for this test; safe to call when nothing was started. */ + public static void stop(String testId) { + if (testId == null) return; + AutoCloseable listener = ACTIVE.remove(testId); + if (listener == null) return; + try { + listener.close(); + } catch (Exception e) { + LOG.log(Level.FINE, "[NetworkFailureRecorder] close failed: " + e.getMessage()); + } + } + + private static AutoCloseable listenCdp(String testId, ChromiumDriver chromium) { + DevTools devTools = chromium.getDevTools(); + devTools.createSession(); + devTools.send(org.openqa.selenium.devtools.v153.network.Network.enable( + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); + Map seen = new ConcurrentHashMap<>(); // requestId (as text; RequestId has no value equality) -> {method, url} + devTools.addListener(org.openqa.selenium.devtools.v153.network.Network.requestWillBeSent(), + (RequestWillBeSent e) -> seen.put(e.getRequestId().toString(), + new String[]{e.getRequest().getMethod(), e.getRequest().getUrl()})); + devTools.addListener(org.openqa.selenium.devtools.v153.network.Network.responseReceived(), + (ResponseReceived e) -> { + int status = e.getResponse().getStatus().intValue(); + if (status < 400) return; + String[] req = seen.get(e.getRequestId().toString()); + ExecutionMetrics.recordNetworkFailure(testId, req != null ? req[0] : "GET", + e.getResponse().getUrl(), status, null); + }); + devTools.addListener(org.openqa.selenium.devtools.v153.network.Network.loadingFailed(), + (LoadingFailed e) -> { + if (Boolean.TRUE.equals(e.getCanceled().orElse(false))) return; // aborted by navigation, not a failure + String[] req = seen.get(e.getRequestId().toString()); + ExecutionMetrics.recordNetworkFailure(testId, req != null ? req[0] : "GET", + req != null ? req[1] : "(unknown)", 0, e.getErrorText()); + }); + return devTools::close; + } + + private static AutoCloseable listenBiDi(String testId, WebDriver driver) { + Network network = new Network(driver); + network.onResponseCompleted(e -> { + int status = (int) e.getResponseData().getStatus(); + if (status >= 400) { + ExecutionMetrics.recordNetworkFailure(testId, e.getRequest().getMethod(), + e.getResponseData().getUrl(), status, null); + } + }); + network.onFetchError(e -> ExecutionMetrics.recordNetworkFailure(testId, e.getRequest().getMethod(), + e.getRequest().getUrl(), 0, e.getErrorText())); + return network; + } +} diff --git a/src/main/java/com/seleniumboot/tracing/TraceRecorder.java b/src/main/java/com/seleniumboot/tracing/TraceRecorder.java index aa908d8..30f7c14 100644 --- a/src/main/java/com/seleniumboot/tracing/TraceRecorder.java +++ b/src/main/java/com/seleniumboot/tracing/TraceRecorder.java @@ -5,7 +5,9 @@ import com.seleniumboot.metrics.ExecutionMetrics; import com.seleniumboot.metrics.TestTiming; import com.seleniumboot.reporting.ScreenshotManager; +import com.seleniumboot.steps.ConsoleEntry; import com.seleniumboot.steps.LocatorUse; +import com.seleniumboot.steps.NetworkFailure; import com.seleniumboot.steps.StepRecord; import org.openqa.selenium.OutputType; import org.openqa.selenium.TakesScreenshot; @@ -29,7 +31,8 @@ *

  • Named step timeline — each step logged via {@code StepLogger} as a clickable block
    • *
  • Per-step screenshots — click a step to reveal its screenshot
    • *
  • Final-state screenshot — taken at the moment of failure
    • - *
  • Browser console errors — when {@code browser.captureConsoleErrors} is on
    • + *
  • Browser console errors, under the step they happened in — when {@code browser.captureConsoleErrors} is on
    • + *
  • Failed requests (4xx/5xx, network errors), under their step — when {@code tracing.captureNetwork} is on
    • *
  • Locators used — each terminal {@code Locator} action, grouped under the step it ran in
    • *
  • Error message + full stack trace
    • * @@ -199,15 +202,32 @@ public static String buildHtml(TestTiming timing, String testName, String finalS sb.append(" \n\n"); } - // ── Console errors ── + // ── Console errors, grouped under the step they happened in ── List consoleErrors = timing.getConsoleErrors(); - if (!consoleErrors.isEmpty()) { + List consoleEntries = timing.getConsoleEntries(); + if (!consoleErrors.isEmpty() || !consoleEntries.isEmpty()) { + java.util.Set timed = new java.util.HashSet<>(); + for (ConsoleEntry c : consoleEntries) timed.add(c.getMessage()); + List untimed = new java.util.ArrayList<>(); + for (String e : consoleErrors) if (!timed.contains(e)) untimed.add(e); + sb.append("
    \n"); sb.append("
    Console Errors"); - sb.append(" (").append(consoleErrors.size()).append(")
    \n"); - sb.append(" 
    ");
-            for (String e : consoleErrors) sb.append(escHtml(e)).append('\n');
-            sb.append("
    \n
    \n"); + sb.append(" (").append(consoleEntries.size() + untimed.size()).append(")\n"); + sb.append("
    \n"); + int current = -2; + for (ConsoleEntry c : consoleEntries) { + current = appendStepHeading(sb, steps, c.getOffsetMs(), current); + sb.append("
    ").append(escHtml(c.getMessage())) + .append("+").append(c.getOffsetMs()).append("ms
    \n"); + } + if (!untimed.isEmpty()) { + if (!consoleEntries.isEmpty()) sb.append("
    Other
    \n"); + for (String e : untimed) { + sb.append("
    ").append(escHtml(e)).append("
    \n"); + } + } + sb.append("
    \n\n"); } // ── Locators used, grouped under the step they ran in ── @@ -219,15 +239,7 @@ public static String buildHtml(TestTiming timing, String testName, String finalS sb.append("
    \n"); int current = -2; for (LocatorUse u : uses) { - int owner = -1; - for (int i = 0; i < steps.size(); i++) { - if (steps.get(i).getOffsetMs() <= u.getOffsetMs()) owner = i; - } - if (owner != current) { - current = owner; - String label = owner < 0 ? "Before first step" : steps.get(owner).getName(); - sb.append("
    ").append(escHtml(label)).append("
    \n"); - } + current = appendStepHeading(sb, steps, u.getOffsetMs(), current); sb.append("
    ") .append("").append(escHtml(u.getLocator())).append("") .append("+").append(u.getOffsetMs()).append("ms") @@ -236,6 +248,24 @@ public static String buildHtml(TestTiming timing, String testName, String finalS sb.append("
    \n
    \n"); } + // ── Failed requests, grouped under the step they happened in ── + List failures = timing.getNetworkFailures(); + if (!failures.isEmpty()) { + sb.append("
    \n"); + sb.append("
    Failed Requests"); + sb.append(" (").append(failures.size()).append(")
    \n"); + sb.append("
    \n"); + int current = -2; + for (NetworkFailure f : failures) { + current = appendStepHeading(sb, steps, f.getOffsetMs(), current); + String what = f.getStatus() > 0 ? String.valueOf(f.getStatus()) : nvl(f.getError(), "failed"); + sb.append("
    ") + .append(escHtml(nvl(f.getMethod(), "GET") + " " + f.getUrl() + " \u2192 " + what)) + .append("+").append(f.getOffsetMs()).append("ms
    \n"); + } + sb.append("
    \n
    \n"); + } + // ── Error section ── if (errorMsg != null || stackTrace != null) { sb.append("
    \n"); @@ -258,6 +288,22 @@ public static String buildHtml(TestTiming timing, String testName, String finalS // Helpers // ------------------------------------------------------------------ + /** + * Writes the step heading when {@code offsetMs} falls under a different step than the previous row; + * a row belongs to the latest step that started at or before it. Returns the step index in effect. + */ + private static int appendStepHeading(StringBuilder sb, List steps, long offsetMs, int current) { + int owner = -1; + for (int i = 0; i < steps.size(); i++) { + if (steps.get(i).getOffsetMs() <= offsetMs) owner = i; + } + if (owner != current) { + String label = owner < 0 ? "Before first step" : steps.get(owner).getName(); + sb.append("
    ").append(escHtml(label)).append("
    \n"); + } + return owner; + } + private static File traceFile(String className, String testName) { return com.seleniumboot.reporting.ReportPaths.resolve("traces/" + className + "/" + sanitize(testName) + "-trace.html"); } diff --git a/src/test/java/com/seleniumboot/tracing/NetworkFailureRecorderLiveTest.java b/src/test/java/com/seleniumboot/tracing/NetworkFailureRecorderLiveTest.java new file mode 100644 index 0000000..e1935a8 --- /dev/null +++ b/src/test/java/com/seleniumboot/tracing/NetworkFailureRecorderLiveTest.java @@ -0,0 +1,160 @@ +package com.seleniumboot.tracing; + +import com.seleniumboot.metrics.ExecutionMetrics; +import com.seleniumboot.metrics.TestTiming; +import com.seleniumboot.steps.NetworkFailure; +import com.sun.net.httpserver.HttpServer; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.openqa.selenium.firefox.FirefoxDriver; +import org.openqa.selenium.firefox.FirefoxOptions; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Supplier; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +/** + * Live check of {@link NetworkFailureRecorder} on real browsers: CDP on Chrome, BiDi on Firefox. + * Each test skips when its browser or driver is not available. + */ +public class NetworkFailureRecorderLiveTest { + + private HttpServer server; + private String base; + + @BeforeClass + public void startServer() throws Exception { + int closedPort; // a port nothing listens on and browsers do not block (unlike port 1) + try (java.net.ServerSocket probe = new java.net.ServerSocket(0)) { closedPort = probe.getLocalPort(); } + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", ex -> { + String path = ex.getRequestURI().getPath(); + int code = path.equals("/fail") ? 500 : path.equals("/missing") ? 404 : 200; + byte[] body = path.equals("/") + ? ("ok").getBytes(StandardCharsets.UTF_8) + : "x".getBytes(StandardCharsets.UTF_8); + ex.getResponseHeaders().set("Content-Type", path.equals("/") ? "text/html" : "text/plain"); + ex.getResponseHeaders().set("Access-Control-Allow-Origin", "*"); + ex.sendResponseHeaders(code, body.length); + ex.getResponseBody().write(body); + ex.close(); + }); + server.start(); + base = "http://127.0.0.1:" + server.getAddress().getPort(); + } + + @AfterClass(alwaysRun = true) + public void stopServer() { + if (server != null) server.stop(0); + } + + @Test + public void chrome_cdp_recordsFailedRequestsOnly() { + assertFailuresRecorded("live.chrome", () -> { + ChromeOptions o = new ChromeOptions(); + o.addArguments("--headless=new", "--no-sandbox"); + return new ChromeDriver(o); + }); + } + + @Test + public void firefox_bidi_recordsFailedRequestsOnly() { + assertFailuresRecorded("live.firefox", () -> { + FirefoxOptions o = new FirefoxOptions(); + o.addArguments("-headless"); + o.setCapability("webSocketUrl", true); + return new FirefoxDriver(o); + }); + } + + @Test + public void chrome_consoleErrors_arriveWithTimestampsForTheTrace() throws Exception { + WebDriver driver; + try { + ChromeOptions o = new ChromeOptions(); + o.addArguments("--headless=new", "--no-sandbox"); + driver = new ChromeDriver(o); + } catch (Throwable t) { + throw new SkipException("Chrome not available: " + t.getMessage(), t); + } + String testId = "live.console"; + try { + com.seleniumboot.driver.DriverManager.pushSessionOverride(driver); + com.seleniumboot.internal.SeleniumBootContext.setCurrentTestId(testId); + ExecutionMetrics.markStart(testId); + driver.get("data:text/html,"); + sleep(300); + List messages = com.seleniumboot.browser.ConsoleErrorCollector.collect(); + + assertTrue(messages.stream().anyMatch(m -> m.contains("boom from page")), messages.toString()); + TestTiming t = ExecutionMetrics.getTiming(testId); + assertTrue(t.getConsoleEntries().stream().anyMatch(c -> c.getMessage().contains("boom from page") + && c.getOffsetMs() >= 0 && c.getOffsetMs() < 60_000), "entry with a sane offset expected"); + } finally { + com.seleniumboot.driver.DriverManager.popSessionOverride(); + driver.quit(); + } + } + + private void assertFailuresRecorded(String testId, Supplier factory) { + WebDriver driver; + try { + driver = factory.get(); + } catch (Throwable t) { + throw new SkipException("browser not available: " + t.getMessage(), t); + } + try { + ExecutionMetrics.markStart(testId); + NetworkFailureRecorder.start(testId, driver); + driver.get(base + "/"); + waitFor(testId, 3); + NetworkFailureRecorder.stop(testId); + + TestTiming t = ExecutionMetrics.getTiming(testId); + List f = t.getNetworkFailures(); + String all = f.stream().map(x -> x.getMethod() + " " + x.getUrl() + " " + x.getStatus() + " " + x.getError()).toList().toString(); + assertTrue(f.stream().anyMatch(x -> x.getUrl().endsWith("/fail") && x.getStatus() == 500 && "POST".equals(x.getMethod())), all); + assertTrue(f.stream().anyMatch(x -> x.getUrl().endsWith("/missing") && x.getStatus() == 404), all); + assertTrue(f.stream().anyMatch(x -> x.getUrl().contains("/refused") && x.getStatus() == 0 && x.getError() != null), all); + assertFalse(f.stream().anyMatch(x -> x.getUrl().endsWith("/fine")), "2xx must not be recorded: " + all); + assertFalse(f.stream().anyMatch(x -> x.getUrl().endsWith("/favicon.ico") && x.getStatus() < 400), all); + + // stopped: later traffic is not recorded + int before = f.size(); + driver.get(base + "/fail"); + sleep(500); + assertEquals(ExecutionMetrics.getTiming(testId).getNetworkFailures().size(), before, "recorder must be stopped"); + } finally { + NetworkFailureRecorder.stop(testId); + driver.quit(); + } + } + + private static void waitFor(String testId, int atLeast) { + long end = System.currentTimeMillis() + 8000; + while (System.currentTimeMillis() < end) { + TestTiming t = ExecutionMetrics.getTiming(testId); + if (t != null && t.getNetworkFailures().size() >= atLeast) return; + sleep(100); + } + } + + private static void sleep(long ms) { + try { Thread.sleep(ms); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } +} diff --git a/src/test/java/com/seleniumboot/unit/TraceCaptureMetricsTest.java b/src/test/java/com/seleniumboot/unit/TraceCaptureMetricsTest.java new file mode 100644 index 0000000..442c764 --- /dev/null +++ b/src/test/java/com/seleniumboot/unit/TraceCaptureMetricsTest.java @@ -0,0 +1,49 @@ +package com.seleniumboot.unit; + +import com.seleniumboot.metrics.ExecutionMetrics; +import com.seleniumboot.metrics.TestTiming; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +/** Per-test recording of console entries and failed requests, as used by the trace. */ +public class TraceCaptureMetricsTest { + + @Test + public void consoleEntry_isStoredWithOffsetFromTestStart() { + String id = "capture.console"; + ExecutionMetrics.markStart(id); + long now = System.currentTimeMillis(); + ExecutionMetrics.recordConsoleEntry(id, "boom", now + 250); + TestTiming t = ExecutionMetrics.getTiming(id); + assertEquals(t.getConsoleEntries().size(), 1); + long off = t.getConsoleEntries().get(0).getOffsetMs(); + assertTrue(off >= 200 && off <= 400, "offset was " + off); + } + + @Test + public void networkFailure_isStored_andCapped() { + String id = "capture.network"; + ExecutionMetrics.markStart(id); + for (int i = 0; i < 300; i++) ExecutionMetrics.recordNetworkFailure(id, "GET", "http://x/" + i, 404, null); + assertEquals(ExecutionMetrics.getTiming(id).getNetworkFailures().size(), 200); + } + + @Test + public void clearSteps_alsoClearsTheCapturedEntries() { + String id = "capture.clear"; + ExecutionMetrics.markStart(id); + ExecutionMetrics.recordNetworkFailure(id, "GET", "http://x", 500, null); + ExecutionMetrics.recordConsoleEntry(id, "e", System.currentTimeMillis()); + ExecutionMetrics.clearSteps(id); + assertTrue(ExecutionMetrics.getTiming(id).getNetworkFailures().isEmpty()); + assertTrue(ExecutionMetrics.getTiming(id).getConsoleEntries().isEmpty()); + } + + @Test + public void nullTestId_isIgnored() { + ExecutionMetrics.recordNetworkFailure(null, "GET", "http://x", 500, null); + ExecutionMetrics.recordConsoleEntry(null, "e", 1L); + } +} diff --git a/src/test/java/com/seleniumboot/unit/TraceRecorderTest.java b/src/test/java/com/seleniumboot/unit/TraceRecorderTest.java index 1d967f2..6091a73 100644 --- a/src/test/java/com/seleniumboot/unit/TraceRecorderTest.java +++ b/src/test/java/com/seleniumboot/unit/TraceRecorderTest.java @@ -1,7 +1,9 @@ package com.seleniumboot.unit; import com.seleniumboot.metrics.TestTiming; +import com.seleniumboot.steps.ConsoleEntry; import com.seleniumboot.steps.LocatorUse; +import com.seleniumboot.steps.NetworkFailure; import com.seleniumboot.steps.StepRecord; import com.seleniumboot.tracing.TraceRecorder; import org.testng.Assert; @@ -235,6 +237,54 @@ public void buildHtml_omitsLocatorSection_whenNoneRecorded() { Assert.assertFalse(TraceRecorder.buildHtml(timing("noLoc"), "noLoc", null).contains("Locators Used")); } + @Test + public void buildHtml_groupsConsoleEntriesUnderTheStepTheyHappenedIn() { + TestTiming t = timing("consoleStepTest"); + t.addStep(new StepRecord("Open page", 0, "INFO", null)); + t.addStep(new StepRecord("Click save", 500, "INFO", null)); + t.setConsoleErrors(List.of("early error", "late error")); + t.addConsoleEntry(new ConsoleEntry("early error", 100)); + t.addConsoleEntry(new ConsoleEntry("late error", 700)); + String section = sectionFrom(TraceRecorder.buildHtml(t, "consoleStepTest", null), "Console Errors"); + int s1 = section.indexOf("Open page"), e1 = section.indexOf("early error"); + int s2 = section.indexOf("Click save"), e2 = section.indexOf("late error"); + Assert.assertTrue(s1 >= 0 && s1 < e1 && e1 < s2 && s2 < e2, section); + Assert.assertEquals(section.split("early error", -1).length, 2, "must not be listed twice"); + } + + @Test + public void buildHtml_consoleErrorsWithoutTimestamps_stillListedFlat() { + TestTiming t = timing("shimOnly"); + t.setConsoleErrors(List.of("shim error")); + String html = TraceRecorder.buildHtml(t, "shimOnly", null); + Assert.assertTrue(html.contains("shim error")); + } + + @Test + public void buildHtml_groupsNetworkFailuresUnderStep_andEscapes() { + TestTiming t = timing("netTest"); + t.addStep(new StepRecord("Submit order", 0, "INFO", null)); + t.addNetworkFailure(new NetworkFailure("POST", "http://x/api/orders?a=", 500, null, 250)); + t.addNetworkFailure(new NetworkFailure("GET", "http://x/img.png", 0, "net::ERR_CONNECTION_REFUSED", 300)); + String html = TraceRecorder.buildHtml(t, "netTest", null); + String section = sectionFrom(html, "Failed Requests"); + Assert.assertTrue(section.contains("Submit order"), section); + Assert.assertTrue(section.contains("POST") && section.contains("500"), section); + Assert.assertTrue(section.contains("net::ERR_CONNECTION_REFUSED"), section); + Assert.assertTrue(section.contains("a=<b>"), "url must be escaped"); + } + + @Test + public void buildHtml_omitsNetworkSection_whenNone() { + Assert.assertFalse(TraceRecorder.buildHtml(timing("noNet"), "noNet", null).contains("Failed Requests")); + } + + private static String sectionFrom(String html, String title) { + int i = html.indexOf(">" + title); + Assert.assertTrue(i > 0, title + " section expected"); + return html.substring(i); + } + private static TestTiming timing(String testName) { TestTiming t = new TestTiming(testName, "test-thread"); t.setStatus("FAILED"); diff --git a/src/test/java/com/seleniumboot/unit/TracingConfigTest.java b/src/test/java/com/seleniumboot/unit/TracingConfigTest.java index 6bf7189..c816d06 100644 --- a/src/test/java/com/seleniumboot/unit/TracingConfigTest.java +++ b/src/test/java/com/seleniumboot/unit/TracingConfigTest.java @@ -33,4 +33,9 @@ public void explicitTrueWinsLocally() { public void captureOnPassStaysOffByDefault() { assertFalse(new SeleniumBootConfig.Tracing().isCaptureOnPass()); } + + @Test + public void captureNetworkStaysOffByDefault() { + assertFalse(new SeleniumBootConfig.Tracing().isCaptureNetwork()); + } }