diff --git a/docs-site/docs/reporting/dashboard-trends.md b/docs-site/docs/reporting/dashboard-trends.md index 90a3755..511c1c9 100644 --- a/docs-site/docs/reporting/dashboard-trends.md +++ b/docs-site/docs/reporting/dashboard-trends.md @@ -41,6 +41,19 @@ A trend requires data from multiple test runs. With only one run, the report has The trend charts remain hidden until metrics from at least two runs are available. Once history contains two or more runs, the charts can show changes in pass rate and duration. +## Opening a Past Run's Report + +Each point on a trend chart can open the full HTML report of that run, if the report was kept. Keeping reports is off by default, because every report embeds its screenshots and can be large. Turn it on with: + +```yaml +flakiness: + keepReports: 10 # keep the 10 newest reports; 0 (the default) keeps none +``` + +After each run, a copy of the report is saved in the history directory next to that run's metrics JSON, as `selenium-boot-report-.html`, and the oldest copies beyond `keepReports` are deleted. Points whose report was kept show "Click to open this run" in their tooltip; clicking opens it in a new tab. Runs from before you turned this on, and the current run, have no link. + +The link is relative to the report, so keep the report and the history directory together when you archive or download them (for example, upload both as one CI artifact). + ## Preserving History in CI CI jobs often use fresh workspaces or disposable containers. If the metrics-history directory is lost after each run, later reports cannot use the earlier metrics to display trends. diff --git a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java index 38ca6ec..94bf959 100644 --- a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java +++ b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java @@ -530,6 +530,7 @@ public static final class Flakiness { private double highRiskThreshold = 33.0; private boolean failOnHighFlakiness = false; private String historyDir = ""; + private int keepReports = 0; public int getHistoryRuns() { return historyRuns; } public void setHistoryRuns(int v) { this.historyRuns = v; } @@ -543,6 +544,13 @@ public static final class Flakiness { /** Where metrics history is kept; blank means {@code /metrics-history}. */ public String getHistoryDir() { return historyDir; } public void setHistoryDir(String v) { this.historyDir = v == null ? "" : v; } + + /** + * How many past HTML reports to keep in the history directory, next to their metrics + * JSON, so a trend-chart point can open that run's report. 0 (the default) keeps none. + */ + public int getKeepReports() { return keepReports; } + public void setKeepReports(int v) { this.keepReports = Math.max(0, v); } } public static final class Tracing { diff --git a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java index f9fe6f1..f310115 100644 --- a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java +++ b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java @@ -17,6 +17,9 @@ public final class ExecutionMetrics { /** History copies written by this JVM; excluded when comparing against earlier runs. */ private static final Set WRITTEN_HISTORY = ConcurrentHashMap.newKeySet(); + /** File name of the newest history copy this JVM wrote; null before the first export. */ + private static volatile String lastHistoryName; + private static final ConcurrentHashMap TIMINGS = new ConcurrentHashMap<>(); @@ -547,6 +550,7 @@ public static void exportToJson() { String historyName = "selenium-boot-metrics-" + timestamp + ".json"; mapper.writeValue(new File(historyDir, historyName), report); WRITTEN_HISTORY.add(historyName); + lastHistoryName = historyName; System.out.println("[Selenium Boot] Metrics exported → " + primary.getPath()); System.out.println("[Selenium Boot] History copy → " @@ -559,6 +563,14 @@ public static void exportToJson() { } } + /** + * File name of the history copy written by the latest export in this JVM, e.g. + * {@code selenium-boot-metrics-20261112-103000.json}; {@code null} if nothing was exported. + */ + public static String lastHistoryName() { + return lastHistoryName; + } + // ========================================================== // Reset // ========================================================== @@ -567,6 +579,7 @@ public static void reset() { START_TIMES.clear(); TIMINGS.clear(); WRITTEN_HISTORY.clear(); + lastHistoryName = null; TOTAL_DURATION.set(0); SUITE_START.set(0); } diff --git a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java index 89dd062..88d348c 100644 --- a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java +++ b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java @@ -48,6 +48,9 @@ public static void generate() { "[Selenium Boot] HTML report generated at " + reportFile.getPath() ); + keepReportCopy(html, keepReportsSetting(), + com.seleniumboot.metrics.ExecutionMetrics.lastHistoryName()); + } catch (Exception e) { System.err.println( "[Selenium Boot] HTML report generation failed: " @@ -163,6 +166,72 @@ static String formatDuration(long ms) { return String.format("%02d:%02d:%02d", totalSeconds / 3600, (totalSeconds % 3600) / 60, totalSeconds % 60); } + private static final String TREND_BASE_PREFIX = "const trendReportBase = "; + + /** {@code flakiness.keepReports}; 0 when no config is loaded. */ + private static int keepReportsSetting() { + try { + SeleniumBootConfig.Flakiness f = SeleniumBootContext.getConfig().getFlakiness(); + return f != null ? f.getKeepReports() : 0; + } catch (RuntimeException e) { + return 0; + } + } + + /** {@code selenium-boot-metrics-.json} to {@code selenium-boot-report-.html}. */ + static String keptReportName(String historyJsonName) { + return historyJsonName.replaceFirst("^selenium-boot-metrics-", "selenium-boot-report-") + .replaceFirst("\\.json$", ".html"); + } + + /** + * Saves this run's report next to its history JSON so later reports can link to it, then + * deletes the oldest kept reports beyond {@code keep}. Does nothing when {@code keep} is 0 + * or no history copy was written. + */ + static void keepReportCopy(String html, int keep, String historyJsonName) { + if (keep <= 0 || historyJsonName == null) return; + File dir = ReportPaths.metricsHistoryDir(); + try { + // The copy sits in the history directory, so its own trend links are plain file names. + int at = html.indexOf(TREND_BASE_PREFIX); + String copy = html; + if (at >= 0) { + int end = html.indexOf(";\n", at); + if (end > at) copy = html.substring(0, at) + TREND_BASE_PREFIX + "\"\"" + html.substring(end); + } + Files.writeString(new File(dir, keptReportName(historyJsonName)).toPath(), copy, + StandardCharsets.UTF_8); + File[] kept = dir.listFiles((d, n) -> n.startsWith("selenium-boot-report-") && n.endsWith(".html")); + if (kept != null && kept.length > keep) { + java.util.Arrays.sort(kept, java.util.Comparator.comparing(File::getName)); + for (int i = 0; i < kept.length - keep; i++) { + if (!kept[i].delete()) kept[i].deleteOnExit(); + } + } + } catch (Exception e) { + System.err.println("[Selenium Boot] Could not keep a copy of the report: " + e.getMessage()); + } + } + + /** + * Where kept reports are, as seen from the main report: a relative path ending in "/" (or + * a file: URI when the two are on different roots), JSON-encoded for the report script. + */ + static String trendReportBase() { + File history = ReportPaths.metricsHistoryDir().getAbsoluteFile(); + File reportDir = ReportPaths.htmlReport().getAbsoluteFile().getParentFile(); + String base; + try { + String rel = reportDir.toPath().normalize().relativize(history.toPath().normalize()) + .toString().replace(File.separatorChar, '/'); + base = rel.isEmpty() ? "" : rel + "/"; + } catch (IllegalArgumentException e) { + base = history.toURI().toString(); + } + return new ObjectMapper().getNodeFactory().textNode(base).toString(); + } + /** * Pass-rate and duration of the last {@code maxRuns} runs from the metrics history, oldest * first, as a JSON array for the trend charts. Empty array when there is no history. @@ -188,6 +257,10 @@ static String buildTrendData(int maxRuns) { run.put("flakyTests", n.path("flakyTests").asInt(0)); long ms = n.path("suiteDurationMs").asLong(0); run.put("durationSec", Math.round((ms > 0 ? ms : n.path("totalTimeMs").asLong(0)) / 100.0) / 10.0); + // Set only when that run's report was kept (flakiness.keepReports), so the + // chart point can open it. + String kept = keptReportName(files[i].getName()); + if (new File(dir, kept).isFile()) run.put("report", kept); } catch (Exception ignored) { // unreadable history file: skip it } @@ -311,6 +384,7 @@ static String buildHtml(JsonNode root) { .replace("{{FLAKINESS_SECTION}}", flakinessSection) .replace("{{DONUT_DATA}}", donutData) .replace("{{TREND_DATA}}", buildTrendData(20)) + .replace("{{TREND_REPORT_BASE}}", trendReportBase()) .replace("{{ROWS}}", rows.toString()) .replace("{{FAILURE_ROWS}}", failureRows) .replace("{{FAILURE_GROUPS}}", failureGroups) diff --git a/src/main/resources/report/report.js b/src/main/resources/report/report.js index 91ea442..f478b97 100644 --- a/src/main/resources/report/report.js +++ b/src/main/resources/report/report.js @@ -105,15 +105,28 @@ if (window.Chart) { // ── Trend over runs ── const trendData = {{TREND_DATA}}; +const trendReportBase = {{TREND_REPORT_BASE}}; if (window.Chart && trendData.length >= 2) { document.getElementById('trendRow').style.display = ''; const labels = trendData.map(r => r.label); + // A point opens that run's report when it was kept (flakiness.keepReports). + const trendRun = els => els.length ? trendData[els[0].index] : null; const line = (el, label, data, color, max) => new Chart(document.getElementById(el), { type: 'line', data: { labels, datasets: [{ label, data, borderColor: color, backgroundColor: color + '33', - fill: true, tension: .3, pointRadius: 3 }] }, + fill: true, tension: .3, pointRadius: trendData.map(r => r.report ? 4 : 3) }] }, options: { responsive: true, maintainAspectRatio: false, - plugins: { legend: { display: false } }, + plugins: { legend: { display: false }, + tooltip: { callbacks: { footer: items => + items.length && trendData[items[0].dataIndex].report ? 'Click to open this run' : '' } } }, + onHover: (e, els) => { + const r = trendRun(els); + e.native.target.style.cursor = r && r.report ? 'pointer' : 'default'; + }, + onClick: (e, els) => { + const r = trendRun(els); + if (r && r.report) window.open(trendReportBase + encodeURIComponent(r.report), '_blank'); + }, scales: { x: { grid: { display: false } }, y: { beginAtZero: true, max } } } }); line('trendRateChart', 'Pass rate (%)', trendData.map(r => r.passRate), color('--good'), 100); diff --git a/src/test/java/com/seleniumboot/reporting/HtmlReportGeneratorTest.java b/src/test/java/com/seleniumboot/reporting/HtmlReportGeneratorTest.java index e3c7dbe..05be02c 100644 --- a/src/test/java/com/seleniumboot/reporting/HtmlReportGeneratorTest.java +++ b/src/test/java/com/seleniumboot/reporting/HtmlReportGeneratorTest.java @@ -813,4 +813,74 @@ public void downloadTest_standaloneExport_isFullySelfContainedAndHasNoExternalFe assertTrue(js.contains("new Blob("), "Export must generate file in-memory using client-side Blob"); assertTrue(js.contains("buildStandaloneTestHtml"), "Export must include standalone HTML builder"); } + + @Test + public void trendData_linksOnlyRunsWhoseReportWasKept() throws Exception { + java.nio.file.Path dir = java.nio.file.Files.createTempDirectory("sb-kept"); + System.setProperty("seleniumboot.history.dir", dir.toString()); + try { + for (String ts : new String[]{"20261110-100000", "20261111-100000"}) { + java.nio.file.Files.writeString(dir.resolve("selenium-boot-metrics-" + ts + ".json"), "{\"passRate\":50.0}"); + } + java.nio.file.Files.writeString(dir.resolve("selenium-boot-report-20261111-100000.html"), ""); + + JsonNode trend = new ObjectMapper().readTree(HtmlReportGenerator.buildTrendData(20)); + assertFalse(trend.get(0).has("report"), "no kept report, no link"); + assertEquals(trend.get(1).get("report").asText(), "selenium-boot-report-20261111-100000.html"); + } finally { + System.clearProperty("seleniumboot.history.dir"); + } + } + + @Test + public void keepReportCopy_writesCopyWithLocalLinks_andPrunesOldest() throws Exception { + java.nio.file.Path dir = java.nio.file.Files.createTempDirectory("sb-keep"); + System.setProperty("seleniumboot.history.dir", dir.toString()); + try { + String html = ""; + HtmlReportGenerator.keepReportCopy(html, 2, "selenium-boot-metrics-20261110-100000.json"); + HtmlReportGenerator.keepReportCopy(html, 2, "selenium-boot-metrics-20261111-100000.json"); + HtmlReportGenerator.keepReportCopy(html, 2, "selenium-boot-metrics-20261112-100000.json"); + + assertFalse(java.nio.file.Files.exists(dir.resolve("selenium-boot-report-20261110-100000.html")), "oldest pruned"); + java.nio.file.Path newest = dir.resolve("selenium-boot-report-20261112-100000.html"); + assertTrue(java.nio.file.Files.exists(dir.resolve("selenium-boot-report-20261111-100000.html"))); + String copy = java.nio.file.Files.readString(newest); + assertTrue(copy.contains("const trendReportBase = \"\";\nconst x = 1;"), "copy links siblings by file name: " + copy); + } finally { + System.clearProperty("seleniumboot.history.dir"); + } + } + + @Test + public void keepReportCopy_offByDefault() throws Exception { + java.nio.file.Path dir = java.nio.file.Files.createTempDirectory("sb-keep-off"); + System.setProperty("seleniumboot.history.dir", dir.toString()); + try { + assertEquals(new SeleniumBootConfig.Flakiness().getKeepReports(), 0); + HtmlReportGenerator.keepReportCopy("", 0, "selenium-boot-metrics-20261110-100000.json"); + HtmlReportGenerator.keepReportCopy("", 5, null); + try (java.util.stream.Stream files = java.nio.file.Files.list(dir)) { + assertEquals(files.count(), 0L); + } + } finally { + System.clearProperty("seleniumboot.history.dir"); + } + } + + @Test + public void trendReportBase_isRelativeToTheReport() throws Exception { + java.nio.file.Path root = java.nio.file.Files.createTempDirectory("sb-base"); + System.setProperty("seleniumboot.reports.dir", root.resolve("target").toString()); + try { + System.setProperty("seleniumboot.history.dir", root.resolve("target/metrics-history").toString()); + assertEquals(HtmlReportGenerator.trendReportBase(), "\"metrics-history/\""); + System.setProperty("seleniumboot.history.dir", root.resolve(".selenium-boot-history").toString()); + assertEquals(HtmlReportGenerator.trendReportBase(), "\"../.selenium-boot-history/\""); + assertTrue(html(1, 0).contains("const trendReportBase = \"../.selenium-boot-history/\";")); + } finally { + System.clearProperty("seleniumboot.history.dir"); + System.clearProperty("seleniumboot.reports.dir"); + } + } }