Skip to content

Commit 2bdaa68

Browse files
shossain786claude
andcommitted
test(ai): cover one-call-per-cause path in AiFailureAnalyzer
Adds an analyze(cfg, ...) test seam and tests against a local fake provider: same-cause sharing, distinct causes, concurrent waiters, and failed calls not cached. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 63b2c01 commit 2bdaa68

2 files changed

Lines changed: 143 additions & 1 deletion

File tree

‎src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,12 @@ static void resetCache() {
7373
* @param pageTitle current page title (may be null)
7474
*/
7575
public static void analyze(String testId, String pageUrl, String pageTitle) {
76+
analyze(config(), testId, pageUrl, pageTitle);
77+
}
78+
79+
/** Test seam: same as {@link #analyze(String, String, String)} with the AI config supplied. */
80+
static void analyze(SeleniumBootConfig.Ai aiCfg, String testId, String pageUrl, String pageTitle) {
7681
try {
77-
SeleniumBootConfig.Ai aiCfg = config();
7882
if (aiCfg == null || !aiCfg.isFailureAnalysis()) return;
7983

8084
String apiKey = resolveApiKey(aiCfg.getApiKey());
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package com.seleniumboot.ai;
2+
3+
import com.seleniumboot.config.SeleniumBootConfig;
4+
import com.seleniumboot.metrics.ExecutionMetrics;
5+
import com.sun.net.httpserver.HttpServer;
6+
import org.testng.annotations.AfterMethod;
7+
import org.testng.annotations.BeforeMethod;
8+
import org.testng.annotations.Test;
9+
10+
import java.net.InetSocketAddress;
11+
import java.nio.charset.StandardCharsets;
12+
import java.util.concurrent.CountDownLatch;
13+
import java.util.concurrent.ExecutorService;
14+
import java.util.concurrent.Executors;
15+
import java.util.concurrent.TimeUnit;
16+
import java.util.concurrent.atomic.AtomicInteger;
17+
18+
import static org.testng.Assert.*;
19+
20+
/** One AI call per distinct failure cause, against a local fake provider. */
21+
public class AiFailureAnalyzerSameCauseTest {
22+
23+
private HttpServer server;
24+
private final AtomicInteger calls = new AtomicInteger();
25+
private volatile int status;
26+
private volatile long delayMs;
27+
28+
@BeforeMethod
29+
public void start() throws Exception {
30+
calls.set(0);
31+
status = 200;
32+
delayMs = 0;
33+
AiFailureAnalyzer.resetCache();
34+
ExecutionMetrics.reset();
35+
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
36+
server.createContext("/", ex -> {
37+
calls.incrementAndGet();
38+
ex.getRequestBody().readAllBytes();
39+
try { Thread.sleep(delayMs); } catch (InterruptedException ignored) { }
40+
byte[] out = "{\"content\":[{\"type\":\"text\",\"text\":\"Root cause: login API down\"}]}"
41+
.getBytes(StandardCharsets.UTF_8);
42+
ex.sendResponseHeaders(status, out.length);
43+
ex.getResponseBody().write(out);
44+
ex.close();
45+
});
46+
server.start();
47+
}
48+
49+
@AfterMethod
50+
public void stop() {
51+
server.stop(0);
52+
AiFailureAnalyzer.resetCache();
53+
ExecutionMetrics.reset();
54+
}
55+
56+
private SeleniumBootConfig.Ai cfg() {
57+
SeleniumBootConfig.Ai ai = new SeleniumBootConfig.Ai();
58+
ai.setFailureAnalysis(true);
59+
ai.setApiKey("k");
60+
ai.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/api");
61+
ai.setTimeoutSeconds(5);
62+
return ai;
63+
}
64+
65+
private static void failed(String id, String message) {
66+
ExecutionMetrics.recordBrowser(id, "chrome");
67+
ExecutionMetrics.recordError(id, new AssertionError(message));
68+
}
69+
70+
@Test
71+
public void sameCauseFailures_shareOneCall_andEachGetTheAnalysis() {
72+
failed("a.One.run", "login API returned 500");
73+
failed("a.Two.run", "login API returned 503");
74+
failed("a.Three.run", "login API returned 502");
75+
76+
for (String id : new String[]{"a.One.run", "a.Two.run", "a.Three.run"}) {
77+
AiFailureAnalyzer.analyze(cfg(), id, null, null);
78+
}
79+
80+
assertEquals(calls.get(), 1, "numbers normalise, so one cause costs one call");
81+
for (String id : new String[]{"a.One.run", "a.Two.run", "a.Three.run"}) {
82+
assertEquals(ExecutionMetrics.getTiming(id).getAiAnalysis(), "Root cause: login API down", id);
83+
}
84+
}
85+
86+
@Test
87+
public void differentCauses_getTheirOwnCall() {
88+
failed("a.One.run", "login API returned 500");
89+
failed("a.Other.run", "cart badge missing");
90+
91+
AiFailureAnalyzer.analyze(cfg(), "a.One.run", null, null);
92+
AiFailureAnalyzer.analyze(cfg(), "a.Other.run", null, null);
93+
94+
assertEquals(calls.get(), 2);
95+
}
96+
97+
@Test
98+
public void concurrentSameCauseFailures_waitForTheFirstCall() throws Exception {
99+
delayMs = 300;
100+
int n = 4;
101+
for (int i = 0; i < n; i++) failed("a.T" + i + ".run", "login API returned 500");
102+
103+
ExecutorService pool = Executors.newFixedThreadPool(n);
104+
CountDownLatch go = new CountDownLatch(1);
105+
for (int i = 0; i < n; i++) {
106+
String id = "a.T" + i + ".run";
107+
pool.submit(() -> {
108+
go.await();
109+
AiFailureAnalyzer.analyze(cfg(), id, null, null);
110+
return null;
111+
});
112+
}
113+
go.countDown();
114+
pool.shutdown();
115+
assertTrue(pool.awaitTermination(15, TimeUnit.SECONDS));
116+
117+
assertEquals(calls.get(), 1, "concurrent same-cause failures share the in-flight call");
118+
for (int i = 0; i < n; i++) {
119+
assertNotNull(ExecutionMetrics.getTiming("a.T" + i + ".run").getAiAnalysis());
120+
}
121+
}
122+
123+
@Test
124+
public void failedCall_isNotCached_soTheNextSameCauseFailureRetries() {
125+
failed("a.One.run", "login API returned 500");
126+
failed("a.Two.run", "login API returned 500");
127+
128+
status = 500;
129+
AiFailureAnalyzer.analyze(cfg(), "a.One.run", null, null);
130+
assertNull(ExecutionMetrics.getTiming("a.One.run").getAiAnalysis());
131+
132+
status = 200;
133+
AiFailureAnalyzer.analyze(cfg(), "a.Two.run", null, null);
134+
135+
assertEquals(calls.get(), 2, "an error must not poison the cause");
136+
assertNotNull(ExecutionMetrics.getTiming("a.Two.run").getAiAnalysis());
137+
}
138+
}

0 commit comments

Comments
 (0)