|
| 1 | +package com.seleniumboot.ai; |
| 2 | + |
| 3 | +import com.seleniumboot.config.SeleniumBootConfig; |
| 4 | +import com.seleniumboot.metrics.ExecutionMetrics; |
| 5 | +import com.sun.net.httpserver.HttpServer; |
| 6 | +import org.testng.annotations.AfterMethod; |
| 7 | +import org.testng.annotations.BeforeMethod; |
| 8 | +import org.testng.annotations.Test; |
| 9 | + |
| 10 | +import java.net.InetSocketAddress; |
| 11 | +import java.nio.charset.StandardCharsets; |
| 12 | +import java.util.concurrent.CountDownLatch; |
| 13 | +import java.util.concurrent.ExecutorService; |
| 14 | +import java.util.concurrent.Executors; |
| 15 | +import java.util.concurrent.TimeUnit; |
| 16 | +import java.util.concurrent.atomic.AtomicInteger; |
| 17 | + |
| 18 | +import static org.testng.Assert.*; |
| 19 | + |
| 20 | +/** One AI call per distinct failure cause, against a local fake provider. */ |
| 21 | +public class AiFailureAnalyzerSameCauseTest { |
| 22 | + |
| 23 | + private HttpServer server; |
| 24 | + private final AtomicInteger calls = new AtomicInteger(); |
| 25 | + private volatile int status; |
| 26 | + private volatile long delayMs; |
| 27 | + |
| 28 | + @BeforeMethod |
| 29 | + public void start() throws Exception { |
| 30 | + calls.set(0); |
| 31 | + status = 200; |
| 32 | + delayMs = 0; |
| 33 | + AiFailureAnalyzer.resetCache(); |
| 34 | + ExecutionMetrics.reset(); |
| 35 | + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); |
| 36 | + server.createContext("/", ex -> { |
| 37 | + calls.incrementAndGet(); |
| 38 | + ex.getRequestBody().readAllBytes(); |
| 39 | + try { Thread.sleep(delayMs); } catch (InterruptedException ignored) { } |
| 40 | + byte[] out = "{\"content\":[{\"type\":\"text\",\"text\":\"Root cause: login API down\"}]}" |
| 41 | + .getBytes(StandardCharsets.UTF_8); |
| 42 | + ex.sendResponseHeaders(status, out.length); |
| 43 | + ex.getResponseBody().write(out); |
| 44 | + ex.close(); |
| 45 | + }); |
| 46 | + server.start(); |
| 47 | + } |
| 48 | + |
| 49 | + @AfterMethod |
| 50 | + public void stop() { |
| 51 | + server.stop(0); |
| 52 | + AiFailureAnalyzer.resetCache(); |
| 53 | + ExecutionMetrics.reset(); |
| 54 | + } |
| 55 | + |
| 56 | + private SeleniumBootConfig.Ai cfg() { |
| 57 | + SeleniumBootConfig.Ai ai = new SeleniumBootConfig.Ai(); |
| 58 | + ai.setFailureAnalysis(true); |
| 59 | + ai.setApiKey("k"); |
| 60 | + ai.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/api"); |
| 61 | + ai.setTimeoutSeconds(5); |
| 62 | + return ai; |
| 63 | + } |
| 64 | + |
| 65 | + private static void failed(String id, String message) { |
| 66 | + ExecutionMetrics.recordBrowser(id, "chrome"); |
| 67 | + ExecutionMetrics.recordError(id, new AssertionError(message)); |
| 68 | + } |
| 69 | + |
| 70 | + @Test |
| 71 | + public void sameCauseFailures_shareOneCall_andEachGetTheAnalysis() { |
| 72 | + failed("a.One.run", "login API returned 500"); |
| 73 | + failed("a.Two.run", "login API returned 503"); |
| 74 | + failed("a.Three.run", "login API returned 502"); |
| 75 | + |
| 76 | + for (String id : new String[]{"a.One.run", "a.Two.run", "a.Three.run"}) { |
| 77 | + AiFailureAnalyzer.analyze(cfg(), id, null, null); |
| 78 | + } |
| 79 | + |
| 80 | + assertEquals(calls.get(), 1, "numbers normalise, so one cause costs one call"); |
| 81 | + for (String id : new String[]{"a.One.run", "a.Two.run", "a.Three.run"}) { |
| 82 | + assertEquals(ExecutionMetrics.getTiming(id).getAiAnalysis(), "Root cause: login API down", id); |
| 83 | + } |
| 84 | + } |
| 85 | + |
| 86 | + @Test |
| 87 | + public void differentCauses_getTheirOwnCall() { |
| 88 | + failed("a.One.run", "login API returned 500"); |
| 89 | + failed("a.Other.run", "cart badge missing"); |
| 90 | + |
| 91 | + AiFailureAnalyzer.analyze(cfg(), "a.One.run", null, null); |
| 92 | + AiFailureAnalyzer.analyze(cfg(), "a.Other.run", null, null); |
| 93 | + |
| 94 | + assertEquals(calls.get(), 2); |
| 95 | + } |
| 96 | + |
| 97 | + @Test |
| 98 | + public void concurrentSameCauseFailures_waitForTheFirstCall() throws Exception { |
| 99 | + delayMs = 300; |
| 100 | + int n = 4; |
| 101 | + for (int i = 0; i < n; i++) failed("a.T" + i + ".run", "login API returned 500"); |
| 102 | + |
| 103 | + ExecutorService pool = Executors.newFixedThreadPool(n); |
| 104 | + CountDownLatch go = new CountDownLatch(1); |
| 105 | + for (int i = 0; i < n; i++) { |
| 106 | + String id = "a.T" + i + ".run"; |
| 107 | + pool.submit(() -> { |
| 108 | + go.await(); |
| 109 | + AiFailureAnalyzer.analyze(cfg(), id, null, null); |
| 110 | + return null; |
| 111 | + }); |
| 112 | + } |
| 113 | + go.countDown(); |
| 114 | + pool.shutdown(); |
| 115 | + assertTrue(pool.awaitTermination(15, TimeUnit.SECONDS)); |
| 116 | + |
| 117 | + assertEquals(calls.get(), 1, "concurrent same-cause failures share the in-flight call"); |
| 118 | + for (int i = 0; i < n; i++) { |
| 119 | + assertNotNull(ExecutionMetrics.getTiming("a.T" + i + ".run").getAiAnalysis()); |
| 120 | + } |
| 121 | + } |
| 122 | + |
| 123 | + @Test |
| 124 | + public void failedCall_isNotCached_soTheNextSameCauseFailureRetries() { |
| 125 | + failed("a.One.run", "login API returned 500"); |
| 126 | + failed("a.Two.run", "login API returned 500"); |
| 127 | + |
| 128 | + status = 500; |
| 129 | + AiFailureAnalyzer.analyze(cfg(), "a.One.run", null, null); |
| 130 | + assertNull(ExecutionMetrics.getTiming("a.One.run").getAiAnalysis()); |
| 131 | + |
| 132 | + status = 200; |
| 133 | + AiFailureAnalyzer.analyze(cfg(), "a.Two.run", null, null); |
| 134 | + |
| 135 | + assertEquals(calls.get(), 2, "an error must not poison the cause"); |
| 136 | + assertNotNull(ExecutionMetrics.getTiming("a.Two.run").getAiAnalysis()); |
| 137 | + } |
| 138 | +} |
0 commit comments