Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ docs/*
# Metric documents are thesis-canonical and belong in the deliverable. Specs, ADRs
# and internal notes stay local, which the docs/* rule above already handles.
!docs/metrics/
# The metric-adding procedure is referenced by the thesis and by future contributors,
# not an internal note, so it ships like docs/metrics/ rather than staying local.
!docs/adding-a-metric.md
.claude/
.agents/
.idea
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.juliashtal.devanalytics.ai.client;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
Expand All @@ -24,14 +25,17 @@ public class OllamaLlmClient implements LlmClient {
private final RestTemplate restTemplate;

private final int seed;
private final String keepAlive;

public OllamaLlmClient(
@Value("${ai.ollama.base-url:http://localhost:11434}") String baseUrl,
@Value("${ai.ollama.num-predict:1024}") int numPredict,
@Value("${ai.ollama.seed:42}") int seed) {
@Value("${ai.ollama.seed:42}") int seed,
@Value("${ai.ollama.keep-alive:0}") String keepAlive) {
this.baseUrl = baseUrl;
this.numPredict = numPredict;
this.seed = seed;
this.keepAlive = keepAlive;
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
factory.setConnectTimeout(5_000);
factory.setReadTimeout(300_000); // 5 minutes — LLM inference can be slow
Expand All @@ -47,6 +51,7 @@ public String complete(String model, String systemPrompt, String userPrompt, boo
req.setStream(false);
if (jsonMode) req.setFormat("json");
req.setOptions(Map.of("num_predict", numPredict, "temperature", 0.0, "seed", seed));
req.setKeepAlive(keepAlive);

log.debug("Sending request to Ollama: model={}, promptLength={}, numPredict={}, seed={}", model, userPrompt.length(), numPredict, seed);
try {
Expand Down Expand Up @@ -76,6 +81,9 @@ static class OllamaRequest {
private boolean stream;
private String format;
private Map<String, Object> options;
/** How long Ollama keeps the model resident after this request; {@code "0"} unloads it immediately. */
@JsonProperty("keep_alive")
private String keepAlive;
}

@Data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.LinkedHashMap;
Expand Down Expand Up @@ -64,6 +65,27 @@ public enum Reduction {
REDUCTIONS = Map.copyOf(m);
}

/**
* Fails application startup when a {@code MetricType.aggregatePeriod} metric has no
* {@link Reduction}, naming the type and this class so the gap is fixed before it can hide
* behind a silent default (see {@link #reductionOrThrow}).
*/
public AggregateWindowResolver() {
validateAggregatePeriodCoverage(REDUCTIONS);
}

static void validateAggregatePeriodCoverage(Map<MetricType, Reduction> reductions) {
List<MetricType> missing = Arrays.stream(MetricType.values())
.filter(t -> t.aggregatePeriod)
.filter(t -> !reductions.containsKey(t))
.toList();
if (!missing.isEmpty()) {
throw new IllegalStateException(
"AggregateWindowResolver.REDUCTIONS has no Reduction for " + missing
+ " — every MetricType.aggregatePeriod metric must be added there.");
}
}

/** One stored window and the value covering it, after cross-repository rows are combined. */
public record WindowValue(LocalDate periodFrom, LocalDate periodTo, double value) {}

Expand Down Expand Up @@ -100,8 +122,6 @@ public boolean isCount(MetricType type) {
* Used where the read side wants a series rather than a single figure.
*/
public List<WindowValue> perWindow(List<MetricSnapshot> aggregateRows, MetricType type) {
Reduction reduction = REDUCTIONS.getOrDefault(type, Reduction.MEDIAN);

Map<List<LocalDate>, List<Double>> byWindow = new LinkedHashMap<>();
aggregateRows.stream()
.sorted(Comparator.comparing(MetricSnapshot::getPeriodFrom)
Expand All @@ -110,6 +130,14 @@ public List<WindowValue> perWindow(List<MetricSnapshot> aggregateRows, MetricTyp
.computeIfAbsent(List.of(s.getPeriodFrom(), s.getPeriodTo()), k -> new ArrayList<>())
.add(s.getValue()));

// Nothing to combine: the reduction lookup is skipped so a type with no rows never
// trips the missing-reduction guard below, regardless of whether it is declared.
if (byWindow.isEmpty()) {
return List.of();
}

Reduction reduction = reductionOrThrow(type);

// Rows sharing a window differ only by repository, so WIDEST_WINDOW can still median them.
Reduction crossRepo = reduction == Reduction.WIDEST_WINDOW ? Reduction.MEDIAN : reduction;

Expand All @@ -127,7 +155,7 @@ public Optional<ResolvedAggregate> resolve(List<MetricSnapshot> aggregateRows, M
List<WindowValue> windows = perWindow(aggregateRows, type);
if (windows.isEmpty()) return Optional.empty();

Reduction reduction = REDUCTIONS.getOrDefault(type, Reduction.MEDIAN);
Reduction reduction = reductionOrThrow(type);

if (reduction == Reduction.WIDEST_WINDOW) {
WindowValue widest = windows.stream()
Expand All @@ -144,6 +172,21 @@ public Optional<ResolvedAggregate> resolve(List<MetricSnapshot> aggregateRows, M
return Optional.of(new ResolvedAggregate(value, from, to));
}

/**
* The declared {@link Reduction}, or a loud failure instead of the median it used to fall
* back to silently — a type reaching here with rows to combine but no entry is exactly the
* gap a new calculator can open by writing a period without one.
*/
private Reduction reductionOrThrow(MetricType type) {
Reduction reduction = REDUCTIONS.get(type);
if (reduction == null) {
throw new IllegalStateException(
"No Reduction declared for " + type + " in AggregateWindowResolver.REDUCTIONS "
+ "— add one before rows for this type can be combined across windows.");
}
return reduction;
}

private double combine(List<Double> values, Reduction reduction) {
return switch (reduction) {
case SUM -> values.stream().mapToDouble(Double::doubleValue).sum();
Expand Down
1 change: 1 addition & 0 deletions dev-analytics/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ ai:
model: ${OLLAMA_MODEL:llama3.2}
num-predict: ${OLLAMA_NUM_PREDICT:1024}
seed: ${OLLAMA_SEED:42}
keep-alive: ${OLLAMA_KEEP_ALIVE:0}

management:
endpoints:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
package com.juliashtal.devanalytics.ai.client;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
import com.github.tomakehurst.wiremock.junit5.WireMockTest;
import com.github.tomakehurst.wiremock.verification.LoggedRequest;
import org.junit.jupiter.api.Test;
import org.springframework.web.server.ResponseStatusException;

import java.util.List;

import static com.github.tomakehurst.wiremock.client.WireMock.*;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE;

/**
* The summarization path's only HTTP boundary: what is sent to Ollama, and how a failure or an
* unusable reply is turned into a {@link ResponseStatusException} the caller can show the user.
*/
@WireMockTest
class OllamaLlmClientTest {

@Test
void complete_success_returnsTheResponseText(WireMockRuntimeInfo wm) {
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(okJson("{\"model\":\"llama3.2\",\"response\":\"hello there\"}")));

OllamaLlmClient client = client(wm);
String result = client.complete("llama3.2", "system", "user prompt", false);

assertThat(result).isEqualTo("hello there");
}

@Test
void complete_requestBody_carriesModelPromptsAndOptions(WireMockRuntimeInfo wm) throws Exception {
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(okJson("{\"model\":\"llama3.2\",\"response\":\"ok\"}")));

client(wm, 256, 7).complete("llama3.2", "be terse", "summarize this", false);

JsonNode body = sentBody();
assertThat(body.get("model").asText()).isEqualTo("llama3.2");
assertThat(body.get("system").asText()).isEqualTo("be terse");
assertThat(body.get("prompt").asText()).isEqualTo("summarize this");
assertThat(body.get("stream").asBoolean()).isFalse();
assertThat(body.get("options").get("num_predict").asInt()).isEqualTo(256);
assertThat(body.get("options").get("temperature").asDouble()).isEqualTo(0.0);
assertThat(body.get("options").get("seed").asInt()).isEqualTo(7);
}

@Test
void complete_requestBody_carriesTheKeepAliveProperty(WireMockRuntimeInfo wm) throws Exception {
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(okJson("{\"model\":\"llama3.2\",\"response\":\"ok\"}")));

new OllamaLlmClient(wm.getHttpBaseUrl(), 1024, 42, "0")
.complete("llama3.2", "system", "prompt", false);

assertThat(sentBody().get("keep_alive").asText()).isEqualTo("0");
}

@Test
void complete_jsonModeTrue_setsFormatJson(WireMockRuntimeInfo wm) throws Exception {
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(okJson("{\"model\":\"llama3.2\",\"response\":\"{}\"}")));

client(wm).complete("llama3.2", "system", "prompt", true);

assertThat(sentBody().get("format").asText()).isEqualTo("json");
}

@Test
void complete_jsonModeFalse_omitsFormatField(WireMockRuntimeInfo wm) throws Exception {
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(okJson("{\"model\":\"llama3.2\",\"response\":\"plain text\"}")));

client(wm).complete("llama3.2", "system", "prompt", false);

assertThat(sentBody().has("format")).isFalse();
}

@Test
void complete_blankResponseField_throwsServiceUnavailable(WireMockRuntimeInfo wm) {
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(okJson("{\"model\":\"llama3.2\",\"response\":\"\"}")));

assertThatThrownBy(() -> client(wm).complete("llama3.2", "system", "prompt", false))
.isInstanceOf(ResponseStatusException.class)
.hasMessageContaining("Empty response")
.hasMessageContaining("llama3.2")
.extracting(e -> ((ResponseStatusException) e).getStatusCode())
.isEqualTo(SERVICE_UNAVAILABLE);
}

@Test
void complete_missingResponseField_throwsServiceUnavailable(WireMockRuntimeInfo wm) {
// No "response" key at all — the field binds to null, same guard as a blank string.
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(okJson("{\"model\":\"llama3.2\"}")));

assertThatThrownBy(() -> client(wm).complete("llama3.2", "system", "prompt", false))
.isInstanceOf(ResponseStatusException.class)
.extracting(e -> ((ResponseStatusException) e).getStatusCode())
.isEqualTo(SERVICE_UNAVAILABLE);
}

@Test
void complete_serverError_wrapsAsServiceUnavailableNamingTheBaseUrl(WireMockRuntimeInfo wm) {
stubFor(post(urlEqualTo("/api/generate")).willReturn(aResponse().withStatus(500)));

assertThatThrownBy(() -> client(wm).complete("llama3.2", "system", "prompt", false))
.isInstanceOf(ResponseStatusException.class)
.hasMessageContaining(wm.getHttpBaseUrl())
.extracting(e -> ((ResponseStatusException) e).getStatusCode())
.isEqualTo(SERVICE_UNAVAILABLE);
}

@Test
void complete_unreachableHost_wrapsAsServiceUnavailable() {
// Exercises the same catch-all path a real read timeout would: nothing answers, the
// client cannot distinguish "slow" from "down", and both fail the same way.
OllamaLlmClient client = new OllamaLlmClient("http://127.0.0.1:1", 1024, 42, "0");

assertThatThrownBy(() -> client.complete("llama3.2", "system", "prompt", false))
.isInstanceOf(ResponseStatusException.class)
.hasMessageContaining("AI model is unavailable")
.extracting(e -> ((ResponseStatusException) e).getStatusCode())
.isEqualTo(SERVICE_UNAVAILABLE);
}

@Test
void complete_invalidJsonResponse_wrapsAsServiceUnavailable(WireMockRuntimeInfo wm) {
stubFor(post(urlEqualTo("/api/generate"))
.willReturn(aResponse().withStatus(200)
.withHeader("Content-Type", "application/json")
.withBody("not json")));

assertThatThrownBy(() -> client(wm).complete("llama3.2", "system", "prompt", false))
.isInstanceOf(ResponseStatusException.class)
.extracting(e -> ((ResponseStatusException) e).getStatusCode())
.isEqualTo(SERVICE_UNAVAILABLE);
}

private static JsonNode sentBody() throws Exception {
List<LoggedRequest> requests = findAll(postRequestedFor(urlEqualTo("/api/generate")));
return new ObjectMapper().readTree(requests.get(requests.size() - 1).getBodyAsString());
}

private static OllamaLlmClient client(WireMockRuntimeInfo wm) {
return client(wm, 1024, 42);
}

private static OllamaLlmClient client(WireMockRuntimeInfo wm, int numPredict, int seed) {
return new OllamaLlmClient(wm.getHttpBaseUrl(), numPredict, seed, "0");
}
}
Loading
Loading