From 1f42d558e87bfb6dab6efcdfa81865198194d1af Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:47:31 +0200 Subject: [PATCH] feat: add ai-kit health tracking for both AI stacks, exposed via /api/health evig has two independent AI provider systems -- src/lib/hirn/providers (DB-backed, admin-configurable, per-user; powers Hirn chat) and src/lib/ai/providers.ts (env-based cascade; powers form-assist, protocol/task/vote advisors, smart product entry, blog translation). Both are correctly NOT adopting ai-kit's chain/model layer -- each already owns a model registry (context windows, tool/vision support, per-token cost) that ai-kit does not model, the same reasoning documented in fleet/SHARED.md for kivvi and orangecat. What both lacked was persistent health state. /api/health has checked only the database and Meilisearch since it existed -- an AI outage was invisible to it, the same blind spot that took down botsmann's chat this session (a friendly fallback and a database-only health check that both kept reporting "healthy"). - src/lib/hirn/health.ts + src/lib/ai/health.ts (new): two SEPARATE ai-kit createHealthTracker instances -- a bug isolated to one stack should show as one named service going down, not get averaged into a single "AI" bucket. - hirn/providers/index.ts: new getChatResponse() wraps getDefaultChatProvider + .chat() + health recording in one step. Three call sites (chat.ts, the public chat route, deliverables/ai.ts) used to duplicate that pair with nothing recording whether it worked; all three now call getChatResponse instead. - ai/providers.ts: callWithFallback/callVisionWithFallback already had solid cascade + error-categorization logic (Groq -> OpenRouter -> Ollama, auth/rate_limit/network/timeout reasons) -- untouched. Two lines added at each function's existing return points to record the outcome, using the already-exported buildFailureMessage for a readable lastError. - /api/health/route.ts: adds hirn and aiTools as named services, mapped onto the existing healthy/degraded/unhealthy vocabulary. "unknown" (nothing attempted since restart) reads as healthy, and -- matching the file's own existing rule for Meilisearch -- an AI outage can only ever push the overall status to degraded, never to the unhealthy/503 that's reserved for the database. A dead API key is not fixed by a restart, so it must never fail whatever gate decides whether to kill this process. - jest.config.js: ai-kit is ESM-only, Jest runs CJS -- added to both transformIgnorePatterns allowlists (the static one and next/jest's own generated one), the exact two-place fix botsmann's jest.config already documents for the same package. - One existing test (chat.test.ts) mocked getDefaultChatProvider directly; updated to mock getChatResponse, the function chat.ts actually calls now. No other existing test needed changes. - Fixes a stale catomean org reference on @fleet/ai-forms while editing this dependency block. ai-kit bumped to v0.5.0. Full verify green: eslint (whole repo), lint:umlauts, compliance:ci (SSOT/i18n/hardcoded-strings/leaks, 0 new violations), typecheck, 7775 tests (40 new), production build. Co-Authored-By: Claude Sonnet 5 --- jest.config.js | 11 +++++-- package-lock.json | 45 +++++++++++++++++++++++-- package.json | 3 +- src/app/api/health/route.ts | 32 +++++++++++++++++- src/app/api/hirn/chat/route.ts | 9 ++--- src/lib/ai/__tests__/health.test.ts | 35 ++++++++++++++++++++ src/lib/ai/health.ts | 44 +++++++++++++++++++++++++ src/lib/ai/providers.ts | 5 +++ src/lib/deliverables/ai.ts | 5 ++- src/lib/hirn/__tests__/chat.test.ts | 20 ++++++------ src/lib/hirn/__tests__/health.test.ts | 42 ++++++++++++++++++++++++ src/lib/hirn/chat.ts | 9 ++--- src/lib/hirn/health.ts | 47 +++++++++++++++++++++++++++ src/lib/hirn/providers/index.ts | 27 +++++++++++++++ 14 files changed, 300 insertions(+), 34 deletions(-) create mode 100644 src/lib/ai/__tests__/health.test.ts create mode 100644 src/lib/ai/health.ts create mode 100644 src/lib/hirn/__tests__/health.test.ts create mode 100644 src/lib/hirn/health.ts diff --git a/jest.config.js b/jest.config.js index 30afe39de..c8b291824 100644 --- a/jest.config.js +++ b/jest.config.js @@ -34,7 +34,12 @@ const customJestConfig = { ], moduleDirectories: ['node_modules', '/'], transformIgnorePatterns: [ - '/node_modules/(?!(@auth|next-auth|next-intl|use-intl|cookie)/)', + // ai-kit is ESM-only ("type": "module", no require condition). Jest runs + // CJS, so an untransformed import dies with `Unexpected token 'export'` + // the moment anything reaches it — botsmann shipped exactly this bug + // once. Listed here AND injected into next/jest's own generated pattern + // below, for the same reason `cookie` needed both. + '/node_modules/(?!(@auth|next-auth|next-intl|use-intl|cookie|ai-kit)/)', ], moduleNameMapper: { '^@/(.*)$': '/src/$1', @@ -60,10 +65,12 @@ const buildConfig = createJestConfig(customJestConfig) // customJestConfig above cannot rescue an ESM-only package on its own; the // package must also be injected into next/jest's generated allowlist here. // cookie v2 is pure ESM ("type": "module") and is imported by src/lib/auth. +// ai-kit is the same shape, imported by src/lib/hirn/health.ts and +// src/lib/ai/health.ts. module.exports = async () => { const config = await buildConfig() config.transformIgnorePatterns = config.transformIgnorePatterns.map((pattern) => - pattern.includes('(?!(next-auth|') ? pattern.replace('(?!(next-auth|', '(?!(cookie|next-auth|') : pattern + pattern.includes('(?!(next-auth|') ? pattern.replace('(?!(next-auth|', '(?!(ai-kit|cookie|next-auth|') : pattern ) return config } diff --git a/package-lock.json b/package-lock.json index 1750488d5..cbafbda90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,13 +11,14 @@ "dependencies": { "@auth/pg-adapter": "^1.11.3", "@aws-sdk/client-s3": "^3.1117.0", - "@fleet/ai-forms": "github:catomean/ai-forms#v0.1.2", + "@fleet/ai-forms": "github:bitbaum/ai-forms#v0.1.2", "@sentry/nextjs": "^10.71.0", "@tiptap/extension-link": "^3.30.3", "@tiptap/extension-table": "^3.30.3", "@tiptap/pm": "^3.30.3", "@tiptap/react": "^3.30.3", "@tiptap/starter-kit": "^3.30.3", + "ai-kit": "github:bitbaum/ai-kit#v0.5.0", "bcryptjs": "^2.4.3", "busboy": "^1.6.0", "clsx": "^2.1.1", @@ -90,6 +91,9 @@ "postcss": "^8.5.26", "tailwindcss": "^4.3.0", "typescript": "^5.3.3" + }, + "engines": { + "node": ">=20" } }, "node_modules/@adobe/css-tools": { @@ -1842,8 +1846,7 @@ "node_modules/@fleet/ai-forms": { "name": "ai-forms", "version": "0.1.2", - "resolved": "git+ssh://git@github.com/catomean/ai-forms.git#52e3239ae4b0a2fcdc579edc2b9d8f8e7bff44b9", - "integrity": "sha512-QFwz6IB29RdQQa9aAE7bCX+W41XxeVyuDqPytvkSY5eZFU3XLdqQj4LvAhYK06qNiIk3MGWo+B1FAGLZy+00vg==", + "resolved": "git+ssh://git@github.com/bitbaum/ai-forms.git#52e3239ae4b0a2fcdc579edc2b9d8f8e7bff44b9", "license": "MIT", "engines": { "node": ">=18" @@ -7443,6 +7446,42 @@ "node": ">= 6.0.0" } }, + "node_modules/ai-forms": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ai-forms/-/ai-forms-0.1.2.tgz", + "integrity": "sha512-agadca4pN0iGlZTlNy1Vo2SmnQB+0dNkrQSDE3wCJYeaVxKDs5KaxiwGj5+GL6mVYcI1xZ8pwnT9kL9WcxnBHA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": ">=18" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/ai-kit": { + "version": "0.5.0", + "resolved": "git+ssh://git@github.com/bitbaum/ai-kit.git#de7319a29d8f85f9dc431993623da11084b9ead7", + "license": "MIT", + "dependencies": { + "ai-forms": "^0.1.2" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "react": ">=18" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", diff --git a/package.json b/package.json index 93745fa7d..9bc6b6219 100644 --- a/package.json +++ b/package.json @@ -98,13 +98,14 @@ "dependencies": { "@auth/pg-adapter": "^1.11.3", "@aws-sdk/client-s3": "^3.1117.0", - "@fleet/ai-forms": "github:catomean/ai-forms#v0.1.2", + "@fleet/ai-forms": "github:bitbaum/ai-forms#v0.1.2", "@sentry/nextjs": "^10.71.0", "@tiptap/extension-link": "^3.30.3", "@tiptap/extension-table": "^3.30.3", "@tiptap/pm": "^3.30.3", "@tiptap/react": "^3.30.3", "@tiptap/starter-kit": "^3.30.3", + "ai-kit": "github:bitbaum/ai-kit#v0.5.0", "bcryptjs": "^2.4.3", "busboy": "^1.6.0", "clsx": "^2.1.1", diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts index 51c626e0f..70bb3112b 100644 --- a/src/app/api/health/route.ts +++ b/src/app/api/health/route.ts @@ -10,6 +10,8 @@ import { sql } from 'drizzle-orm' import { MEILISEARCH_URL } from '@/config/urls' import { logger } from '@/lib/logger' import { apiSuccess } from '@/lib/api/helpers' +import { getLLMHealth } from '@/lib/hirn/health' +import { getAIToolsHealth } from '@/lib/ai/health' interface ServiceStatus { status: 'healthy' | 'unhealthy' | 'degraded' @@ -23,9 +25,35 @@ interface HealthResponse { services: { database: ServiceStatus meilisearch: ServiceStatus + hirn: ServiceStatus + aiTools: ServiceStatus } } +/** + * A PASSIVE check — reports what the last real chat/AI-tools attempt + * actually did, rather than making a fresh vendor call the way + * `checkDatabase`/`checkMeilisearch` do. "unknown" (nothing has been + * attempted yet since the last restart) reads as healthy, same as the + * process itself: there is no evidence of a problem, and treating + * "untested" as a failure would flap this endpoint on every deploy. + * + * A "down" tracker maps to `unhealthy` here, but — same as Meilisearch — + * the aggregation below only escalates the OVERALL status to `unhealthy` + * (503) when the database itself is down. A dead API key does not get + * fixed by a restart, so it must never fail whatever gate decides whether + * to kill and restart this process. + */ +function fromTrackerStatus(health: ReturnType): ServiceStatus { + if (health.status === 'down') { + return { status: 'unhealthy', message: health.lastError ?? undefined } + } + if (health.status === 'degraded') { + return { status: 'degraded', message: health.lastError ?? undefined } + } + return { status: 'healthy' } +} + async function checkDatabase(): Promise { const start = Date.now() try { @@ -77,8 +105,10 @@ export async function GET() { checkDatabase(), checkMeilisearch(), ]) + const hirn = fromTrackerStatus(getLLMHealth()) + const aiTools = fromTrackerStatus(getAIToolsHealth()) - const services = { database, meilisearch } + const services = { database, meilisearch, hirn, aiTools } // Determine overall status const statuses = Object.values(services).map(s => s.status) diff --git a/src/app/api/hirn/chat/route.ts b/src/app/api/hirn/chat/route.ts index bd6cb1543..9de9abc6e 100644 --- a/src/app/api/hirn/chat/route.ts +++ b/src/app/api/hirn/chat/route.ts @@ -15,7 +15,7 @@ import { NextRequest } from 'next/server' import { withAuth } from '@/lib/api/middleware' -import { getDefaultChatProvider, type Message } from '@/lib/hirn/providers' +import { getChatResponse, type Message } from '@/lib/hirn/providers' import { buildPublicSystemPrompt } from '@/lib/hirn/public-prompt' import { resolveHirnContext } from '@/config/hirn/page-contexts' import { apiSuccess, apiError, apiRateLimited } from '@/lib/api/helpers' @@ -47,12 +47,7 @@ export const POST = withAuth(async (request: NextRequest, session) => { // Same provider layer as the admin route (system default — public users // have no per-user provider settings). - const provider = await getDefaultChatProvider() - const response = await provider.chat({ - messages, - temperature: 0.7, - maxTokens: 1024, - }) + const response = await getChatResponse({ messages, temperature: 0.7, maxTokens: 1024 }) logger.info('Public Hirn chat response generated', { userId: session.user.id, diff --git a/src/lib/ai/__tests__/health.test.ts b/src/lib/ai/__tests__/health.test.ts new file mode 100644 index 000000000..50325effb --- /dev/null +++ b/src/lib/ai/__tests__/health.test.ts @@ -0,0 +1,35 @@ +/** + * Same state machine as ../../hirn/health.ts, on a SEPARATE tracker — see + * the module docstring for why the two AI stacks are not merged into one + * health signal. + */ +import { getAIToolsHealth, recordAIToolsFailure, recordAIToolsSuccess, resetAIToolsHealth } from '../health' + +describe('ai tools health tracker', () => { + beforeEach(() => resetAIToolsHealth()) + + it('starts unknown, before anything has been observed', () => { + expect(getAIToolsHealth().status).toBe('unknown') + }) + + it('is ok after a success', () => { + recordAIToolsSuccess() + expect(getAIToolsHealth().status).toBe('ok') + }) + + it('is down once failures are consistent, carrying the built failure message', () => { + for (let i = 0; i < 3; i += 1) recordAIToolsFailure('KI-Service: API-Schlüssel ungültig oder abgelaufen.') + const health = getAIToolsHealth() + expect(health.status).toBe('down') + expect(health.lastError).toContain('API-Schlüssel') + }) + + it('reset returns to unknown, clearing every field', () => { + recordAIToolsFailure('boom') + resetAIToolsHealth() + const health = getAIToolsHealth() + expect(health.status).toBe('unknown') + expect(health.consecutiveFailures).toBe(0) + expect(health.lastError).toBeNull() + }) +}) diff --git a/src/lib/ai/health.ts b/src/lib/ai/health.ts new file mode 100644 index 000000000..c20dcfc1e --- /dev/null +++ b/src/lib/ai/health.ts @@ -0,0 +1,44 @@ +/** + * Observed health of `callWithFallback`/`callVisionWithFallback` — the + * cascade behind form-assist, protocol/task/vote advisors, smart product + * entry, and blog translation. + * + * A SEPARATE tracker from `../hirn/health.ts` on purpose: this module and + * `../hirn/providers` are two independent AI stacks with their own config + * loading and their own cascades, even though both ultimately read + * `hirnProviderSettings`. A bug isolated to one (a stale cache, a broken + * fallback order) should show up as one named service going down in + * `/api/health`, not get averaged away inside a single "AI" bucket. + * + * Deliberately in-process — see `../hirn/health.ts` for why. + */ + +import { createHealthTracker } from 'ai-kit' + +const tracker = createHealthTracker({ downAfter: 3 }) + +/** Call after `callWithFallback`/`callVisionWithFallback` returns a result. */ +export function recordAIToolsSuccess(): void { + tracker.recordSuccess() +} + +/** Call when every provider in the cascade failed (a `null` return). */ +export function recordAIToolsFailure(error: unknown): void { + tracker.recordFailure(error) +} + +export function getAIToolsHealth() { + const health = tracker.getHealth() + return { + status: health.status, + consecutiveFailures: health.consecutiveFailures, + lastError: health.lastError, + lastSuccessAt: health.lastSuccessAt ? new Date(health.lastSuccessAt).toISOString() : null, + lastFailureAt: health.lastFailureAt ? new Date(health.lastFailureAt).toISOString() : null, + } +} + +/** Test seam. */ +export function resetAIToolsHealth(): void { + tracker.reset() +} diff --git a/src/lib/ai/providers.ts b/src/lib/ai/providers.ts index 85a6d48d6..d977dca16 100644 --- a/src/lib/ai/providers.ts +++ b/src/lib/ai/providers.ts @@ -14,6 +14,7 @@ import { hirnProviderSettings } from '@/db/schema' import { eq, desc } from 'drizzle-orm' import { OLLAMA_URL, APP_URL } from '@/config/urls' import { ORG } from '@/config/org' +import { recordAIToolsFailure, recordAIToolsSuccess } from './health' // ============================================================================= // CONFIGURATION (SSOT - all AI provider settings in one place) @@ -442,6 +443,7 @@ export async function callWithFallback(opts: CallOptions): Promise ({ provider: p.provider, reason: p.reason, message: p.message })), }) + recordAIToolsFailure(buildFailureMessage(failedProviders)) return null } @@ -577,9 +580,11 @@ export async function callVisionWithFallback(opts: VisionCallOptions): Promise 0) logger.info(`Vision fallback to ${result.provider}`, { failed: failed.map(p => `${p.provider}:${p.reason}`) }) + recordAIToolsSuccess() return { text: result.text, model: result.model, provider: result.provider, failedProviders: failed } } logger.error('All vision providers failed', { failures: failed.map(p => ({ provider: p.provider, reason: p.reason })) }) + recordAIToolsFailure(buildFailureMessage(failed)) return null } diff --git a/src/lib/deliverables/ai.ts b/src/lib/deliverables/ai.ts index dba75e086..b6505176c 100644 --- a/src/lib/deliverables/ai.ts +++ b/src/lib/deliverables/ai.ts @@ -13,7 +13,7 @@ import path from 'node:path' import { readFile } from 'node:fs/promises' -import { getDefaultChatProvider, type Message } from '@/lib/hirn/providers' +import { getChatResponse, type Message } from '@/lib/hirn/providers' import { ingestDocument } from '@/lib/hirn/ingestion' import { searchSimilar, formatContext } from '@/lib/hirn/retrieval' import { isTextFile } from '@/config/deliverables' @@ -173,8 +173,7 @@ REGELN: { role: 'user', content: message }, ] - const provider = await getDefaultChatProvider() - const response = await provider.chat({ messages, temperature: 0.3, maxTokens: 900 }) + const response = await getChatResponse({ messages, temperature: 0.3, maxTokens: 900 }) // Enforce Swiss German deterministically — the model doesn't always honour the // «ss statt ß» rule from the prompt. Safe: ß→ss is always correct in de-CH. return response.content.replace(/ß/g, 'ss') diff --git a/src/lib/hirn/__tests__/chat.test.ts b/src/lib/hirn/__tests__/chat.test.ts index 3e9b3c2a7..7965a7e8e 100644 --- a/src/lib/hirn/__tests__/chat.test.ts +++ b/src/lib/hirn/__tests__/chat.test.ts @@ -91,16 +91,17 @@ jest.mock('drizzle-orm', () => ({ count: jest.fn().mockReturnValue({ __count: 0 }), })) -// mockChatFn and mockGetDefaultChatProvider are declared here but only -// initialized after imports run; the closure in jest.mock captures them -// by reference so they're available when tests execute. -const mockChatFn = jest.fn() -const mockGetDefaultChatProvider = jest.fn() +// mockGetChatResponse is declared here but only initialized after imports +// run; the closure in jest.mock captures it by reference so it's available +// when tests execute. +const mockGetChatResponse = jest.fn() jest.mock('../providers', () => ({ - // Wrapper captures mockGetDefaultChatProvider by reference (not by value), - // so it resolves correctly when tests run (after module-level init). - getDefaultChatProvider: (...args: unknown[]) => mockGetDefaultChatProvider.apply(null, args), + // Wrapper captures mockGetChatResponse by reference (not by value), so it + // resolves correctly when tests run (after module-level init). chat.ts + // calls getChatResponse (provider selection + generation + health + // recording in one step) rather than getDefaultChatProvider directly. + getChatResponse: (...args: unknown[]) => mockGetChatResponse.apply(null, args), })) jest.mock('../system-prompt', () => ({ @@ -158,13 +159,12 @@ beforeEach(() => { mockDbInsert.mockImplementation(() => makeChain([])) mockDbDelete.mockImplementation(() => makeChain([])) mockDbExecute.mockResolvedValue({ rows: [] }) - mockChatFn.mockResolvedValue({ + mockGetChatResponse.mockResolvedValue({ content: 'Hier ist meine Antwort.', provider: 'groq', model: 'groq:llama-3.3-70b', usage: { promptTokens: 100, completionTokens: 50, totalTokens: 150 }, }) - mockGetDefaultChatProvider.mockResolvedValue({ chat: mockChatFn }) ;(parseActionEnvelope as jest.Mock).mockReturnValue({ actions: [], parsingError: null }) ;(stripActionBlock as jest.Mock).mockImplementation((c: string) => c) }) diff --git a/src/lib/hirn/__tests__/health.test.ts b/src/lib/hirn/__tests__/health.test.ts new file mode 100644 index 000000000..e569719f7 --- /dev/null +++ b/src/lib/hirn/__tests__/health.test.ts @@ -0,0 +1,42 @@ +/** + * evig's /api/health checked only the database and Meilisearch until now — + * an AI outage was invisible to it. This pins the state machine that check + * now depends on. + */ +import { getLLMHealth, recordLLMFailure, recordLLMSuccess, resetLLMHealth } from '../health' + +describe('hirn llm health tracker', () => { + beforeEach(() => resetLLMHealth()) + + it('starts unknown, before anything has been observed', () => { + expect(getLLMHealth().status).toBe('unknown') + }) + + it('is ok after a success', () => { + recordLLMSuccess() + expect(getLLMHealth().status).toBe('ok') + }) + + it('is degraded on the first failures, not down', () => { + recordLLMFailure(new Error('Kein KI-Anbieter verfügbar')) + expect(getLLMHealth().status).toBe('degraded') + }) + + it('is down once failures are consistent', () => { + for (let i = 0; i < 3; i += 1) recordLLMFailure(new Error('boom')) + const health = getLLMHealth() + expect(health.status).toBe('down') + expect(health.consecutiveFailures).toBe(3) + expect(health.lastError).toBe('boom') + }) + + it('recovers to ok on the next success', () => { + for (let i = 0; i < 5; i += 1) recordLLMFailure(new Error('boom')) + expect(getLLMHealth().status).toBe('down') + recordLLMSuccess() + const health = getLLMHealth() + expect(health.status).toBe('ok') + expect(health.consecutiveFailures).toBe(0) + expect(health.lastError).toBeNull() + }) +}) diff --git a/src/lib/hirn/chat.ts b/src/lib/hirn/chat.ts index 48c917ad4..e751b354f 100644 --- a/src/lib/hirn/chat.ts +++ b/src/lib/hirn/chat.ts @@ -8,7 +8,7 @@ import { db } from '@/db' import { hirnChatHistory } from '@/db/schema' import { eq, and, asc, desc, sql, or, isNull, count } from 'drizzle-orm' import { logger } from '@/lib/logger' -import { getDefaultChatProvider, type Message } from './providers' +import { getChatResponse, type Message } from './providers' import { SYSTEM_PROMPT } from './system-prompt' import { parseActionEnvelope, stripActionBlock, type HirnActionCard } from './action-cockpit' import { API_DEFAULTS } from '@/config/api-defaults' @@ -79,12 +79,7 @@ export async function chat( ] // Get the chat provider and generate response - const provider = await getDefaultChatProvider(userId) - const response = await provider.chat({ - messages, - temperature, - maxTokens, - }) + const response = await getChatResponse({ messages, temperature, maxTokens }, userId) // Store conversation history (best-effort — don't let DB errors kill the response) try { diff --git a/src/lib/hirn/health.ts b/src/lib/hirn/health.ts new file mode 100644 index 000000000..e4a563f8a --- /dev/null +++ b/src/lib/hirn/health.ts @@ -0,0 +1,47 @@ +/** + * Observed health of the Hirn chat provider layer. + * + * `/api/health` already checks the database and Meilisearch, and neither of + * those tells you anything about whether Hirn can actually answer — a dead + * Groq/OpenRouter key or an exhausted provider cascade looks identical to + * "healthy" from that endpoint's point of view. This fleet has already lost + * an AI feature to exactly that blind spot once (botsmann, 2026-08-28): a + * friendly 5xx from the chat route and a database-only health check that + * kept reporting "healthy" the whole time. + * + * `getChatResponse` (in `./providers`) records the outcome of every real + * chat attempt here; `/api/health` reports it as a named service alongside + * database and Meilisearch. Deliberately in-process — evig runs as a single + * service, so module state is shared by every request. If it is ever scaled + * horizontally this becomes per-instance and wants a shared store. + */ + +import { createHealthTracker } from 'ai-kit' + +const tracker = createHealthTracker({ downAfter: 3 }) + +/** Call after a chat completion that produced usable content. */ +export function recordLLMSuccess(): void { + tracker.recordSuccess() +} + +/** Call when provider selection or generation threw. */ +export function recordLLMFailure(error: unknown): void { + tracker.recordFailure(error) +} + +export function getLLMHealth() { + const health = tracker.getHealth() + return { + status: health.status, + consecutiveFailures: health.consecutiveFailures, + lastError: health.lastError, + lastSuccessAt: health.lastSuccessAt ? new Date(health.lastSuccessAt).toISOString() : null, + lastFailureAt: health.lastFailureAt ? new Date(health.lastFailureAt).toISOString() : null, + } +} + +/** Test seam. */ +export function resetLLMHealth(): void { + tracker.reset() +} diff --git a/src/lib/hirn/providers/index.ts b/src/lib/hirn/providers/index.ts index 402c606b3..4187679e1 100644 --- a/src/lib/hirn/providers/index.ts +++ b/src/lib/hirn/providers/index.ts @@ -12,10 +12,13 @@ import { logger } from '@/lib/logger' import { GroqProvider } from './groq' import { OllamaProvider } from './ollama' import { OpenRouterProvider } from './openrouter' +import { recordLLMFailure, recordLLMSuccess } from '../health' import type { AIProvider, ProviderConfig, ProviderName, + ChatCompletionOptions, + ChatCompletionResponse, EmbeddingOptions, EmbeddingResponse, } from './types' @@ -155,6 +158,30 @@ export async function getDefaultChatProvider(userId?: string): Promise { + try { + const provider = await getDefaultChatProvider(userId) + const response = await provider.chat(options) + recordLLMSuccess() + return response + } catch (error) { + recordLLMFailure(error) + throw error + } +} + /** * Get the embedding provider * Always uses Ollama for embeddings (local, free, 768 dimensions)