From 68a947182c5f78d5292e3a821f82fc9b83085ba8 Mon Sep 17 00:00:00 2001
From: Radu Raicea <radu@posthog.com>
Date: Thu, 30 Jul 2026 11:11:50 -0400
Subject: [PATCH] feat(aio): align AI observability signal source

---
 .../components/config/agentRosterMeta.ts      |  6 ++---
 .../scenes/inbox/signalSourcesLogic.test.ts   | 24 +++++++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/frontend/src/scenes/inbox/components/config/agentRosterMeta.ts b/frontend/src/scenes/inbox/components/config/agentRosterMeta.ts
index 99e302d32558..2d60eb197145 100644
--- a/frontend/src/scenes/inbox/components/config/agentRosterMeta.ts
+++ b/frontend/src/scenes/inbox/components/config/agentRosterMeta.ts
@@ -71,9 +71,9 @@ export const AGENT_ROSTER_GROUPS: AgentRosterGroup[] = [
                 source: 'llm_analytics',
                 sourceProduct: SignalSourceProduct.LlmAnalytics,
                 label: 'AI observability',
-                description: 'Findings from evaluation reports on your LLM traffic.',
-                docsUrl: 'https://posthog.com/docs/ai-evals/evaluations',
-                docsLabel: 'AI observability',
+                description: 'Quality problems in your AI features. Set up evaluations to start getting signals.',
+                docsUrl: 'https://posthog.com/docs/ai-evals',
+                docsLabel: 'evaluations',
             },
             {
                 source: 'analytics',
diff --git a/frontend/src/scenes/inbox/signalSourcesLogic.test.ts b/frontend/src/scenes/inbox/signalSourcesLogic.test.ts
index 49206f86140e..cc3f7a4270cd 100644
--- a/frontend/src/scenes/inbox/signalSourcesLogic.test.ts
+++ b/frontend/src/scenes/inbox/signalSourcesLogic.test.ts
@@ -168,4 +168,28 @@ describe('signalSourcesLogic', () => {
         expect(createSourceConfig).toHaveBeenCalledTimes(1)
         expect(logic.values.isGithubIssuesToggling).toBe(false)
     })
+
+    it('creates an AI observability config for evaluation reports', async () => {
+        const createSourceConfig = jest.spyOn(api.signalSourceConfigs, 'create').mockResolvedValue({
+            id: 'config-evaluation-reports',
+            source_product: SignalSourceProduct.LlmAnalytics,
+            source_type: SignalSourceType.EvaluationReport,
+            enabled: true,
+            config: {},
+            created_at: '2026-07-30T00:00:00Z',
+            updated_at: '2026-07-30T00:00:00Z',
+            status: null,
+        })
+        logic.actions.loadSourceConfigsSuccess([])
+
+        logic.actions.toggleEvalReports()
+        await expectLogic(logic).toFinishAllListeners()
+
+        expect(createSourceConfig).toHaveBeenCalledWith({
+            source_product: SignalSourceProduct.LlmAnalytics,
+            source_type: SignalSourceType.EvaluationReport,
+            enabled: true,
+            config: {},
+        })
+    })
 })