diff --git a/packages/core/src/inbox/signalSourceService.test.ts b/packages/core/src/inbox/signalSourceService.test.ts index 1d7c0c9d87..7bbbad130f 100644 --- a/packages/core/src/inbox/signalSourceService.test.ts +++ b/packages/core/src/inbox/signalSourceService.test.ts @@ -124,6 +124,17 @@ describe("SignalSourceService.toggleSource", () => { }); }); + it("creates an llm_analytics config with the evaluation_report source type", async () => { + const client = fakeClient(); + const service = new SignalSourceService(); + await service.toggleSource(client, 1, "llm_analytics", true, [], []); + expect(client.createSignalSourceConfig).toHaveBeenCalledWith(1, { + source_product: "llm_analytics", + source_type: "evaluation_report", + enabled: true, + }); + }); + it("ensures the issues table syncs with full_refresh for github before enabling", async () => { const client = fakeClient(); const service = new SignalSourceService(); diff --git a/packages/core/src/inbox/signalSourceService.ts b/packages/core/src/inbox/signalSourceService.ts index 92fb48ce63..7a740170ec 100644 --- a/packages/core/src/inbox/signalSourceService.ts +++ b/packages/core/src/inbox/signalSourceService.ts @@ -37,6 +37,8 @@ const SOURCE_TYPE_MAP: Partial> = { conversations: "ticket", health_checks: "health_issue", session_replay: "session_analysis_cluster", + // AI observability signals are always emitted per evaluation report. + llm_analytics: "evaluation_report", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]), ), @@ -67,6 +69,7 @@ const ALL_SOURCE_PRODUCTS: SignalSourceProduct[] = [ "error_tracking", "health_checks", "session_replay", + "llm_analytics", ...EXTERNAL_INBOX_SOURCES.map((s) => s.product), ]; diff --git a/packages/shared/src/inbox-types.ts b/packages/shared/src/inbox-types.ts index 89501ced1c..58b3f7c547 100644 --- a/packages/shared/src/inbox-types.ts +++ b/packages/shared/src/inbox-types.ts @@ -424,13 +424,10 @@ export type SourceProduct = | ExternalInboxSourceProduct; /** - * Products that render as a toggle in the Self-driving sources modal. Excludes non-toggle - * products (`llm_analytics`, `signals_scout`) that appear only as signal origins. + * Products that render as a toggle in the Self-driving sources modal. Excludes `signals_scout`, + * which appears only as a signal origin and is always on rather than user-toggled. */ -export type ToggleableSourceProduct = Exclude< - SourceProduct, - "llm_analytics" | "signals_scout" ->; +export type ToggleableSourceProduct = Exclude; /** * Every backend signal `source_type`: the PostHog-native types (alphabetical) plus the @@ -439,6 +436,7 @@ export type ToggleableSourceProduct = Exclude< export type SourceType = | "cross_source_issue" | "evaluation" + | "evaluation_report" | "health_issue" | "issue_created" | "issue_reopened" diff --git a/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx b/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx index 3f98471ae2..5dc82aab4d 100644 --- a/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx +++ b/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx @@ -84,7 +84,6 @@ export function ConfigureAgentsSection() { handleSetupCancel, userAutonomyConfig, userAutonomyConfigLoading, - evaluationsUrl, } = useSignalSourceManager(); const { hasGithubIntegration, isLoadingIntegrations } = useRepositoryIntegration(); @@ -188,7 +187,6 @@ export function ConfigureAgentsSection() { disabled={!hasGithubIntegration} sourceStates={sourceStates} onSetup={handleSetup} - evaluationsUrl={evaluationsUrl} /> )} diff --git a/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx b/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx index 05ad62bbc0..a93c745f62 100644 --- a/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx +++ b/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx @@ -14,7 +14,7 @@ import type { SignalSourceValues } from "@posthog/ui/features/inbox/components/S import { InboxBadge } from "@posthog/ui/features/inbox/components/utils/InboxBadge"; import { getSourceProductMeta } from "@posthog/ui/features/inbox/components/utils/source-product-icons"; import { Badge } from "@posthog/ui/primitives/Badge"; -import { Box, Flex, Spinner, Switch, Text, Tooltip } from "@radix-ui/themes"; +import { Box, Flex, Spinner, Switch, Text } from "@radix-ui/themes"; import { type ComponentType, memo, useCallback } from "react"; type AgentRosterStatus = "standby" | "watching" | "syncing" | "sync_failed"; @@ -61,7 +61,6 @@ interface ResponderAgentRosterProps { > >; onSetup?: (source: ResponderAgentSource) => void; - evaluationsUrl?: string; } export function ResponderAgentRoster({ @@ -70,7 +69,6 @@ export function ResponderAgentRoster({ disabled, sourceStates, onSetup, - evaluationsUrl, }: ResponderAgentRosterProps) { return ( @@ -91,9 +89,6 @@ export function ResponderAgentRoster({ onSetup={onSetup} /> ))} - {group.label === "PostHog data" && evaluationsUrl ? ( - - ) : null} ))} @@ -264,80 +259,6 @@ function AgentIcon({ ); } -const EvaluationsAgentCard = memo(function EvaluationsAgentCard({ - evaluationsUrl, -}: { - evaluationsUrl: string; -}) { - return ( - window.open(evaluationsUrl, "_blank", "noopener")} - className={[ - AGENT_CARD_BASE_CLASS, - "border-border", - AGENT_CARD_INTERACTIVE_IDLE_CLASS, - ].join(" ")} - > - - - - - - - AI Observability - - - - Internal - - - - - Quality problems in your AI features. - - - { - e.stopPropagation(); - e.preventDefault(); - window.open( - "https://posthog.com/docs/ai-observability", - "_blank", - "noopener", - ); - }} - className="inline-flex items-center gap-1 text-(--accent-11) no-underline" - > - Learn about AI observability - - - - - - - - - ); -}); - function ResponderAgentCardSkeleton() { return ( diff --git a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx index a7d9816c3a..bb0d7f3a3b 100644 --- a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx +++ b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx @@ -16,7 +16,7 @@ import { } from "@posthog/shared"; import { getSourceProductMeta } from "@posthog/ui/features/inbox/components/utils/source-product-icons"; import { Badge } from "@posthog/ui/primitives/Badge"; -import { Box, Flex, Spinner, Switch, Text, Tooltip } from "@radix-ui/themes"; +import { Box, Flex, Spinner, Switch, Text } from "@radix-ui/themes"; import { memo, useCallback } from "react"; export type SignalSourceValues = Record; @@ -210,75 +210,6 @@ const ExternalSourceCard = memo(function ExternalSourceCard({ ); }); -interface EvaluationsSectionProps { - evaluationsUrl: string; -} - -export const EvaluationsSection = memo(function EvaluationsSection({ - evaluationsUrl, -}: EvaluationsSectionProps) { - return ( - window.open(evaluationsUrl, "_blank", "noopener")} - className="cursor-pointer rounded-(--radius-3) border border-border bg-(--color-panel-solid) transition duration-150 hover:border-(--gray-6) hover:bg-(--gray-2) hover:shadow-sm" - > - - - - - - - - - AI observability - - - Internal - - - - Monitor how your AI features are performing - - - { - e.stopPropagation(); - e.preventDefault(); - window.open( - "https://posthog.com/docs/ai-observability", - "_blank", - "noopener", - ); - }} - className="inline-flex items-center gap-[4px] text-(--accent-11) no-underline" - > - Learn about AI observability - - - - - - - - - ); -}); - function SourceRunningIndicator({ status, message, @@ -303,7 +234,6 @@ interface SignalSourceTogglesProps { disabled?: boolean; sourceStates?: Partial>; onSetup?: (source: ToggleableSourceProduct) => void; - evaluationsUrl?: string; } export function SignalSourceToggles({ @@ -312,7 +242,6 @@ export function SignalSourceToggles({ disabled, sourceStates, onSetup, - evaluationsUrl, }: SignalSourceTogglesProps) { const toggleSessionReplay = useCallback( (checked: boolean) => onToggle("session_replay", checked), @@ -330,6 +259,10 @@ export function SignalSourceToggles({ (checked: boolean) => onToggle("health_checks", checked), [onToggle], ); + const toggleLlmAnalytics = useCallback( + (checked: boolean) => onToggle("llm_analytics", checked), + [onToggle], + ); return ( @@ -390,9 +323,16 @@ export function SignalSourceToggles({ ) : undefined } /> - {evaluationsUrl && ( - - )} + } + label="AI observability" + description="Quality problems in your AI features. Set up evaluations to start getting signals." + checked={value.llm_analytics} + onCheckedChange={toggleLlmAnalytics} + disabled={disabled} + docsUrl="https://posthog.com/docs/ai-evals" + docsLabel="evaluations" + /> diff --git a/packages/ui/src/features/inbox/components/responderAgentMeta.ts b/packages/ui/src/features/inbox/components/responderAgentMeta.ts index 76844498f8..b47d1faa59 100644 --- a/packages/ui/src/features/inbox/components/responderAgentMeta.ts +++ b/packages/ui/src/features/inbox/components/responderAgentMeta.ts @@ -56,6 +56,15 @@ export const RESPONDER_AGENT_GROUPS: ResponderAgentGroup[] = [ docsLabel: "Session Replay", alpha: true, }, + { + source: "llm_analytics", + sourceProduct: "llm_analytics", + label: "AI observability", + description: + "Quality problems in your AI features. Set up evaluations to start getting signals.", + docsUrl: "https://posthog.com/docs/ai-evals", + docsLabel: "evaluations", + }, ], }, { diff --git a/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts b/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts index f47c0d2177..69ae012443 100644 --- a/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts +++ b/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts @@ -24,6 +24,8 @@ const SOURCE_TYPE_MAP: Partial> = { conversations: "ticket", health_checks: "health_issue", session_replay: "session_analysis_cluster", + // AI observability signals are always emitted per evaluation report. + llm_analytics: "evaluation_report", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]), ), @@ -40,6 +42,7 @@ const SOURCE_LABELS: Partial> = { error_tracking: "Error tracking", health_checks: "Health checks", session_replay: "Session replay", + llm_analytics: "AI observability", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.label]), ), @@ -64,6 +67,7 @@ const ALL_SOURCE_PRODUCTS: SourceKey[] = [ "error_tracking", "health_checks", "session_replay", + "llm_analytics", ...EXTERNAL_INBOX_SOURCES.map((s) => s.product), ];