diff --git a/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodInfo.tsx b/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodInfo.tsx index bc7eae9196..439d22d13c 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodInfo.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodInfo.tsx @@ -1,6 +1,6 @@ 'use client'; -import { Dispatch, FC, SetStateAction, useCallback, useMemo, useState } from 'react'; +import { Dispatch, FC, SetStateAction, useCallback, useEffect, useMemo, useState } from 'react'; import { DialInput, DialNoDataContent } from '@epam/ai-dial-ui-kit'; import { ColDef } from 'ag-grid-community'; @@ -32,7 +32,6 @@ const MethodInfo: FC = ({ testSuite, onChangeTestSuite }) => { const PARAMETERS_COLUMNS: ColDef[] = useMemo(() => PARAMETERS_SCHEMA_COLUMNS(t), [t]); const [view, setView] = useState(ParamsView.TABLE); - const [finalPathError, setFinalPathError] = useState(undefined); const inputSchema = useMemo(() => { return convertSchemaToTable(testSuite?.endpointRef?.requestBodySchema?.schema); @@ -46,31 +45,31 @@ const MethodInfo: FC = ({ testSuite, onChangeTestSuite }) => { return testSuite?.endpointRef?.parameters || []; }, [testSuite?.endpointRef?.parameters]); - const validateFinalPath = useCallback( - (urlTemplate?: string) => { - const relativeUrlPattern = testSuite.endpointRef?.relativeUrlPattern; - - if (!urlTemplate || !relativeUrlPattern) { - return undefined; - } - - if (isContainRegexSymbols(relativeUrlPattern)) { - try { - const regex = new RegExp(relativeUrlPattern); - - if (!regex.test(urlTemplate)) { - dispatch({ type: ValidationActionType.SetField, field: 'urlTemplate', isValid: false }); - return `Not matches with ${relativeUrlPattern} regex`; - } - } catch (error) { - console.error('Invalid regex pattern:', error); - } - } - dispatch({ type: ValidationActionType.SetField, field: 'urlTemplate', isValid: true }); + /** + * Derived rather than stored, so switching method re-validates the freshly seeded path instead of + * leaving the previous method's error — and its disabled Save — standing. + */ + const finalPathError = useMemo(() => { + const relativeUrlPattern = testSuite.endpointRef?.relativeUrlPattern; + const urlTemplate = testSuite.requestTemplate?.urlTemplate; + + if (!urlTemplate || !relativeUrlPattern || !isContainRegexSymbols(relativeUrlPattern)) { return undefined; - }, - [dispatch, testSuite.endpointRef?.relativeUrlPattern], - ); + } + + try { + return new RegExp(relativeUrlPattern).test(urlTemplate) + ? undefined + : `Not matches with ${relativeUrlPattern} regex`; + } catch (error) { + console.error('Invalid regex pattern:', error); + return undefined; + } + }, [testSuite.endpointRef?.relativeUrlPattern, testSuite.requestTemplate?.urlTemplate]); + + useEffect(() => { + dispatch({ type: ValidationActionType.SetField, field: 'urlTemplate', isValid: !finalPathError }); + }, [dispatch, finalPathError]); const onChangeEndpointRef = useCallback( (endpointRef: TestSuiteEndpointRef) => { @@ -88,9 +87,8 @@ const MethodInfo: FC = ({ testSuite, onChangeTestSuite }) => { ...testSuite, requestTemplate: { ...testSuite.requestTemplate, urlTemplate: finalPath }, }); - setFinalPathError(validateFinalPath(finalPath)); }, - [onChangeTestSuite, testSuite, validateFinalPath], + [onChangeTestSuite, testSuite], ); return (testSuite?.endpointRef && !!Object.keys(testSuite?.endpointRef).length) || testSuite ? ( diff --git a/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodItem.tsx b/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodItem.tsx index 89a7259b8e..3e995079b2 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodItem.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/Methods/MethodItem.tsx @@ -12,26 +12,29 @@ interface Props { item: TestSuiteEndpointRef; isActive: boolean; onClick: (index: number) => void; + label?: string; } -const MethodItem: FC = ({ index, item, isActive, onClick }) => { +const MethodItem: FC = ({ index, item, isActive, onClick, label }) => { const onMethodClick = useCallback(() => { onClick(index); }, [onClick, index]); return ( -
{item.method} - - -
+ {' '} + + ); }; diff --git a/apps/ai-dial-admin/src/components/TestSuites/Methods/Methods.tsx b/apps/ai-dial-admin/src/components/TestSuites/Methods/Methods.tsx index 9cc3697864..7220e4f8ea 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/Methods/Methods.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/Methods/Methods.tsx @@ -1,18 +1,26 @@ 'use client'; -import { Dispatch, FC, ReactNode, SetStateAction, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + Dispatch, + FC, + ReactNode, + SetStateAction, + useCallback, + useEffect, + useId, + useMemo, + useRef, + useState, +} from 'react'; import { DialCollapsibleSidebar, DialConditionalResizableContainer, DialLoader } from '@epam/ai-dial-ui-kit'; -import { getDeployment } from '@/src/app/[lang]/test-suites/actions'; -import { CHAT_COMPLETION_METHOD } from '@/src/components/TestSuites/constants/chat-completion-method'; -import { CHAT_COMPLETION_SUITE, DEFAULT_SUITE } from '@/src/components/TestSuites/constants/methods'; -import { generateMethodPathCombinations } from '@/src/components/TestSuites/utils/method'; +import { getDeploymentById } from '@/src/app/[lang]/test-suites/actions'; +import { buildMethodGroups, flattenMethodGroups } from '@/src/components/TestSuites/utils/method-groups'; import { TestSuitesI18nKey } from '@/src/constants/i18n'; import { useI18n } from '@/src/locales/client'; import { Deployment } from '@/src/models/evaluation/deployment'; -import { TestSuite, TestSuiteEndpointRef } from '@/src/models/evaluation/test-suite'; -import { uniquifyResponseColumns } from '@/src/utils/evaluation/request-chain'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; import MethodInfo from './MethodInfo'; import MethodItem from './MethodItem'; @@ -27,17 +35,38 @@ interface Props { const Methods: FC = ({ testSuite, selectedTarget, onChange, isCreate, takenColumnNames = [], children }) => { const t = useI18n(); + const groupHeadingId = useId(); const [activeMethodIndex, setActiveMethodIndex] = useState(); const [fullApplication, setFullApplication] = useState(); - const [methods, setMethods] = useState([]); const [isLoading, setIsLoading] = useState(true); + const buildGroupsFor = useCallback( + (deployment?: Deployment | null) => + buildMethodGroups({ deployment, endpointRef: testSuite.endpointRef, takenColumnNames }), + // `takenColumnNames` is a fresh array each render; its contents are what matter here. + // eslint-disable-next-line react-hooks/exhaustive-deps + [testSuite.endpointRef, takenColumnNames.join(',')], + ); + + const groups = useMemo(() => buildGroupsFor(fullApplication), [buildGroupsFor, fullApplication]); + + const options = useMemo(() => flattenMethodGroups(groups), [groups]); + + /** Each group's first index into `options`, so items stay addressable by a single flat index. */ + const groupOffsets = useMemo( + () => + groups.reduce((offsets, group, groupIndex) => { + offsets.push(groupIndex === 0 ? 0 : offsets[groupIndex - 1] + groups[groupIndex - 1].options.length); + return offsets; + }, []), + [groups], + ); + const methodInfo = useMemo(() => { if (activeMethodIndex == null) return {}; - if (activeMethodIndex === 0) return CHAT_COMPLETION_METHOD; - return methods[activeMethodIndex - 1] ?? {}; - }, [activeMethodIndex, methods]); + return options[activeMethodIndex]?.ref ?? {}; + }, [activeMethodIndex, options]); const onMethodClick = useCallback( (index: number) => { @@ -45,38 +74,30 @@ const Methods: FC = ({ testSuite, selectedTarget, onChange, isCreate, tak return; } + const option = options[index]; + if (!option) return; + setActiveMethodIndex(index); - if (index === 0) { - onChange((prev: TestSuite) => ({ - ...prev, - ...CHAT_COMPLETION_SUITE, - responseColumns: uniquifyResponseColumns(CHAT_COMPLETION_SUITE.responseColumns, takenColumnNames), - })); - } else { - const route = methods[index - 1]; - if (!route) return; - onChange((prev: TestSuite) => ({ - ...prev, - ...DEFAULT_SUITE(route), - })); - } + onChange((prev: TestSuite) => ({ + ...prev, + ...option.seed, + })); }, - [activeMethodIndex, methods, onChange, takenColumnNames], + [activeMethodIndex, options, onChange], ); useEffect(() => { if (!fullApplication && selectedTarget) { - const { deploymentId, $type } = selectedTarget; - getDeployment(deploymentId, $type) + const { deploymentId } = selectedTarget; + getDeploymentById(deploymentId) .then((data) => { setFullApplication(data); - const loadedMethods = generateMethodPathCombinations(data?.routes); - setMethods(loadedMethods); - const selectedIndex = [CHAT_COMPLETION_METHOD, ...loadedMethods].findIndex( - (method) => - method.method === testSuite.endpointRef?.method && - method.relativeUrlPattern === testSuite.endpointRef?.relativeUrlPattern, + const loadedOptions = flattenMethodGroups(buildGroupsFor(data)); + const selectedIndex = loadedOptions.findIndex( + ({ ref }) => + ref.method === testSuite.endpointRef?.method && + ref.relativeUrlPattern === testSuite.endpointRef?.relativeUrlPattern, ); if (selectedIndex !== -1) { setActiveMethodIndex(selectedIndex); @@ -102,6 +123,7 @@ const Methods: FC = ({ testSuite, selectedTarget, onChange, isCreate, tak }); } }; + return isLoading ? ( ) : ( @@ -125,30 +147,40 @@ const Methods: FC = ({ testSuite, selectedTarget, onChange, isCreate, tak onToggle={setIsSidebarOpened} >
-
- {t(TestSuitesI18nKey.ChatInterface)} - -
-
- {!!methods.length && ( - {t(TestSuitesI18nKey.Other)} - )} - {methods.map((method, routeIndex) => ( - - ))} -
+ {groups.map((group, groupIndex) => { + if (!group.options.length) { + return null; + } + + const headingId = `${groupHeadingId}-${group.titleKey}`; + + return ( +
+ + {t(group.titleKey)} + + {group.options.map((option, optionIndex) => { + const index = groupOffsets[groupIndex] + optionIndex; + + return ( + + ); + })} +
+ ); + })}
diff --git a/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/MethodInfo.spec.tsx b/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/MethodInfo.spec.tsx new file mode 100644 index 0000000000..cda46d4a38 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/MethodInfo.spec.tsx @@ -0,0 +1,88 @@ +import { render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +import { useSaveValidationContext } from '@/src/context/SaveValidationContext'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; +import MethodInfo from '../MethodInfo'; + +vi.mock('@epam/ai-dial-ui-kit', async (importOriginal) => ({ + ...((await importOriginal()) as object), + DialInput: ({ value, onChange, error, invalid }: any) => ( +
+ onChange(e.target.value)} + /> + {error ? {error} : null} +
+ ), + DialNoDataContent: ({ title }: any) =>
{title}
, +})); + +vi.mock('@/src/components/Common/ViewSelector/ViewSelector', () => ({ __esModule: true, default: () => null })); +vi.mock('@/src/components/Common/ViewSelector/TableView', () => ({ __esModule: true, default: () => null })); +vi.mock('@/src/components/EntityTabs/JsonEditor/JsonEditor', () => ({ __esModule: true, default: () => null })); +vi.mock('../Endpoint', () => ({ __esModule: true, default: () => null })); + +const suite = (relativeUrlPattern: string, urlTemplate: string): TestSuite => + ({ + endpointRef: { method: 'POST', relativeUrlPattern }, + requestTemplate: { urlTemplate }, + }) as TestSuite; + +describe('MethodInfo final-path validation', () => { + const { dispatch } = useSaveValidationContext(); + + beforeEach(() => { + vi.mocked(dispatch).mockClear(); + }); + + test('reports no error for a path matching the regex pattern', () => { + render( + , + ); + + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); + expect(dispatch).toHaveBeenCalledWith(expect.objectContaining({ field: 'urlTemplate', isValid: true })); + }); + + test('reports an error for a path the pattern rejects', () => { + render( + , + ); + + expect(screen.getByRole('alert')).toHaveTextContent('Not matches with /openai/v1/responses/[^/]+/cancel regex'); + expect(dispatch).toHaveBeenCalledWith(expect.objectContaining({ field: 'urlTemplate', isValid: false })); + }); + + test('clears a standing error once the method changes', () => { + const { rerender } = render( + , + ); + expect(screen.getByRole('alert')).toBeInTheDocument(); + + rerender( + , + ); + + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); + expect(dispatch).toHaveBeenLastCalledWith(expect.objectContaining({ field: 'urlTemplate', isValid: true })); + }); + + test('skips validation for a pattern carrying no regex symbols', () => { + render(); + + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); + }); +}); diff --git a/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/MethodItem.spec.tsx b/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/MethodItem.spec.tsx new file mode 100644 index 0000000000..7d5cdec0cc --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/MethodItem.spec.tsx @@ -0,0 +1,62 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, expect, test, vi } from 'vitest'; + +import MethodItem from '../MethodItem'; + +vi.mock('@epam/ai-dial-ui-kit', () => ({ + DialEllipsisTooltip: ({ text }: any) => {text}, +})); + +const item = { method: 'GET', relativeUrlPattern: '/openai/v1/responses/[^/]+' }; + +describe('MethodItem', () => { + test('renders a button carrying the method and the readable label', () => { + render( + , + ); + + expect(screen.getByRole('button', { name: 'GET /openai/v1/responses/{response_id}' })).toBeInTheDocument(); + }); + + test('falls back to the URL pattern when no label is given', () => { + render(); + + expect(screen.getByRole('button', { name: 'GET /openai/v1/responses/[^/]+' })).toBeInTheDocument(); + }); + + test('exposes the active state through aria-current', () => { + render(); + + expect(screen.getByRole('button')).toHaveAttribute('aria-current', 'true'); + }); + + test('reports its index on click', async () => { + const onClick = vi.fn(); + const user = userEvent.setup(); + + render(); + await user.click(screen.getByRole('button')); + + expect(onClick).toHaveBeenCalledWith(3); + }); + + test('is reachable by keyboard and activates on Enter', async () => { + const onClick = vi.fn(); + const user = userEvent.setup(); + + render(); + + await user.tab(); + expect(screen.getByRole('button')).toHaveFocus(); + + await user.keyboard('{Enter}'); + expect(onClick).toHaveBeenCalledWith(2); + }); +}); diff --git a/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/Methods.spec.tsx b/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/Methods.spec.tsx index ddb97aef29..a2005da5be 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/Methods.spec.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/Methods/tests/Methods.spec.tsx @@ -3,7 +3,7 @@ import userEvent from '@testing-library/user-event'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import Methods from '../Methods'; -const mockGetDeployment = vi.fn(); +const mockGetDeploymentById = vi.fn(); const mockGenerateMethodPathCombinations = vi.fn(); vi.mock('@/src/components/TestSuites/utils/method', () => ({ @@ -11,7 +11,7 @@ vi.mock('@/src/components/TestSuites/utils/method', () => ({ })); vi.mock('@/src/app/[lang]/test-suites/actions', () => ({ - getDeployment: (...args: any[]) => mockGetDeployment(...args), + getDeploymentById: (...args: any[]) => mockGetDeploymentById(...args), })); vi.mock('@epam/ai-dial-ui-kit', () => ({ @@ -22,10 +22,10 @@ vi.mock('@epam/ai-dial-ui-kit', () => ({ vi.mock('../MethodItem', () => ({ __esModule: true, - default: ({ item, index, onClick, isActive }: any) => ( + default: ({ item, index, onClick, isActive, label }: any) => (
-
), @@ -59,9 +59,9 @@ describe('Methods component', () => { beforeEach(() => { onChange.mockClear(); - mockGetDeployment.mockClear(); + mockGetDeploymentById.mockClear(); mockGenerateMethodPathCombinations.mockClear(); - mockGetDeployment.mockResolvedValue(mockDeployment); + mockGetDeploymentById.mockResolvedValue(mockDeployment); mockGenerateMethodPathCombinations.mockReturnValue(mockMethods); }); @@ -83,7 +83,8 @@ describe('Methods component', () => { render(); await waitFor(() => { - expect(mockGetDeployment).toHaveBeenCalledWith('test-deployment', 'application'); + expect(mockGetDeploymentById).toHaveBeenCalledOnce(); + expect(mockGetDeploymentById).toHaveBeenCalledWith('test-deployment'); expect(mockGenerateMethodPathCombinations).toHaveBeenCalledWith(mockDeployment.routes); }); }); @@ -140,12 +141,12 @@ describe('Methods component', () => { ); await waitFor(() => { - expect(mockGetDeployment).toHaveBeenCalledTimes(1); + expect(mockGetDeploymentById).toHaveBeenCalledOnce(); }); rerender(); - expect(mockGetDeployment).toHaveBeenCalledTimes(1); + expect(mockGetDeploymentById).toHaveBeenCalledOnce(); }); test('keeps the default chat-completion column name when it is not taken', async () => { @@ -200,4 +201,219 @@ describe('Methods component', () => { expect.objectContaining({ name: 'answer2', displayName: 'answer2' }), ); }); + + describe('Responses group', () => { + const selectedApplication: any = { deploymentId: 'gpt-4o', $type: 'dial-model' }; + + const renderWithInterfaces = (interfaces?: string[], testSuite: any = { endpointRef: {} }) => { + mockGetDeploymentById.mockResolvedValue({ ...mockDeployment, deploymentId: 'gpt-4o', interfaces }); + + return render(); + }; + + test('is absent when the deployment reports no interfaces', async () => { + renderWithInterfaces(); + + await screen.findByRole('button', { name: 'POST /chat/completions' }); + expect(screen.queryByRole('group', { name: 'TestSuites.OpenAIResponses' })).not.toBeInTheDocument(); + }); + + test('is absent when the reported interfaces omit openaiResponses', async () => { + renderWithInterfaces(['chat', 'openaiChatCompletions']); + + await screen.findByRole('button', { name: 'POST /chat/completions' }); + expect(screen.queryByRole('group', { name: 'TestSuites.OpenAIResponses' })).not.toBeInTheDocument(); + }); + + test('renders the four operations when openaiResponses is reported', async () => { + renderWithInterfaces(['chat', 'openaiResponses']); + + expect(await screen.findByRole('group', { name: 'TestSuites.OpenAIResponses' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'POST /openai/v1/responses' })).toBeInTheDocument(); + expect( + screen.getByRole('button', { name: 'POST /openai/v1/responses/{response_id}/cancel' }), + ).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'GET /openai/v1/responses/{response_id}' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'DELETE /openai/v1/responses/{response_id}' })).toBeInTheDocument(); + }); + + test('renders groups in order: chat interface, responses, other', async () => { + renderWithInterfaces(['openaiResponses']); + + await screen.findByRole('group', { name: 'TestSuites.OpenAIResponses' }); + const groupNames = screen + .getAllByRole('group') + .map((group) => group.getAttribute('aria-labelledby')) + .map((id) => document.getElementById(id ?? '')?.textContent); + + expect(groupNames).toEqual([ + 'TestSuites.OpenAIChatCompletions', + 'TestSuites.OpenAIResponses', + 'TestSuites.Other', + ]); + }); + + test('renders for the full declared interface list', async () => { + renderWithInterfaces(['chat', 'openaiChatCompletions', 'openaiResponses', 'anthropicMessages']); + + expect(await screen.findByRole('group', { name: 'TestSuites.OpenAIResponses' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'POST /openai/v1/responses' })).toBeInTheDocument(); + }); + + test('is absent for a target declaring only anthropicMessages', async () => { + renderWithInterfaces(['chat', 'anthropicMessages']); + + await screen.findByRole('button', { name: 'POST /chat/completions' }); + expect(screen.queryByRole('group', { name: 'TestSuites.OpenAIResponses' })).not.toBeInTheDocument(); + }); + + test('is absent when the declared interface list is empty', async () => { + renderWithInterfaces([]); + + await screen.findByRole('button', { name: 'POST /chat/completions' }); + expect(screen.queryByRole('group', { name: 'TestSuites.OpenAIResponses' })).not.toBeInTheDocument(); + }); + + test.each([ + ['no interfaces are reported', undefined], + ['the declared list is empty', []], + ['the declared list omits openaiResponses', ['chat', 'anthropicMessages']], + ['the declared list includes openaiResponses', ['chat', 'openaiResponses']], + ])('keeps the custom routes group when %s', async (_label, interfaces) => { + renderWithInterfaces(interfaces as string[] | undefined); + + expect(await screen.findByRole('group', { name: 'TestSuites.Other' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'GET /api/users' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'POST /api/users' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'GET /api/data' })).toBeInTheDocument(); + }); + + test('stays visible for a suite already selecting a Responses method', async () => { + renderWithInterfaces(undefined, { + endpointRef: { method: 'POST', relativeUrlPattern: '/openai/v1/responses' }, + }); + + expect(await screen.findByRole('group', { name: 'TestSuites.OpenAIResponses' })).toBeInTheDocument(); + }); + + test('marks the saved Responses method as current', async () => { + renderWithInterfaces(['openaiResponses'], { + endpointRef: { method: 'POST', relativeUrlPattern: '^/openai/v1/responses/[^/]+/cancel$' }, + }); + + const cancel = await screen.findByRole('button', { + name: 'POST /openai/v1/responses/{response_id}/cancel', + }); + + expect(cancel).toHaveAttribute('aria-current', 'true'); + expect(screen.getByRole('button', { name: 'POST /chat/completions' })).toHaveAttribute('aria-current', 'false'); + }); + + test('seeds the create-response suite with the target deployment id', async () => { + const user = userEvent.setup(); + renderWithInterfaces(['openaiResponses']); + + await user.click(await screen.findByRole('button', { name: 'POST /openai/v1/responses' })); + + const updater = onChange.mock.calls.at(-1)?.[0]; + expect(updater({}).requestTemplate.body.content).toEqual({ model: 'gpt-4o', input: '${{user_message}}' }); + expect(updater({}).responseColumns[0]).toEqual(expect.objectContaining({ name: 'answer' })); + }); + + test('seeds a response-scoped operation with a placeholder path and clears the previous columns', async () => { + const user = userEvent.setup(); + renderWithInterfaces(['openaiResponses']); + + await user.click(await screen.findByRole('button', { name: 'GET /openai/v1/responses/{response_id}' })); + + const previous: any = { + responseColumns: [{ name: 'answer', displayName: 'answer', expression: 'choices[0].message.content' }], + }; + const updater = onChange.mock.calls.at(-1)?.[0]; + expect(updater(previous).requestTemplate.urlTemplate).toBe('/openai/v1/responses/${{response_id}}'); + expect(updater(previous).requestTemplate.body.content).toEqual({}); + expect(updater(previous).responseColumns).toEqual([]); + }); + }); + + describe('Anthropic Messages group', () => { + const selectedApplication: any = { deploymentId: 'claude-3', $type: 'dial-model' }; + + const renderWithInterfaces = (interfaces?: string[], testSuite: any = { endpointRef: {} }) => { + mockGetDeploymentById.mockResolvedValue({ ...mockDeployment, deploymentId: 'claude-3', interfaces }); + + return render(); + }; + + test('is absent when the deployment reports no interfaces', async () => { + renderWithInterfaces(); + + await screen.findByRole('button', { name: 'POST /chat/completions' }); + expect(screen.queryByRole('group', { name: 'TestSuites.AnthropicMessages' })).not.toBeInTheDocument(); + }); + + test('is absent when the reported interfaces omit anthropicMessages', async () => { + renderWithInterfaces(['chat', 'openaiChatCompletions']); + + await screen.findByRole('button', { name: 'POST /chat/completions' }); + expect(screen.queryByRole('group', { name: 'TestSuites.AnthropicMessages' })).not.toBeInTheDocument(); + }); + + test('renders the create-message operation when anthropicMessages is reported', async () => { + renderWithInterfaces(['chat', 'anthropicMessages']); + + expect(await screen.findByRole('group', { name: 'TestSuites.AnthropicMessages' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'POST /anthropic/v1/messages' })).toBeInTheDocument(); + }); + + test('is absent when a features property is truthy, since there is no features-flag equivalent', async () => { + mockGetDeploymentById.mockResolvedValue({ + ...mockDeployment, + deploymentId: 'claude-3', + interfaces: undefined, + features: { chat_completion: true, responses_api: true }, + }); + + render( + , + ); + + await screen.findByRole('button', { name: 'POST /chat/completions' }); + expect(screen.queryByRole('group', { name: 'TestSuites.AnthropicMessages' })).not.toBeInTheDocument(); + }); + + test('stays visible for a suite already selecting the Anthropic Messages method', async () => { + renderWithInterfaces(undefined, { + endpointRef: { method: 'POST', relativeUrlPattern: '/anthropic/v1/messages' }, + }); + + expect(await screen.findByRole('group', { name: 'TestSuites.AnthropicMessages' })).toBeInTheDocument(); + }); + + test('marks the saved Anthropic Messages method as current', async () => { + renderWithInterfaces(['anthropicMessages'], { + endpointRef: { method: 'POST', relativeUrlPattern: '/anthropic/v1/messages' }, + }); + + const create = await screen.findByRole('button', { name: 'POST /anthropic/v1/messages' }); + + expect(create).toHaveAttribute('aria-current', 'true'); + expect(screen.getByRole('button', { name: 'POST /chat/completions' })).toHaveAttribute('aria-current', 'false'); + }); + + test('seeds the create-message suite with the target deployment id', async () => { + const user = userEvent.setup(); + renderWithInterfaces(['anthropicMessages']); + + await user.click(await screen.findByRole('button', { name: 'POST /anthropic/v1/messages' })); + + const updater = onChange.mock.calls.at(-1)?.[0]; + expect(updater({}).requestTemplate.body.content).toEqual({ + model: 'claude-3', + max_tokens: 1024, + messages: [{ role: 'user', content: '${{user_message}}' }], + }); + expect(updater({}).responseColumns[0]).toEqual(expect.objectContaining({ name: 'answer' })); + }); + }); }); diff --git a/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/Target.tsx b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/Target.tsx index ed6b5f727a..c169b29c21 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/Target.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/Target.tsx @@ -7,12 +7,13 @@ import { DialTabs } from '@epam/ai-dial-ui-kit'; import { getDeployments } from '@/src/app/[lang]/test-suites/actions'; import RadioSelectGrid from '@/src/components/Grid/GridView/RadioSelectGrid'; import { EVALUATION_DEPLOYMENTS_COLUMNS, MCP_DEPLOYMENTS_COLUMNS } from '@/src/constants/grid-columns/grid-columns'; +import { MCP_INTERFACE_FILTER } from '@/src/constants/deployment-interfaces'; import { EntitiesI18nKey, MenuI18nKey } from '@/src/constants/i18n'; import { useI18n } from '@/src/locales/client'; import { Deployment, DeploymentType } from '@/src/models/evaluation/deployment'; import { SuiteType, TestSuite } from '@/src/models/evaluation/test-suite'; import { TargetTab } from './types'; -import { buildDeploymentUpdate, buildMcpDeploymentUpdate, getInitialTab } from './utils'; +import { applyTargetSelection, getInitialTab } from './utils'; interface Props { selectedTargetId?: string; @@ -44,10 +45,7 @@ const Target: FC = ({ selectedTargetId, suiteType, onChangeTarget, onChan const onSelect = useCallback( (data: Deployment) => { onChangeTarget(data); - onChange((prev: TestSuite) => ({ - ...prev, - ...(activeTab === TargetTab.Mcp ? buildMcpDeploymentUpdate(data) : buildDeploymentUpdate(data)), - })); + onChange((prev: TestSuite) => applyTargetSelection(prev, data, activeTab)); }, [onChangeTarget, onChange, activeTab], ); @@ -69,7 +67,7 @@ const Target: FC = ({ selectedTargetId, suiteType, onChangeTarget, onChan setIsLoading(true); - getDeployments(type, activeTab === TargetTab.Mcp ? 'mcp' : void 0).then((res) => { + getDeployments(type, activeTab === TargetTab.Mcp ? MCP_INTERFACE_FILTER : void 0).then((res) => { const filtered = res?.response || []; deploymentsByTabRef.current.set(activeTab, filtered); setDeployments(filtered); diff --git a/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/tests/Target.spec.tsx b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/tests/Target.spec.tsx new file mode 100644 index 0000000000..c2eacb5bfe --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/tests/Target.spec.tsx @@ -0,0 +1,57 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +import { CREATE_RESPONSE_METHOD } from '@/src/components/TestSuites/constants/responses-method'; +import { Deployment } from '@/src/models/evaluation/deployment'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; +import Target from '../Target'; + +const deployment: Deployment = { + $type: 'dial-application', + deploymentId: 'new-target', + displayName: 'New target', +}; +const getDeploymentsMock = vi.fn(); + +vi.mock('@/src/app/[lang]/test-suites/actions', () => ({ + getDeployments: (...args: unknown[]) => getDeploymentsMock(...args), +})); + +vi.mock('@/src/components/Grid/GridView/RadioSelectGrid', () => ({ + default: ({ onSelect }: { onSelect: (selected: Deployment) => void }) => ( + + ), +})); + +describe('Target', () => { + beforeEach(() => { + getDeploymentsMock.mockReset(); + getDeploymentsMock.mockResolvedValue({ response: [] }); + }); + + test('updates the target and reseeds its request model in one state update', async () => { + const user = userEvent.setup(); + const onChangeTarget = vi.fn(); + const onChange = vi.fn(); + const suite = { + endpointRef: CREATE_RESPONSE_METHOD, + requestTemplate: { body: { content: { model: 'old-target', input: 'hello' } } }, + } as TestSuite; + + render(); + + await user.click(screen.getByRole('button', { name: 'Select target' })); + + expect(onChangeTarget).toHaveBeenCalledWith(deployment); + expect(onChange).toHaveBeenCalledOnce(); + + const update = onChange.mock.calls[0][0] as (previous: TestSuite) => TestSuite; + const result = update(suite); + + expect(result.deploymentRef?.id).toBe('new-target'); + expect(result.requestTemplate?.body?.content).toEqual({ model: 'new-target', input: 'hello' }); + }); +}); diff --git a/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/tests/utils.spec.ts b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/tests/utils.spec.ts index fdbdddd6ab..72c0475bc6 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/tests/utils.spec.ts +++ b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/tests/utils.spec.ts @@ -1,9 +1,11 @@ import { describe, expect, test } from 'vitest'; +import { CREATE_MESSAGE_METHOD } from '@/src/components/TestSuites/constants/anthropic-messages-method'; +import { CREATE_RESPONSE_METHOD } from '@/src/components/TestSuites/constants/responses-method'; import { Deployment } from '@/src/models/evaluation/deployment'; -import { SuiteType } from '@/src/models/evaluation/test-suite'; +import { SuiteType, TestSuite } from '@/src/models/evaluation/test-suite'; import { TargetTab } from '../types'; -import { buildDeploymentUpdate, buildMcpDeploymentUpdate, getInitialTab } from '../utils'; +import { applyTargetSelection, buildDeploymentUpdate, buildMcpDeploymentUpdate, getInitialTab } from '../utils'; describe('Target utils', () => { describe('buildDeploymentUpdate', () => { @@ -81,6 +83,86 @@ describe('Target utils', () => { }); }); + describe('applyTargetSelection', () => { + const deployment: Deployment = { + $type: 'dial-model', + deploymentId: 'new-model', + displayName: 'New model', + }; + + test('updates the deployment and reseeds matching requests throughout the chain', () => { + const suite = { + endpointRef: CREATE_RESPONSE_METHOD, + requestTemplate: { body: { content: { model: 'old-model', input: 'hello', store: true } } }, + additionalRequests: [ + { + name: 'anthropic', + endpointRef: CREATE_MESSAGE_METHOD, + requestTemplate: { + body: { content: { model: 'old-model', max_tokens: 1024, messages: [] } }, + }, + }, + { + name: 'route', + endpointRef: { method: 'POST', relativeUrlPattern: '/custom' }, + requestTemplate: { body: { content: { model: 'custom-model' } } }, + }, + ], + } as TestSuite; + + const result = applyTargetSelection(suite, deployment, TargetTab.Models); + + expect(result.deploymentRef).toEqual({ + id: 'new-model', + name: 'New model', + version: undefined, + type: 'dial-model', + }); + expect(result.requestTemplate?.body?.content).toEqual({ model: 'new-model', input: 'hello', store: true }); + expect(result.additionalRequests?.[0].requestTemplate?.body?.content).toEqual({ + model: 'new-model', + max_tokens: 1024, + messages: [], + }); + expect(result.additionalRequests?.[1].requestTemplate?.body?.content).toEqual({ model: 'custom-model' }); + }); + + test('leaves unrelated request bodies unchanged', () => { + const suite = { + endpointRef: { method: 'POST', relativeUrlPattern: '/chat/completions' }, + requestTemplate: { body: { content: { model: 'hand-edited', messages: [] } } }, + } as TestSuite; + + const result = applyTargetSelection(suite, deployment, TargetTab.Applications); + + expect(result.requestTemplate).toBe(suite.requestTemplate); + }); + + test('uses the existing MCP transition without reseeding HTTP requests', () => { + const suite = { + endpointRef: CREATE_RESPONSE_METHOD, + requestTemplate: { body: { content: { model: 'old-model', input: 'hello' } } }, + additionalRequests: [{ endpointRef: CREATE_MESSAGE_METHOD }], + } as TestSuite; + const mcpDeployment: Deployment = { + $type: 'dial-toolset', + deploymentId: 'mcp-1', + displayName: 'MCP target', + }; + + const result = applyTargetSelection(suite, mcpDeployment, TargetTab.Mcp); + + expect(result).toMatchObject({ + suiteType: SuiteType.McpTool, + mcpDeploymentRef: { id: 'mcp-1', name: 'MCP target', type: 'dial-toolset' }, + deploymentRef: undefined, + endpointRef: undefined, + requestTemplate: undefined, + }); + expect(result.additionalRequests).toBe(suite.additionalRequests); + }); + }); + describe('getInitialTab', () => { test('should return Mcp tab for MCP_TOOL suite type', () => { expect(getInitialTab(SuiteType.McpTool)).toBe(TargetTab.Mcp); diff --git a/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/utils.ts b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/utils.ts index 0dd8148d9e..3ad17c7a2b 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/utils.ts +++ b/apps/ai-dial-admin/src/components/TestSuites/Modals/Create/utils.ts @@ -1,7 +1,12 @@ +import { CREATE_MESSAGE_METHOD } from '@/src/components/TestSuites/constants/anthropic-messages-method'; +import { CREATE_RESPONSE_METHOD } from '@/src/components/TestSuites/constants/responses-method'; +import { reseedRequestModels } from '@/src/components/TestSuites/utils/model-reseeding'; import { Deployment } from '@/src/models/evaluation/deployment'; import { SuiteType, TestSuite } from '@/src/models/evaluation/test-suite'; import { TargetTab } from './types'; +const MODEL_TARGET_ENDPOINTS = [CREATE_MESSAGE_METHOD, CREATE_RESPONSE_METHOD]; + export function buildDeploymentUpdate(data: Deployment): Partial { return { suiteType: SuiteType.Deployment, @@ -32,6 +37,17 @@ export function buildMcpDeploymentUpdate(deployment: Deployment): Partial ({ })); vi.mock('@/src/components/TestSuites/Modals/Create/CreateTestSuite', () => ({ - default: () =>
CreateTestSuite
, + default: ({ currentEntity, onCreate }: { currentEntity: TestSuite; onCreate: (suite: TestSuite) => void }) => ( + + ), })); vi.mock('@epam/ai-dial-ui-kit', async (importOriginal) => { @@ -193,6 +197,21 @@ describe('TestSuiteProperties', () => { expect(getAllDeploymentsMock).not.toHaveBeenCalled(); }); + test('finishing the target picker without a target change preserves a hand-edited model', async () => { + const user = userEvent.setup(); + const suite = { + deploymentRef: { id: 'app-1', name: 'My App', type: DeploymentType.Application }, + endpointRef: { method: 'POST', relativeUrlPattern: '/openai/v1/responses' }, + requestTemplate: { body: { content: { model: 'hand-edited', input: 'hello' } } }, + } as TestSuite; + + render(); + + await user.click(screen.getByRole('button', { name: 'Finish update' })); + + expect(onChangeMock).toHaveBeenCalledWith(suite); + }); + test('Open for MCP asset toolset uses mcpDeploymentRef without by-id lookup', async () => { const user = userEvent.setup(); diff --git a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOut.tsx b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOut.tsx index 8d0283cdab..0c2837fbef 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOut.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOut.tsx @@ -23,12 +23,20 @@ import { BasicI18nKey, ButtonsI18nKey, TestSuitesI18nKey } from '@/src/constants import { BASE_BUTTON_ICON_PROPS } from '@/src/constants/main-layout'; import { useAppContext } from '@/src/context/AppContext'; import { useI18n } from '@/src/locales/client'; -import { SuiteType, TestCase, TestCaseSchema, TestSuite, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; +import { + SuiteType, + TestCase, + TestCaseSchema, + TestSuite, + TryOutCoreResponse, + TryOutResponse, +} from '@/src/models/evaluation/test-suite'; import { columnsTab, EntityViewTab, responseTab } from '@/src/utils/tabs/utils'; import { normalizeResponseBodyForColumns, unwrapJsonRequestBody, } from '@/src/components/TestSuites/utils/column-eval-context'; +import { TryOutInvocation } from '@/src/components/TestSuites/utils/models'; import { getRequestTurnCounts, getTryOutSectionShape } from '@/src/utils/evaluation/tryout-sections'; import CollapsibleSection from './CollapsibleSection'; import TryOutColumns from './TryOutColumns'; @@ -36,10 +44,6 @@ import TryOutRequestPreview from './TryOutRequestPreview'; import TryOutRequestTabs from './TryOutRequestTabs'; import TryOutResponsePreview from './TryOutResponse'; -export interface TryOutResponse { - statusCode: number; - [key: string]: unknown; -} interface Props { testSuite: TestSuite; testCaseId?: string; @@ -48,7 +52,7 @@ interface Props { } const TryOutResponseBody: FC<{ - response: TryOutResponse | null; + response: TryOutCoreResponse | null; isRequestSend: boolean; growOnOpen: boolean; }> = ({ response, isRequestSend, growOnOpen }) => { @@ -84,13 +88,24 @@ const TryOut: FC = ({ testSuite, testCaseId, schema, initialTestCase }) = const tabs = [responseTab(t), columnsTab(t)]; const [activeTab, setActiveTab] = useState(tabs[0].id as EntityViewTab); const [requestBody, setRequestBody] = useState>({}); - const [response, setResponse] = useState(null); - const [resolvedRequest, setResolvedRequest] = useState>({}); - const [history, setHistory] = useState(undefined); + const [tryOutResult, setTryOutResult] = useState(null); const [isRequestSend, setIsRequestSend] = useState(false); - const [grafanaTraceUrl, setGrafanaTraceUrl] = useState(undefined); const [selectedRequestIndex, setSelectedRequestIndex] = useState(0); - const [isPreviewLoading, setIsPreviewLoading] = useState(() => !response && !!testCaseId); + const [isPreviewLoading, setIsPreviewLoading] = useState(() => !!testCaseId); + + const response = tryOutResult?.response ?? null; + const resolvedRequest = tryOutResult?.resolvedRequest ?? {}; + const history = tryOutResult?.history; + const grafanaTraceUrl = tryOutResult?.grafanaTraceUrl; + + const invocation = useMemo( + () => ({ + response: tryOutResult?.response, + extractedColumns: tryOutResult?.extractedColumns, + extractionWarnings: tryOutResult?.extractionWarnings, + }), + [tryOutResult], + ); const multiTurnLength = initialTestCase?.multiTurnData?.length ?? 0; const turnCounts = useMemo( @@ -126,18 +141,15 @@ const TryOut: FC = ({ testSuite, testCaseId, schema, initialTestCase }) = const testSuiteId = testSuite.id || ''; if (res?.success) { - const tryoutResponse = (res.response?.response as TryOutResponse) || null; - setResolvedRequest(res.response?.resolvedRequest || {}); - setResponse(tryoutResponse); - setGrafanaTraceUrl(res.response?.grafanaTraceUrl); - setHistory(res.response?.history); + setTryOutResult(res.response ?? null); if (!testCaseId) saveTryoutResponseToStorage(testSuiteId, res.response); } else { - const errorResponse = { response: { error: res?.errorMessage || 'Unknown error', statusCode: 500 } }; - setResolvedRequest({ body: requestBody || {} }); - setResponse(errorResponse.response); - setHistory(undefined); - if (!testCaseId) saveTryoutResponseToStorage(testSuiteId, errorResponse as any); + const errorResult: TryOutResponse = { + resolvedRequest: { body: requestBody || {} }, + response: { error: res?.errorMessage || 'Unknown error', statusCode: 500 }, + }; + setTryOutResult(errorResult); + if (!testCaseId) saveTryoutResponseToStorage(testSuiteId, errorResult); } } finally { setIsRequestSend(false); @@ -148,10 +160,7 @@ const TryOut: FC = ({ testSuite, testCaseId, schema, initialTestCase }) = if (!testCaseId) { const responseFromStorage = getTryoutResponseFromStorage(testSuite.id || ''); if (responseFromStorage) { - setResponse(responseFromStorage.response as TryOutResponse); - setResolvedRequest(responseFromStorage.resolvedRequest || {}); - setGrafanaTraceUrl(responseFromStorage.grafanaTraceUrl); - setHistory(responseFromStorage.history); + setTryOutResult(responseFromStorage); } } @@ -187,7 +196,7 @@ const TryOut: FC = ({ testSuite, testCaseId, schema, initialTestCase }) = iconBefore={} label={t(ButtonsI18nKey.Change)} onClick={() => { - setResponse(null); + setTryOutResult(null); setActiveTab(EntityViewTab.Response); }} /> @@ -267,6 +276,7 @@ const TryOut: FC = ({ testSuite, testCaseId, schema, initialTestCase }) = schema={schema} multiTurnData={initialTestCase?.multiTurnData} columns={testSuite.responseColumns} + invocation={invocation} response={normalizeResponseBodyForColumns(response?.body as Record)} request={unwrapJsonRequestBody(resolvedRequest.body as Record | undefined)} isLoading={isRequestSend} diff --git a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutColumns.tsx b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutColumns.tsx index c825b5abbf..5a1b5c8543 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutColumns.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutColumns.tsx @@ -7,12 +7,15 @@ import { capitalize } from 'lodash'; import CopyButton from '@/src/components/Common/CopyButton/CopyButton'; import JsonEditor from '@/src/components/EntityTabs/JsonEditor/JsonEditor'; +import { evaluateTryOutColumnSections } from '@/src/components/TestSuites/utils/evaluate-columns'; import { - evaluateTryOutColumnSections, + ColumnExtractionStatus, EvaluatedColumn, - TryOutColumnTurnResult, + NotExtractedReason, TryOutColumnResults, -} from '@/src/components/TestSuites/utils/evaluate-columns'; + TryOutColumnTurnResult, + TryOutInvocation, +} from '@/src/components/TestSuites/utils/models'; import { BasicI18nKey, TestSuitesI18nKey, ValidityStatusI18nKey } from '@/src/constants/i18n'; import { useI18n } from '@/src/locales/client'; import { ResponseColumn, TestCaseSchema, TestSuite, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; @@ -26,46 +29,84 @@ interface Props { schema?: TestCaseSchema[]; multiTurnData?: Record[]; columns?: ResponseColumn[]; + invocation?: TryOutInvocation; + /** Normalized response and request bodies — used only by the MCP fallback. */ response?: Record; request?: Record; selectedRequestIndex?: number; } -const ColumnResultsList: FC<{ columns: EvaluatedColumn[] }> = ({ columns }) => { +interface ColumnStatusStyle { + cardClass: string; + badgeClass: string; + labelKey: string; +} + +const COLUMN_STATUS_STYLE: Record = { + [ColumnExtractionStatus.Extracted]: { + cardClass: 'border-success bg-success', + badgeClass: 'border-success bg-controls-accent-success-alpha-hover', + labelKey: ValidityStatusI18nKey.Valid, + }, + [ColumnExtractionStatus.Failed]: { + cardClass: 'border-error bg-error', + badgeClass: 'border-error bg-controls-error-alpha-hover', + labelKey: ValidityStatusI18nKey.Invalid, + }, + [ColumnExtractionStatus.NotExtracted]: { + cardClass: 'border-primary bg-layer-2', + badgeClass: 'border-primary bg-layer-3', + labelKey: TestSuitesI18nKey.ColumnNotExtracted, + }, +}; + +/** Stated per reason rather than through a lookup, so each key keeps its own interpolation params. */ +const getNotExtractedReason = (t: ReturnType, column: EvaluatedColumn): string => { + if (column.reason === NotExtractedReason.RequestFailed) { + return t(TestSuitesI18nKey.ColumnNotExtractedRequestFailed, { statusCode: column.statusCode ?? '' }); + } + if (column.reason === NotExtractedReason.StreamIncomplete) { + return t(TestSuitesI18nKey.ColumnNotExtractedStreamIncomplete); + } + + return t(TestSuitesI18nKey.ColumnNotExtractedNoneReported); +}; + +const ColumnResultCard: FC<{ column: EvaluatedColumn }> = ({ column }) => { const t = useI18n(); + const isNotExtracted = column.status === ColumnExtractionStatus.NotExtracted; + const statusStyle = COLUMN_STATUS_STYLE[column.status]; + const statusLabel = t(statusStyle.labelKey); + const reason = isNotExtracted ? getNotExtractedReason(t, column) : column.error; return ( -
- {columns.map((column, index) => ( -
-
-
-
{column.name}
- -
- -
-
{column.expression}
-
{column.result !== null ? column.result : 'Null'}
+
+
+
+
{column.name}
+
- ))} + +
+
{column.expression}
+ {reason ?
{reason}
: null} + {isNotExtracted ? null :
{column.result}
}
); }; +const ColumnResultsList: FC<{ columns: EvaluatedColumn[] }> = ({ columns }) => ( +
+ {columns.map((column, index) => ( + + ))} +
+); + const TurnColumnSection: FC<{ turn: TryOutColumnTurnResult; showTurnLabel: boolean }> = ({ turn, showTurnLabel }) => { const t = useI18n(); const copyText = useMemo( @@ -107,6 +148,7 @@ const TryOutColumns: FC = ({ schema, multiTurnData, columns, + invocation, response, request, selectedRequestIndex = 0, @@ -125,6 +167,7 @@ const TryOutColumns: FC = ({ schema, multiTurnLength: multiTurnData?.length ?? 0, fallbackColumns: columns || [], + fallbackInvocation: invocation || {}, fallbackResponse: response || {}, fallbackRequest: request, }) @@ -142,7 +185,7 @@ const TryOutColumns: FC = ({ return () => { cancelled = true; }; - }, [testSuite, history, schema, multiTurnData, columns, response, request]); + }, [testSuite, history, schema, multiTurnData, columns, invocation, response, request]); const renderGrouped = () => { const group = results.groups?.find((item) => item.requestIndex === selectedRequestIndex); diff --git a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutResponse.tsx b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutResponse.tsx index 33dfe943e5..ea4321751f 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutResponse.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/components/TryOutResponse.tsx @@ -14,7 +14,7 @@ import CopyButton from '@/src/components/Common/CopyButton/CopyButton'; import JsonEditor from '@/src/components/EntityTabs/JsonEditor/JsonEditor'; import { BasicI18nKey, RunsI18nKey, TestSuitesI18nKey } from '@/src/constants/i18n'; import { useI18n } from '@/src/locales/client'; -import { TestCaseSchema, TestSuite, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; +import { TestCaseSchema, TestSuite, TryOutCoreResponse, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; import { getRequestTurnCounts, getTryOutSectionShape, @@ -22,10 +22,9 @@ import { shouldShowTurnLabels, } from '@/src/utils/evaluation/tryout-sections'; import CollapsibleSection from './CollapsibleSection'; -import { TryOutResponse } from './TryOut'; interface Props { - response: TryOutResponse; + response: TryOutCoreResponse; resolvedRequest: Record; history?: TryOutHistoryEntry[]; grafanaTraceUrl?: string; @@ -76,7 +75,7 @@ const HistoryEntryPair: FC<{ }> = ({ entry, isRequestSend, sectionTitle }) => { const t = useI18n(); const turnRequestBody = (entry.resolvedRequest?.body as object) ?? {}; - const turnResponseBody = (entry.response as { body?: object })?.body as object | undefined; + const turnResponseBody = entry.response?.body as object | undefined; return (
diff --git a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOut.spec.tsx b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOut.spec.tsx index 7ebfddc505..ccb7568a00 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOut.spec.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOut.spec.tsx @@ -5,9 +5,10 @@ import { describe, expect, test, vi } from 'vitest'; import { getTestCaseTemplateVariables, tryOutTestCase, tryOutTestSuite } from '@/src/app/[lang]/test-suites/actions'; import { convertVariableIntoInitialRequest } from '@/src/components/TestSuites/utils/template-variables'; -import { ButtonsI18nKey, TabsI18nKey, TestSuitesI18nKey } from '@/src/constants/i18n'; +import { ButtonsI18nKey, TabsI18nKey, TestSuitesI18nKey, ValidityStatusI18nKey } from '@/src/constants/i18n'; import { SuiteType, TestCase, TestCaseSchema, TestSuite, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; import { TestCaseItemType } from '@/src/types/evaluation'; +import { getTryoutResponseFromStorage } from '@/src/components/TestSuites/utils/tryout-storage'; import TryOut from '../components/TryOut'; vi.mock('@/src/app/[lang]/test-suites/actions', () => ({ @@ -137,11 +138,16 @@ const deploymentSuite: TestSuite = { endpointRef: { method: 'POST', relativeUrlPattern: '/api/search' }, }; -const deploymentSuiteWithRequestColumn: TestSuite = { - ...deploymentSuite, +const mcpSuiteWithRequestColumn: TestSuite = { + ...mcpSuite, responseColumns: [{ name: 'reqFoo', displayName: 'reqFoo', expression: '$request.foo', type: 'STRING' }], }; +const deploymentSuiteWithAnswerColumn: TestSuite = { + ...deploymentSuite, + responseColumns: [{ name: 'answer', displayName: 'answer', expression: 'output', type: 'STRING' }], +}; + const multiRequestSchema: TestCaseSchema[] = [ { name: 'shared', type: TestCaseItemType.STRING, required: false, description: '', perTurn: false }, ]; @@ -219,7 +225,7 @@ describe('TryOut Columns tab request binding', () => { expect(screen.queryByRole('tab', { name: TabsI18nKey.Columns })).not.toBeInTheDocument(); }); - test('binds $request to the request body, not the request envelope', async () => { + test('binds $request to the request body, not the request envelope, for an MCP suite', async () => { vi.mocked(tryOutTestSuite).mockResolvedValueOnce({ success: true, response: { @@ -229,7 +235,7 @@ describe('TryOut Columns tab request binding', () => { }); const user = userEvent.setup(); - render(); + render(); const sendButton = await screen.findByRole('button', { name: ButtonsI18nKey.SendRequest }); await user.click(sendButton); @@ -241,6 +247,85 @@ describe('TryOut Columns tab request binding', () => { expect(screen.getByText('bar')).toBeInTheDocument(); }); }); + + test("renders the backend's reported extraction for a deployment suite", async () => { + vi.mocked(tryOutTestSuite).mockResolvedValueOnce({ + success: true, + response: { + resolvedRequest: { url: '/openai/v1/responses', body: {} }, + response: { statusCode: 200, body: { events: [] } }, + extractedColumns: { answer: 'Hi there, friend!' }, + extractionWarnings: [], + }, + }); + + const user = userEvent.setup(); + render(); + + await user.click(await screen.findByRole('button', { name: ButtonsI18nKey.SendRequest })); + await user.click(await screen.findByRole('tab', { name: TabsI18nKey.Columns })); + + await waitFor(() => { + expect(screen.getByText('Hi there, friend!')).toBeInTheDocument(); + }); + expect(screen.getByText(ValidityStatusI18nKey.Valid)).toBeInTheDocument(); + }); + + test('reports not extracted when the invocation failed', async () => { + vi.mocked(tryOutTestSuite).mockResolvedValueOnce({ + success: true, + response: { + resolvedRequest: { url: '/openai/v1/responses', body: {} }, + response: { statusCode: 401, body: 'At least API-KEY or Authorization header must be provided' }, + }, + }); + + const user = userEvent.setup(); + render(); + + await user.click(await screen.findByRole('button', { name: ButtonsI18nKey.SendRequest })); + await user.click(await screen.findByRole('tab', { name: TabsI18nKey.Columns })); + + await waitFor(() => { + expect(screen.getByText(TestSuitesI18nKey.ColumnNotExtracted)).toBeInTheDocument(); + }); + expect(screen.getByText(TestSuitesI18nKey.ColumnNotExtractedRequestFailed)).toBeInTheDocument(); + expect(screen.queryByText(ValidityStatusI18nKey.Invalid)).not.toBeInTheDocument(); + }); + + test('a restored result shows the same extraction as the original', async () => { + vi.mocked(getTryoutResponseFromStorage).mockReturnValueOnce({ + resolvedRequest: { url: '/openai/v1/responses', body: {} }, + response: { statusCode: 200, body: { events: [] } }, + extractedColumns: { answer: 'Hi there, friend!' }, + extractionWarnings: [], + }); + + const user = userEvent.setup(); + render(); + + await user.click(await screen.findByRole('tab', { name: TabsI18nKey.Columns })); + + await waitFor(() => { + expect(screen.getByText('Hi there, friend!')).toBeInTheDocument(); + }); + }); + + test('a restored result recorded before extraction was captured reports not extracted', async () => { + vi.mocked(getTryoutResponseFromStorage).mockReturnValueOnce({ + resolvedRequest: { url: '/openai/v1/responses', body: {} }, + response: { statusCode: 200, body: { events: [] } }, + }); + + const user = userEvent.setup(); + render(); + + await user.click(await screen.findByRole('tab', { name: TabsI18nKey.Columns })); + + await waitFor(() => { + expect(screen.getByText(TestSuitesI18nKey.ColumnNotExtractedNoneReported)).toBeInTheDocument(); + }); + }); }); describe('TryOut request tabs', () => { diff --git a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOutColumns.spec.tsx b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOutColumns.spec.tsx index 19542116ea..4398f0dbe9 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOutColumns.spec.tsx +++ b/apps/ai-dial-admin/src/components/TestSuites/RequestTemplate/tests/TryOutColumns.spec.tsx @@ -1,12 +1,14 @@ import { render, screen, waitFor } from '@testing-library/react'; import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { evaluateTryOutColumnSections } from '@/src/components/TestSuites/utils/evaluate-columns'; import { + ColumnExtractionStatus, EvaluatedColumn, - evaluateTryOutColumnSections, + NotExtractedReason, TryOutColumnResults, -} from '@/src/components/TestSuites/utils/evaluate-columns'; -import { TestSuitesI18nKey } from '@/src/constants/i18n'; +} from '@/src/components/TestSuites/utils/models'; +import { TestSuitesI18nKey, ValidityStatusI18nKey } from '@/src/constants/i18n'; import { ResponseColumn, SuiteType, TestSuite, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; import TryOutColumns from '../components/TryOutColumns'; @@ -48,7 +50,7 @@ const makeEvaluatedColumn = (overrides: Partial = {}): Evaluate expression: 'foo', type: 'STRING', result: 'bar', - valid: true, + status: ColumnExtractionStatus.Extracted, ...overrides, }); @@ -246,6 +248,94 @@ describe('TryOutColumns', () => { expect(screen.queryByText('JsonEditor:{"out":"c"}')).not.toBeInTheDocument(); }); + test('passes the reported extraction through as the invocation', async () => { + vi.mocked(evaluateTryOutColumnSections).mockResolvedValueOnce({ shape: 'single', flatColumns: [] }); + + const invocation = { + response: { statusCode: 200 }, + extractedColumns: { testCol: 'bar' }, + extractionWarnings: [], + }; + + render( + , + ); + + await waitFor(() => { + expect(evaluateTryOutColumnSections).toHaveBeenCalledWith( + expect.objectContaining({ fallbackInvocation: invocation }), + ); + }); + }); + + describe('column result cards', () => { + const renderColumns = async (columns: EvaluatedColumn[]) => { + vi.mocked(evaluateTryOutColumnSections).mockResolvedValueOnce({ shape: 'single', flatColumns: columns }); + + render(); + + await waitFor(() => { + expect(screen.getByText(columns[0].name)).toBeInTheDocument(); + }); + }; + + test('an extracted column shows its value and the valid badge', async () => { + await renderColumns([makeEvaluatedColumn({ result: 'Hi there, friend!' })]); + + expect(screen.getByRole('group', { name: TestSuitesI18nKey.ColumnResultLabel })).toBeInTheDocument(); + expect(screen.getByText('Hi there, friend!')).toBeInTheDocument(); + expect(screen.getByText(ValidityStatusI18nKey.Valid)).toBeInTheDocument(); + }); + + test('a failed column shows the backend error and no value', async () => { + await renderColumns([ + makeEvaluatedColumn({ + name: 'summary', + result: '', + status: ColumnExtractionStatus.Failed, + error: 'Expression matched nothing', + }), + ]); + + expect(screen.getByText('Expression matched nothing')).toBeInTheDocument(); + expect(screen.getByText(ValidityStatusI18nKey.Invalid)).toBeInTheDocument(); + }); + + test.each([ + [NotExtractedReason.RequestFailed, TestSuitesI18nKey.ColumnNotExtractedRequestFailed], + [NotExtractedReason.StreamIncomplete, TestSuitesI18nKey.ColumnNotExtractedStreamIncomplete], + [NotExtractedReason.NoExtractionReported, TestSuitesI18nKey.ColumnNotExtractedNoneReported], + ])('a not-extracted column states its reason (%s)', async (reason, reasonKey) => { + await renderColumns([ + makeEvaluatedColumn({ + result: '', + status: ColumnExtractionStatus.NotExtracted, + reason, + statusCode: 401, + }), + ]); + + expect(screen.getByText(TestSuitesI18nKey.ColumnNotExtracted)).toBeInTheDocument(); + expect(screen.getByText(reasonKey)).toBeInTheDocument(); + expect(screen.getByRole('group', { name: TestSuitesI18nKey.ColumnResultLabel })).toBeInTheDocument(); + }); + + test('all three kinds render together, each addressable by role', async () => { + await renderColumns([ + makeEvaluatedColumn({ name: 'answer' }), + makeEvaluatedColumn({ name: 'summary', status: ColumnExtractionStatus.Failed, result: '' }), + makeEvaluatedColumn({ + name: 'id', + status: ColumnExtractionStatus.NotExtracted, + reason: NotExtractedReason.RequestFailed, + result: '', + }), + ]); + + expect(screen.getAllByRole('group', { name: TestSuitesI18nKey.ColumnResultLabel })).toHaveLength(3); + }); + }); + test('shows Turn labels for single-request multi-turn suites', async () => { vi.mocked(evaluateTryOutColumnSections).mockResolvedValueOnce({ shape: 'turns', diff --git a/apps/ai-dial-admin/src/components/TestSuites/constants/anthropic-messages-body.ts b/apps/ai-dial-admin/src/components/TestSuites/constants/anthropic-messages-body.ts new file mode 100644 index 0000000000..bb10907324 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/constants/anthropic-messages-body.ts @@ -0,0 +1,22 @@ +/** + * `model` carries the target's deployment id rather than a template variable. + * `reseedAnthropicMessagesModel` keeps it in step when the suite's target changes. + */ +export const ANTHROPIC_MESSAGES_BODY = (deploymentId: string) => ({ + model: deploymentId, + max_tokens: 1024, + messages: [{ role: 'user', content: '${{user_message}}' }], +}); + +/** + * JSONata expression reaching the assistant's text in a create-message result. + * + * The response carries no top-level text field: `content` is an ordered array of blocks + * discriminated by `type`, and the generated text lives in the `text` field of blocks whose type is + * `text`, alongside any tool-use or thinking blocks. + * + * `$join` collapses the result to a single string: a model may split its answer across several text + * blocks, and the column is declared as a string, so an unjoined multi-match would hand the column + * an array. + */ +export const ANTHROPIC_MESSAGES_ANSWER_EXPRESSION = "$join(content[type='text'].text)"; diff --git a/apps/ai-dial-admin/src/components/TestSuites/constants/anthropic-messages-method.ts b/apps/ai-dial-admin/src/components/TestSuites/constants/anthropic-messages-method.ts new file mode 100644 index 0000000000..7028601374 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/constants/anthropic-messages-method.ts @@ -0,0 +1,268 @@ +/** + * DIAL's Anthropic Messages passthrough, shaped like `CREATE_RESPONSE_METHOD`. + * + * Every URL keeps DIAL's `/anthropic/v1` prefix, in the stored pattern, the seeded path, and the + * displayed label alike. It is what tells the Evaluation Framework backend that a request targets + * DIAL's Anthropic Messages passthrough rather than a `/messages` route the deployment happens to + * expose itself, which would otherwise be routed to the wrong host. + * + * The schemas below are mapped from Anthropic's own Messages API document (`CreateMessageRequest` + * → request, `Message` → response), with the same deviation `responses-method.ts` documents for + * Responses: `model` is typed as a plain string described as a deployment id, rather than an enum of + * Anthropic model names — the value that belongs here is a DIAL deployment id. + * + * The content-block unions on both sides are represented by their discriminator plus the variants a + * test suite actually exercises, not the document's full expansion — `convertSchemaToTable` renders + * only top-level properties, so a full expansion would be invisible in the table and unreadable in + * the JSON view. + * + * Every top-level property carries an explicit `type`, including the union-valued ones, because a + * property with only `oneOf` renders a blank Type cell in the schema table. + */ + +export const ANTHROPIC_MESSAGES_URL_PREFIX = '/anthropic/v1'; + +export const ANTHROPIC_MESSAGES_RELATIVE_URL = `${ANTHROPIC_MESSAGES_URL_PREFIX}/messages`; + +const CONTENT_TYPE_PARAMETER = { + name: 'Content-Type', + in: 'header', + required: true, + description: 'Must be application/json', + schema: { + type: 'string', + }, +}; + +const REQUEST_CONTENT_BLOCK = { + oneOf: [ + { + type: 'object', + title: 'Text', + required: ['type', 'text'], + properties: { + type: { type: 'string', enum: ['text'] }, + text: { type: 'string' }, + }, + }, + { + type: 'object', + title: 'Image', + required: ['type', 'source'], + properties: { + type: { type: 'string', enum: ['image'] }, + source: { type: 'object', description: 'A base64, URL, or file source for the image.' }, + }, + }, + { + type: 'object', + title: 'Tool use', + description: "A model-produced tool call, replayed back on an assistant message's content.", + required: ['type', 'id', 'name', 'input'], + properties: { + type: { type: 'string', enum: ['tool_use'] }, + id: { type: 'string' }, + name: { type: 'string' }, + input: { type: 'object' }, + }, + }, + { + type: 'object', + title: 'Tool result', + description: "The result of a tool call, sent back on a user message's content.", + required: ['type', 'tool_use_id'], + properties: { + type: { type: 'string', enum: ['tool_result'] }, + tool_use_id: { type: 'string' }, + content: { type: 'string' }, + is_error: { type: 'boolean' }, + }, + }, + ], +}; + +const MESSAGE = { + type: 'object', + required: ['role', 'content'], + properties: { + role: { type: 'string', enum: ['user', 'assistant'] }, + content: { + type: 'string', + description: 'Message text, or an array of content blocks for images and tool use/results.', + oneOf: [{ type: 'string' }, { type: 'array', items: REQUEST_CONTENT_BLOCK }], + }, + }, +}; + +const TOOL = { + type: 'object', + required: ['name', 'input_schema'], + properties: { + name: { type: 'string' }, + description: { type: 'string' }, + input_schema: { type: 'object', description: 'JSON Schema describing the tool input.' }, + }, +}; + +const TOOL_CHOICE = { + type: 'string', + description: 'auto, any, or none — or {"type":"tool","name":...} to force a specific tool.', + oneOf: [ + { type: 'string', enum: ['auto', 'any', 'none'] }, + { + type: 'object', + required: ['type', 'name'], + properties: { + type: { type: 'string', enum: ['tool'] }, + name: { type: 'string' }, + }, + }, + ], +}; + +export const CREATE_MESSAGE_METHOD = { + method: 'POST', + operationId: 'createMessage', + summary: ANTHROPIC_MESSAGES_RELATIVE_URL, + relativeUrlPattern: ANTHROPIC_MESSAGES_RELATIVE_URL, + description: + 'Creates a model response for the given conversation. Unlike chat completions, this endpoint is not parameterised on the deployment id, so the target deployment is selected by the `model` field in the request body.', + parameters: [CONTENT_TYPE_PARAMETER], + requestBodySchema: { + contentType: 'application/json', + schema: { + type: 'object', + required: ['model', 'messages', 'max_tokens'], + properties: { + model: { + type: 'string', + description: 'The id of the deployment to invoke.', + }, + messages: { + type: 'array', + description: 'Input messages, alternating user and assistant turns.', + items: MESSAGE, + }, + max_tokens: { + type: 'integer', + description: 'The maximum number of tokens to generate before stopping.', + }, + system: { + type: 'string', + description: 'System prompt, prepended before the first message.', + oneOf: [{ type: 'string' }, { type: 'array', items: REQUEST_CONTENT_BLOCK }], + }, + stop_sequences: { + type: 'array', + description: 'Custom sequences that, if generated, stop the response.', + items: { type: 'string' }, + }, + temperature: { + type: 'number', + minimum: 0, + maximum: 1, + description: 'Sampling temperature, between 0 and 1.', + }, + top_p: { + type: 'number', + description: 'Nucleus sampling probability mass.', + }, + top_k: { + type: 'integer', + description: 'Only sample from the top K options for each token.', + }, + tools: { + type: 'array', + description: 'Tools the model may call.', + items: TOOL, + }, + tool_choice: TOOL_CHOICE, + stream: { + type: 'boolean', + description: + 'If true, the response is streamed as server-sent events. Test suites read the JSON response, so leave this unset.', + }, + metadata: { + type: 'object', + description: 'Metadata about the request, e.g. an end-user identifier.', + }, + }, + }, + }, + responseBodySchema: { + type: 'object', + required: ['id', 'type', 'role', 'content', 'model', 'stop_reason', 'usage'], + properties: { + id: { type: 'string', description: 'Unique identifier for this message.' }, + type: { type: 'string', enum: ['message'] }, + role: { type: 'string', enum: ['assistant'] }, + content: { + type: 'array', + description: + 'The generated content, in order. Assistant text lives in the text field of blocks whose type is text — there is no top-level text field on the wire.', + items: { + oneOf: [ + { + type: 'object', + title: 'Text', + required: ['type', 'text'], + properties: { + type: { type: 'string', enum: ['text'] }, + text: { type: 'string', description: 'The generated text.' }, + citations: { type: 'array', items: { type: 'object' } }, + }, + }, + { + type: 'object', + title: 'Tool use', + required: ['type', 'id', 'name', 'input'], + properties: { + type: { type: 'string', enum: ['tool_use'] }, + id: { type: 'string' }, + name: { type: 'string' }, + input: { type: 'object' }, + }, + }, + { + type: 'object', + title: 'Thinking', + required: ['type', 'thinking'], + properties: { + type: { type: 'string', enum: ['thinking'] }, + thinking: { type: 'string' }, + signature: { type: 'string' }, + }, + }, + { + type: 'object', + title: 'Redacted thinking', + description: 'Encrypted reasoning that was flagged by safety systems.', + required: ['type', 'data'], + properties: { + type: { type: 'string', enum: ['redacted_thinking'] }, + data: { type: 'string' }, + }, + }, + ], + }, + }, + model: { type: 'string', description: 'The deployment that generated the message.' }, + stop_reason: { + type: 'string', + enum: ['end_turn', 'max_tokens', 'stop_sequence', 'tool_use', 'pause_turn', 'refusal'], + }, + stop_sequence: { type: 'string', description: 'The stop sequence that triggered stopping, if any.' }, + usage: { + type: 'object', + description: 'Token counts for the request and the generated output.', + required: ['input_tokens', 'output_tokens'], + properties: { + input_tokens: { type: 'integer' }, + output_tokens: { type: 'integer' }, + cache_creation_input_tokens: { type: 'integer' }, + cache_read_input_tokens: { type: 'integer' }, + }, + }, + }, + }, +}; diff --git a/apps/ai-dial-admin/src/components/TestSuites/constants/methods.ts b/apps/ai-dial-admin/src/components/TestSuites/constants/methods.ts index b8540152eb..12fdc4a67c 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/constants/methods.ts +++ b/apps/ai-dial-admin/src/components/TestSuites/constants/methods.ts @@ -1,7 +1,11 @@ import { APPLICATION_JSON_TYPE } from '@/src/constants/request-headers'; +import { ANTHROPIC_MESSAGES_ANSWER_EXPRESSION, ANTHROPIC_MESSAGES_BODY } from './anthropic-messages-body'; +import { ANTHROPIC_MESSAGES_RELATIVE_URL, CREATE_MESSAGE_METHOD } from './anthropic-messages-method'; import { CHAT_COMPLETION_BODY } from './chat-completion-body'; import { TestSuiteEndpointRef } from '@/src/models/evaluation/test-suite'; import { CHAT_COMPLETION_METHOD } from './chat-completion-method'; +import { RESPONSES_ANSWER_EXPRESSION, RESPONSES_BODY } from './responses-body'; +import { CREATE_RESPONSE_METHOD, RESPONSES_RELATIVE_URL } from './responses-method'; import { TestCaseItemType } from '@/src/types/evaluation'; export const CHAT_COMPLETION_RELATIVE_URL = '/chat/completions'; @@ -24,6 +28,58 @@ export const CHAT_COMPLETION_SUITE = { ], }; +export const RESPONSES_SUITE = (deploymentId: string) => ({ + endpointRef: CREATE_RESPONSE_METHOD, + requestTemplate: { + urlTemplate: RESPONSES_RELATIVE_URL, + body: { + contentType: APPLICATION_JSON_TYPE, + content: RESPONSES_BODY(deploymentId), + }, + }, + responseColumns: [ + { + name: 'answer', + displayName: 'answer', + expression: RESPONSES_ANSWER_EXPRESSION, + type: TestCaseItemType.STRING, + }, + ], +}); + +export const ANTHROPIC_MESSAGES_SUITE = (deploymentId: string) => ({ + endpointRef: CREATE_MESSAGE_METHOD, + requestTemplate: { + urlTemplate: ANTHROPIC_MESSAGES_RELATIVE_URL, + body: { + contentType: APPLICATION_JSON_TYPE, + content: ANTHROPIC_MESSAGES_BODY(deploymentId), + }, + }, + responseColumns: [ + { + name: 'answer', + displayName: 'answer', + expression: ANTHROPIC_MESSAGES_ANSWER_EXPRESSION, + type: TestCaseItemType.STRING, + }, + ], +}); + +export const RESPONSE_ITEM_SUITE = (route: TestSuiteEndpointRef, urlTemplate: string) => ({ + endpointRef: route, + requestTemplate: { + urlTemplate, + body: { + contentType: APPLICATION_JSON_TYPE, + content: {}, + }, + }, + // Explicitly empty rather than omitted: a seed is merged over the previous configuration, so + // leaving the key out would keep the previous method's extraction expressions. + responseColumns: [], +}); + export const DEFAULT_SUITE = (route: TestSuiteEndpointRef) => ({ endpointRef: { method: route.method, diff --git a/apps/ai-dial-admin/src/components/TestSuites/constants/responses-body.ts b/apps/ai-dial-admin/src/components/TestSuites/constants/responses-body.ts new file mode 100644 index 0000000000..45152ea867 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/constants/responses-body.ts @@ -0,0 +1,22 @@ +/** + * `model` carries the target's deployment id rather than a template variable. + * `reseedResponsesModel` keeps it in step when the suite's target changes. + */ +export const RESPONSES_BODY = (deploymentId: string) => ({ + model: deploymentId, + input: '${{user_message}}', +}); + +/** + * JSONata expression reaching the assistant's text in a create-response result. + * + * The response carries no top-level `output_text` — that is an SDK convenience accessor, not a wire + * field. `Response.output` is an ordered array of items whose `type` discriminates them, and the + * generated text lives in the `output_text` content parts of the items whose type is `message`. + * Reasoning items and tool calls share the array, hence both filters. + * + * `$join` collapses the result to a single string: a model may split its answer across several text + * parts or messages, and the column is declared as a string, so an unjoined multi-match would hand + * the column an array. + */ +export const RESPONSES_ANSWER_EXPRESSION = "$join(output[type='message'].content[type='output_text'].text)"; diff --git a/apps/ai-dial-admin/src/components/TestSuites/constants/responses-method.ts b/apps/ai-dial-admin/src/components/TestSuites/constants/responses-method.ts new file mode 100644 index 0000000000..aff612d657 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/constants/responses-method.ts @@ -0,0 +1,768 @@ +/** + * DIAL's OpenAI Responses API operations, shaped like `CHAT_COMPLETION_METHOD`. + * + * `relativeUrlPattern` is expressed as a regex, because `MethodInfo` validates the user-editable + * final path against it: a literal `{response_id}` pattern would reject every real response id. The + * readable form lives in `summary`. + * + * Every URL keeps DIAL's `/openai/v1` prefix, in the stored pattern, the seeded path, and the + * displayed label alike. It is not decoration: it is what tells the Evaluation Framework backend that + * a request targets DIAL's Responses API rather than a `/responses` route the deployment happens to + * expose itself, which would otherwise be routed to the wrong host. `/chat/completions` needs no + * equivalent because its own URL is parameterised on the deployment. + * + * DIAL's own OpenAPI declares `ResponsesApiRequest` as a bare `type: object`, so the create + * operation's schemas below are mapped from the OpenAI Responses API document (`CreateResponseRequest` + * → request, `Response` → response). Three deliberate deviations from that document: + * + * - `model` is typed as a plain string and described as a deployment id, rather than carrying the + * document's enum of OpenAI model names — the value that belongs here is a DIAL deployment id. + * - `model` and `input` are marked required. The document marks neither (`model` can arrive via + * `prompt`, and `input` via `conversation`), but here `model` is the only deployment selector (see + * `reseedResponsesModel`), and a test suite with no input does nothing. + * - The deep unions — `ResponseInputItem` (33 variants), `Tool` (16), `ResponseOutputItem` (28) — are + * represented by their discriminator plus the variants a test suite actually exercises, not + * inlined whole. `convertSchemaToTable` renders only top-level properties, so the full expansion + * would be invisible in the table and unreadable in the JSON view. + * + * Every top-level property carries an explicit `type`, including the union-valued ones, because a + * property with only `oneOf` renders a blank Type cell in the schema table. + */ + +export const RESPONSES_URL_PREFIX = '/openai/v1'; + +export const RESPONSES_RELATIVE_URL = `${RESPONSES_URL_PREFIX}/responses`; +export const RESPONSE_ITEM_RELATIVE_URL_PATTERN = `^${RESPONSES_RELATIVE_URL}/[^/]+$`; +export const RESPONSE_CANCEL_RELATIVE_URL_PATTERN = `^${RESPONSES_RELATIVE_URL}/[^/]+/cancel$`; + +export const RESPONSE_ITEM_DISPLAY_URL = `${RESPONSES_RELATIVE_URL}/{response_id}`; +export const RESPONSE_CANCEL_DISPLAY_URL = `${RESPONSE_ITEM_DISPLAY_URL}/cancel`; + +export const RESPONSE_ID_VARIABLE = 'response_id'; +export const RESPONSE_ITEM_URL_TEMPLATE = `${RESPONSES_RELATIVE_URL}/\${{${RESPONSE_ID_VARIABLE}}}`; +export const RESPONSE_CANCEL_URL_TEMPLATE = `${RESPONSE_ITEM_URL_TEMPLATE}/cancel`; + +const CACHE_POLICY_PARAMETER = { + name: 'X-DIAL-CACHE-POLICY', + in: 'header', + required: false, + schema: { + type: 'string', + enum: ['availability-priority', 'cache-priority'], + description: 'Upstream selection policy for prompt-caching deployments (availability-priority or cache-priority).', + }, +}; + +const CONTENT_TYPE_PARAMETER = { + name: 'Content-Type', + in: 'header', + required: true, + description: 'Must be application/json', + schema: { + type: 'string', + }, +}; + +const RESPONSE_ID_PARAMETER = { + name: 'response_id', + in: 'path', + required: true, + schema: { + type: 'string', + }, +}; + +const INPUT_CONTENT_PART = { + oneOf: [ + { + type: 'object', + title: 'Input text', + required: ['type', 'text'], + properties: { + type: { type: 'string', enum: ['input_text'] }, + text: { type: 'string' }, + }, + }, + { + type: 'object', + title: 'Input image', + required: ['type'], + properties: { + type: { type: 'string', enum: ['input_image'] }, + detail: { type: 'string', enum: ['low', 'high', 'auto', 'original'] }, + file_id: { type: 'string' }, + image_url: { type: 'string', description: 'A fully qualified URL or a base64 data URL.' }, + }, + }, + { + type: 'object', + title: 'Input file', + required: ['type'], + properties: { + type: { type: 'string', enum: ['input_file'] }, + detail: { type: 'string', enum: ['auto', 'low', 'high'] }, + file_data: { type: 'string', description: 'Base64-encoded file content.' }, + file_id: { type: 'string' }, + file_url: { type: 'string' }, + filename: { type: 'string' }, + }, + }, + ], +}; + +const INPUT_ITEM = { + oneOf: [ + { + type: 'object', + title: 'Message', + description: 'A message from the user, assistant, system, or developer.', + required: ['role', 'content'], + properties: { + type: { type: 'string', enum: ['message'] }, + role: { type: 'string', enum: ['user', 'assistant', 'system', 'developer'] }, + content: { + type: 'string', + description: 'Message text, or an array of content parts for images and files.', + oneOf: [{ type: 'string' }, { type: 'array', items: INPUT_CONTENT_PART }], + }, + phase: { + type: 'string', + enum: ['commentary', 'final_answer'], + description: 'Labels an assistant message as intermediate commentary or the final answer.', + }, + }, + }, + { + type: 'object', + title: 'Item reference', + description: 'References an item that already exists on the conversation.', + required: ['id'], + properties: { + type: { type: 'string', enum: ['item_reference'] }, + id: { type: 'string' }, + }, + }, + { + type: 'object', + title: 'Model-produced item', + description: + 'Any other item from the response `output` array — a tool call, a tool output, a reasoning item, a compaction item — replayed back to the model. Discriminated on `type`; see the response schema.', + required: ['type'], + properties: { + type: { type: 'string' }, + id: { type: 'string' }, + }, + }, + ], +}; + +const TOOL = { + oneOf: [ + { + type: 'object', + title: 'Function tool', + required: ['type', 'name'], + properties: { + type: { type: 'string', enum: ['function'] }, + name: { type: 'string' }, + description: { type: 'string' }, + parameters: { type: 'object', description: 'JSON Schema describing the function parameters.' }, + output_schema: { type: 'object' }, + strict: { type: 'boolean', description: 'Whether strict parameter validation is enforced.' }, + async: { type: 'boolean' }, + defer_loading: { type: 'boolean', description: 'Whether the function is loaded via tool search.' }, + }, + }, + { + type: 'object', + title: 'Custom tool', + required: ['type', 'name'], + properties: { + type: { type: 'string', enum: ['custom'] }, + name: { type: 'string' }, + description: { type: 'string' }, + format: { type: 'object', description: 'Free-text input, or a lark/regex grammar.' }, + }, + }, + { + type: 'object', + title: 'Built-in tool', + description: + 'A tool hosted by the provider, configured by its own fields: file_search, web_search, web_search_preview, computer, computer_use_preview, mcp, code_interpreter, image_generation, local_shell, shell, namespace, tool_search, apply_patch, programmatic_tool_calling.', + required: ['type'], + properties: { + type: { type: 'string' }, + }, + }, + ], +}; + +const REASONING = { + type: 'object', + description: 'Configuration for reasoning models.', + properties: { + effort: { + type: 'string', + enum: ['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'], + description: 'How much reasoning effort to spend before answering.', + }, + context: { + type: 'string', + enum: ['auto', 'current_turn', 'all_turns'], + description: 'Which reasoning items are rendered back to the model on later turns.', + }, + mode: { type: 'string', description: 'Reasoning execution mode, e.g. standard or pro.' }, + summary: { type: 'string', enum: ['auto', 'concise', 'detailed'] }, + generate_summary: { + type: 'string', + enum: ['auto', 'concise', 'detailed'], + description: 'Deprecated: use summary.', + }, + }, +}; + +const TEXT_CONFIG = { + type: 'object', + description: 'Configuration for the textual output, including structured-output formats.', + properties: { + format: { + type: 'object', + description: 'text, json_object, or json_schema. Discriminated on `type`.', + required: ['type'], + properties: { + type: { type: 'string', enum: ['text', 'json_object', 'json_schema'] }, + name: { type: 'string', description: 'Required for json_schema.' }, + schema: { type: 'object', description: 'Required for json_schema.' }, + description: { type: 'string' }, + strict: { type: 'boolean' }, + }, + }, + verbosity: { type: 'string', enum: ['low', 'medium', 'high'] }, + }, +}; + +const TOOL_CHOICE = { + type: 'string', + description: + 'none, auto, or required — or an object forcing a specific tool: {"type":"function","name":...}, {"type":"mcp","server_label":...}, {"type":"custom","name":...}, {"type":"allowed_tools","mode":...,"tools":[...]}, or a bare built-in tool type.', + oneOf: [ + { type: 'string', enum: ['none', 'auto', 'required'] }, + { + type: 'object', + required: ['type'], + properties: { + type: { type: 'string' }, + name: { type: 'string' }, + mode: { type: 'string', enum: ['auto', 'required'] }, + server_label: { type: 'string' }, + tools: { type: 'array', items: { type: 'object' } }, + }, + }, + ], +}; + +const PROMPT_REFERENCE = { + type: 'object', + description: 'Reference to a prompt template and its variables.', + required: ['id'], + properties: { + id: { type: 'string' }, + version: { type: 'string' }, + variables: { + type: 'object', + description: 'Template variable values: a string, or an input text / image / file content part.', + }, + }, +}; + +const METADATA = { + type: 'object', + description: 'Up to 16 key-value pairs. Keys are at most 64 characters, values at most 512.', +}; + +export const CREATE_RESPONSE_METHOD = { + method: 'POST', + operationId: 'createResponse', + summary: RESPONSES_RELATIVE_URL, + relativeUrlPattern: RESPONSES_RELATIVE_URL, + description: + 'Creates a model response for the given input. Unlike chat completions, this endpoint is not parameterised on the deployment id, so the target deployment is selected by the `model` field in the request body.', + parameters: [CONTENT_TYPE_PARAMETER, CACHE_POLICY_PARAMETER], + requestBodySchema: { + contentType: 'application/json', + schema: { + type: 'object', + required: ['model', 'input'], + properties: { + model: { + type: 'string', + description: 'The id of the deployment to invoke.', + }, + input: { + type: 'string', + description: + 'A text prompt, equivalent to a single user message — or an array of input items for multi-turn input, images, files, and replayed tool calls.', + oneOf: [ + { type: 'string', description: 'A text input to the model, equivalent to a user-role text message.' }, + { type: 'array', items: INPUT_ITEM }, + ], + }, + instructions: { + type: 'string', + description: + "A system (or developer) message inserted into the model's context. Not carried over when used with previous_response_id.", + oneOf: [{ type: 'string' }, { type: 'array', items: INPUT_ITEM }], + }, + conversation: { + type: 'string', + description: + 'The conversation this response belongs to, as an id or {"id":...}. Items are prepended to the input and outputs appended afterwards. Cannot be combined with previous_response_id.', + oneOf: [ + { type: 'string', description: 'The unique id of the conversation.' }, + { type: 'object', required: ['id'], properties: { id: { type: 'string' } } }, + ], + }, + previous_response_id: { + type: 'string', + description: 'Id of the previous response, to continue from it. Cannot be combined with conversation.', + }, + background: { + type: 'boolean', + description: 'Whether to run the model response in the background.', + }, + stream: { + type: 'boolean', + description: + 'If true, the response is streamed as server-sent events. Test suites read the JSON response, so leave this unset.', + }, + stream_options: { + type: 'object', + description: 'Only meaningful when stream is true.', + properties: { + include_obfuscation: { type: 'boolean' }, + }, + }, + store: { + type: 'boolean', + description: + 'Whether to store the generated response for later retrieval. Required for the retrieve, delete, and cancel operations to find it.', + }, + include: { + type: 'array', + description: + 'Additional output data to include, e.g. reasoning.encrypted_content or web_search_call.results.', + items: { + type: 'string', + enum: [ + 'file_search_call.results', + 'web_search_call.results', + 'web_search_call.action.sources', + 'message.input_image.image_url', + 'computer_call_output.output.image_url', + 'code_interpreter_call.outputs', + 'reasoning.encrypted_content', + 'message.output_text.logprobs', + ], + }, + }, + max_output_tokens: { + type: 'integer', + description: 'Upper bound for generated tokens, including visible output and reasoning tokens.', + }, + max_tool_calls: { + type: 'integer', + description: 'Maximum total number of built-in tool calls processed in a response.', + }, + temperature: { + type: 'number', + minimum: 0, + maximum: 2, + description: 'Sampling temperature, between 0 and 2.', + }, + top_p: { + type: 'number', + description: 'Nucleus sampling probability mass.', + }, + top_logprobs: { + type: 'integer', + minimum: 0, + maximum: 20, + description: 'Number of most likely tokens to return log probabilities for.', + }, + text: TEXT_CONFIG, + reasoning: REASONING, + tools: { + type: 'array', + description: 'Tools the model may call. Discriminated on `type`.', + items: TOOL, + }, + tool_choice: TOOL_CHOICE, + parallel_tool_calls: { + type: 'boolean', + description: 'Whether to allow the model to run tool calls in parallel.', + }, + truncation: { + type: 'string', + enum: ['auto', 'disabled'], + description: + 'auto drops items from the beginning to fit the context window; disabled (the default) fails with 400 instead.', + }, + context_management: { + type: 'array', + description: 'Context compaction settings.', + items: { + type: 'object', + required: ['type'], + properties: { + type: { type: 'string', description: 'Currently only compaction is supported.' }, + compact_threshold: { type: 'integer', description: 'Token threshold at which compaction is triggered.' }, + }, + }, + }, + prompt: PROMPT_REFERENCE, + prompt_cache_key: { + type: 'string', + description: 'Used to optimize prompt cache hit rates. Replaces the deprecated user field.', + }, + prompt_cache_options: { + type: 'object', + description: 'Prompt caching options. Supported for gpt-5.6 and later models.', + properties: { + mode: { + type: 'string', + enum: ['implicit', 'explicit'], + description: + 'implicit (default) adds one implicit breakpoint plus up to three explicit; explicit uses only up to four explicit breakpoints.', + }, + ttl: { type: 'string', enum: ['30m'] }, + comparison_response_id: { + type: 'string', + description: 'Response id to compare against when diagnosing prompt cache reuse.', + }, + }, + }, + prompt_cache_retention: { + type: 'string', + enum: ['in_memory', '24h'], + description: 'Deprecated: use prompt_cache_options.ttl.', + }, + moderation: { + type: 'object', + description: 'Moderation model and policy applied to the input and the output.', + required: ['model'], + properties: { + model: { type: 'string', description: 'The moderation model to use, e.g. omni-moderation-latest.' }, + policy: { + type: 'object', + properties: { + input: { + type: 'object', + required: ['mode'], + properties: { mode: { type: 'string', enum: ['score', 'block'] } }, + }, + output: { + type: 'object', + required: ['mode'], + properties: { mode: { type: 'string', enum: ['score', 'block'] } }, + }, + }, + }, + }, + }, + service_tier: { + type: 'string', + enum: ['auto', 'default', 'flex', 'scale', 'priority', 'fast', 'ultrafast'], + description: 'Processing tier used to serve the request.', + }, + safety_identifier: { + type: 'string', + maxLength: 64, + description: 'Stable, hashed identifier of the end user, for abuse detection.', + }, + metadata: METADATA, + user: { + type: 'string', + description: 'Deprecated: replaced by safety_identifier and prompt_cache_key.', + }, + }, + }, + }, + responseBodySchema: { + type: 'object', + required: [ + 'id', + 'object', + 'created_at', + 'error', + 'incomplete_details', + 'instructions', + 'metadata', + 'model', + 'output', + 'parallel_tool_calls', + 'temperature', + 'tool_choice', + 'tools', + 'top_p', + ], + properties: { + id: { type: 'string', description: 'Unique identifier for this response.' }, + object: { type: 'string', enum: ['response'] }, + created_at: { type: 'number', description: 'Unix timestamp, in seconds, of when the response was created.' }, + completed_at: { + type: 'number', + description: 'Unix timestamp, in seconds. Only present when status is completed.', + }, + status: { + type: 'string', + enum: ['completed', 'failed', 'in_progress', 'cancelled', 'queued', 'incomplete'], + }, + output: { + type: 'array', + description: + 'The items the model generated, in order. Assistant text lives in the content of the items whose type is message — there is no top-level output_text field on the wire.', + items: { + oneOf: [ + { + type: 'object', + title: 'Output message', + description: 'An assistant message. Carries the generated text.', + required: ['id', 'type', 'role', 'content', 'status'], + properties: { + id: { type: 'string' }, + type: { type: 'string', enum: ['message'] }, + role: { type: 'string', enum: ['assistant'] }, + status: { type: 'string', enum: ['in_progress', 'completed', 'incomplete'] }, + content: { + type: 'array', + description: 'Output text parts and refusals, discriminated on `type`.', + items: { + oneOf: [ + { + type: 'object', + title: 'Output text', + required: ['type', 'text', 'annotations'], + properties: { + type: { type: 'string', enum: ['output_text'] }, + text: { type: 'string', description: 'The generated text.' }, + annotations: { + type: 'array', + description: + 'File citations, URL citations, container file citations, and file paths, discriminated on `type`.', + items: { type: 'object', required: ['type'], properties: { type: { type: 'string' } } }, + }, + logprobs: { type: 'array', items: { type: 'object' } }, + }, + }, + { + type: 'object', + title: 'Refusal', + required: ['type', 'refusal'], + properties: { + type: { type: 'string', enum: ['refusal'] }, + refusal: { type: 'string' }, + }, + }, + ], + }, + }, + phase: { type: 'string', enum: ['commentary', 'final_answer'] }, + }, + }, + { + type: 'object', + title: 'Reasoning item', + required: ['id', 'type', 'summary'], + properties: { + id: { type: 'string' }, + type: { type: 'string', enum: ['reasoning'] }, + summary: { + type: 'array', + items: { + type: 'object', + required: ['type', 'text'], + properties: { type: { type: 'string', enum: ['summary_text'] }, text: { type: 'string' } }, + }, + }, + content: { + type: 'array', + items: { + type: 'object', + required: ['type', 'text'], + properties: { type: { type: 'string', enum: ['reasoning_text'] }, text: { type: 'string' } }, + }, + }, + encrypted_content: { + type: 'string', + description: 'Encrypted reasoning; resend in later turns when stateless or under ZDR.', + }, + status: { type: 'string', enum: ['in_progress', 'completed', 'incomplete'] }, + }, + }, + { + type: 'object', + title: 'Tool call or tool output', + description: + 'One of function_call, function_call_output, file_search_call, web_search_call, computer_call, computer_call_output, code_interpreter_call, image_generation_call, mcp_call, mcp_list_tools, mcp_approval_request, mcp_approval_response, custom_tool_call, custom_tool_call_output, local_shell_call, local_shell_call_output, shell_call, shell_call_output, apply_patch_call, apply_patch_call_output, tool_search_call, tool_search_output, additional_tools, compaction, program, program_output. Discriminated on `type`.', + required: ['type'], + properties: { + type: { type: 'string' }, + id: { type: 'string' }, + call_id: { type: 'string' }, + name: { type: 'string' }, + arguments: { type: 'string', description: 'JSON string of the call arguments.' }, + output: { type: 'string' }, + status: { type: 'string' }, + }, + }, + ], + }, + }, + error: { + type: 'object', + description: 'Set when the model failed to generate a response. Null on success.', + required: ['code', 'message'], + properties: { + code: { type: 'string', description: 'e.g. server_error, rate_limit_exceeded, invalid_prompt.' }, + message: { type: 'string' }, + misalignment: { + type: 'object', + properties: { + detailed_explanation: { type: 'string' }, + error_type: { type: 'string', description: 'Clients must accept values beyond those documented.' }, + steer: { type: 'object', required: ['message'], properties: { message: { type: 'string' } } }, + }, + }, + }, + }, + incomplete_details: { + type: 'object', + description: 'Why the response is incomplete. Null when it is not.', + properties: { + reason: { type: 'string', enum: ['max_output_tokens', 'max_messages', 'content_filter', 'steered'] }, + }, + }, + model: { type: 'string', description: 'The deployment that generated the response.' }, + instructions: { + type: 'string', + description: 'The system or developer message inserted into the context, as sent.', + oneOf: [{ type: 'string' }, { type: 'array', items: INPUT_ITEM }], + }, + conversation: { + type: 'object', + description: 'The conversation this response belongs to.', + required: ['id'], + properties: { id: { type: 'string' } }, + }, + previous_response_id: { type: 'string' }, + background: { type: 'boolean' }, + max_output_tokens: { type: 'integer' }, + max_tool_calls: { type: 'integer' }, + parallel_tool_calls: { type: 'boolean' }, + temperature: { type: 'number' }, + top_p: { type: 'number' }, + top_logprobs: { type: 'integer' }, + truncation: { type: 'string', enum: ['auto', 'disabled'] }, + text: TEXT_CONFIG, + reasoning: REASONING, + tools: { type: 'array', description: 'The tools the model could call.', items: TOOL }, + tool_choice: TOOL_CHOICE, + prompt: PROMPT_REFERENCE, + prompt_cache_key: { type: 'string' }, + prompt_cache_options: { + type: 'object', + properties: { + mode: { type: 'string', enum: ['implicit', 'explicit'] }, + ttl: { type: 'string', enum: ['30m'] }, + comparison_response_id: { type: 'string' }, + }, + }, + prompt_cache_retention: { + type: 'string', + enum: ['in_memory', '24h'], + description: 'Deprecated: use prompt_cache_options.ttl.', + }, + prompt_cache_diagnostics: { + type: 'object', + description: 'Why the prompt cache hit or missed. Discriminated on `type`.', + required: ['type'], + properties: { + type: { type: 'string', enum: ['cache_hit', 'cache_miss', 'comparison_response_not_found', 'unavailable'] }, + cache_missed_tokens: { type: 'integer' }, + comparison_reusable_tokens: { type: 'integer' }, + reason: { + type: 'string', + description: + 'On a miss: model_changed, prompt_cache_key_changed, tools_changed, text_format_changed, reasoning_effort_changed, verbosity_changed, context_compacted, input_changed, or service_tier_changed.', + }, + }, + }, + moderation: { + type: 'object', + description: 'Moderation results for the input and the output.', + required: ['input', 'output'], + properties: { + input: { type: 'object' }, + output: { type: 'object' }, + }, + }, + service_tier: { + type: 'string', + enum: ['auto', 'default', 'flex', 'scale', 'priority', 'fast', 'ultrafast'], + }, + safety_identifier: { type: 'string' }, + metadata: METADATA, + usage: { + type: 'object', + description: 'Token counts for the request and the generated output.', + required: ['input_tokens', 'input_tokens_details', 'output_tokens', 'output_tokens_details', 'total_tokens'], + properties: { + input_tokens: { type: 'integer' }, + input_tokens_details: { + type: 'object', + required: ['cached_tokens', 'cache_write_tokens'], + properties: { + cached_tokens: { type: 'integer' }, + cache_write_tokens: { type: 'integer' }, + }, + }, + output_tokens: { type: 'integer' }, + output_tokens_details: { + type: 'object', + required: ['reasoning_tokens'], + properties: { reasoning_tokens: { type: 'integer' } }, + }, + total_tokens: { type: 'integer' }, + }, + }, + user: { type: 'string', description: 'Deprecated: replaced by safety_identifier and prompt_cache_key.' }, + }, + }, +}; + +export const GET_RESPONSE_METHOD = { + method: 'GET', + operationId: 'getResponseItem', + summary: RESPONSE_ITEM_DISPLAY_URL, + relativeUrlPattern: RESPONSE_ITEM_RELATIVE_URL_PATTERN, + description: 'Retrieves a previously created response by its id.', + parameters: [RESPONSE_ID_PARAMETER], +}; + +export const DELETE_RESPONSE_METHOD = { + method: 'DELETE', + operationId: 'deleteResponseItem', + summary: RESPONSE_ITEM_DISPLAY_URL, + relativeUrlPattern: RESPONSE_ITEM_RELATIVE_URL_PATTERN, + description: 'Deletes a previously created response by its id.', + parameters: [RESPONSE_ID_PARAMETER], +}; + +export const CANCEL_RESPONSE_METHOD = { + method: 'POST', + operationId: 'cancelResponseItem', + summary: RESPONSE_CANCEL_DISPLAY_URL, + relativeUrlPattern: RESPONSE_CANCEL_RELATIVE_URL_PATTERN, + description: 'Cancels a response that is still in progress.', + parameters: [RESPONSE_ID_PARAMETER], +}; diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/anthropic-messages-model.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/anthropic-messages-model.ts new file mode 100644 index 0000000000..d7e1eacccd --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/anthropic-messages-model.ts @@ -0,0 +1,16 @@ +import { CREATE_MESSAGE_METHOD } from '@/src/components/TestSuites/constants/anthropic-messages-method'; +import { reseedRequestModels } from '@/src/components/TestSuites/utils/model-reseeding'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; + +/** + * Rewrites `model` in every create-message request body so it names the suite's current target. + * + * DIAL's Anthropic Messages endpoint carries no deployment segment, so `model` is what selects the + * deployment. Changing a suite's target otherwise leaves the old id in place and the suite keeps + * invoking the previous deployment — silently, because that id still names a real one. + * + * Requests on any other method, and bodies that are form-data parts, are returned untouched. + */ +export const reseedAnthropicMessagesModel = (suite: TestSuite, deploymentId: string): TestSuite => { + return reseedRequestModels(suite, deploymentId, [CREATE_MESSAGE_METHOD]); +}; diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/column-extraction.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/column-extraction.ts new file mode 100644 index 0000000000..c1a680e186 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/column-extraction.ts @@ -0,0 +1,99 @@ +import { + ColumnExtractionStatus, + EvaluatedColumn, + NotExtractedReason, + TryOutInvocation, +} from '@/src/components/TestSuites/utils/models'; +import { ExtractionWarning, ResponseColumn, StreamingStatus } from '@/src/models/evaluation/test-suite'; + +/** + * A reported value rendered for display. The only failure signal is an explicit `null` in the + * extraction, so `false`, `0` and `''` render as themselves rather than reading as "nothing extracted". + */ +export const formatExtractedValue = (value: unknown): string => + typeof value === 'string' ? value : (JSON.stringify(value) ?? ''); + +const isSuccessStatus = (statusCode?: number): boolean => statusCode == null || (statusCode >= 200 && statusCode < 300); + +/** + * Why the invocation reported no extraction. Read from what the response says rather than re-deriving + * the backend's own condition: a stream can terminate abnormally while still carrying a 200. + */ +const notExtractedReason = (invocation: TryOutInvocation): NotExtractedReason => { + const { statusCode, streamingStatus } = invocation.response ?? {}; + + if (streamingStatus && streamingStatus !== StreamingStatus.Success) { + return NotExtractedReason.StreamIncomplete; + } + if (!isSuccessStatus(statusCode)) { + return NotExtractedReason.RequestFailed; + } + + return NotExtractedReason.NoExtractionReported; +}; + +const notExtractedColumn = ( + column: ResponseColumn, + reason: NotExtractedReason, + statusCode?: number, +): EvaluatedColumn => ({ + name: column.name, + expression: column.expression, + type: column.type, + status: ColumnExtractionStatus.NotExtracted, + result: '', + reason, + ...(reason === NotExtractedReason.RequestFailed && statusCode != null ? { statusCode } : {}), +}); + +const warningFor = (warnings: ExtractionWarning[] | undefined, name: string): ExtractionWarning | undefined => + warnings?.find((warning) => warning.column === name); + +/** + * One invocation's column results, taken from the extraction it reported. + * + * The frontend classifies rather than re-derives: it never decides whether extraction should have + * happened, only reports what the response says about it. A column present with a value is + * `Extracted`; present as an explicit `null` is `Failed`, carrying the warning's error and the + * expression the backend actually evaluated; declared but absent from the mapping is `NotExtracted`. + */ +export const resolveInvocationColumns = ( + columns: ResponseColumn[], + invocation: TryOutInvocation, +): EvaluatedColumn[] => { + if (!columns.length) { + return []; + } + + const { extractedColumns, extractionWarnings } = invocation; + + if (!extractedColumns) { + const reason = notExtractedReason(invocation); + return columns.map((column) => notExtractedColumn(column, reason, invocation.response?.statusCode)); + } + + return columns.map((column) => { + if (!Object.prototype.hasOwnProperty.call(extractedColumns, column.name)) { + return notExtractedColumn(column, NotExtractedReason.NoExtractionReported); + } + + const value = extractedColumns[column.name]; + const warning = warningFor(extractionWarnings, column.name); + const base = { name: column.name, expression: warning?.expression || column.expression, type: column.type }; + + if (value === null) { + return { + ...base, + status: ColumnExtractionStatus.Failed, + result: '', + ...(warning?.error ? { error: warning.error } : {}), + }; + } + + return { + ...base, + status: ColumnExtractionStatus.Extracted, + result: formatExtractedValue(value), + }; + }); +}; diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/evaluate-columns.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/evaluate-columns.ts index 09904bd5a9..de69ee6782 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/utils/evaluate-columns.ts +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/evaluate-columns.ts @@ -1,166 +1,124 @@ import jsonata from 'jsonata'; +import { formatExtractedValue, resolveInvocationColumns } from '@/src/components/TestSuites/utils/column-extraction'; import { - normalizeResponseBodyForColumns, - unwrapJsonRequestBody, -} from '@/src/components/TestSuites/utils/column-eval-context'; -import { ResponseColumn, TestCaseSchema, TestSuite, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; + ColumnExtractionStatus, + EvaluatedColumn, + EvaluateTryOutColumnSectionsParams, + TryOutColumnGroupResult, + TryOutColumnResults, + TryOutColumnTurnResult, + TryOutInvocation, +} from '@/src/components/TestSuites/utils/models'; +import { ResponseColumn, SuiteType, TryOutHistoryEntry } from '@/src/models/evaluation/test-suite'; import { toRequestView } from '@/src/utils/evaluation/request-chain'; import { getRequestTurnCounts, getTryOutSectionShape, groupTryOutSections, shouldShowTurnLabels, - TryOutSectionShape, } from '@/src/utils/evaluation/tryout-sections'; -export interface EvaluatedColumn { - name: string; - expression: string; - type: string; - result: string; - valid: boolean; -} - -export interface TryOutColumnTurnResult { - turnIndex: number; - columns: EvaluatedColumn[]; - responseBody?: unknown; -} - -export interface TryOutColumnGroupResult { - requestIndex: number; - showTurnLabels: boolean; - turns: TryOutColumnTurnResult[]; -} - -export interface TryOutColumnResults { - shape: TryOutSectionShape; - flatColumns?: EvaluatedColumn[]; - groups?: TryOutColumnGroupResult[]; -} - const hasContent = (value?: Record): boolean => !!value && Object.keys(value).length > 0; -const parseColumnBindingValue = (result: string): unknown => { - if (result === 'true') { - return true; - } - if (result === 'false') { - return false; - } - if (result === '') { - return ''; - } - - try { - return JSON.parse(result); - } catch { - return result; - } -}; - -const mergeColumnBindings = ( - bindings: Record, - evaluated: EvaluatedColumn[], -): Record => { - const next = { ...bindings }; +const historyEntryDisplayBody = (entry: TryOutHistoryEntry): unknown => entry.response?.body; - for (const column of evaluated) { - if (!column.name || !column.valid) { - continue; - } - next[column.name] = parseColumnBindingValue(column.result); - } - - return next; -}; - -const historyEntryDisplayBody = (entry: TryOutHistoryEntry): unknown => (entry.response as { body?: unknown })?.body; - -const historyEntryResponse = (entry: TryOutHistoryEntry): Record => - normalizeResponseBodyForColumns((entry.response as { body?: Record })?.body) || {}; - -const historyEntryRequest = (entry: TryOutHistoryEntry): Record | undefined => - unwrapJsonRequestBody(entry.resolvedRequest?.body as Record | undefined); +const historyEntryInvocation = (entry: TryOutHistoryEntry): TryOutInvocation => ({ + response: entry.response, + extractedColumns: entry.extractedColumns, + extractionWarnings: entry.extractionWarnings, +}); +/** + * Client-side evaluation of column expressions. Reached only for MCP-tool suites, whose try-out + * performs no extraction — for every other suite the backend's own extraction is what is displayed. + */ export const evaluateColumns = async ( columns: ResponseColumn[], response: Record, request?: Record, - extraBindings?: Record, ): Promise => { return Promise.all( columns.map(async (column) => { - let result: string = ''; - let valid = false; + let result = ''; + let status = ColumnExtractionStatus.Failed; try { const expr = jsonata(column.expression); // Backend/eval column expressions use $_request / $_response; FE docs/examples use $request / $response. const bindings = { - ...extraBindings, request, response, _request: request, _response: response, }; const evaluated = await expr.evaluate(response, bindings); - valid = evaluated != null; - if (!valid) { - result = ''; - } else { - result = typeof evaluated === 'object' ? JSON.stringify(evaluated) : String(evaluated); + + if (evaluated != null) { + status = ColumnExtractionStatus.Extracted; + result = formatExtractedValue(evaluated); } } catch { result = ''; - valid = false; + status = ColumnExtractionStatus.Failed; } return { name: column.name, expression: column.expression, type: column.type, + status, result, - valid, }; }), ); }; +/** + * Column results for every section the Try Out panel shows. + * + * Values come from the extraction each invocation reported — per invocation, never accumulated across + * them: the backend reports each one already reconciled against the frame carried between requests, so + * re-deriving a later request's values from an earlier one's would reimplement those chaining rules a + * second time. + */ export const evaluateTryOutColumnSections = async ({ testSuite, history, schema, multiTurnLength = 0, fallbackColumns = [], + fallbackInvocation = {}, fallbackResponse = {}, fallbackRequest, -}: { - testSuite: TestSuite; - history?: TryOutHistoryEntry[]; - schema?: TestCaseSchema[]; - multiTurnLength?: number; - fallbackColumns?: ResponseColumn[]; - fallbackResponse?: Record; - fallbackRequest?: Record; -}): Promise => { +}: EvaluateTryOutColumnSectionsParams): Promise => { const turnCounts = getRequestTurnCounts(testSuite, schema, multiTurnLength); const shape = getTryOutSectionShape(turnCounts); + const isMcp = testSuite.suiteType === SuiteType.McpTool; const useGroupedHistory = !!history?.length && (shape === 'requests' || shape === 'combined' || shape === 'turns'); if (!useGroupedHistory) { - if (!hasContent(fallbackResponse) && !hasContent(fallbackRequest)) { - return { shape, flatColumns: [] }; + if (isMcp) { + const flatColumns = + hasContent(fallbackResponse) || hasContent(fallbackRequest) + ? await evaluateColumns(fallbackColumns, fallbackResponse, fallbackRequest) + : []; + + return { shape, flatColumns }; } - const flatColumns = await evaluateColumns(fallbackColumns, fallbackResponse, fallbackRequest); - return { shape, flatColumns }; + // No invocation at all is distinct from an invocation that reported no extraction: the first shows + // nothing, the second shows why each column has no value. + const hasInvocation = !!fallbackInvocation.response || !!fallbackInvocation.extractedColumns; + + return { + shape, + flatColumns: hasInvocation ? resolveInvocationColumns(fallbackColumns, fallbackInvocation) : [], + }; } const groups = groupTryOutSections(history, turnCounts); - let accumulatedBindings: Record = {}; const groupResults: TryOutColumnGroupResult[] = []; for (const group of groups) { @@ -168,10 +126,8 @@ export const evaluateTryOutColumnSections = async ({ const turns: TryOutColumnTurnResult[] = []; for (const { turnIndex, item } of group.turns) { - const response = historyEntryResponse(item); - const request = historyEntryRequest(item); - const columns = await evaluateColumns(requestColumns, response, request, accumulatedBindings); - accumulatedBindings = mergeColumnBindings(accumulatedBindings, columns); + const columns = resolveInvocationColumns(requestColumns, historyEntryInvocation(item)); + turns.push({ turnIndex, columns, responseBody: historyEntryDisplayBody(item) }); } diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/method-groups.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/method-groups.ts new file mode 100644 index 0000000000..2e63fa2e12 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/method-groups.ts @@ -0,0 +1,159 @@ +import { + ANTHROPIC_MESSAGES_RELATIVE_URL, + CREATE_MESSAGE_METHOD, +} from '@/src/components/TestSuites/constants/anthropic-messages-method'; +import { CHAT_COMPLETION_METHOD } from '@/src/components/TestSuites/constants/chat-completion-method'; +import { + ANTHROPIC_MESSAGES_SUITE, + CHAT_COMPLETION_RELATIVE_URL, + CHAT_COMPLETION_SUITE, + DEFAULT_SUITE, + RESPONSES_SUITE, + RESPONSE_ITEM_SUITE, +} from '@/src/components/TestSuites/constants/methods'; +import { + CANCEL_RESPONSE_METHOD, + CREATE_RESPONSE_METHOD, + DELETE_RESPONSE_METHOD, + GET_RESPONSE_METHOD, + RESPONSES_RELATIVE_URL, + RESPONSE_CANCEL_URL_TEMPLATE, + RESPONSE_ITEM_URL_TEMPLATE, +} from '@/src/components/TestSuites/constants/responses-method'; +import { generateMethodPathCombinations } from '@/src/components/TestSuites/utils/method'; +import { BuildMethodGroupsParams, MethodGroup, MethodOption } from '@/src/components/TestSuites/utils/models'; +import { TestSuitesI18nKey } from '@/src/constants/i18n'; +import { DeploymentApiInterface } from '@/src/models/dial/interfaces'; +import { Deployment } from '@/src/models/evaluation/deployment'; +import { TestSuiteEndpointRef } from '@/src/models/evaluation/test-suite'; +import { uniquifyResponseColumns } from '@/src/utils/evaluation/request-chain'; + +const RESPONSES_URL_PATTERNS = new Set([ + CREATE_RESPONSE_METHOD.relativeUrlPattern, + GET_RESPONSE_METHOD.relativeUrlPattern, + CANCEL_RESPONSE_METHOD.relativeUrlPattern, +]); + +const isResponsesEndpoint = (endpointRef?: TestSuiteEndpointRef): boolean => + !!endpointRef?.relativeUrlPattern && RESPONSES_URL_PATTERNS.has(endpointRef.relativeUrlPattern); + +/** + * An unreported `interfaces` list means "not reported" rather than "supports nothing", so the group + * is also kept for a suite already configured against a Responses method — otherwise reopening such + * a suite would leave its selected method unreachable. + */ +const shouldOfferResponses = (deployment?: Deployment | null, endpointRef?: TestSuiteEndpointRef): boolean => + !!deployment?.interfaces?.includes(DeploymentApiInterface.OpenAIResponses) || isResponsesEndpoint(endpointRef); + +const buildChatCompletionsGroup = (takenColumnNames: string[]): MethodGroup => ({ + titleKey: TestSuitesI18nKey.OpenAIChatCompletions, + options: [ + { + ref: CHAT_COMPLETION_METHOD, + displayUrl: CHAT_COMPLETION_RELATIVE_URL, + seed: { + ...CHAT_COMPLETION_SUITE, + responseColumns: uniquifyResponseColumns(CHAT_COMPLETION_SUITE.responseColumns, takenColumnNames), + }, + }, + ], +}); + +const buildResponsesGroup = (deploymentId: string, takenColumnNames: string[]): MethodGroup => { + const createSuite = RESPONSES_SUITE(deploymentId); + + return { + titleKey: TestSuitesI18nKey.OpenAIResponses, + options: [ + { + ref: CREATE_RESPONSE_METHOD, + displayUrl: RESPONSES_RELATIVE_URL, + seed: { + ...createSuite, + responseColumns: uniquifyResponseColumns(createSuite.responseColumns, takenColumnNames), + }, + }, + { + ref: GET_RESPONSE_METHOD, + displayUrl: GET_RESPONSE_METHOD.summary, + seed: RESPONSE_ITEM_SUITE(GET_RESPONSE_METHOD, RESPONSE_ITEM_URL_TEMPLATE), + }, + { + ref: DELETE_RESPONSE_METHOD, + displayUrl: DELETE_RESPONSE_METHOD.summary, + seed: RESPONSE_ITEM_SUITE(DELETE_RESPONSE_METHOD, RESPONSE_ITEM_URL_TEMPLATE), + }, + { + ref: CANCEL_RESPONSE_METHOD, + displayUrl: CANCEL_RESPONSE_METHOD.summary, + seed: RESPONSE_ITEM_SUITE(CANCEL_RESPONSE_METHOD, RESPONSE_CANCEL_URL_TEMPLATE), + }, + ], + }; +}; + +const isAnthropicMessagesEndpoint = (endpointRef?: TestSuiteEndpointRef): boolean => + endpointRef?.method === CREATE_MESSAGE_METHOD.method && + endpointRef?.relativeUrlPattern === CREATE_MESSAGE_METHOD.relativeUrlPattern; + +/** + * Unlike Responses, Anthropic Messages support has no features-flag equivalent — Core reports it + * only through `interfaces` — so this is a 2-way OR (interfaces + sticky) rather than a 3-way OR. + */ +const shouldOfferAnthropicMessages = (deployment?: Deployment | null, endpointRef?: TestSuiteEndpointRef): boolean => + !!deployment?.interfaces?.includes(DeploymentApiInterface.AnthropicMessages) || + isAnthropicMessagesEndpoint(endpointRef); + +const buildAnthropicMessagesGroup = (deploymentId: string, takenColumnNames: string[]): MethodGroup => { + const createSuite = ANTHROPIC_MESSAGES_SUITE(deploymentId); + + return { + titleKey: TestSuitesI18nKey.AnthropicMessages, + options: [ + { + ref: CREATE_MESSAGE_METHOD, + displayUrl: ANTHROPIC_MESSAGES_RELATIVE_URL, + seed: { + ...createSuite, + responseColumns: uniquifyResponseColumns(createSuite.responseColumns, takenColumnNames), + }, + }, + ], + }; +}; + +const buildRoutesGroup = (deployment?: Deployment | null): MethodGroup => ({ + titleKey: TestSuitesI18nKey.Other, + options: generateMethodPathCombinations(deployment?.routes).map((route) => ({ + ref: route, + displayUrl: route.relativeUrlPattern ?? '', + seed: DEFAULT_SUITE(route), + })), +}); + +/** + * Method options offered for a deployment target, grouped for the method sidebar. Groups are + * returned in display order; a group with no options is still returned so callers decide whether to + * render its heading. + */ +export const buildMethodGroups = ({ + deployment, + endpointRef, + takenColumnNames = [], +}: BuildMethodGroupsParams): MethodGroup[] => { + const groups: MethodGroup[] = [buildChatCompletionsGroup(takenColumnNames)]; + + if (shouldOfferResponses(deployment, endpointRef)) { + groups.push(buildResponsesGroup(deployment?.deploymentId ?? '', takenColumnNames)); + } + + if (shouldOfferAnthropicMessages(deployment, endpointRef)) { + groups.push(buildAnthropicMessagesGroup(deployment?.deploymentId ?? '', takenColumnNames)); + } + + groups.push(buildRoutesGroup(deployment)); + + return groups; +}; + +export const flattenMethodGroups = (groups: MethodGroup[]): MethodOption[] => groups.flatMap((group) => group.options); diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/model-reseeding.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/model-reseeding.ts new file mode 100644 index 0000000000..65cc02e82b --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/model-reseeding.ts @@ -0,0 +1,67 @@ +import { + TestSuite, + TestSuiteAdditionalRequest, + TestSuiteEndpointRef, + TestSuiteRequestTemplate, +} from '@/src/models/evaluation/test-suite'; + +const matchesEndpoint = ( + endpointRef: TestSuiteEndpointRef | undefined, + targetEndpoints: readonly TestSuiteEndpointRef[], +): boolean => + targetEndpoints.some( + (target) => endpointRef?.method === target.method && endpointRef?.relativeUrlPattern === target.relativeUrlPattern, + ); + +const withModel = ( + template: TestSuiteRequestTemplate | undefined, + deploymentId: string, +): TestSuiteRequestTemplate | undefined => { + const content = template?.body?.content; + + if (!content || Array.isArray(content)) { + return template; + } + + return { + ...template, + body: { + ...template.body, + content: { ...content, model: deploymentId }, + }, + }; +}; + +const reseedRequest = ( + request: T, + deploymentId: string, + targetEndpoints: readonly TestSuiteEndpointRef[], +): T => + matchesEndpoint(request.endpointRef, targetEndpoints) + ? { ...request, requestTemplate: withModel(request.requestTemplate, deploymentId) } + : request; + +/** Rewrites `model` for matching top-level and chained requests without mutating the suite. */ +export const reseedRequestModels = ( + suite: TestSuite, + deploymentId: string, + targetEndpoints: readonly TestSuiteEndpointRef[], +): TestSuite => { + if (!deploymentId) { + return suite; + } + + const reseeded = reseedRequest(suite, deploymentId, targetEndpoints); + const additionalRequests = suite.additionalRequests?.map((request: TestSuiteAdditionalRequest) => + reseedRequest(request, deploymentId, targetEndpoints), + ); + + if ( + reseeded === suite && + !additionalRequests?.some((request, index) => request !== suite.additionalRequests?.[index]) + ) { + return suite; + } + + return additionalRequests ? { ...reseeded, additionalRequests } : reseeded; +}; diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/models.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/models.ts index bbdc12a3ad..2d3680473c 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/utils/models.ts +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/models.ts @@ -1,5 +1,103 @@ +import { TestSuitesI18nKey } from '@/src/constants/i18n'; +import { Deployment } from '@/src/models/evaluation/deployment'; +import { + ExtractionWarning, + ResponseColumn, + TestCaseSchema, + TestSuite, + TestSuiteEndpointRef, + TryOutCoreResponse, + TryOutHistoryEntry, +} from '@/src/models/evaluation/test-suite'; +import { TryOutSectionShape } from '@/src/utils/evaluation/tryout-sections'; + export interface ParsedTemplateParam { name: string; hasDefault: boolean; defaultValue?: string; } + +export interface MethodOption { + ref: TestSuiteEndpointRef; + /** + * Readable URL for the sidebar. Kept out of `ref` because `relativeUrlPattern` is a regex the + * final path is validated against, not something a user should have to read. + */ + displayUrl: string; + seed: Partial; +} + +export interface MethodGroup { + titleKey: TestSuitesI18nKey; + options: MethodOption[]; +} + +export interface BuildMethodGroupsParams { + deployment?: Deployment | null; + endpointRef?: TestSuiteEndpointRef; + takenColumnNames?: string[]; +} + +/** What a Try Out column result says happened to that column on one invocation. */ +export enum ColumnExtractionStatus { + Extracted = 'EXTRACTED', + Failed = 'FAILED', + NotExtracted = 'NOT_EXTRACTED', +} + +/** Why an invocation reported no extraction at all. */ +export enum NotExtractedReason { + RequestFailed = 'REQUEST_FAILED', + StreamIncomplete = 'STREAM_INCOMPLETE', + NoExtractionReported = 'NO_EXTRACTION_REPORTED', +} + +export interface EvaluatedColumn { + name: string; + /** The expression that produced this result — the backend's when it reported one. */ + expression: string; + type: string; + status: ColumnExtractionStatus; + /** Formatted extracted value; empty unless `status` is `Extracted`. */ + result: string; + /** Backend error text, set only for `Failed` and only when a warning named the column. */ + error?: string; + reason?: NotExtractedReason; + /** Response status behind a `RequestFailed` reason. */ + statusCode?: number; +} + +export interface TryOutInvocation { + response?: TryOutCoreResponse; + extractedColumns?: Record; + extractionWarnings?: ExtractionWarning[]; +} + +export interface TryOutColumnTurnResult { + turnIndex: number; + columns: EvaluatedColumn[]; + responseBody?: unknown; +} + +export interface TryOutColumnGroupResult { + requestIndex: number; + showTurnLabels: boolean; + turns: TryOutColumnTurnResult[]; +} + +export interface TryOutColumnResults { + shape: TryOutSectionShape; + flatColumns?: EvaluatedColumn[]; + groups?: TryOutColumnGroupResult[]; +} + +export interface EvaluateTryOutColumnSectionsParams { + testSuite: TestSuite; + history?: TryOutHistoryEntry[]; + schema?: TestCaseSchema[]; + multiTurnLength?: number; + fallbackColumns?: ResponseColumn[]; + fallbackInvocation?: TryOutInvocation; + fallbackResponse?: Record; + fallbackRequest?: Record; +} diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/responses-model.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/responses-model.ts new file mode 100644 index 0000000000..f86f4edfdc --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/responses-model.ts @@ -0,0 +1,16 @@ +import { CREATE_RESPONSE_METHOD } from '@/src/components/TestSuites/constants/responses-method'; +import { reseedRequestModels } from '@/src/components/TestSuites/utils/model-reseeding'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; + +/** + * Rewrites `model` in every create-response request body so it names the suite's current target. + * + * DIAL's Responses API endpoint carries no deployment segment, so `model` is what selects the + * deployment. Changing a suite's target otherwise leaves the old id in place and the suite keeps + * invoking the previous deployment — silently, because that id still names a real one. + * + * Requests on any other method, and bodies that are form-data parts, are returned untouched. + */ +export const reseedResponsesModel = (suite: TestSuite, deploymentId: string): TestSuite => { + return reseedRequestModels(suite, deploymentId, [CREATE_RESPONSE_METHOD]); +}; diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/tests/anthropic-messages-model.spec.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/anthropic-messages-model.spec.ts new file mode 100644 index 0000000000..87f9512dbb --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/anthropic-messages-model.spec.ts @@ -0,0 +1,128 @@ +import { describe, expect, test } from 'vitest'; + +import { reseedAnthropicMessagesModel } from '@/src/components/TestSuites/utils/anthropic-messages-model'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; + +const createMessageSuite = ( + content: Record = { model: 'claude-3-opus', max_tokens: 1024, messages: [] }, +): TestSuite => + ({ + endpointRef: { method: 'POST', relativeUrlPattern: '/anthropic/v1/messages' }, + requestTemplate: { + urlTemplate: '/anthropic/v1/messages', + body: { contentType: 'application/json', content }, + }, + }) as TestSuite; + +describe('reseedAnthropicMessagesModel', () => { + test('rewrites model to the new deployment id', () => { + const result = reseedAnthropicMessagesModel(createMessageSuite(), 'claude-3-sonnet'); + + expect(result.requestTemplate?.body?.content).toEqual({ + model: 'claude-3-sonnet', + max_tokens: 1024, + messages: [], + }); + }); + + test('preserves hand-added body fields', () => { + const suite = createMessageSuite({ + model: 'claude-3-opus', + max_tokens: 1024, + messages: [], + system: 'be terse', + temperature: 0.5, + }); + + expect(reseedAnthropicMessagesModel(suite, 'claude-3-sonnet').requestTemplate?.body?.content).toEqual({ + model: 'claude-3-sonnet', + max_tokens: 1024, + messages: [], + system: 'be terse', + temperature: 0.5, + }); + }); + + test('adds model when the body has none', () => { + const result = reseedAnthropicMessagesModel( + createMessageSuite({ max_tokens: 1024, messages: [] }), + 'claude-3-sonnet', + ); + + expect(result.requestTemplate?.body?.content).toEqual({ + model: 'claude-3-sonnet', + max_tokens: 1024, + messages: [], + }); + }); + + test('leaves a chat-completion suite untouched', () => { + const suite = { + endpointRef: { method: 'POST', relativeUrlPattern: '/chat/completions' }, + requestTemplate: { body: { content: { model: 'claude-3-opus', messages: [] } } }, + } as TestSuite; + + expect(reseedAnthropicMessagesModel(suite, 'claude-3-sonnet')).toBe(suite); + }); + + test('leaves a route-derived suite untouched', () => { + const suite = { + endpointRef: { method: 'GET', relativeUrlPattern: '/api/users' }, + requestTemplate: { body: { content: { model: 'claude-3-opus' } } }, + } as TestSuite; + + expect(reseedAnthropicMessagesModel(suite, 'claude-3-sonnet')).toBe(suite); + }); + + test('leaves a form-data body untouched', () => { + const suite = { + endpointRef: { method: 'POST', relativeUrlPattern: '/anthropic/v1/messages' }, + requestTemplate: { body: { contentType: 'multipart/form-data', content: [{ key: 'a', value: 'b' }] } }, + } as unknown as TestSuite; + + expect(reseedAnthropicMessagesModel(suite, 'claude-3-sonnet').requestTemplate?.body?.content).toEqual([ + { key: 'a', value: 'b' }, + ]); + }); + + test('returns the suite unchanged when there is no deployment id', () => { + const suite = createMessageSuite(); + + expect(reseedAnthropicMessagesModel(suite, '')).toBe(suite); + }); + + test('rewrites model in a chained create-message request', () => { + const suite = { + endpointRef: { method: 'POST', relativeUrlPattern: '/chat/completions' }, + requestTemplate: { body: { content: { messages: [] } } }, + additionalRequests: [ + { + name: 'create', + endpointRef: { method: 'POST', relativeUrlPattern: '/anthropic/v1/messages' }, + requestTemplate: { body: { content: { model: 'claude-3-opus', max_tokens: 1024, messages: [] } } }, + }, + ], + } as TestSuite; + + const result = reseedAnthropicMessagesModel(suite, 'claude-3-sonnet'); + + expect(result.additionalRequests?.[0].requestTemplate?.body?.content).toEqual({ + model: 'claude-3-sonnet', + max_tokens: 1024, + messages: [], + }); + expect(result.requestTemplate?.body?.content).toEqual({ messages: [] }); + }); + + test('does not mutate its input', () => { + const suite = createMessageSuite(); + + reseedAnthropicMessagesModel(suite, 'claude-3-sonnet'); + + expect(suite.requestTemplate?.body?.content).toEqual({ + model: 'claude-3-opus', + max_tokens: 1024, + messages: [], + }); + }); +}); diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/tests/column-extraction.spec.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/column-extraction.spec.ts new file mode 100644 index 0000000000..7a6f38dfac --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/column-extraction.spec.ts @@ -0,0 +1,255 @@ +import { describe, expect, test } from 'vitest'; + +import { formatExtractedValue, resolveInvocationColumns } from '../column-extraction'; +import { ColumnExtractionStatus, NotExtractedReason, TryOutInvocation } from '../models'; +import { ResponseColumn, StreamingStatus, TryOutResponse } from '@/src/models/evaluation/test-suite'; + +const column = (overrides: Partial = {}): ResponseColumn => ({ + name: 'answer', + displayName: 'answer', + expression: "$join(output[type='message'].content[type='output_text'].text)", + type: 'STRING', + ...overrides, +}); + +const invocation = (overrides: Partial = {}): TryOutInvocation => ({ + response: { statusCode: 200 }, + ...overrides, +}); + +describe('formatExtractedValue', () => { + test('returns a string verbatim', () => { + expect(formatExtractedValue('Hi there, friend!')).toBe('Hi there, friend!'); + }); + + test.each([ + ['a number', 42, '42'], + ['zero', 0, '0'], + ['false', false, 'false'], + ['true', true, 'true'], + ['an empty string', '', ''], + ['an object', { a: 1 }, '{"a":1}'], + ['an array', [1, 'two'], '[1,"two"]'], + ])('formats %s', (_label, value, expected) => { + expect(formatExtractedValue(value)).toBe(expected); + }); +}); + +describe('resolveInvocationColumns', () => { + test('a reported value is extracted', () => { + const results = resolveInvocationColumns( + [column()], + invocation({ extractedColumns: { answer: 'Hi there, friend!' } }), + ); + + expect(results).toEqual([ + expect.objectContaining({ + name: 'answer', + status: ColumnExtractionStatus.Extracted, + result: 'Hi there, friend!', + }), + ]); + }); + + test.each([ + ['false', false, 'false'], + ['zero', 0, '0'], + ['an empty string', '', ''], + ])('%s is extracted, not a failure', (_label, value, expected) => { + const [result] = resolveInvocationColumns([column()], invocation({ extractedColumns: { answer: value } })); + + expect(result.status).toBe(ColumnExtractionStatus.Extracted); + expect(result.result).toBe(expected); + }); + + test('an explicit null is a failure carrying the warning error', () => { + const results = resolveInvocationColumns( + [column({ name: 'summary' })], + invocation({ + extractedColumns: { summary: null }, + extractionWarnings: [{ column: 'summary', expression: '$.missing.path', error: 'Expression matched nothing' }], + }), + ); + + expect(results).toEqual([ + expect.objectContaining({ + name: 'summary', + status: ColumnExtractionStatus.Failed, + result: '', + error: 'Expression matched nothing', + }), + ]); + }); + + test('the expression displayed is the one the backend evaluated', () => { + const [result] = resolveInvocationColumns( + [column({ name: 'summary', expression: 'locally.edited.path' })], + invocation({ + extractedColumns: { summary: null }, + extractionWarnings: [{ column: 'summary', expression: 'saved.path', error: 'boom' }], + }), + ); + + expect(result.expression).toBe('saved.path'); + }); + + test('a failure with no matching warning invents no reason', () => { + const [result] = resolveInvocationColumns( + [column({ name: 'summary' })], + invocation({ + extractedColumns: { summary: null }, + extractionWarnings: [{ column: 'other', expression: 'x', error: 'not this one' }], + }), + ); + + expect(result.status).toBe(ColumnExtractionStatus.Failed); + expect(result.error).toBeUndefined(); + }); + + test('success and failure coexist in one invocation', () => { + const results = resolveInvocationColumns( + [column(), column({ name: 'summary' })], + invocation({ + extractedColumns: { answer: 'Hi!', summary: null }, + extractionWarnings: [{ column: 'summary', expression: 'x', error: 'Expression matched nothing' }], + }), + ); + + expect(results.map(({ name, status }) => [name, status])).toEqual([ + ['answer', ColumnExtractionStatus.Extracted], + ['summary', ColumnExtractionStatus.Failed], + ]); + }); + + test('a declared column absent from the mapping is not extracted', () => { + const results = resolveInvocationColumns( + [column(), column({ name: 'id', expression: 'id' })], + invocation({ extractedColumns: { answer: 'Hi!' } }), + ); + + expect(results[1]).toEqual( + expect.objectContaining({ + name: 'id', + status: ColumnExtractionStatus.NotExtracted, + reason: NotExtractedReason.NoExtractionReported, + }), + ); + }); + + test('keeps the declared name and type for every column', () => { + const results = resolveInvocationColumns( + [column({ name: 'score', type: 'NUMBER' })], + invocation({ extractedColumns: { score: 0.5 } }), + ); + + expect(results[0]).toEqual(expect.objectContaining({ name: 'score', type: 'NUMBER' })); + }); + + describe('no extraction reported', () => { + test('a non-success status reports the request failed', () => { + const results = resolveInvocationColumns([column()], { response: { statusCode: 401 } }); + + expect(results).toEqual([ + expect.objectContaining({ + status: ColumnExtractionStatus.NotExtracted, + reason: NotExtractedReason.RequestFailed, + statusCode: 401, + result: '', + }), + ]); + }); + + test.each([StreamingStatus.Timeout, StreamingStatus.Error, StreamingStatus.Failed])( + 'a %s stream reports the stream did not complete', + (streamingStatus) => { + const results = resolveInvocationColumns([column()], { + response: { statusCode: 200, streaming: true, streamingStatus }, + }); + + expect(results[0].reason).toBe(NotExtractedReason.StreamIncomplete); + }, + ); + + test('a successful stream is not treated as incomplete', () => { + const results = resolveInvocationColumns([column()], { + response: { statusCode: 200, streaming: true, streamingStatus: StreamingStatus.Success }, + }); + + expect(results[0].reason).toBe(NotExtractedReason.NoExtractionReported); + }); + + // A result stored before the backend reported extraction restores as an envelope without it. + test('a successful invocation with columns but no extraction reports the neutral reason', () => { + const results = resolveInvocationColumns([column()], { response: { statusCode: 200 } }); + + expect(results[0]).toEqual( + expect.objectContaining({ + status: ColumnExtractionStatus.NotExtracted, + reason: NotExtractedReason.NoExtractionReported, + }), + ); + expect(results[0].statusCode).toBeUndefined(); + }); + + test('an invocation with no response at all reports the neutral reason', () => { + const results = resolveInvocationColumns([column()], {}); + + expect(results[0].reason).toBe(NotExtractedReason.NoExtractionReported); + }); + + test('a suite declaring no columns yields no results', () => { + expect(resolveInvocationColumns([], { response: { statusCode: 401 } })).toEqual([]); + expect(resolveInvocationColumns([], invocation({ extractedColumns: {} }))).toEqual([]); + }); + }); + + test('a streaming Responses API response shows both reported columns', () => { + const results = resolveInvocationColumns([column(), column({ name: 'id', expression: 'id' })], { + response: { + statusCode: 200, + streaming: true, + body: { + events: [ + { event: 'response.created', data: { response: { id: 'dial_gpt-5.6-sol', output: [] } } }, + { event: 'response.output_text.delta', data: { delta: 'Hi ' } }, + { event: 'response.completed', data: { response: { id: 'dial_gpt-5.6-sol' } } }, + ], + }, + }, + extractedColumns: { answer: 'Hi there, friend!', id: 'dial_gpt-5.6-sol' }, + extractionWarnings: [], + }); + + expect(results).toEqual([ + expect.objectContaining({ + name: 'answer', + status: ColumnExtractionStatus.Extracted, + result: 'Hi there, friend!', + }), + expect.objectContaining({ + name: 'id', + status: ColumnExtractionStatus.Extracted, + result: 'dial_gpt-5.6-sol', + }), + ]); + }); + + test('a full try-out envelope satisfies the invocation shape', () => { + const envelope: TryOutResponse = { + resolvedRequest: { url: '/openai/v1/responses', body: {} }, + response: { statusCode: 200, streaming: true, events: [] }, + grafanaTraceUrl: 'http://grafana:3000/explore?x', + history: [], + extractedColumns: { answer: 'Hi there, friend!', id: 'dial_gpt' }, + extractionWarnings: [], + }; + + const results = resolveInvocationColumns([column(), column({ name: 'id', expression: 'id' })], { + response: envelope.response, + extractedColumns: envelope.extractedColumns, + extractionWarnings: envelope.extractionWarnings, + }); + + expect(results.map(({ result }) => result)).toEqual(['Hi there, friend!', 'dial_gpt']); + }); +}); diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/tests/evaluate-columns.spec.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/evaluate-columns.spec.ts index bfa701c023..c696d3d21a 100644 --- a/apps/ai-dial-admin/src/components/TestSuites/utils/tests/evaluate-columns.spec.ts +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/evaluate-columns.spec.ts @@ -2,13 +2,16 @@ import { describe, expect, test } from 'vitest'; import { ResponseColumn, + StreamingStatus, SuiteType, TestCaseSchema, TestSuite, TryOutHistoryEntry, } from '@/src/models/evaluation/test-suite'; import { TestCaseItemType } from '@/src/types/evaluation'; -import { evaluateColumns, evaluateTryOutColumnSections, EvaluatedColumn } from '../evaluate-columns'; +import { normalizeResponseBodyForColumns } from '../column-eval-context'; +import { evaluateColumns, evaluateTryOutColumnSections } from '../evaluate-columns'; +import { ColumnExtractionStatus, EvaluatedColumn, NotExtractedReason } from '../models'; const makeColumn = (overrides: Partial = {}): ResponseColumn => ({ name: 'answer', @@ -40,6 +43,9 @@ const chatResponse = { model: 'gpt-4.1-2025-04-14', }; +const mcpSuite: TestSuite = { suiteType: SuiteType.McpTool }; +const deploymentSuite: TestSuite = { suiteType: SuiteType.Deployment }; + describe('evaluateColumns', () => { test('should resolve a simple nested path expression', async () => { const columns = [makeColumn()]; @@ -52,7 +58,7 @@ describe('evaluateColumns', () => { expression: 'choices[0].message.content', type: 'STRING', result: 'The capital of Belarus is Minsk.', - valid: true, + status: ColumnExtractionStatus.Extracted, }, ]); }); @@ -64,7 +70,7 @@ describe('evaluateColumns', () => { expect(results).toHaveLength(1); expect(results[0].result).toBe('739'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); expect(results[0].type).toBe('NUMBER'); }); @@ -74,25 +80,25 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe('gpt-4.1-2025-04-14'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); - test('should return valid=false and result=empty string for non-existent path', async () => { + test('should fail with an empty result for a non-existent path', async () => { const columns = [makeColumn({ expression: 'nonexistent.path' })]; const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe(''); - expect(results[0].valid).toBe(false); + expect(results[0].status).toBe(ColumnExtractionStatus.Failed); }); - test('should return valid=false and result=empty string for invalid expression syntax', async () => { + test('should fail with an empty result for invalid expression syntax', async () => { const columns = [makeColumn({ expression: '[[[invalid' })]; const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe(''); - expect(results[0].valid).toBe(false); + expect(results[0].status).toBe(ColumnExtractionStatus.Failed); }); test('should handle multiple columns in parallel', async () => { @@ -105,9 +111,12 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse); expect(results).toHaveLength(3); - expect(results[0]).toMatchObject({ name: 'answer', result: 'The capital of Belarus is Minsk.', valid: true }); - expect(results[1]).toMatchObject({ name: 'model', result: 'gpt-4.1-2025-04-14', valid: true }); - expect(results[2]).toMatchObject({ name: 'tokens', result: '739', valid: true }); + expect(results.map(({ name, result }) => [name, result])).toEqual([ + ['answer', 'The capital of Belarus is Minsk.'], + ['model', 'gpt-4.1-2025-04-14'], + ['tokens', '739'], + ]); + expect(results.every(({ status }) => status === ColumnExtractionStatus.Extracted)).toBe(true); }); test('should return empty array when columns array is empty', async () => { @@ -122,7 +131,7 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, {}); expect(results[0].result).toBe(''); - expect(results[0].valid).toBe(false); + expect(results[0].status).toBe(ColumnExtractionStatus.Failed); }); test('should handle JSONata function expressions', async () => { @@ -131,7 +140,7 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe('1'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); test('should handle JSONata string function expressions', async () => { @@ -140,7 +149,7 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe('ASSISTANT'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); test('should handle JSONata arithmetic expressions', async () => { @@ -151,7 +160,7 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe('739'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); test('should preserve name, expression, and type from column even on failure', async () => { @@ -159,40 +168,23 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse); - expect(results[0].name).toBe('broken'); - expect(results[0].expression).toBe('!!!'); - expect(results[0].type).toBe('CUSTOM'); - expect(results[0].valid).toBe(false); - }); - - test('should handle expression that evaluates to boolean false as valid', async () => { - const response = { flag: false }; - const columns = [makeColumn({ expression: 'flag', type: 'BOOLEAN' })]; - - const results = await evaluateColumns(columns, response); - - expect(results[0].result).toBe('false'); - expect(results[0].valid).toBe(true); - }); - - test('should handle expression that evaluates to 0 as valid', async () => { - const response = { count: 0 }; - const columns = [makeColumn({ expression: 'count', type: 'NUMBER' })]; - - const results = await evaluateColumns(columns, response); - - expect(results[0].result).toBe('0'); - expect(results[0].valid).toBe(true); + expect(results[0]).toMatchObject({ + name: 'broken', + expression: '!!!', + type: 'CUSTOM', + status: ColumnExtractionStatus.Failed, + }); }); - test('should handle expression that evaluates to empty string as valid', async () => { - const response = { text: '' }; - const columns = [makeColumn({ expression: 'text', type: 'STRING' })]; - - const results = await evaluateColumns(columns, response); + test.each([ + ['boolean false', { flag: false }, 'flag', 'false'], + ['0', { count: 0 }, 'count', '0'], + ['an empty string', { text: '' }, 'text', ''], + ])('should treat %s as extracted', async (_label, response, expression, expected) => { + const results = await evaluateColumns([makeColumn({ expression })], response); - expect(results[0].result).toBe(''); - expect(results[0].valid).toBe(true); + expect(results[0].result).toBe(expected); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); test('should still resolve a body-relative expression when a request is also supplied (regression guard)', async () => { @@ -202,7 +194,7 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse, request); expect(results[0].result).toBe('The capital of Belarus is Minsk.'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); test('should resolve $response. to the same value as the bare field', async () => { @@ -211,7 +203,7 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe('The capital of Belarus is Minsk.'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); test('should resolve $request to the request body verbatim, and a nested path within it', async () => { @@ -224,18 +216,17 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse, request); expect(results[0].result).toBe(JSON.stringify(request)); - expect(results[0].valid).toBe(true); expect(results[1].result).toBe('Hi there'); - expect(results[1].valid).toBe(true); + expect(results.every(({ status }) => status === ColumnExtractionStatus.Extracted)).toBe(true); }); - test('should fall into the invalid/empty-result path for $request when no request was supplied', async () => { + test('should fail for $request when no request was supplied', async () => { const columns = [makeColumn({ name: 'reqField', expression: '$request.messages[0].content' })]; const results = await evaluateColumns(columns, chatResponse); expect(results[0].result).toBe(''); - expect(results[0].valid).toBe(false); + expect(results[0].status).toBe(ColumnExtractionStatus.Failed); }); test('should support function composition over the $request binding', async () => { @@ -245,7 +236,7 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse, request); expect(results[0].result).toBe('2'); - expect(results[0].valid).toBe(true); + expect(results[0].status).toBe(ColumnExtractionStatus.Extracted); }); test('should resolve $_request / $_response aliases used by backend column expressions', async () => { @@ -260,86 +251,253 @@ describe('evaluateColumns', () => { const results = await evaluateColumns(columns, chatResponse, request); - expect(results[0].valid).toBe(true); expect(JSON.parse(results[0].result)).toEqual([ { role: 'user', content: 'Hi' }, { role: 'assistant', content: 'The capital of Belarus is Minsk.' }, ]); expect(results[1].result).toBe('The capital of Belarus is Minsk.'); - expect(results[1].valid).toBe(true); + expect(results.every(({ status }) => status === ColumnExtractionStatus.Extracted)).toBe(true); }); - test('should resolve $answer from extraBindings passed by a prior request evaluation', async () => { - const columns = [makeColumn({ name: 'followUp', expression: '$answer', type: 'STRING' })]; - const response = { choices: [{ message: { content: 'later' } }] }; + test('cannot resolve a Responses API SSE envelope, not even a top-level field', async () => { + const sseBody = { + events: [ + { event: 'response.created', data: { response: { id: 'dial_gpt-5.6-sol' } } }, + { event: 'response.output_text.delta', data: { delta: 'Hi ' } }, + ], + }; + const columns = [ + makeColumn({ expression: "$join(output[type='message'].content[type='output_text'].text)" }), + makeColumn({ name: 'id', expression: 'id' }), + ]; - const results = await evaluateColumns(columns, response, undefined, { answer: 'from request 0' }); + const results = await evaluateColumns(columns, normalizeResponseBodyForColumns(sseBody) ?? {}); - expect(results[0].result).toBe('from request 0'); - expect(results[0].valid).toBe(true); + expect(results.map(({ name, status }) => [name, status])).toEqual([ + ['answer', ColumnExtractionStatus.Failed], + ['id', ColumnExtractionStatus.Failed], + ]); }); }); describe('evaluateTryOutColumnSections', () => { - const chatResponse = { - choices: [{ message: { content: 'Paris' } }], + const entry = (overrides: Partial): TryOutHistoryEntry => ({ + resolvedRequest: { body: { contentType: 'application/json', content: {} } }, + response: { statusCode: 200, body: {} }, + ...overrides, + }); + + const chainSuite: TestSuite = { + suiteType: SuiteType.Deployment, + responseColumns: [makeColumn({ name: 'answer' })], + additionalRequests: [ + { responseColumns: [makeColumn({ name: 'is_correct', expression: '$answer = "Paris"' })] }, + { responseColumns: [makeColumn({ name: 'result', expression: '$answer' })] }, + ], }; - test('returns grouped results for a three-request chain with history', async () => { - const suite: TestSuite = { - responseColumns: [makeColumn({ name: 'answer', expression: 'choices[0].message.content' })], + describe('a multi-request chain', () => { + test("shows each request's own reported extraction", async () => { + const history = [ + entry({ extractedColumns: { answer: 'Paris' } }), + entry({ extractedColumns: { is_correct: true } }), + entry({ extractedColumns: { result: 'Paris' } }), + ]; + + const results = await evaluateTryOutColumnSections({ + testSuite: chainSuite, + history, + schema: [], + multiTurnLength: 1, + }); + + expect(results.shape).toBe('requests'); + expect(results.groups).toHaveLength(3); + expect(results.groups?.map((group) => group.turns[0].columns[0].result)).toEqual(['Paris', 'true', 'Paris']); + expect( + results.groups?.every((group) => group.turns[0].columns[0].status === ColumnExtractionStatus.Extracted), + ).toBe(true); + }); + + // `$answer` refers to request #0's column; the backend already reconciled it, so nothing here does. + test("takes a later request's cross-request column value from its own reported extraction", async () => { + const history = [ + entry({ extractedColumns: { answer: 'Paris' } }), + entry({ extractedColumns: { is_correct: true } }), + entry({ extractedColumns: { result: 'Paris' } }), + ]; + + const results = await evaluateTryOutColumnSections({ + testSuite: chainSuite, + history, + schema: [], + multiTurnLength: 1, + }); + + expect(results.groups?.[1].turns[0]).toMatchObject({ + columns: [expect.objectContaining({ name: 'is_correct', result: 'true' })], + }); + }); + + test('shows results only for the invocations that ran when a chain stopped early', async () => { + const history = [ + entry({ extractedColumns: { answer: 'Paris' } }), + entry({ response: { statusCode: 500, body: { error: 'boom' } } }), + ]; + + const results = await evaluateTryOutColumnSections({ + testSuite: chainSuite, + history, + schema: [], + multiTurnLength: 1, + }); + + expect(results.groups).toHaveLength(2); + expect(results.groups?.map(({ requestIndex }) => requestIndex)).toEqual([0, 1]); + expect(results.groups?.[1].turns[0].columns[0]).toMatchObject({ + status: ColumnExtractionStatus.NotExtracted, + reason: NotExtractedReason.RequestFailed, + statusCode: 500, + }); + }); + }); + + describe('per-turn sections', () => { + const combinedSuite: TestSuite = { + suiteType: SuiteType.Deployment, + responseColumns: [makeColumn({ name: 'answer' })], + inputBindings: [{ templateVariable: 'prompt', dataField: 'prompt' }], additionalRequests: [ { - responseColumns: [makeColumn({ name: 'is_correct', expression: '$answer = "Paris"' })], - }, - { - responseColumns: [makeColumn({ name: 'result', expression: '$answer' })], + responseColumns: [makeColumn({ name: 'is_correct' })], + inputBindings: [{ templateVariable: 'prompt', dataField: 'prompt' }], }, ], }; - const history: TryOutHistoryEntry[] = [ - { - resolvedRequest: { body: { contentType: 'application/json', content: { q: 1 } } }, - response: { body: chatResponse }, - }, - { - resolvedRequest: { body: { contentType: 'application/json', content: { q: 2 } } }, - response: { body: { ok: true } }, - }, - { - resolvedRequest: { body: { contentType: 'application/json', content: { q: 3 } } }, - response: { body: { done: true } }, - }, - ]; + test("each turn shows that turn's own extracted values", async () => { + const history = [ + entry({ requestIndex: 0, turnIndex: 0, extractedColumns: { answer: 'first' } }), + entry({ requestIndex: 0, turnIndex: 1, extractedColumns: { answer: 'second' } }), + entry({ requestIndex: 1, turnIndex: 0, extractedColumns: { is_correct: false } }), + ]; + + const results = await evaluateTryOutColumnSections({ + testSuite: combinedSuite, + history, + schema: [{ name: 'prompt', perTurn: true } as never], + multiTurnLength: 2, + }); + + expect(results.shape).toBe('combined'); + expect(results.groups?.[0].turns.map(({ columns }) => columns[0].result)).toEqual(['first', 'second']); + expect(results.groups?.[1].turns[0].columns[0].result).toBe('false'); + }); + }); - const results = await evaluateTryOutColumnSections({ - testSuite: suite, - history, - schema: [], - multiTurnLength: 1, + describe('the single-invocation case', () => { + test("takes its values from the envelope's own extraction", async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: deploymentSuite, + fallbackColumns: [makeColumn(), makeColumn({ name: 'id', expression: 'id' })], + fallbackInvocation: { + response: { statusCode: 200 }, + extractedColumns: { answer: 'Hi there, friend!', id: 'dial_gpt' }, + extractionWarnings: [], + }, + }); + + expect(results.shape).toBe('single'); + expect(results.flatColumns?.map(({ result }) => result)).toEqual(['Hi there, friend!', 'dial_gpt']); }); - expect(results.shape).toBe('requests'); - expect(results.groups).toHaveLength(3); - expect(results.groups?.[0].turns[0].columns[0].result).toBe('Paris'); - expect(results.groups?.[1].turns[0].columns[0].valid).toBe(true); - expect(results.groups?.[2].turns[0].columns[0].result).toBe('Paris'); + test('reports a failed invocation as not extracted', async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: deploymentSuite, + fallbackColumns: [makeColumn()], + fallbackInvocation: { response: { statusCode: 401 } }, + }); + + expect(results.flatColumns?.[0]).toMatchObject({ + status: ColumnExtractionStatus.NotExtracted, + reason: NotExtractedReason.RequestFailed, + statusCode: 401, + }); + }); + + test('reports an abnormally terminated stream as not extracted', async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: deploymentSuite, + fallbackColumns: [makeColumn()], + fallbackInvocation: { + response: { statusCode: 200, streaming: true, streamingStatus: StreamingStatus.Timeout }, + }, + }); + + expect(results.flatColumns?.[0].reason).toBe(NotExtractedReason.StreamIncomplete); + }); + + test('renders nothing before a request has been sent', async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: deploymentSuite, + fallbackColumns: [makeColumn()], + }); + + expect(results.flatColumns).toEqual([]); + }); }); - test('falls back to flat request #0 columns when history is absent', async () => { - const suite: TestSuite = { - responseColumns: [makeColumn({ name: 'answer', expression: 'choices[0].message.content' })], - }; + describe('client-side evaluation', () => { + test('an MCP suite still evaluates its expressions locally', async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: mcpSuite, + fallbackColumns: [makeColumn()], + fallbackResponse: chatResponse, + }); - const results = await evaluateTryOutColumnSections({ - testSuite: suite, - fallbackColumns: suite.responseColumns, - fallbackResponse: chatResponse, + expect(results.flatColumns?.[0]).toMatchObject({ + result: 'The capital of Belarus is Minsk.', + status: ColumnExtractionStatus.Extracted, + }); + }); + + test('an MCP suite renders nothing before a request has been sent', async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: mcpSuite, + fallbackColumns: [makeColumn()], + }); + + expect(results.flatColumns).toEqual([]); + }); + + test('a non-MCP suite never evaluates locally, even when the expression would resolve', async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: deploymentSuite, + fallbackColumns: [makeColumn()], + fallbackInvocation: { response: { statusCode: 200, body: chatResponse } }, + fallbackResponse: chatResponse, + }); + + expect(results.flatColumns?.[0]).toMatchObject({ + result: '', + status: ColumnExtractionStatus.NotExtracted, + }); }); - expect(results.shape).toBe('single'); - expect(results.flatColumns?.[0].result).toBe('Paris'); + test('a non-MCP failed invocation never evaluates locally against the error body', async () => { + const results = await evaluateTryOutColumnSections({ + testSuite: deploymentSuite, + fallbackColumns: [makeColumn({ name: 'err', expression: 'error' })], + fallbackInvocation: { response: { statusCode: 500, body: { error: 'boom' } } }, + fallbackResponse: { error: 'boom' }, + }); + + expect(results.flatColumns?.[0]).toMatchObject({ + result: '', + status: ColumnExtractionStatus.NotExtracted, + reason: NotExtractedReason.RequestFailed, + }); + }); }); test('returns grouped per-turn results for a single-request multi-turn history', async () => { @@ -355,12 +513,14 @@ describe('evaluateTryOutColumnSections', () => { { turnIndex: 0, resolvedRequest: { body: { contentType: 'application/json', content: { q: 1 } } }, - response: { body: { choices: [{ message: { content: 'Paris' } }] } }, + response: { statusCode: 200, body: { choices: [{ message: { content: 'Paris' } }] } }, + extractedColumns: { answer: 'Paris' }, }, { turnIndex: 1, resolvedRequest: { body: { contentType: 'application/json', content: { q: 2 } } }, - response: { body: { choices: [{ message: { content: 'London' } }] } }, + response: { statusCode: 200, body: { choices: [{ message: { content: 'London' } }] } }, + extractedColumns: { answer: 'London' }, }, ]; diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/tests/method-groups.spec.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/method-groups.spec.ts new file mode 100644 index 0000000000..e237368c5f --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/method-groups.spec.ts @@ -0,0 +1,347 @@ +import { describe, expect, test } from 'vitest'; + +import { buildMethodGroups, flattenMethodGroups } from '@/src/components/TestSuites/utils/method-groups'; +import { TestSuitesI18nKey } from '@/src/constants/i18n'; +import { DeploymentApiInterface } from '@/src/models/dial/interfaces'; +import { Deployment } from '@/src/models/evaluation/deployment'; + +const deployment = (interfaces?: DeploymentApiInterface[], routes?: Deployment['routes']): Deployment => + ({ + $type: 'dial-model', + deploymentId: 'gpt-4o', + interfaces, + routes, + }) as Deployment; + +const ROUTES = { 'route-1': { paths: ['/api/users'], methods: ['GET'] } } as Deployment['routes']; + +const routeOptions = (params: Parameters[0]) => + buildMethodGroups(params) + .find((group) => group.titleKey === TestSuitesI18nKey.Other) + ?.options.map(({ ref, displayUrl, seed }) => [ + ref.method, + ref.relativeUrlPattern, + displayUrl, + seed.requestTemplate?.urlTemplate, + ]); + +const titles = (params: Parameters[0]) => + buildMethodGroups(params) + .filter((group) => group.options.length) + .map((group) => group.titleKey); + +describe('buildMethodGroups', () => { + describe('Responses group gating', () => { + test('omits the group when interfaces are not reported', () => { + expect(titles({ deployment: deployment() })).toEqual([TestSuitesI18nKey.OpenAIChatCompletions]); + }); + + test('omits the group when the reported interfaces do not include openaiResponses', () => { + const interfaces = [DeploymentApiInterface.Chat, DeploymentApiInterface.OpenAIChatCompletions]; + + expect(titles({ deployment: deployment(interfaces) })).toEqual([TestSuitesI18nKey.OpenAIChatCompletions]); + }); + + test('includes the group when openaiResponses is reported', () => { + const interfaces = [DeploymentApiInterface.Chat, DeploymentApiInterface.OpenAIResponses]; + + expect(titles({ deployment: deployment(interfaces) })).toEqual([ + TestSuitesI18nKey.OpenAIChatCompletions, + TestSuitesI18nKey.OpenAIResponses, + ]); + }); + + test('includes the group for a suite already selecting a Responses method, without the interface', () => { + const titleKeys = titles({ + deployment: deployment(), + endpointRef: { method: 'POST', relativeUrlPattern: '/openai/v1/responses' }, + }); + + expect(titleKeys).toContain(TestSuitesI18nKey.OpenAIResponses); + }); + + test('includes the group for a suite selecting a response-scoped method', () => { + const titleKeys = titles({ + deployment: deployment(), + endpointRef: { method: 'POST', relativeUrlPattern: '^/openai/v1/responses/[^/]+/cancel$' }, + }); + + expect(titleKeys).toContain(TestSuitesI18nKey.OpenAIResponses); + }); + + test('omits the group for a suite selecting an unrelated method', () => { + const titleKeys = titles({ + deployment: deployment(), + endpointRef: { method: 'POST', relativeUrlPattern: '/chat/completions' }, + }); + + expect(titleKeys).not.toContain(TestSuitesI18nKey.OpenAIResponses); + }); + + test("omits the group for a suite selecting a deployment's own unprefixed /responses route", () => { + const titleKeys = titles({ + deployment: deployment(), + endpointRef: { method: 'POST', relativeUrlPattern: '/responses' }, + }); + + expect(titleKeys).not.toContain(TestSuitesI18nKey.OpenAIResponses); + }); + + test('omits the group when the reported interfaces are empty', () => { + expect(titles({ deployment: deployment([]) })).toEqual([TestSuitesI18nKey.OpenAIChatCompletions]); + }); + + test('includes the group for the full declared interface list', () => { + const interfaces = [ + DeploymentApiInterface.Chat, + DeploymentApiInterface.OpenAIChatCompletions, + DeploymentApiInterface.OpenAIResponses, + DeploymentApiInterface.AnthropicMessages, + ]; + + expect(titles({ deployment: deployment(interfaces) })).toEqual([ + TestSuitesI18nKey.OpenAIChatCompletions, + TestSuitesI18nKey.OpenAIResponses, + TestSuitesI18nKey.AnthropicMessages, + ]); + }); + + test('omits the group for a target declaring only anthropicMessages', () => { + const interfaces = [DeploymentApiInterface.Chat, DeploymentApiInterface.AnthropicMessages]; + + expect(titles({ deployment: deployment(interfaces) })).toEqual([ + TestSuitesI18nKey.OpenAIChatCompletions, + TestSuitesI18nKey.AnthropicMessages, + ]); + }); + + test('omits the group when there is no deployment at all', () => { + expect(titles({})).toEqual([TestSuitesI18nKey.OpenAIChatCompletions]); + }); + }); + + describe('Anthropic Messages group gating', () => { + test('omits the group when interfaces are not reported', () => { + expect(titles({ deployment: deployment() })).toEqual([TestSuitesI18nKey.OpenAIChatCompletions]); + }); + + test('omits the group when the reported interfaces do not include anthropicMessages', () => { + const interfaces = [DeploymentApiInterface.Chat, DeploymentApiInterface.OpenAIChatCompletions]; + + expect(titles({ deployment: deployment(interfaces) })).toEqual([TestSuitesI18nKey.OpenAIChatCompletions]); + }); + + test('includes the group when anthropicMessages is reported', () => { + const interfaces = [DeploymentApiInterface.Chat, DeploymentApiInterface.AnthropicMessages]; + + expect(titles({ deployment: deployment(interfaces) })).toEqual([ + TestSuitesI18nKey.OpenAIChatCompletions, + TestSuitesI18nKey.AnthropicMessages, + ]); + }); + + test('includes the group for a suite already selecting the create-message method, without the interface', () => { + const titleKeys = titles({ + deployment: deployment(), + endpointRef: { method: 'POST', relativeUrlPattern: '/anthropic/v1/messages' }, + }); + + expect(titleKeys).toContain(TestSuitesI18nKey.AnthropicMessages); + }); + + test('omits the group for a suite selecting an unrelated method', () => { + const titleKeys = titles({ + deployment: deployment(), + endpointRef: { method: 'POST', relativeUrlPattern: '/chat/completions' }, + }); + + expect(titleKeys).not.toContain(TestSuitesI18nKey.AnthropicMessages); + }); + + test('features have no effect: a truthy features property does not enable the group on its own', () => { + const withFeatures = { ...deployment(), features: { responses_api: true } } as Deployment; + + expect(titles({ deployment: withFeatures })).not.toContain(TestSuitesI18nKey.AnthropicMessages); + }); + + test('omits the group when there is no deployment at all', () => { + expect(titles({})).toEqual([TestSuitesI18nKey.OpenAIChatCompletions]); + }); + }); + + describe('group order and contents', () => { + test('orders chat interface, responses, then routes', () => { + expect(titles({ deployment: deployment([DeploymentApiInterface.OpenAIResponses], ROUTES) })).toEqual([ + TestSuitesI18nKey.OpenAIChatCompletions, + TestSuitesI18nKey.OpenAIResponses, + TestSuitesI18nKey.Other, + ]); + }); + + test('lists the four Responses operations in order', () => { + const groups = buildMethodGroups({ deployment: deployment([DeploymentApiInterface.OpenAIResponses]) }); + const responses = groups.find((group) => group.titleKey === TestSuitesI18nKey.OpenAIResponses); + + expect(responses?.options.map(({ ref }) => [ref.method, ref.relativeUrlPattern])).toEqual([ + ['POST', '/openai/v1/responses'], + ['GET', '^/openai/v1/responses/[^/]+$'], + ['DELETE', '^/openai/v1/responses/[^/]+$'], + ['POST', '^/openai/v1/responses/[^/]+/cancel$'], + ]); + }); + + test('shows the readable URL rather than the regex pattern', () => { + const groups = buildMethodGroups({ deployment: deployment([DeploymentApiInterface.OpenAIResponses]) }); + const responses = groups.find((group) => group.titleKey === TestSuitesI18nKey.OpenAIResponses); + + expect(responses?.options.map(({ displayUrl }) => displayUrl)).toEqual([ + '/openai/v1/responses', + '/openai/v1/responses/{response_id}', + '/openai/v1/responses/{response_id}', + '/openai/v1/responses/{response_id}/cancel', + ]); + }); + + test('returns an empty routes group when the deployment declares no routes', () => { + const groups = buildMethodGroups({ deployment: deployment() }); + + expect(groups.find((group) => group.titleKey === TestSuitesI18nKey.Other)?.options).toEqual([]); + }); + }); + + describe('custom routes group', () => { + const expected = [['GET', '/api/users', '/api/users', '/api/users']]; + + test.each([ + ['interfaces are not reported', undefined], + ['the reported interfaces are empty', []], + ['the reported interfaces omit openaiResponses', [DeploymentApiInterface.OpenAIChatCompletions]], + ['the reported interfaces include openaiResponses', [DeploymentApiInterface.OpenAIResponses]], + ])('derives routes from the deployment when %s', (_label, interfaces) => { + expect( + routeOptions({ deployment: deployment(interfaces as DeploymentApiInterface[] | undefined, ROUTES) }), + ).toEqual(expected); + }); + + test('offers chat interface and routes for a target declaring no API interfaces', () => { + expect(titles({ deployment: deployment(undefined, ROUTES) })).toEqual([ + TestSuitesI18nKey.OpenAIChatCompletions, + TestSuitesI18nKey.Other, + ]); + }); + + test('keeps routes addressable after the Responses group', () => { + const groups = buildMethodGroups({ deployment: deployment([DeploymentApiInterface.OpenAIResponses], ROUTES) }); + + expect(flattenMethodGroups(groups).at(-1)?.displayUrl).toBe('/api/users'); + }); + }); + + describe('create-response seed', () => { + const createSeed = (takenColumnNames?: string[]) => { + const groups = buildMethodGroups({ + deployment: deployment([DeploymentApiInterface.OpenAIResponses]), + takenColumnNames, + }); + + return groups.find((group) => group.titleKey === TestSuitesI18nKey.OpenAIResponses)?.options[0]?.seed; + }; + + test('seeds model from the deployment id and input from the user_message variable', () => { + expect(createSeed()?.requestTemplate?.body?.content).toEqual({ + model: 'gpt-4o', + input: '${{user_message}}', + }); + }); + + test('seeds the request path without the DIAL prefix', () => { + expect(createSeed()?.requestTemplate?.urlTemplate).toBe('/openai/v1/responses'); + }); + + test('seeds an answer response column extracting the message output text', () => { + expect(createSeed()?.responseColumns?.[0]).toEqual( + expect.objectContaining({ + name: 'answer', + displayName: 'answer', + expression: "$join(output[type='message'].content[type='output_text'].text)", + }), + ); + }); + + test('uniquifies the answer column against taken names', () => { + expect(createSeed(['answer', 'history'])?.responseColumns?.[0]).toEqual( + expect.objectContaining({ name: 'answer2', displayName: 'answer2' }), + ); + }); + }); + + describe('response-scoped seeds', () => { + const responseScopedSeeds = () => { + const groups = buildMethodGroups({ deployment: deployment([DeploymentApiInterface.OpenAIResponses]) }); + const responses = groups.find((group) => group.titleKey === TestSuitesI18nKey.OpenAIResponses); + + return responses?.options.slice(1).map(({ seed }) => seed) ?? []; + }; + + test('seed a response_id placeholder into the request path', () => { + expect(responseScopedSeeds().map((seed) => seed.requestTemplate?.urlTemplate)).toEqual([ + '/openai/v1/responses/${{response_id}}', + '/openai/v1/responses/${{response_id}}', + '/openai/v1/responses/${{response_id}}/cancel', + ]); + }); + + test('seed an empty body and clear any response columns', () => { + responseScopedSeeds().forEach((seed) => { + expect(seed.requestTemplate?.body?.content).toEqual({}); + expect(seed.responseColumns).toEqual([]); + }); + }); + }); + + describe('path patterns', () => { + test('reject a path that omits the DIAL Responses prefix', () => { + const groups = buildMethodGroups({ deployment: deployment([DeploymentApiInterface.OpenAIResponses]) }); + const item = groups + .find((group) => group.titleKey === TestSuitesI18nKey.OpenAIResponses) + ?.options.find(({ ref }) => ref.method === 'GET'); + + const pattern = new RegExp(item?.ref.relativeUrlPattern ?? ''); + + expect(pattern.test('/openai/v1/responses/resp_abc123')).toBe(true); + expect(pattern.test('/responses/resp_abc123')).toBe(false); + expect(pattern.test('/prefix/openai/v1/responses/resp_abc123')).toBe(false); + expect(pattern.test('/openai/v1/responses/resp_abc123/trailing')).toBe(false); + }); + + test('accept the seeded placeholder path and a concrete response id, and reject an unrelated path', () => { + const groups = buildMethodGroups({ deployment: deployment([DeploymentApiInterface.OpenAIResponses]) }); + const cancel = groups + .find((group) => group.titleKey === TestSuitesI18nKey.OpenAIResponses) + ?.options.find(({ displayUrl }) => displayUrl.endsWith('/cancel')); + + const pattern = new RegExp(cancel?.ref.relativeUrlPattern ?? ''); + + expect(pattern.test('/openai/v1/responses/${{response_id}}/cancel')).toBe(true); + expect(pattern.test('/openai/v1/responses/resp_abc123/cancel')).toBe(true); + expect(pattern.test('/openai/v1/responses/resp_abc123')).toBe(false); + expect(pattern.test('/prefix/openai/v1/responses/resp_abc123/cancel')).toBe(false); + expect(pattern.test('/openai/v1/responses/resp_abc123/cancel/trailing')).toBe(false); + }); + }); +}); + +describe('flattenMethodGroups', () => { + test('flattens options in group order, so an index addresses one option', () => { + const groups = buildMethodGroups({ deployment: deployment([DeploymentApiInterface.OpenAIResponses], ROUTES) }); + + expect(flattenMethodGroups(groups).map(({ displayUrl }) => displayUrl)).toEqual([ + '/chat/completions', + '/openai/v1/responses', + '/openai/v1/responses/{response_id}', + '/openai/v1/responses/{response_id}', + '/openai/v1/responses/{response_id}/cancel', + '/api/users', + ]); + }); +}); diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/tests/model-reseeding.spec.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/model-reseeding.spec.ts new file mode 100644 index 0000000000..dcf168b7d5 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/model-reseeding.spec.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from 'vitest'; + +import { CREATE_MESSAGE_METHOD } from '@/src/components/TestSuites/constants/anthropic-messages-method'; +import { CREATE_RESPONSE_METHOD } from '@/src/components/TestSuites/constants/responses-method'; +import { reseedRequestModels } from '@/src/components/TestSuites/utils/model-reseeding'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; + +describe('reseedRequestModels', () => { + test('rewrites each matching request using one traversal', () => { + const suite = { + endpointRef: CREATE_MESSAGE_METHOD, + requestTemplate: { body: { content: { model: 'old-anthropic', messages: [] } } }, + additionalRequests: [ + { + name: 'response', + endpointRef: CREATE_RESPONSE_METHOD, + requestTemplate: { body: { content: { model: 'old-response', input: 'hi' } } }, + }, + ], + } as TestSuite; + + const result = reseedRequestModels(suite, 'new-deployment', [CREATE_MESSAGE_METHOD, CREATE_RESPONSE_METHOD]); + + expect(result.requestTemplate?.body?.content).toEqual({ model: 'new-deployment', messages: [] }); + expect(result.additionalRequests?.[0].requestTemplate?.body?.content).toEqual({ + model: 'new-deployment', + input: 'hi', + }); + }); +}); diff --git a/apps/ai-dial-admin/src/components/TestSuites/utils/tests/responses-model.spec.ts b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/responses-model.spec.ts new file mode 100644 index 0000000000..4bee4c9535 --- /dev/null +++ b/apps/ai-dial-admin/src/components/TestSuites/utils/tests/responses-model.spec.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from 'vitest'; + +import { reseedResponsesModel } from '@/src/components/TestSuites/utils/responses-model'; +import { TestSuite } from '@/src/models/evaluation/test-suite'; + +const createResponseSuite = (content: Record = { model: 'gpt-4o', input: 'hi' }): TestSuite => + ({ + endpointRef: { method: 'POST', relativeUrlPattern: '/openai/v1/responses' }, + requestTemplate: { + urlTemplate: '/openai/v1/responses', + body: { contentType: 'application/json', content }, + }, + }) as TestSuite; + +describe('reseedResponsesModel', () => { + test('rewrites model to the new deployment id', () => { + const result = reseedResponsesModel(createResponseSuite(), 'claude-3'); + + expect(result.requestTemplate?.body?.content).toEqual({ model: 'claude-3', input: 'hi' }); + }); + + test('preserves hand-added body fields', () => { + const suite = createResponseSuite({ model: 'gpt-4o', input: 'hi', store: true, conversation_id: 'c1' }); + + expect(reseedResponsesModel(suite, 'claude-3').requestTemplate?.body?.content).toEqual({ + model: 'claude-3', + input: 'hi', + store: true, + conversation_id: 'c1', + }); + }); + + test('adds model when the body has none', () => { + const result = reseedResponsesModel(createResponseSuite({ input: 'hi' }), 'claude-3'); + + expect(result.requestTemplate?.body?.content).toEqual({ model: 'claude-3', input: 'hi' }); + }); + + test('leaves a chat-completion suite untouched', () => { + const suite = { + endpointRef: { method: 'POST', relativeUrlPattern: '/chat/completions' }, + requestTemplate: { body: { content: { model: 'gpt-4o', messages: [] } } }, + } as TestSuite; + + expect(reseedResponsesModel(suite, 'claude-3')).toBe(suite); + }); + + test('leaves a route-derived suite untouched', () => { + const suite = { + endpointRef: { method: 'GET', relativeUrlPattern: '/api/users' }, + requestTemplate: { body: { content: { model: 'gpt-4o' } } }, + } as TestSuite; + + expect(reseedResponsesModel(suite, 'claude-3')).toBe(suite); + }); + + test('leaves a response-scoped suite untouched, since it carries no model', () => { + const suite = { + endpointRef: { method: 'GET', relativeUrlPattern: '^/openai/v1/responses/[^/]+$' }, + requestTemplate: { urlTemplate: '/openai/v1/responses/${{response_id}}', body: { content: {} } }, + } as TestSuite; + + expect(reseedResponsesModel(suite, 'claude-3')).toBe(suite); + }); + + test('leaves a form-data body untouched', () => { + const suite = { + endpointRef: { method: 'POST', relativeUrlPattern: '/openai/v1/responses' }, + requestTemplate: { body: { contentType: 'multipart/form-data', content: [{ key: 'a', value: 'b' }] } }, + } as unknown as TestSuite; + + expect(reseedResponsesModel(suite, 'claude-3').requestTemplate?.body?.content).toEqual([{ key: 'a', value: 'b' }]); + }); + + test('returns the suite unchanged when there is no deployment id', () => { + const suite = createResponseSuite(); + + expect(reseedResponsesModel(suite, '')).toBe(suite); + }); + + test('rewrites model in a chained create-response request', () => { + const suite = { + endpointRef: { method: 'POST', relativeUrlPattern: '/chat/completions' }, + requestTemplate: { body: { content: { messages: [] } } }, + additionalRequests: [ + { + name: 'create', + endpointRef: { method: 'POST', relativeUrlPattern: '/openai/v1/responses' }, + requestTemplate: { body: { content: { model: 'gpt-4o', input: 'hi' } } }, + }, + ], + } as TestSuite; + + const result = reseedResponsesModel(suite, 'claude-3'); + + expect(result.additionalRequests?.[0].requestTemplate?.body?.content).toEqual({ + model: 'claude-3', + input: 'hi', + }); + expect(result.requestTemplate?.body?.content).toEqual({ messages: [] }); + }); + + test('does not mutate its input', () => { + const suite = createResponseSuite(); + + reseedResponsesModel(suite, 'claude-3'); + + expect(suite.requestTemplate?.body?.content).toEqual({ model: 'gpt-4o', input: 'hi' }); + }); +}); diff --git a/apps/ai-dial-admin/src/constants/deployment-interfaces.ts b/apps/ai-dial-admin/src/constants/deployment-interfaces.ts index 00736d751d..756f74a8b2 100644 --- a/apps/ai-dial-admin/src/constants/deployment-interfaces.ts +++ b/apps/ai-dial-admin/src/constants/deployment-interfaces.ts @@ -1,5 +1,8 @@ import { DeploymentInterfaceType } from '@/src/models/dial/interfaces'; +/** `interface` query value narrowing a deployment listing to MCP servers. */ +export const MCP_INTERFACE_FILTER = 'mcp'; + export const MODEL_INTERFACE_TYPES: DeploymentInterfaceType[] = [ DeploymentInterfaceType.OpenAIChatCompletions, DeploymentInterfaceType.OpenAIResponses, diff --git a/apps/ai-dial-admin/src/constants/i18n.ts b/apps/ai-dial-admin/src/constants/i18n.ts index 51a38898c0..29e00e258c 100644 --- a/apps/ai-dial-admin/src/constants/i18n.ts +++ b/apps/ai-dial-admin/src/constants/i18n.ts @@ -1845,7 +1845,9 @@ export enum TestSuitesI18nKey { ImportFromPC = 'TestSuites.ImportFromPC', FromDial = 'TestSuites.FromDial', Other = 'TestSuites.Other', - ChatInterface = 'TestSuites.ChatInterface', + OpenAIChatCompletions = 'TestSuites.OpenAIChatCompletions', + OpenAIResponses = 'TestSuites.OpenAIResponses', + AnthropicMessages = 'TestSuites.AnthropicMessages', MethodChangeWarning = 'TestSuites.MethodChangeWarning', ImportSuccess = 'TestSuites.ImportSuccess', ImportFailed = 'TestSuites.ImportFailed', @@ -2040,6 +2042,11 @@ export enum TestSuitesI18nKey { TrendsTooltipDate = 'TestSuites.TrendsTooltipDate', TrendsTooltipRun = 'TestSuites.TrendsTooltipRun', TrendsTooltipScore = 'TestSuites.TrendsTooltipScore', + ColumnNotExtracted = 'TestSuites.ColumnNotExtracted', + ColumnNotExtractedRequestFailed = 'TestSuites.ColumnNotExtractedRequestFailed', + ColumnNotExtractedStreamIncomplete = 'TestSuites.ColumnNotExtractedStreamIncomplete', + ColumnNotExtractedNoneReported = 'TestSuites.ColumnNotExtractedNoneReported', + ColumnResultLabel = 'TestSuites.ColumnResultLabel', } export enum DatasetsI18nKey { diff --git a/apps/ai-dial-admin/src/locales/en.ts b/apps/ai-dial-admin/src/locales/en.ts index 05ff50a321..c10a31a3e0 100644 --- a/apps/ai-dial-admin/src/locales/en.ts +++ b/apps/ai-dial-admin/src/locales/en.ts @@ -1887,7 +1887,9 @@ export default { FromDial: 'From DIAL files system', ImportFromPC: 'Import from PC storage', Other: 'Other', - ChatInterface: 'Chat interface', + OpenAIChatCompletions: 'OpenAI Chat Completions', + OpenAIResponses: 'OpenAI Responses', + AnthropicMessages: 'Anthropic Messages', ChangeMethod: 'Change method', ChangeMethodDisabledWhileTryOutOpen: 'Close Try out to change the method', RequestBodyPreview: 'Request body preview', @@ -2063,6 +2065,11 @@ export default { RunConditionSelect: 'Select', ViewOnlyIncludedInRun: 'View only included in run ({count})', TurnLabel: 'Turn {index}', + ColumnNotExtracted: 'Not extracted', + ColumnNotExtractedRequestFailed: 'Request failed (HTTP {statusCode}) — no extraction was performed', + ColumnNotExtractedStreamIncomplete: 'The response stream did not complete — no extraction was performed', + ColumnNotExtractedNoneReported: 'No extraction was reported for this request', + ColumnResultLabel: '{name} — {status}', RequestLabel: 'Request {index}', TurnCountBadge: '{count} turns', ImportWarnings: 'Warnings', diff --git a/apps/ai-dial-admin/src/models/dial/interfaces.ts b/apps/ai-dial-admin/src/models/dial/interfaces.ts index 1306146457..acd597d4f8 100644 --- a/apps/ai-dial-admin/src/models/dial/interfaces.ts +++ b/apps/ai-dial-admin/src/models/dial/interfaces.ts @@ -1,3 +1,23 @@ +/** + * The APIs a deployment declares support for, as reported in `Deployment.interfaces`. Read-only and + * authoritative: a deployment serves an API listed here and no other. + * + * Distinct from `DeploymentInterfaceType`, which is the *configurable* map keying an interface to a + * base URL. Three values coincide, but the two are different fields with different lifecycles — one + * is edited here, the other is declared by the deployment. + */ +export enum DeploymentApiInterface { + Chat = 'chat', + OpenAIChatCompletions = 'openaiChatCompletions', + OpenAIResponses = 'openaiResponses', + AnthropicMessages = 'anthropicMessages', +} + +/** + * Interfaces configurable per entity through the Core config `interfaces` map. Which of these an + * entity type may declare is set by the `*_INTERFACE_TYPES` allowlists in + * `@/src/constants/deployment-interfaces`; every member carries a label in `InterfacesField`. + */ export enum DeploymentInterfaceType { OpenAIChatCompletions = 'openaiChatCompletions', OpenAIResponses = 'openaiResponses', diff --git a/apps/ai-dial-admin/src/models/evaluation/deployment.ts b/apps/ai-dial-admin/src/models/evaluation/deployment.ts index 63872682f8..dbee7419a6 100644 --- a/apps/ai-dial-admin/src/models/evaluation/deployment.ts +++ b/apps/ai-dial-admin/src/models/evaluation/deployment.ts @@ -1,3 +1,4 @@ +import { DeploymentApiInterface } from '@/src/models/dial/interfaces'; import { DialRoute } from '@/src/models/dial/route'; export enum DeploymentType { @@ -15,6 +16,12 @@ export interface Deployment { createdAt?: string; updatedAt?: string; routes?: Record; + /** + * The APIs this deployment declares support for. Populated only by the single-deployment + * endpoints; the deployment listing returns a short projection without it, so absent means + * "not reported" rather than "supports nothing". + */ + interfaces?: DeploymentApiInterface[]; } export interface ToolsetDeployment extends Deployment { diff --git a/apps/ai-dial-admin/src/models/evaluation/test-suite.ts b/apps/ai-dial-admin/src/models/evaluation/test-suite.ts index 869e069bd0..2caf7ca6e8 100644 --- a/apps/ai-dial-admin/src/models/evaluation/test-suite.ts +++ b/apps/ai-dial-admin/src/models/evaluation/test-suite.ts @@ -192,9 +192,44 @@ export interface InputBindingRowData extends InputBinding { defaultValue?: unknown; } -export interface TryOutHistoryEntry { +/** Terminal parse status of a streamed response; anything but `Success` means the invocation failed. */ +export enum StreamingStatus { + Success = 'SUCCESS', + Failed = 'FAILED', + Timeout = 'TIMEOUT', + Error = 'ERROR', +} + +/** One column whose backend extraction failed, carrying the expression that was actually evaluated. */ +export interface ExtractionWarning { + column: string; + expression: string; + error: string; +} + +export interface TryOutCoreResponse { + statusCode: number; + body?: unknown; + streaming?: boolean; + events?: unknown[]; + streamingStatus?: StreamingStatus; + truncationWarning?: string; + [key: string]: unknown; +} + +/** + * `extractedColumns` is the backend's own reconciled extraction for this one invocation — a column + * whose extraction failed appears with an explicit `null`. Both fields are absent when no extraction + * was performed: the suite declares no response columns, the invocation failed, or the try-out is MCP. + */ +interface TryOutExtraction { + extractedColumns?: Record; + extractionWarnings?: ExtractionWarning[]; +} + +export interface TryOutHistoryEntry extends TryOutExtraction { resolvedRequest: Record; - response: Record; + response: TryOutCoreResponse; durationMs?: number; traceId?: string; grafanaTraceUrl?: string; @@ -202,9 +237,9 @@ export interface TryOutHistoryEntry { turnIndex?: number; } -export interface TryOutResponse { +export interface TryOutResponse extends TryOutExtraction { resolvedRequest: Record; - response: Record; + response: TryOutCoreResponse; grafanaTraceUrl?: string; history?: TryOutHistoryEntry[]; } diff --git a/openspec/changes/archive/2026-09-04-test-suite-responses-api-methods/.openspec.yaml b/openspec/changes/archive/2026-09-04-test-suite-responses-api-methods/.openspec.yaml new file mode 100644 index 0000000000..1d9aeef992 --- /dev/null +++ b/openspec/changes/archive/2026-09-04-test-suite-responses-api-methods/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-04 diff --git a/openspec/changes/archive/2026-09-04-test-suite-responses-api-methods/design.md b/openspec/changes/archive/2026-09-04-test-suite-responses-api-methods/design.md new file mode 100644 index 0000000000..a4a96b314a --- /dev/null +++ b/openspec/changes/archive/2026-09-04-test-suite-responses-api-methods/design.md @@ -0,0 +1,306 @@ +## Context + +See `proposal.md` — Why. The constraints that shape the approach: + +- **Method selection is index-arithmetic today.** `Methods.tsx` treats index `0` as chat completions + and `index - 1` as an offset into route-derived methods, in three places (`methodInfo`, + `onMethodClick`, and the selection-restore `findIndex`). A third group has no place in that scheme. +- **`relativeUrlPattern` is a regex, not a path template.** `MethodInfo.tsx` validates the editable + final path against it whenever it contains regex meta symbols, and `path-error.ts` counts `{` and + `}` among those. `new RegExp('/responses/{response_id}')` matches only that literal text, so a + `{response_id}`-style pattern would reject every real response id a user types. +- **DIAL's Responses API endpoint is not deployment-parameterised.** `/chat/completions` is reached at + `/openai/deployments/{deployment}/chat/completions`; `/openai/v1/responses` has no deployment + segment, so `model` in the request body is the only deployment selector. +- **DIAL's OpenAPI defines `ResponsesApiRequest` as a bare `type: object`.** It cannot supply the + request or response schema. +- **The Evaluation Framework contract was confirmed against the running service** (`/v3/api-docs` on the local evaluation-framework backend): `DeploymentInfoDto.interfaces` is a + `string[]` whose item enum is exactly the eight values below, documented there as "Populated on + single-deployment responses only; absent from listing entries". The enum in D1 and the model + field match it value for value. +- **The needed fetch already happens.** `Methods.tsx` calls `getDeployment(deploymentId, $type)` on + mount — the single-deployment endpoint that now returns `interfaces`. No new request is required. + Surfaces backed by the listing (`MethodTabContent`'s `selectedApplication`, target pickers) cannot + see `interfaces` at all. + +## Goals / Non-Goals + +**Goals:** + +- Keep grouping and gating logic out of the component, as a pure function with unit tests. +- Retire the index arithmetic rather than extend it, so a fourth group later costs nothing. +- Make the seeded configuration for each operation immediately runnable where that is possible, and + honestly chain-only where it is not. +- Keep `model` and the suite's target from diverging. + +**Non-Goals:** + +- No general refactor of `Methods.tsx` beyond what a third group requires. +- No change to how route-derived methods are discovered (`generateMethodPathCombinations`). +- No new server action, API method, or request. + +## Decisions + +> **Superseded before merge (2026-09-09).** DIAL confirmed that `interfaces` is the definitive +> declaration of a deployment's supported APIs, drawn from `chat`, `openaiChatCompletions`, +> `openaiResponses` and `anthropicMessages`. D1 and D1a below record the decisions as originally +> taken; D1b states what the branch actually ships. The delta spec under `specs/` reflects D1b. + +### D1. Extend `DeploymentInterfaceType` to all eight wire values + +The Evaluation Framework's `interfaces` array and the existing `interface` query parameter draw on the +same DIAL vocabulary: `chat`, `embedding`, `mcp`, `custom_ui`, `openaiChatCompletions`, +`openaiResponses`, `openaiEmbeddings`, `anthropicMessages`. The enum currently holds the last four. + +Extending it keeps one runtime-usable source for these strings, as `code-standards.md` requires, and +removes an existing raw string — `Target.tsx` passes the literal `'mcp'` as an interface filter, which +becomes `DeploymentInterfaceType.Mcp`. + +*Alternative rejected:* a second enum for the Evaluation Framework's vocabulary. It duplicates four +wire strings across two enums that must then be kept in step, for no gain — the vocabularies coincide +because both come from DIAL Core. + +*Consequence:* `getInterfaceTypeLabel` in `InterfacesField.tsx` is an exhaustive switch with no +`default` returning `string`, so widening the enum breaks its return type. It gains +`default: return type`. This weakens the compile-time guarantee that a newly configurable type gets a +label — accepted because the configurable set is not the enum but the explicit `*_INTERFACE_TYPES` +allowlists in `constants/deployment-interfaces.ts`, which this change does not touch. + +### D1a. Gate on `features.responses_api`, keeping `interfaces` as a secondary signal + +The `interfaces` array is the documented field and the Evaluation Framework DTO carries it, but it is +not the field that arrives in practice: DIAL Core does not report `interfaces` for deployments fetched +through its `/openai/...` API, so a Responses-capable model surfaces its support only through Core's +per-deployment feature map, as `features.responses_api: true`. That was confirmed on the wire — the +single-deployment response for a Responses-capable model came back with `capabilities`, `owner`, +`reference` and `inputAttachmentTypes` but no `interfaces` at all. + +So the gate is `features.responses_api === true` **or** `interfaces` containing `openaiResponses`, +**or** an already-selected Responses method. Both reported signals are honoured rather than one +replacing the other: `features` is what models actually send, `interfaces` is the documented contract +and is authoritative wherever Core populates it, and neither is expensive to check. + +The Evaluation Framework types `features` as a free-form object (`additionalProperties: {}`), so Core's +keys pass through verbatim. `DeploymentFeatures` therefore declares Core's snake_case wire names and +keeps Core's spelling rather than this repo's `is`/`has` boolean convention, and declares only the +flags this app reads. + +*Alternative rejected:* replacing `interfaces` with `features` outright. It would discard a field the +backend documents and populates for non-model deployment types, for no saving. + +### D1b. `interfaces` is definitive: two enums, one gate + +`interfaces` is authoritative, so the gate is `interfaces` containing `openaiResponses` **or** an +already-selected Responses method. `features.responses_api` and `DeploymentFeatures` are removed — +a single reader of a signal the backend no longer treats as primary is dead weight, and keeping a +second signal would let a stale `features` flag contradict a definitive `interfaces`. + +D1's rejected alternative becomes the accepted one, because the premise changed. The two fields are +not one vocabulary in two places: + +- `DeploymentApiInterface` — the APIs a deployment *declares support for*, read-only, exactly the + four values above. +- `DeploymentInterfaceType` — the interfaces *configurable* per entity in the Core config map, keyed + to a base URL: `openaiChatCompletions`, `openaiResponses`, `anthropicMessages`, + `openaiEmbeddings`. + +Three values coincide, and TypeScript keeping the two unassignable is a feature, not the cost D1 +feared: nothing compares across them (`supportsResponsesInterface` reads the config map, +`shouldOfferResponses` reads the declared list). Splitting them returns `DeploymentInterfaceType` to +its four labelled members, so `getInterfaceTypeLabel` is exhaustive again and D1's `default: return +type` consequence is reverted — `InterfacesField.tsx` is untouched by this change. + +`Target.tsx`'s `'mcp'` becomes `MCP_INTERFACE_FILTER` in `constants/deployment-interfaces.ts`. It +narrows a deployment *listing*; it is neither a declared API nor a configurable interface row, so it +belongs to neither enum. + +*Unchanged:* the "Other" group. `buildRoutesGroup` reads only `deployment.routes`, and `DEFAULT_SUITE` +is untouched, so custom endpoints are independent of `interfaces` in both directions. Pinned by unit +and component cases covering `interfaces` absent, empty, without the Responses value, and with it. + +### D2. Grouping as a pure, data-only helper + +New `src/components/TestSuites/utils/method-groups.ts`: + +``` +buildMethodGroups({ deployment, endpointRef, takenColumnNames }) -> MethodGroup[] +``` + +Each `MethodGroup` carries a heading i18n key and its options; each option carries the +`TestSuiteEndpointRef` to display and the `Partial` to seed on selection. Types go in an +adjacent `models.ts`, per `code-standards.md`. + +Seeds are **data, not callbacks** — `takenColumnNames` is a parameter, so column-name uniquification +happens inside the helper and the result stays plain data. That keeps `utils.md`'s purity rule +satisfied and makes every gating and seeding rule in the spec testable without rendering React. + +`Methods.tsx` then keeps a flat list derived from the groups for index-based selection, and renders +headings by iterating groups. `methodInfo`, `onMethodClick`, and the selection-restore `findIndex` all +read the flat list — no `=== 0` and no `- 1`. + +*Alternative rejected:* keeping a second hardcoded block in `Methods.tsx` alongside the chat one. It +would leave three index-arithmetic sites to hand-maintain and put the gating rule inside a component +that already carries fetch, resize, and sidebar state. + +### D3. Regex-form URL patterns for the response-scoped operations + +Given the validation constraint in Context: + +| Operation | `relativeUrlPattern` | seeded final path | +| --------- | -------------------- | ----------------- | +| create | `/openai/v1/responses` | `/openai/v1/responses` | +| retrieve, delete | `/openai/v1/responses/[^/]+` | `/openai/v1/responses/${{response_id}}` | +| cancel | `/openai/v1/responses/[^/]+/cancel` | `/openai/v1/responses/${{response_id}}/cancel` | + +`[^/]+` matches both the seeded `${{response_id}}` placeholder and any concrete id a user substitutes, +which is what the spec's two path-validation scenarios require. The readable `{response_id}` form +lives in the descriptor's `summary`, as `CHAT_COMPLETION_METHOD` does with +`/openai/deployments/{Deployment Name}/chat/completions`. + +The `/openai/v1` prefix is stated once, as `RESPONSES_URL_PREFIX`, and every pattern, template, and +display form is built from it — so the decision has one place to change. + +Verified against `isContainRegexSymbols`: `/openai/v1/responses` contains no regex meta symbol, so the +create operation's path is not regex-validated (unchanged from the unprefixed form); the two +parameterised patterns are, and they accept the placeholder and a concrete id while rejecting both a +wrong-shape path and an unprefixed one. + +*Alternative rejected:* literal `{response_id}` patterns. They read better in the sidebar but make the +final path unvalidatable against a real id, which breaks the operation. + +### D3a. The `/openai/v1` prefix is kept, not stripped + +Reverses the original decision to strip it. The prefix is not cosmetic: it is what tells the +Evaluation Framework backend that a request targets DIAL's Responses API. Without it, a deployment +that exposes its own unrelated `/responses` route is indistinguishable from the DIAL Responses API, +and the backend would route the request to the wrong host. `/chat/completions` has no equivalent +problem because its URL names the deployment. + +Applied to all three forms — stored pattern, seeded path, displayed label. Keeping the label +unprefixed while the Final path showed the prefix was considered and rejected: the Final path is the +`urlTemplate` that is actually sent, so it must carry the prefix, and a sidebar label that disagreed +with it would misdescribe the request. + +*Consequence:* the gating helper no longer recognises a bare `/responses` pattern, which is the +intended effect — a deployment's own `/responses` route stays in the routes group and does not +summon the Responses group. Covered by two tests. + +### D4. Schemas mapped from the supplied OpenAI Responses document + +DIAL's own schema is a stub (Context), so `requestBodySchema` is mapped from the document's +`CreateResponseRequest` and `responseBodySchema` from its `Response`, both covering every top-level +property with an explicit `type` and `description`. + +**The response has no `output_text`.** `Response` requires `output`, an ordered array of +`ResponseOutputItem`, and the generated text sits in the `output_text` content parts of the items whose +`type` is `message`; reasoning items and tool calls share that array. `output_text` is an SDK +convenience accessor, not a wire field. So the `answer` column extracts: + +``` +$join(output[type='message'].content[type='output_text'].text) +``` + +`$join` collapses the match to a single string, because a model may split its answer across several +text parts or messages and the column is declared `string` — an unjoined multi-match would hand the +column an array. Evaluated with the repo's `jsonata` against five shapes: reasoning-then-message +returns the text, multiple parts concatenate, and refusal-only / tool-call-only / empty `output` each +yield `undefined` rather than throwing. + +This is corroborated inside the repo. `src/utils/analytics/hop-inspector/responses.ts` already parses +DIAL Responses traffic for the trace inspector and walks the identical path — `message` items, then +`output_text` parts, then `text` joined with `''` — with comments citing counts measured over 199 real +hops. Document and observed DIAL traffic agree. + +Three deliberate deviations from the document, recorded in the constants file's header: + +- `model` is a plain string described as a DIAL deployment id, not the document's enum of ~90 OpenAI + model names. The value that belongs here is a deployment id. +- `model` and `input` are marked required. The document marks neither — `model` can arrive via + `prompt`, `input` via `conversation` — but DIAL's endpoint has no deployment segment in its URL, so + `model` is the only deployment selector, and a suite with no input does nothing. +- The deep unions (`ResponseInputItem` 33 variants, `Tool` 16, `ResponseOutputItem` 28) are + represented by their `type` discriminator plus the variants a suite exercises, not inlined whole. + `convertSchemaToTable` renders only top-level properties, so a full expansion would be invisible in + the table and unreadable in the JSON view. + +Every top-level property carries an explicit `type` even where the value is a union, because a +property with only `oneOf` renders a blank Type cell in the schema table. + +Operation-level parameters follow DIAL's own operation definition, which is reliable even where its +component schemas are not: `Content-Type` and `X-DIAL-CACHE-POLICY`. No `api-version` parameter — DIAL +does not declare one on this operation, unlike `/chat/completions`. Note `X-DIAL-CACHE-POLICY` differs +from the `X-CACHE-POLICY` in the existing chat-completion descriptor; each descriptor mirrors its own +operation. + +Constants split per `code-standards.md`: the descriptor and the body template as separate files under +`TestSuites/constants/`, mirroring the existing `chat-completion-method.ts` / `chat-completion-body.ts` +pair, with `RESPONSES_SUITE` joining `CHAT_COMPLETION_SUITE` in `methods.ts`. + +### D5. Seed `model` literally, and re-seed it when the target changes + +`model` is set to the target's deployment id at selection time. On its own that goes stale: +`Properties.tsx`'s `onUpdate` replaces `deploymentRef` without touching `requestTemplate.body`, so the +suite would invoke the previous deployment while displaying the new one — and silently, because the +stale value still names a real deployment. + +So `onUpdate` also rewrites `body.model`, guarded on the suite's method being `POST /responses`, and +merging rather than replacing the body so hand-added fields survive. The rewrite logic is a pure helper +next to the grouping one, for the same testability reason. + +*Alternatives rejected:* a defaulted template variable `${{model:}}` makes the value visible in the +bindings UI but bakes the same stale default, so it renames the problem; leaving the staleness +undocumented was declined by the requester. + +### D6. Targeted accessibility fix in the code being changed + +`MethodItem` is a clickable `
` with no role, no keyboard handler, and selection conveyed only by a +background class. The spec requires the selected method to be "shown as active", and `a11y.md` requires +that state to be programmatic and the control to be a real button. Since this change renders a third +group of these items and asserts activeness in tests, `MethodItem` becomes a +`