From 8f02dda238e6bba218fcd628fcaaac160aeab4b4 Mon Sep 17 00:00:00 2001 From: Dushyant Garg Date: Sat, 11 Apr 2026 23:06:17 -0400 Subject: [PATCH 1/2] Add config conventions --- CHANGELOG.md | 5 +- README.md | 36 ++++ examples/config-cli/README.md | 8 + examples/config-cli/index.mjs | 67 +++++++ .../changes/add-config-conventions/tasks.md | 16 -- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/command-blocks/spec.md | 0 .../specs/config-conventions/spec.md | 0 .../tasks.md | 16 ++ openspec/specs/command-blocks/spec.md | 11 +- openspec/specs/config-conventions/spec.md | 31 +++ package.json | 2 +- src/blocks/config.ts | 176 ++++++++++++++++++ src/blocks/output.ts | 21 +++ src/errors/codes.ts | 3 +- src/index.ts | 16 ++ test/smoke.test.ts | 115 ++++++++++++ 19 files changed, 495 insertions(+), 28 deletions(-) create mode 100644 examples/config-cli/README.md create mode 100644 examples/config-cli/index.mjs delete mode 100644 openspec/changes/add-config-conventions/tasks.md rename openspec/changes/{add-config-conventions => archive/2026-04-12-add-config-conventions}/.openspec.yaml (100%) rename openspec/changes/{add-config-conventions => archive/2026-04-12-add-config-conventions}/design.md (100%) rename openspec/changes/{add-config-conventions => archive/2026-04-12-add-config-conventions}/proposal.md (100%) rename openspec/changes/{add-config-conventions => archive/2026-04-12-add-config-conventions}/specs/command-blocks/spec.md (100%) rename openspec/changes/{add-config-conventions => archive/2026-04-12-add-config-conventions}/specs/config-conventions/spec.md (100%) create mode 100644 openspec/changes/archive/2026-04-12-add-config-conventions/tasks.md create mode 100644 openspec/specs/config-conventions/spec.md create mode 100644 src/blocks/config.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 30bd0eb..52c6a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,12 @@ All notable changes to this project will be documented in this file. - auth preflight helper for provider-backed commands - lookup-resolution helpers for id-or-query flows, not-found outcomes, and ambiguous matches - adapter contract helpers for capability checks and normalized adapter failures +- config convention helpers for env/profile loading and structured config diagnostics ### Changed - command and tool metadata now support descriptions, aliases, usage, and examples -- human failure output can now render structured auth guidance details, adapter failure metadata, and lookup candidate summaries -- README examples now show help-oriented command metadata, auth preflight usage, adapter contract patterns, and lookup resolution patterns +- human failure output can now render structured auth guidance details, adapter failure metadata, config diagnostics, and lookup candidate summaries +- README examples now show help-oriented command metadata, auth preflight usage, adapter contract patterns, config conventions, and lookup resolution patterns ## [0.1.3] - 2026-04-11 diff --git a/README.md b/README.md index ac21ad7..35f4320 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,12 @@ Helpers: - `requireCapability` - `adapterFailure` - `unsupportedCapability` +- `defineProfile` +- `loadConfig` +- `validateConfig` +- `missingConfig` +- `malformedConfig` +- `selectProfile` - `notFound` - `ambiguousMatch` - `resolveById` @@ -219,6 +225,34 @@ const tool = createTool({ Provider-specific SDK calls and object models should stay in the downstream repo. AgentTK only owns the contract boundary, capability checks, and normalized failure shapes. +## Config conventions + +```ts +import { createTool, defineCommand, isFailure, loadConfig, ok } from 'agenttk' +import { z } from 'zod' + +const schema = z.object({ + account: z.string().min(1), + apiBaseUrl: z.string().url() +}) + +const config = loadConfig(schema, { + env: { apiBaseUrl: process.env.API_BASE_URL }, + profiles: { + work: { account: 'team@example.com', apiBaseUrl: 'https://api.example.com' } + }, + profile: 'work' +}, { + expected: '{"account":"team@example.com","apiBaseUrl":"https://api.example.com"}', + nextStep: 'Set API_BASE_URL or choose a valid profile' +}) + +if (isFailure(config)) return config +return ok({ type: 'config', record: config }) +``` + +Load order is predictable: base config, then selected profile, then env overrides. AgentTK stays out of secrets management and provider-specific config policy. + ## Lookup resolution ```ts @@ -252,6 +286,7 @@ const tool = createTool({ - `examples/minimal-cli/` - `examples/tasks-cli/` - `examples/adapter-cli/` +- `examples/config-cli/` ```bash node examples/minimal-cli/index.mjs hello --json @@ -259,6 +294,7 @@ node examples/minimal-cli/index.mjs help node examples/tasks-cli/index.mjs add --title "Send estimate" --dry-run --json node examples/tasks-cli/index.mjs add node examples/adapter-cli/index.mjs status --json +node examples/config-cli/index.mjs show --profile work ``` ## Development diff --git a/examples/config-cli/README.md b/examples/config-cli/README.md new file mode 100644 index 0000000..4a60a93 --- /dev/null +++ b/examples/config-cli/README.md @@ -0,0 +1,8 @@ +# config-cli + +Minimal example showing environment-backed config loading, named profiles, and structured config diagnostics. + +```bash +node examples/config-cli/index.mjs show --json +node examples/config-cli/index.mjs show --profile work +``` diff --git a/examples/config-cli/index.mjs b/examples/config-cli/index.mjs new file mode 100644 index 0000000..10497a6 --- /dev/null +++ b/examples/config-cli/index.mjs @@ -0,0 +1,67 @@ +import { z } from 'zod' +import { createTool, defineCommand, isFailure, loadConfig, ok } from 'agenttk' + +const schema = z.object({ + account: z.string().min(1), + region: z.string().min(1), + apiBaseUrl: z.string().url() +}) + +function parseFlags(rawArgs) { + const input = { profile: undefined } + + for (let i = 0; i < rawArgs.length; i += 1) { + const arg = rawArgs[i] + if (arg === '--profile') { + input.profile = rawArgs[i + 1] + i += 1 + } + } + + return input +} + +const profiles = { + work: { + account: 'team@example.com', + region: 'us', + apiBaseUrl: 'https://api.example.com' + }, + personal: { + account: 'me@example.com', + region: 'us', + apiBaseUrl: 'https://personal.example.com' + } +} + +const envConfig = { + account: process.env.CONFIG_DEMO_ACCOUNT, + region: process.env.CONFIG_DEMO_REGION, + apiBaseUrl: process.env.CONFIG_DEMO_API_BASE_URL +} + +const tool = createTool({ + name: 'config-demo', + commands: [ + defineCommand({ + name: 'show', + description: 'Load config from env and optional named profiles', + handler: async ({ rawArgs }) => { + const flags = parseFlags(rawArgs) + const config = loadConfig(schema, { + env: envConfig, + profiles, + profile: flags.profile + }, { + expected: '{"account":"team@example.com","region":"us","apiBaseUrl":"https://api.example.com"}', + nextStep: 'Set CONFIG_DEMO_* env vars or run config-demo show --profile work' + }) + + if (isFailure(config)) return config + return ok({ type: 'config', record: config }) + } + }) + ] +}) + +await tool.run(process.argv.slice(2)) diff --git a/openspec/changes/add-config-conventions/tasks.md b/openspec/changes/add-config-conventions/tasks.md deleted file mode 100644 index be7c25b..0000000 --- a/openspec/changes/add-config-conventions/tasks.md +++ /dev/null @@ -1,16 +0,0 @@ -## 1. Config primitives - -- [ ] 1.1 Define config loading helper patterns. -- [ ] 1.2 Add validation helpers for missing or malformed config. -- [ ] 1.3 Add profile/account naming conventions that stay provider-agnostic. - -## 2. Docs and output - -- [ ] 2.1 Ensure config diagnostics are actionable in human mode. -- [ ] 2.2 Preserve config diagnostic detail in JSON mode. -- [ ] 2.3 Add examples for environment-backed and profile-backed config. - -## 3. Verification - -- [ ] 3.1 Add tests for missing-config and malformed-config flows. -- [ ] 3.2 Validate the OpenSpec change. diff --git a/openspec/changes/add-config-conventions/.openspec.yaml b/openspec/changes/archive/2026-04-12-add-config-conventions/.openspec.yaml similarity index 100% rename from openspec/changes/add-config-conventions/.openspec.yaml rename to openspec/changes/archive/2026-04-12-add-config-conventions/.openspec.yaml diff --git a/openspec/changes/add-config-conventions/design.md b/openspec/changes/archive/2026-04-12-add-config-conventions/design.md similarity index 100% rename from openspec/changes/add-config-conventions/design.md rename to openspec/changes/archive/2026-04-12-add-config-conventions/design.md diff --git a/openspec/changes/add-config-conventions/proposal.md b/openspec/changes/archive/2026-04-12-add-config-conventions/proposal.md similarity index 100% rename from openspec/changes/add-config-conventions/proposal.md rename to openspec/changes/archive/2026-04-12-add-config-conventions/proposal.md diff --git a/openspec/changes/add-config-conventions/specs/command-blocks/spec.md b/openspec/changes/archive/2026-04-12-add-config-conventions/specs/command-blocks/spec.md similarity index 100% rename from openspec/changes/add-config-conventions/specs/command-blocks/spec.md rename to openspec/changes/archive/2026-04-12-add-config-conventions/specs/command-blocks/spec.md diff --git a/openspec/changes/add-config-conventions/specs/config-conventions/spec.md b/openspec/changes/archive/2026-04-12-add-config-conventions/specs/config-conventions/spec.md similarity index 100% rename from openspec/changes/add-config-conventions/specs/config-conventions/spec.md rename to openspec/changes/archive/2026-04-12-add-config-conventions/specs/config-conventions/spec.md diff --git a/openspec/changes/archive/2026-04-12-add-config-conventions/tasks.md b/openspec/changes/archive/2026-04-12-add-config-conventions/tasks.md new file mode 100644 index 0000000..29de0c6 --- /dev/null +++ b/openspec/changes/archive/2026-04-12-add-config-conventions/tasks.md @@ -0,0 +1,16 @@ +## 1. Config primitives + +- [x] 1.1 Define config loading helper patterns. +- [x] 1.2 Add validation helpers for missing or malformed config. +- [x] 1.3 Add profile/account naming conventions that stay provider-agnostic. + +## 2. Docs and output + +- [x] 2.1 Ensure config diagnostics are actionable in human mode. +- [x] 2.2 Preserve config diagnostic detail in JSON mode. +- [x] 2.3 Add examples for environment-backed and profile-backed config. + +## 3. Verification + +- [x] 3.1 Add tests for missing-config and malformed-config flows. +- [x] 3.2 Validate the OpenSpec change. diff --git a/openspec/specs/command-blocks/spec.md b/openspec/specs/command-blocks/spec.md index 305c004..c59c95e 100644 --- a/openspec/specs/command-blocks/spec.md +++ b/openspec/specs/command-blocks/spec.md @@ -6,14 +6,9 @@ TBD - created by archiving change add-agenttk-v0. Update Purpose after archive. ### Requirement: Validation helpers The system SHALL provide reusable validation helpers for command inputs. -#### Scenario: Valid input passes through -- **WHEN** a command validates input against a declared schema and the input is valid -- **THEN** the helper returns the typed parsed value - -#### Scenario: Invalid input returns a structured validation failure -- **WHEN** a command validates input against a declared schema and the input is invalid -- **THEN** the helper returns a structured failure with code `VALIDATION_ERROR` -- **AND** the failure message describes the validation problem without throwing an uncaught exception into the caller +#### Scenario: Config diagnostics reuse structured validation style +- **WHEN** a downstream tool validates environment or config inputs before command execution +- **THEN** AgentTK can return a structured failure with actionable guidance using the same predictable envelope style as other validation helpers ### Requirement: Corrective guidance in validation failures The system SHALL support validation failures that include corrective guidance. diff --git a/openspec/specs/config-conventions/spec.md b/openspec/specs/config-conventions/spec.md new file mode 100644 index 0000000..6c71b73 --- /dev/null +++ b/openspec/specs/config-conventions/spec.md @@ -0,0 +1,31 @@ +# config-conventions Specification + +## Purpose +TBD - created by archiving change add-config-conventions. Update Purpose after archive. +## Requirements +### Requirement: Config loading conventions +The system SHALL provide reusable conventions for loading config in AgentTK-based tools. + +#### Scenario: Load config from declared inputs +- **WHEN** a downstream CLI loads configuration from environment variables, local config objects, or profile selection +- **THEN** AgentTK provides a narrow helper pattern that keeps the loading path predictable + +### Requirement: Config diagnostics +The system SHALL support structured diagnostics for missing or malformed config. + +#### Scenario: Required config is missing +- **WHEN** a downstream tool cannot find a required config value +- **THEN** AgentTK returns a structured failure that identifies the missing config input +- **AND** the payload can include corrective guidance for how to provide it + +#### Scenario: Config value is malformed +- **WHEN** a downstream tool finds a config value but it fails validation +- **THEN** AgentTK returns a structured diagnostic failure rather than an uncaught exception + +### Requirement: Provider-agnostic profile conventions +The system SHALL support generic profile or account naming conventions. + +#### Scenario: Tool selects a named profile +- **WHEN** a downstream CLI chooses among named profiles or accounts +- **THEN** AgentTK provides a reusable convention that does not depend on any single provider domain + diff --git a/package.json b/package.json index 05bb041..a6c6cfe 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "build": "tsc -p tsconfig.json", "check": "tsc -p tsconfig.json --noEmit", "test": "node --import tsx --test test/*.test.ts", - "examples:smoke": "node examples/minimal-cli/index.mjs hello --json && node examples/tasks-cli/index.mjs add --title \"Send estimate\" --dry-run --json && node examples/tasks-cli/index.mjs add --title \"Send estimate\"", + "examples:smoke": "node examples/minimal-cli/index.mjs hello --json && node examples/tasks-cli/index.mjs add --title \"Send estimate\" --dry-run --json && node examples/tasks-cli/index.mjs add --title \"Send estimate\" && node examples/config-cli/index.mjs show --profile work --json && node examples/config-cli/index.mjs show --profile work", "verify": "npm run build && npm test && npm run examples:smoke", "prepublishOnly": "npm run verify" }, diff --git a/src/blocks/config.ts b/src/blocks/config.ts new file mode 100644 index 0000000..4057df9 --- /dev/null +++ b/src/blocks/config.ts @@ -0,0 +1,176 @@ +import { ZodError, type ZodSchema } from 'zod' +import { fail } from '../core/result.js' +import { ErrorCodes } from '../errors/codes.js' +import type { CommandFailure } from '../core/types.js' + +export type ConfigSource = 'env' | 'config' | 'profile' | 'merged' + +export type ConfigDiagnosticDetails = { + kind: 'config' + reason: 'missing' | 'malformed' + source?: ConfigSource + key?: string + profile?: string + account?: string + expected?: string + issues?: string[] + nextStep?: string +} + +export type ConfigFailureOptions = Omit + +export type ConfigProfile = Record> = { + name: string + account?: string + values: Partial +} + +export type ConfigLoadInput = Record> = { + env?: Partial + config?: Partial + profiles?: Record> + profile?: string + account?: string +} + +export type ConfigLoadOptions = { + expected?: string + nextStep?: string +} + +function configFailure( + reason: ConfigDiagnosticDetails['reason'], + message: string, + options?: ConfigFailureOptions +): CommandFailure { + return fail({ + error: { + code: ErrorCodes.ConfigError, + message, + details: { + kind: 'config', + reason, + source: options?.source, + key: options?.key, + profile: options?.profile, + account: options?.account, + expected: options?.expected, + issues: options?.issues, + nextStep: options?.nextStep + } + } + }) +} + +export function missingConfig(key: string, options?: ConfigFailureOptions): CommandFailure { + return configFailure('missing', `Missing config: ${key}`, { + ...options, + key + }) +} + +export function malformedConfig(message: string, options?: ConfigFailureOptions): CommandFailure { + return configFailure('malformed', message, options) +} + +export function defineProfile>( + name: string, + values: Partial, + options?: { account?: string } +): ConfigProfile { + return { + name, + account: options?.account, + values + } +} + +export function selectProfile>( + profiles: Record>, + name: string, + options?: Omit +): Partial | CommandFailure { + const profile = profiles[name] + if (profile) return profile + + return missingConfig(`profile:${name}`, { + source: 'profile', + profile: name, + account: options?.account, + expected: options?.expected, + nextStep: options?.nextStep + }) +} + +function formatIssues(error: ZodError): string[] { + return error.issues.map((issue) => { + const path = issue.path.length ? `${issue.path.join('.')}: ` : '' + return `${path}${issue.message}` + }) +} + +function definedEntries>(input?: Partial): Partial { + if (!input) return {} + return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)) as Partial +} + +function isCommandFailure(value: unknown): value is CommandFailure { + if (!value || typeof value !== 'object') return false + return 'ok' in value && 'error' in value && value.ok === false +} + +export function validateConfig( + schema: ZodSchema, + input: unknown, + options?: ConfigFailureOptions +): TConfig | CommandFailure { + try { + return schema.parse(input) + } catch (error) { + if (error instanceof ZodError) { + const issues = formatIssues(error) + return malformedConfig(issues.join('; ') || 'Config is malformed', { + ...options, + issues + }) + } + + return malformedConfig('Config is malformed', options) + } +} + +export function loadConfig>( + schema: ZodSchema, + input: ConfigLoadInput, + options?: ConfigLoadOptions +): TConfig | CommandFailure { + let selectedProfile: Partial | undefined + + if (input.profile) { + const resolvedProfile = selectProfile(input.profiles ?? {}, input.profile, { + account: input.account, + expected: options?.expected, + nextStep: options?.nextStep + }) + + if (isCommandFailure(resolvedProfile)) { + return resolvedProfile + } + + selectedProfile = resolvedProfile + } + + const merged = { + ...definedEntries(input.config), + ...definedEntries(selectedProfile), + ...definedEntries(input.env) + } + + return validateConfig(schema, merged, { + source: 'merged', + profile: input.profile, + account: input.account, + expected: options?.expected, + nextStep: options?.nextStep + }) +} diff --git a/src/blocks/output.ts b/src/blocks/output.ts index 4db86e5..18dc808 100644 --- a/src/blocks/output.ts +++ b/src/blocks/output.ts @@ -10,13 +10,27 @@ function renderFailureDetails(result: Extract): st const expectedAccount = typeof details.expectedAccount === 'string' ? details.expectedAccount : undefined const nextStep = typeof details.nextStep === 'string' ? details.nextStep : undefined const query = typeof details.query === 'string' ? details.query : undefined + const reason = typeof details.reason === 'string' ? details.reason : undefined + const source = typeof details.source === 'string' ? details.source : undefined + const key = typeof details.key === 'string' ? details.key : undefined + const profile = typeof details.profile === 'string' ? details.profile : undefined + const account = typeof details.account === 'string' ? details.account : undefined + const expected = typeof details.expected === 'string' ? details.expected : undefined const category = typeof details.category === 'string' ? details.category : undefined const operation = typeof details.operation === 'string' ? details.operation : undefined const capability = typeof details.capability === 'string' ? details.capability : undefined const retryable = typeof details.retryable === 'boolean' ? details.retryable : undefined const causeCode = typeof details.causeCode === 'string' ? details.causeCode : undefined + const issues = Array.isArray(details.issues) ? details.issues : undefined const candidates = Array.isArray(details.candidates) ? details.candidates : undefined + if (source) lines.push(`Source: ${source}`) + if (key) lines.push(`Key: ${key}`) + if (profile) lines.push(`Profile: ${profile}`) + if (account) lines.push(`Account: ${account}`) + if (reason) lines.push(`Reason: ${reason}`) + if (expected) lines.push(`Expected: ${expected}`) + if (provider) lines.push(`Provider: ${provider}`) if (currentAccount) lines.push(`Current account: ${currentAccount}`) if (expectedAccount) lines.push(`Expected account: ${expectedAccount}`) @@ -26,6 +40,13 @@ function renderFailureDetails(result: Extract): st if (retryable !== undefined) lines.push(`Retryable: ${retryable ? 'yes' : 'no'}`) if (causeCode) lines.push(`Cause code: ${causeCode}`) if (query) lines.push(`Query: ${query}`) + if (issues?.length) { + lines.push('Issues:') + for (const issue of issues) { + if (typeof issue !== 'string') continue + lines.push(`- ${issue}`) + } + } if (candidates?.length) { lines.push('Candidates:') for (const candidate of candidates) { diff --git a/src/errors/codes.ts b/src/errors/codes.ts index 72dd512..9055340 100644 --- a/src/errors/codes.ts +++ b/src/errors/codes.ts @@ -7,7 +7,8 @@ export const ErrorCodes = { NotFound: 'NOT_FOUND', AmbiguousMatch: 'AMBIGUOUS_MATCH', AdapterError: 'ADAPTER_ERROR', - UnsupportedCapability: 'UNSUPPORTED_CAPABILITY' + UnsupportedCapability: 'UNSUPPORTED_CAPABILITY', + ConfigError: 'CONFIG_ERROR' } as const export type ErrorCode = (typeof ErrorCodes)[keyof typeof ErrorCodes] diff --git a/src/index.ts b/src/index.ts index 9037149..ca91299 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,6 +25,22 @@ export { supportsCapability, unsupportedCapability } from './blocks/adapter.js' +export { + defineProfile, + loadConfig, + malformedConfig, + missingConfig, + selectProfile, + validateConfig +} from './blocks/config.js' +export type { + ConfigDiagnosticDetails, + ConfigFailureOptions, + ConfigLoadInput, + ConfigLoadOptions, + ConfigProfile, + ConfigSource +} from './blocks/config.js' export type { AdapterCapability, AdapterContract, diff --git a/test/smoke.test.ts b/test/smoke.test.ts index 5ab4b67..78ec973 100644 --- a/test/smoke.test.ts +++ b/test/smoke.test.ts @@ -11,9 +11,13 @@ import { createTool, defineAdapter, defineCommand, + defineProfile, expectFailure, expectOk, fakeAdapter, + loadConfig, + malformedConfig, + missingConfig, nextStepGuidance, notFound, ok, @@ -23,8 +27,10 @@ import { resolveByQuery, resolveOne, runTool, + selectProfile, supportsCapability, unsupportedCapability, + validateConfig, validateInput } from '../src/index.js' @@ -435,6 +441,87 @@ test('lookup failures render candidates and next steps in human mode', async () assert.match(result.stderr, /Next step: Run demo pick --id task-1/) }) +test('config helpers can define and select named profiles', () => { + const work = defineProfile('work', { account: 'team@example.com', region: 'us' }, { account: 'team@example.com' }) + assert.equal(work.name, 'work') + assert.equal(work.account, 'team@example.com') + + const selected = selectProfile( + { + work: work.values, + personal: { account: 'me@example.com', region: 'us' } + }, + 'work' + ) + + if ('ok' in selected) throw new Error('expected selected profile values') + assert.equal(selected.account, 'team@example.com') +}) + +test('loadConfig merges base config, selected profile, and env overrides', () => { + const schema = z.object({ + account: z.string().min(1), + region: z.string().min(1), + apiBaseUrl: z.string().url() + }) + + const config = loadConfig(schema, { + config: { region: 'eu', apiBaseUrl: 'https://base.example.com' }, + profiles: { + work: { account: 'team@example.com', apiBaseUrl: 'https://work.example.com' } + }, + profile: 'work', + env: { region: 'us' } + }) + + if ('ok' in config && config.ok === false) throw new Error('expected config to load successfully') + assert.equal(config.account, 'team@example.com') + assert.equal(config.region, 'us') + assert.equal(config.apiBaseUrl, 'https://work.example.com') +}) + +test('missingConfig and malformed config paths preserve structured diagnostics', () => { + const missing = expectFailure( + missingConfig('API_BASE_URL', { + source: 'env', + nextStep: 'Set API_BASE_URL and try again' + }), + 'CONFIG_ERROR' + ) + assert.equal(missing.error.details?.reason, 'missing') + assert.equal(missing.error.details?.source, 'env') + + const malformed = expectFailure( + malformedConfig('apiBaseUrl: Invalid url', { + source: 'merged', + expected: '{"apiBaseUrl":"https://api.example.com"}', + issues: ['apiBaseUrl: Invalid url'] + }), + 'CONFIG_ERROR' + ) + assert.equal(malformed.error.details?.reason, 'malformed') + assert.equal(malformed.error.details?.source, 'merged') +}) + +test('validateConfig returns CONFIG_ERROR for malformed config', () => { + const schema = z.object({ + account: z.string().min(1), + apiBaseUrl: z.string().url() + }) + + const result = validateConfig(schema, { + account: 'team@example.com', + apiBaseUrl: 'not-a-url' + }, { + source: 'merged', + nextStep: 'Fix apiBaseUrl and try again' + }) + + const failure = expectFailure(result, 'CONFIG_ERROR') + assert.equal(failure.error.details?.reason, 'malformed') + assert.match(failure.error.message, /Invalid url/) +}) + test('adapter failures render normalized details in human mode', async () => { const tool = createTool({ name: 'demo', @@ -461,6 +548,34 @@ test('adapter failures render normalized details in human mode', async () => { assert.match(result.stderr, /Next step: Reconnect with write scopes/) }) +test('config failures render structured diagnostics in human mode', async () => { + const tool = createTool({ + name: 'demo', + commands: [ + defineCommand({ + name: 'config', + handler: async () => + missingConfig('API_BASE_URL', { + source: 'env', + profile: 'work', + expected: '{"apiBaseUrl":"https://api.example.com"}', + nextStep: 'Set API_BASE_URL or run demo config --profile work' + }) + }) + ] + }) + + const result = await runTool(tool, ['config']) + expectFailure(result.result, 'CONFIG_ERROR') + assert.match(result.stderr, /Error \[CONFIG_ERROR\]: Missing config: API_BASE_URL/) + assert.match(result.stderr, /Source: env/) + assert.match(result.stderr, /Key: API_BASE_URL/) + assert.match(result.stderr, /Profile: work/) + assert.match(result.stderr, /Reason: missing/) + assert.match(result.stderr, /Expected: \{"apiBaseUrl":"https:\/\/api.example.com"\}/) + assert.match(result.stderr, /Next step: Set API_BASE_URL or run demo config --profile work/) +}) + test('notFound helper preserves structured lookup guidance', () => { const result = notFound('Task not found', { query: 'invoice', From 8353ec7ee51b95a46a3b91419ef4ec11aec9075a Mon Sep 17 00:00:00 2001 From: Dushyant Garg Date: Sun, 12 Apr 2026 04:21:37 -0400 Subject: [PATCH 2/2] Add testing fixtures --- CHANGELOG.md | 3 +- README.md | 34 +++++++ examples/testing-fixtures/README.md | 23 +++++ .../changes/add-testing-fixtures/tasks.md | 15 --- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/testing-kit/spec.md | 4 +- .../2026-04-12-add-testing-fixtures/tasks.md | 15 +++ openspec/specs/testing-kit/spec.md | 13 +++ src/index.ts | 11 ++- src/testing/assertions.ts | 98 +++++++++++++++++++ src/testing/fixtures.ts | 20 ++++ test/smoke.test.ts | 88 +++++++++++------ 14 files changed, 274 insertions(+), 50 deletions(-) create mode 100644 examples/testing-fixtures/README.md delete mode 100644 openspec/changes/add-testing-fixtures/tasks.md rename openspec/changes/{add-testing-fixtures => archive/2026-04-12-add-testing-fixtures}/.openspec.yaml (100%) rename openspec/changes/{add-testing-fixtures => archive/2026-04-12-add-testing-fixtures}/design.md (100%) rename openspec/changes/{add-testing-fixtures => archive/2026-04-12-add-testing-fixtures}/proposal.md (100%) rename openspec/changes/{add-testing-fixtures => archive/2026-04-12-add-testing-fixtures}/specs/testing-kit/spec.md (90%) create mode 100644 openspec/changes/archive/2026-04-12-add-testing-fixtures/tasks.md create mode 100644 src/testing/fixtures.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c6a54..9fc591e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,12 @@ All notable changes to this project will be documented in this file. - lookup-resolution helpers for id-or-query flows, not-found outcomes, and ambiguous matches - adapter contract helpers for capability checks and normalized adapter failures - config convention helpers for env/profile loading and structured config diagnostics +- richer testing helpers for auth, lookup, adapter, config, and dry-run assertions ### Changed - command and tool metadata now support descriptions, aliases, usage, and examples - human failure output can now render structured auth guidance details, adapter failure metadata, config diagnostics, and lookup candidate summaries -- README examples now show help-oriented command metadata, auth preflight usage, adapter contract patterns, config conventions, and lookup resolution patterns +- README examples now show help-oriented command metadata, auth preflight usage, adapter contract patterns, config conventions, testing helpers, and lookup resolution patterns ## [0.1.3] - 2026-04-11 diff --git a/README.md b/README.md index 35f4320..84b3148 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,13 @@ Testing: - `runTool` - `expectOk` - `expectFailure` +- `expectDryRun` +- `expectAuthFailure` +- `expectLookupFailure` +- `expectAdapterFailure` +- `expectConfigFailure` +- `authFailureFixture` +- `lookupCandidatesFixture` - `fakeAdapter` ## Auth preflight @@ -281,12 +288,39 @@ const tool = createTool({ }) ``` +## Testing helpers + +Use the richer assertions when you want cheap confidence around framework primitives, not shell-heavy rechecks of every field by hand. + +```ts +import test from 'node:test' +import { authFailureFixture, expectAuthFailure, requireAuth } from 'agenttk' + +test('auth failure stays structured', async () => { + const auth = await requireAuth(authFailureFixture({ + code: 'ACCOUNT_MISMATCH', + currentAccount: 'personal@example.com', + expectedAccount: 'team@example.com' + })) + + expectAuthFailure(auth, { + code: 'ACCOUNT_MISMATCH', + provider: 'google', + currentAccount: 'personal@example.com', + expectedAccount: 'team@example.com' + }) +}) +``` + +Keep using plain `runTool(...)` tests for command wiring and output mode checks. Use the richer helpers when you want fast, data-first checks around auth, lookup, dry-run, adapter, and config behavior. + ## Examples - `examples/minimal-cli/` - `examples/tasks-cli/` - `examples/adapter-cli/` - `examples/config-cli/` +- `examples/testing-fixtures/` ```bash node examples/minimal-cli/index.mjs hello --json diff --git a/examples/testing-fixtures/README.md b/examples/testing-fixtures/README.md new file mode 100644 index 0000000..7b36f95 --- /dev/null +++ b/examples/testing-fixtures/README.md @@ -0,0 +1,23 @@ +# testing-fixtures + +Minimal downstream-style example of using AgentTK's richer testing helpers. + +```ts +import test from 'node:test' +import { accountMismatch, authFailureFixture, expectAuthFailure, requireAuth } from 'agenttk' + +test('auth preflight stays cheap to test', async () => { + const auth = await requireAuth(authFailureFixture({ + code: 'ACCOUNT_MISMATCH', + currentAccount: 'personal@example.com', + expectedAccount: 'team@example.com' + })) + + expectAuthFailure(auth, { + code: 'ACCOUNT_MISMATCH', + provider: 'google', + currentAccount: 'personal@example.com', + expectedAccount: 'team@example.com' + }) +}) +``` diff --git a/openspec/changes/add-testing-fixtures/tasks.md b/openspec/changes/add-testing-fixtures/tasks.md deleted file mode 100644 index 2f1ed9b..0000000 --- a/openspec/changes/add-testing-fixtures/tasks.md +++ /dev/null @@ -1,15 +0,0 @@ -## 1. Fixture coverage - -- [ ] 1.1 Add reusable fixtures or assertions for auth failure scenarios. -- [ ] 1.2 Add reusable fixtures or assertions for lookup ambiguity and not-found flows. -- [ ] 1.3 Add richer helpers for dry-run and adapter-backed command verification. - -## 2. Examples and docs - -- [ ] 2.1 Document when to use the richer test helpers versus plain command tests. -- [ ] 2.2 Add at least one downstream-style example using the richer fixtures. - -## 3. Verification - -- [ ] 3.1 Ensure the testing-kit spec reflects the new helper surface. -- [ ] 3.2 Validate the OpenSpec change. diff --git a/openspec/changes/add-testing-fixtures/.openspec.yaml b/openspec/changes/archive/2026-04-12-add-testing-fixtures/.openspec.yaml similarity index 100% rename from openspec/changes/add-testing-fixtures/.openspec.yaml rename to openspec/changes/archive/2026-04-12-add-testing-fixtures/.openspec.yaml diff --git a/openspec/changes/add-testing-fixtures/design.md b/openspec/changes/archive/2026-04-12-add-testing-fixtures/design.md similarity index 100% rename from openspec/changes/add-testing-fixtures/design.md rename to openspec/changes/archive/2026-04-12-add-testing-fixtures/design.md diff --git a/openspec/changes/add-testing-fixtures/proposal.md b/openspec/changes/archive/2026-04-12-add-testing-fixtures/proposal.md similarity index 100% rename from openspec/changes/add-testing-fixtures/proposal.md rename to openspec/changes/archive/2026-04-12-add-testing-fixtures/proposal.md diff --git a/openspec/changes/add-testing-fixtures/specs/testing-kit/spec.md b/openspec/changes/archive/2026-04-12-add-testing-fixtures/specs/testing-kit/spec.md similarity index 90% rename from openspec/changes/add-testing-fixtures/specs/testing-kit/spec.md rename to openspec/changes/archive/2026-04-12-add-testing-fixtures/specs/testing-kit/spec.md index 11f08ae..869548b 100644 --- a/openspec/changes/add-testing-fixtures/specs/testing-kit/spec.md +++ b/openspec/changes/archive/2026-04-12-add-testing-fixtures/specs/testing-kit/spec.md @@ -1,6 +1,6 @@ -## MODIFIED Requirements +## ADDED Requirements -### Requirement: Lightweight testing kit +### Requirement: Rich primitive-aware testing helpers The system SHALL provide lightweight testing helpers for AgentTK-based CLIs. #### Scenario: Framework primitives have matching test helpers diff --git a/openspec/changes/archive/2026-04-12-add-testing-fixtures/tasks.md b/openspec/changes/archive/2026-04-12-add-testing-fixtures/tasks.md new file mode 100644 index 0000000..6fb70f0 --- /dev/null +++ b/openspec/changes/archive/2026-04-12-add-testing-fixtures/tasks.md @@ -0,0 +1,15 @@ +## 1. Fixture coverage + +- [x] 1.1 Add reusable fixtures or assertions for auth failure scenarios. +- [x] 1.2 Add reusable fixtures or assertions for lookup ambiguity and not-found flows. +- [x] 1.3 Add richer helpers for dry-run and adapter-backed command verification. + +## 2. Examples and docs + +- [x] 2.1 Document when to use the richer test helpers versus plain command tests. +- [x] 2.2 Add at least one downstream-style example using the richer fixtures. + +## 3. Verification + +- [x] 3.1 Ensure the testing-kit spec reflects the new helper surface. +- [x] 3.2 Validate the OpenSpec change. diff --git a/openspec/specs/testing-kit/spec.md b/openspec/specs/testing-kit/spec.md index 4099063..fbf3f6a 100644 --- a/openspec/specs/testing-kit/spec.md +++ b/openspec/specs/testing-kit/spec.md @@ -50,3 +50,16 @@ The system SHALL keep the testing helpers lightweight in v0. - **WHEN** AgentTK v0 is implemented - **THEN** the testing kit does not require live UAT runners, network recording, snapshot orchestration, or process-level CLI spawning to satisfy the v0 contract +### Requirement: Rich primitive-aware testing helpers +The system SHALL provide lightweight testing helpers for AgentTK-based CLIs. + +#### Scenario: Framework primitives have matching test helpers +- **WHEN** AgentTK introduces reusable primitives such as auth blocks, lookup resolution, dry-run helpers, or adapter contracts +- **THEN** the testing kit can expose matching lightweight fixtures or assertions +- **AND** downstream tools do not need to rebuild the same harness patterns repeatedly + +#### Scenario: Data-first testing remains possible +- **WHEN** a downstream developer uses the richer testing kit helpers +- **THEN** they can still test command behavior through explicit inputs and outputs +- **AND** the testing kit does not require a heavyweight integration framework or snapshot system + diff --git a/src/index.ts b/src/index.ts index ca91299..d8c8091 100644 --- a/src/index.ts +++ b/src/index.ts @@ -86,5 +86,14 @@ export { asDryRun } from './blocks/dry-run.js' export * from './errors/index.js' export { runTool } from './testing/run-cli.js' -export { expectFailure, expectOk } from './testing/assertions.js' +export { + expectAdapterFailure, + expectAuthFailure, + expectConfigFailure, + expectDryRun, + expectFailure, + expectLookupFailure, + expectOk +} from './testing/assertions.js' +export { authFailureFixture, lookupCandidatesFixture } from './testing/fixtures.js' export { fakeAdapter } from './testing/fake-adapter.js' diff --git a/src/testing/assertions.ts b/src/testing/assertions.ts index 2ee0758..c29f993 100644 --- a/src/testing/assertions.ts +++ b/src/testing/assertions.ts @@ -14,3 +14,101 @@ export function expectFailure(result: CommandResult, code?: string): CommandFail } return failure } + +export function expectDryRun(result: CommandResult): CommandSuccess { + const success = expectOk(result) + assert.equal(success.dryRun, true) + return success +} + +export function expectAuthFailure( + result: CommandResult, + options?: { + code?: 'AUTH_REQUIRED' | 'AUTH_INVALID' | 'ACCOUNT_MISMATCH' + provider?: string + currentAccount?: string + expectedAccount?: string + } +): CommandFailure { + const failure = expectFailure(result, options?.code) + if (options?.provider !== undefined) { + assert.equal(failure.error.details?.provider, options.provider) + } + if (options?.currentAccount !== undefined) { + assert.equal(failure.error.details?.currentAccount, options.currentAccount) + } + if (options?.expectedAccount !== undefined) { + assert.equal(failure.error.details?.expectedAccount, options.expectedAccount) + } + return failure +} + +export function expectLookupFailure( + result: CommandResult, + options?: { + code?: 'NOT_FOUND' | 'AMBIGUOUS_MATCH' + query?: string + candidateCount?: number + } +): CommandFailure { + const failure = expectFailure(result, options?.code) + if (options?.query !== undefined) { + assert.equal(failure.error.details?.query, options.query) + } + if (options?.candidateCount !== undefined) { + const candidates = Array.isArray(failure.error.details?.candidates) ? failure.error.details?.candidates : [] + assert.equal(candidates.length, options.candidateCount) + } + return failure +} + +export function expectAdapterFailure( + result: CommandResult, + options?: { + code?: 'ADAPTER_ERROR' | 'UNSUPPORTED_CAPABILITY' + provider?: string + category?: string + retryable?: boolean + capability?: string + } +): CommandFailure { + const failure = expectFailure(result, options?.code) + if (options?.provider !== undefined) { + assert.equal(failure.error.details?.provider, options.provider) + } + if (options?.category !== undefined) { + assert.equal(failure.error.details?.category, options.category) + } + if (options?.retryable !== undefined) { + assert.equal(failure.error.details?.retryable, options.retryable) + } + if (options?.capability !== undefined) { + assert.equal(failure.error.details?.capability, options.capability) + } + return failure +} + +export function expectConfigFailure( + result: CommandResult, + options?: { + reason?: 'missing' | 'malformed' + source?: string + key?: string + profile?: string + } +): CommandFailure { + const failure = expectFailure(result, 'CONFIG_ERROR') + if (options?.reason !== undefined) { + assert.equal(failure.error.details?.reason, options.reason) + } + if (options?.source !== undefined) { + assert.equal(failure.error.details?.source, options.source) + } + if (options?.key !== undefined) { + assert.equal(failure.error.details?.key, options.key) + } + if (options?.profile !== undefined) { + assert.equal(failure.error.details?.profile, options.profile) + } + return failure +} diff --git a/src/testing/fixtures.ts b/src/testing/fixtures.ts new file mode 100644 index 0000000..2b81f30 --- /dev/null +++ b/src/testing/fixtures.ts @@ -0,0 +1,20 @@ +import type { AuthCheckFailure } from '../blocks/auth.js' +import type { CandidateSummary } from '../blocks/lookup.js' + +export function authFailureFixture(overrides: Partial = {}): AuthCheckFailure { + return { + ok: false, + code: 'AUTH_REQUIRED', + provider: 'google', + nextStep: 'Run demo auth login', + ...overrides + } +} + +export function lookupCandidatesFixture(candidates?: CandidateSummary[]): CandidateSummary[] { + if (candidates) return candidates + return [ + { id: 'task-1', label: 'Invoice follow-up', description: 'Daily Focus' }, + { id: 'task-2', label: 'Invoice draft', description: 'Backlog' } + ] +} diff --git a/test/smoke.test.ts b/test/smoke.test.ts index 78ec973..d27cd86 100644 --- a/test/smoke.test.ts +++ b/test/smoke.test.ts @@ -12,7 +12,12 @@ import { defineAdapter, defineCommand, defineProfile, + expectAdapterFailure, + expectAuthFailure, + expectConfigFailure, + expectDryRun, expectFailure, + expectLookupFailure, expectOk, fakeAdapter, loadConfig, @@ -31,7 +36,9 @@ import { supportsCapability, unsupportedCapability, validateConfig, - validateInput + validateInput, + authFailureFixture, + lookupCandidatesFixture } from '../src/index.js' test('tool runs a simple command and emits json', async () => { @@ -272,17 +279,21 @@ test('auth invalid and account mismatch helpers use stable auth codes', async () }) test('requireAuth short-circuits auth failures from a preflight check', async () => { - const gated = await requireAuth(async () => ({ - ok: false as const, + const gated = await requireAuth(async () => + authFailureFixture({ + code: 'ACCOUNT_MISMATCH', + currentAccount: 'personal@example.com', + expectedAccount: 'team@example.com', + nextStep: 'Reauthenticate with the team account' + }) + ) + + const failure = expectAuthFailure(gated, { code: 'ACCOUNT_MISMATCH', provider: 'google', - currentAccount: 'personal@example.com', expectedAccount: 'team@example.com', - nextStep: 'Reauthenticate with the team account' - })) - - const failure = expectFailure(gated, 'ACCOUNT_MISMATCH') - assert.equal(failure.error.details?.provider, 'google') + currentAccount: 'personal@example.com' + }) assert.equal(failure.error.details?.expectedAccount, 'team@example.com') }) @@ -291,6 +302,17 @@ test('requireAuth returns true for a passing auth preflight', async () => { assert.equal(result, true) }) +test('testing fixtures provide reusable auth and lookup defaults', () => { + const auth = authFailureFixture() + assert.equal(auth.ok, false) + assert.equal(auth.code, 'AUTH_REQUIRED') + assert.equal(auth.provider, 'google') + + const candidates = lookupCandidatesFixture() + assert.equal(candidates.length, 2) + assert.equal(candidates[0]?.id, 'task-1') +}) + test('human auth output renders provider, account context, and next step', async () => { const tool = createTool({ name: 'demo', @@ -344,18 +366,16 @@ test('resolveOne returns NOT_FOUND with guidance for misses', () => { }) test('resolveOne returns AMBIGUOUS_MATCH with compact candidates', () => { - const result = resolveOne( - { query: 'invoice' }, - [ - { id: 'task-1', label: 'Invoice follow-up', description: 'Daily Focus' }, - { id: 'task-2', label: 'Invoice draft', description: 'Backlog' } - ], - { nextStep: 'Retry with an explicit id' } - ) + const result = resolveOne({ query: 'invoice' }, lookupCandidatesFixture(), { + nextStep: 'Retry with an explicit id' + }) - const failure = expectFailure(result, 'AMBIGUOUS_MATCH') + const failure = expectLookupFailure(result, { + code: 'AMBIGUOUS_MATCH', + query: 'invoice', + candidateCount: 2 + }) const candidates = failure.error.details?.candidates as Array<{ id: string; label: string }> - assert.equal(candidates.length, 2) assert.equal(candidates[0]?.id, 'task-1') assert.equal(failure.error.details?.nextStep, 'Retry with an explicit id') }) @@ -382,11 +402,13 @@ test('requireCapability returns a structured unsupported capability failure', () nextStep: 'Use an adapter with tasks.write support' }) - const failure = expectFailure(result, 'UNSUPPORTED_CAPABILITY') - assert.equal(failure.error.details?.provider, 'google') - assert.equal(failure.error.details?.capability, 'tasks.write') + const failure = expectAdapterFailure(result, { + code: 'UNSUPPORTED_CAPABILITY', + provider: 'google', + capability: 'tasks.write', + retryable: false + }) assert.equal(failure.error.details?.operation, 'createTask') - assert.equal(failure.error.details?.retryable, false) }) test('adapterFailure preserves normalized category and retryability hints', () => { @@ -481,25 +503,30 @@ test('loadConfig merges base config, selected profile, and env overrides', () => }) test('missingConfig and malformed config paths preserve structured diagnostics', () => { - const missing = expectFailure( + const missing = expectConfigFailure( missingConfig('API_BASE_URL', { source: 'env', nextStep: 'Set API_BASE_URL and try again' }), - 'CONFIG_ERROR' + { + reason: 'missing', + source: 'env', + key: 'API_BASE_URL' + } ) - assert.equal(missing.error.details?.reason, 'missing') assert.equal(missing.error.details?.source, 'env') - const malformed = expectFailure( + const malformed = expectConfigFailure( malformedConfig('apiBaseUrl: Invalid url', { source: 'merged', expected: '{"apiBaseUrl":"https://api.example.com"}', issues: ['apiBaseUrl: Invalid url'] }), - 'CONFIG_ERROR' + { + reason: 'malformed', + source: 'merged' + } ) - assert.equal(malformed.error.details?.reason, 'malformed') assert.equal(malformed.error.details?.source, 'merged') }) @@ -624,8 +651,7 @@ test('dry-run preserves payload and marks successful results', async () => { }) ) - const success = expectOk(result) - assert.equal(success.dryRun, true) + const success = expectDryRun(result) assert.equal(success.id, 'task-123') assert.equal(success.destination, 'google_tasks') assert.deepEqual(success.record, { title: 'Send estimate' })