From dfa039b8a31e1a7947c2679bea5bada8f8708f9c Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 11:39:51 +0100 Subject: [PATCH 01/28] feat(cli): publish narrative documents from workspaces --- cli/README.md | 14 ++- cli/smoke/harness/hub-api.ts | 5 ++ cli/smoke/workspace-documents.smoke.spec.ts | 62 +++++++++++++ .../command-helpers/workspace/bump.spec.ts | 26 +++++- cli/src/command-helpers/workspace/bump.ts | 60 +++++++++++-- cli/src/command-helpers/workspace/bundle.ts | 14 ++- .../workspace/commands.spec.ts | 1 + cli/src/command-helpers/workspace/commands.ts | 27 +++++- .../workspace/narrative-document.spec.ts | 30 +++++++ .../workspace/narrative-document.ts | 87 ++++++++++++++++++ .../command-helpers/workspace/push.spec.ts | 35 +++++++- cli/src/command-helpers/workspace/push.ts | 88 ++++++++++++++++++- shared/src/hub/calm-hub-client.spec.ts | 35 ++++++++ shared/src/hub/calm-hub-client.ts | 86 ++++++++++++++++++ shared/src/template/front-matter.ts | 11 +++ 15 files changed, 565 insertions(+), 16 deletions(-) create mode 100644 cli/smoke/workspace-documents.smoke.spec.ts create mode 100644 cli/src/command-helpers/workspace/narrative-document.spec.ts create mode 100644 cli/src/command-helpers/workspace/narrative-document.ts diff --git a/cli/README.md b/cli/README.md index dc68647f3..5a55ea275 100644 --- a/cli/README.md +++ b/cli/README.md @@ -863,7 +863,9 @@ where `$TYPE` is one of `patterns`, `architectures`, `standards`, `interfaces`. #### `calm workspace push` -Push every document in the workspace manifest to a CalmHub instance. Each document's identity — namespace, type, mapping id and **version** — comes from its `$id` (of the form `$BASE_URL/calm/namespaces/$NAMESPACE/$TYPE/$MAPPING_ID/versions/$VERSION`). Push **does not auto-bump**: it creates exactly the version each document declares. Documents without a well-formed mapping `$id` (or whose type has no CalmHub resource type) are skipped with a warning. +Push every document in the workspace manifest to a CalmHub instance. JSON mapping documents derive their identity — namespace, type, mapping id and **version** — from `$id` (of the form `$BASE_URL/calm/namespaces/$NAMESPACE/$TYPE/$MAPPING_ID/versions/$VERSION`). Push **does not auto-bump**: it creates exactly the version each document declares. Documents without a well-formed mapping `$id` (or whose type has no CalmHub resource type) are skipped with a warning. + +Narrative Markdown documents use `--type knowledge` or `--type sad`. They require YAML frontmatter with a `title` and `--namespace`. The first push stores the Hub numeric document id, location, and version (`1.0.0`) in `workspace-manifest.json`. Later changes require `workspace bump`; the command updates the manifest version without rewriting the Markdown. ``` calm workspace push [--calm-hub-url ] [--fail-if-modified] @@ -887,6 +889,16 @@ calm workspace push --calm-hub-url https://calmhub.example.com calm workspace push --fail-if-modified # strict merge-time mode ``` +```shell +# First-class document POC: add, publish, inspect, edit, bump, and publish again +calm workspace add ./docs/payments-sad.md --type sad --namespace finos +calm workspace push --calm-hub-url http://localhost:8080 +calm workspace show # shows the published Hub location +calm workspace check --calm-hub-url http://localhost:8080 +calm workspace bump --minor --calm-hub-url http://localhost:8080 +calm workspace push --calm-hub-url http://localhost:8080 +``` + #### `calm workspace check` Check whether any tracked document has changed on disk relative to CalmHub but has **not** been version-bumped. Intended as a CI/PR gate — it **exits non-zero** when a bump is required, so a PR cannot merge with unversioned changes. diff --git a/cli/smoke/harness/hub-api.ts b/cli/smoke/harness/hub-api.ts index 323874a01..a1e7a5d5d 100644 --- a/cli/smoke/harness/hub-api.ts +++ b/cli/smoke/harness/hub-api.ts @@ -39,5 +39,10 @@ export function hubApi(baseUrl: string = SMOKE_HUB_URL) { `${baseUrl}/calm/namespaces/${namespace}/${type}/${mapping}/versions/${version}` ); }, + async getNarrativeDocument(namespace: string, type: string, id: number, version: string): Promise { + const body = await getJson(`${baseUrl}/api/calm/namespaces/${namespace}/documents/${type}/${id}/versions/${version}`); + if (typeof body.documentMarkdown !== 'string') throw new Error('Narrative document response has no documentMarkdown'); + return body.documentMarkdown; + }, }; } diff --git a/cli/smoke/workspace-documents.smoke.spec.ts b/cli/smoke/workspace-documents.smoke.spec.ts new file mode 100644 index 000000000..1146ac224 --- /dev/null +++ b/cli/smoke/workspace-documents.smoke.spec.ts @@ -0,0 +1,62 @@ +import path from 'path'; +import * as fs from 'fs'; +import { execSync } from 'child_process'; +import { afterAll, beforeAll, describe, expect, test } from 'vitest'; +import { installPackedCli, type CliInstall } from '../src/test_helpers/cli-runner'; +import { SMOKE_HUB_URL } from './global-setup'; +import { hubApi } from './harness/hub-api'; +import { hubDocId } from './harness/fixtures'; + +const CLI_ROOT = path.resolve(__dirname, '..'); +const NS = 'smoke-workspace-documents'; +const api = hubApi(); + +describe('workspace narrative-document POC', () => { + let cli: CliInstall; + let wsDir: string; + let documentPath: string; + let architecturePath: string; + let documentId: number; + const initial = '---\ntitle: Payments SAD\ndescription: Smoke document\n---\n# Payments\n'; + + async function run(args: string[]) { + return cli.run(args, { cwd: wsDir }); + } + + beforeAll(async () => { + cli = installPackedCli(CLI_ROOT, 'calm-smoke-workspace-documents'); + wsDir = path.join(cli.tempDir, 'repo'); + fs.mkdirSync(wsDir, { recursive: true }); + execSync('git init', { cwd: wsDir, stdio: 'inherit' }); + documentPath = path.join(wsDir, 'payments-sad.md'); + fs.writeFileSync(documentPath, initial); + architecturePath = path.join(wsDir, 'payments.architecture.json'); + fs.writeFileSync(architecturePath, JSON.stringify({ + $schema: 'https://calm.finos.org/release/1.0/meta/calm.json', + $id: hubDocId(NS, 'architectures', 'payments', '1.0.0'), + title: 'Payments', nodes: [], relationships: [], + }, null, 2)); + await cli.run(['hub', 'create', 'namespace', '--name', NS, '--description', 'workspace documents smoke', '-c', SMOKE_HUB_URL]); + }, 120_000); + + afterAll(() => cli?.cleanup()); + + test('publishes, retrieves, bumps, and republishes a Markdown document', async () => { + await run(['workspace', 'init', 'documents']); + await run(['workspace', 'add', architecturePath, '--type', 'architecture', '--namespace', NS]); + await run(['workspace', 'add', documentPath, '--type', 'sad', '--namespace', NS]); + await run(['workspace', 'push', '--calm-hub-url', SMOKE_HUB_URL]); + expect(await api.listVersions(NS, 'architectures', 'payments')).toContain('1.0.0'); + + const manifestPath = path.join(wsDir, '.calm-workspace', 'bundles', 'documents', 'workspace-manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as Record; + documentId = manifest['Payments SAD'].calmHubDocumentId; + expect(await api.getNarrativeDocument(NS, 'sad', documentId, '1.0.0')).toBe(initial); + + fs.writeFileSync(documentPath, initial.replace('# Payments', '# Updated payments')); + await expect(run(['workspace', 'check', '--calm-hub-url', SMOKE_HUB_URL])).rejects.toHaveProperty('exitCode', 1); + await run(['workspace', 'bump', '--minor', '--calm-hub-url', SMOKE_HUB_URL]); + await run(['workspace', 'push', '--calm-hub-url', SMOKE_HUB_URL]); + expect(await api.getNarrativeDocument(NS, 'sad', documentId, '1.1.0')).toContain('# Updated payments'); + }); +}); diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index c0490e1a1..03f60ef44 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import { detectChangedResources, bumpWorkspace, canonicalEqual, maxIncrement } from './bump'; -import { saveManifest } from './bundle'; +import { loadManifest, saveManifest } from './bundle'; import { CalmHubClient, ResourceChangeType } from '@finos/calm-shared/src/hub/calm-hub-client'; import { mkdir, writeFile, rm, readFile } from 'fs/promises'; import path from 'path'; @@ -16,10 +16,14 @@ const idAt = (resource: string, version: string, type = 'architectures', ns = 'c interface ClientOpts { versions?: Record; remote?: Record; + narrativeVersions?: string[]; + narrativeMarkdown?: string; } const makeClient = (opts: ClientOpts = {}): CalmHubClient => ({ getMappedResourceVersions: vi.fn(async (_ns: string, mappingId: string) => opts.versions?.[mappingId] ?? []), getMappedResourceByVersion: vi.fn(async (_ns: string, mappingId: string, version: string) => opts.remote?.[`${mappingId}@${version}`] ?? {}), + getNarrativeDocumentVersions: vi.fn(async () => opts.narrativeVersions ?? []), + getNarrativeDocumentVersion: vi.fn(async () => ({ documentMarkdown: opts.narrativeMarkdown ?? '' })), }) as unknown as CalmHubClient; describe('bump', () => { @@ -48,6 +52,26 @@ describe('bump', () => { }); describe('detectChangedResources', () => { + it('detects and bumps changed narrative Markdown without rewriting it', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Changed\n'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', + version: '1.0.0', calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const client = makeClient({ narrativeVersions: ['1.0.0'], narrativeMarkdown: markdown.replace('Changed', 'Published') }); + + const changed = await detectChangedResources(bundlePath, client); + expect(changed).toHaveLength(1); + await bumpWorkspace(bundlePath, client, { increment: 'MINOR', preDetectedChanges: changed }); + + expect((await loadManifest(bundlePath)).payments.version).toBe('1.1.0'); + expect(await readFile(path.join(filesPath, 'payments.md'), 'utf8')).toBe(markdown); + }); + it('skips a brand-new resource with no versions in CalmHub', async () => { await write('a.json', { $id: idAt('a', '1.0.0'), title: 'A' }); await saveManifest(bundlePath, { 'a': { path: 'files/a.json', type: 'architecture' } }); diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index 473dc5cd5..fde261ac9 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -1,6 +1,6 @@ import { readFile, writeFile } from 'fs/promises'; import { existsSync } from 'fs'; -import { loadManifest, resolveFilePath } from './bundle'; +import { loadManifest, resolveFilePath, saveManifest } from './bundle'; import { buildRefRulesFromDiskIds, syncReferences, RefUpdateResult } from './ref-rewrite'; import { CalmHubClient, ResourceChangeType } from '@finos/calm-shared/src/hub/calm-hub-client'; import { @@ -11,6 +11,7 @@ import { import { computeSemVerBump, sortSemVer } from '@finos/calm-shared/src/hub/semver'; import { canonicalEqual } from '@finos/calm-shared/src/hub/canonical'; import { initLogger, Logger } from '@finos/calm-shared/src/logger'; +import { NarrativeDocumentIdentity, isNarrativeDocumentType, parseNarrativeDocument, validateNarrativeIdentity } from './narrative-document'; // Re-exported for existing consumers (push.ts, tests) that import it from here. export { canonicalEqual }; @@ -37,14 +38,17 @@ function bumpDocumentContent(raw: string, metadata: DocumentMetadata): string { return JSON.stringify(json, null, 2); } -export interface ChangedResource { +interface ChangedResourceBase { id: string; filePath: string; - metadata: DocumentMetadata; currentVersion: string; latestHubVersion: string; } +export type ChangedResource = + | (ChangedResourceBase & { kind: 'mapping'; metadata: DocumentMetadata }) + | (ChangedResourceBase & { kind: 'narrative'; narrativeIdentity: NarrativeDocumentIdentity }); + export interface BumpResult { bumped: Array<{ id: string; filePath: string; fromVersion: string; toVersion: string; triggeredBy?: string; increment?: ResourceChangeType }>; refUpdates: RefUpdateResult[]; @@ -90,6 +94,7 @@ export async function detectChangedResources( for (const [id, entry] of Object.entries(manifest)) { const filePath = resolveFilePath(bundlePath, entry.path); if (!existsSync(filePath)) { + if (isNarrativeDocumentType(entry.type)) throw new Error(`Narrative document '${id}' file not found: ${filePath}`); logger.warn(`File not found for id '${id}': ${filePath}`); continue; } @@ -98,10 +103,39 @@ export async function detectChangedResources( try { raw = await readFile(filePath, 'utf8'); } catch (e) { + if (isNarrativeDocumentType(entry.type)) throw new Error(`Narrative document '${id}' could not be read: ${e instanceof Error ? e.message : String(e)}`); logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); continue; } + if (isNarrativeDocumentType(entry.type)) { + const version = entry.version; + if (!version) throw new Error(`Narrative document '${id}' has no manifest version.`); + if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { + throw new Error(`Narrative document '${id}' has incomplete Hub identity. Re-add the document to repair it.`); + } + const identity: NarrativeDocumentIdentity = { + namespace: entry.namespace ?? '', type: entry.type, version, calmHubDocumentId: entry.calmHubDocumentId, + }; + parseNarrativeDocument(raw, id); + if (entry.calmHubDocumentId === undefined) { + validateNarrativeIdentity(identity, false); + continue; + } + validateNarrativeIdentity(identity, true); + const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId!); + if (versions.length === 0 || !versions.includes(version)) continue; + const remote = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId!, version + ); + if (remote.documentMarkdown === raw) continue; + changed.push({ + id, filePath, currentVersion: version, + latestHubVersion: sortSemVer(versions)[versions.length - 1], kind: 'narrative', narrativeIdentity: identity, + }); + continue; + } + let metadata: DocumentMetadata; try { metadata = extractDocumentMetadata(raw); @@ -141,6 +175,7 @@ export async function detectChangedResources( metadata, currentVersion: metadata.version, latestHubVersion: sortSemVer(versions)[versions.length - 1], + kind: 'mapping', }); } @@ -171,6 +206,18 @@ export async function bumpWorkspace( for (const c of changed) { const docIncrement = options.perDocIncrements?.get(c.id) ?? options.increment; const toVersion = computeSemVerBump(c.latestHubVersion, docIncrement); + if (c.kind === 'narrative') { + const manifest = await loadManifest(bundlePath); + const entry = manifest[c.id]; + if (!entry) throw new Error(`Narrative document '${c.id}' is no longer in the manifest.`); + manifest[c.id] = { ...entry, version: toVersion }; + await saveManifest(bundlePath, manifest); + bumped.push({ id: c.id, filePath: c.filePath, fromVersion: c.currentVersion, toVersion, increment: docIncrement }); + appliedIncrements.set(c.id, docIncrement); + bumpedIds.add(c.id); + logger.info(`Bumped '${c.id}' ${c.currentVersion} -> ${toVersion}`); + continue; + } const raw = await readFile(c.filePath, 'utf8'); const updated = bumpDocumentContent(raw, { ...c.metadata, version: toVersion }); await writeFile(c.filePath, updated, 'utf8'); @@ -191,8 +238,11 @@ export async function bumpWorkspace( for (let depth = 0; depth < MAX_CASCADE_DEPTH; depth++) { const manifest = await loadManifest(bundlePath); - const rules = await buildRefRulesFromDiskIds(manifest, bundlePath); - const refUpdates = await syncReferences(bundlePath, manifest, rules); + const jsonManifest = Object.fromEntries( + Object.entries(manifest).filter(([, entry]) => !isNarrativeDocumentType(entry.type)) + ); + const rules = await buildRefRulesFromDiskIds(jsonManifest, bundlePath); + const refUpdates = await syncReferences(bundlePath, jsonManifest, rules); allRefUpdates.push(...refUpdates); const cascadeCandidates = refUpdates.filter(r => r.changeCount > 0 && !bumpedIds.has(r.docId)); diff --git a/cli/src/command-helpers/workspace/bundle.ts b/cli/src/command-helpers/workspace/bundle.ts index 3cf21c4d1..253ee4fa2 100644 --- a/cli/src/command-helpers/workspace/bundle.ts +++ b/cli/src/command-helpers/workspace/bundle.ts @@ -4,6 +4,7 @@ import { existsSync } from 'fs'; import { JSONPath } from 'jsonpath-plus'; import { printBundleTreeFromGraph } from './tree'; import type { CalmDocumentType } from '@finos/calm-models/types'; +import type { NarrativeDocumentType } from '@finos/calm-shared/src/hub/calm-hub-client'; /** * Property names that can contain document references (URLs or paths) in CALM JSON. @@ -63,13 +64,15 @@ export function extractAllReferences(json: object): string[] { return Array.from(new Set(allRefs)); } -export type WorkspaceDocumentType = CalmDocumentType | 'unknown'; +export type WorkspaceDocumentType = CalmDocumentType | NarrativeDocumentType | 'unknown'; export type WorkspaceManifestEntry = { path: string; type: WorkspaceDocumentType; namespace?: string; calmHubId?: string; + version?: string; + calmHubDocumentId?: number; }; export type WorkspaceManifest = Record; @@ -167,7 +170,7 @@ export async function determineDocumentId(srcPath: string, explicitId?: string): export async function addFileToBundle( bundlePath: string, srcPath: string, - opts?: { id?: string; destName?: string; copy?: boolean; type?: WorkspaceDocumentType; namespace?: string } + opts?: { id?: string; destName?: string; copy?: boolean; type?: WorkspaceDocumentType; namespace?: string; version?: string } ): Promise<{ id: string; destPath: string; rel: string }> { const id = await determineDocumentId(srcPath, opts?.id); @@ -190,7 +193,12 @@ export async function addFileToBundle( } const manifest = await loadManifest(bundlePath); - manifest[id] = { path: rel, type: opts?.type ?? 'unknown', ...(opts?.namespace ? { namespace: opts.namespace } : {}) }; + manifest[id] = { + path: rel, + type: opts?.type ?? 'unknown', + ...(opts?.namespace ? { namespace: opts.namespace } : {}), + ...(opts?.version ? { version: opts.version } : {}), + }; await saveManifest(bundlePath, manifest); return { id, destPath, rel }; diff --git a/cli/src/command-helpers/workspace/commands.spec.ts b/cli/src/command-helpers/workspace/commands.spec.ts index a9d75bf57..103539e19 100644 --- a/cli/src/command-helpers/workspace/commands.spec.ts +++ b/cli/src/command-helpers/workspace/commands.spec.ts @@ -95,6 +95,7 @@ vi.mock('../../cli-config', () => ({ vi.mock('@finos/calm-shared/src/hub/calm-hub-client', () => ({ CalmHubClient: mocks.CalmHubClient, + NARRATIVE_DOCUMENT_TYPES: ['knowledge', 'sad'], })); vi.mock('./document-id-prompt', () => ({ diff --git a/cli/src/command-helpers/workspace/commands.ts b/cli/src/command-helpers/workspace/commands.ts index 62144d0a4..ed53916a8 100644 --- a/cli/src/command-helpers/workspace/commands.ts +++ b/cli/src/command-helpers/workspace/commands.ts @@ -18,6 +18,8 @@ import { CalmHubClient, ResourceChangeType } from '@finos/calm-shared/src/hub/ca import { isConformantDocumentId, namespaceFromDocumentId } from '@finos/calm-shared/src/hub/document-id-utils'; import { loadCliConfig } from '../../cli-config'; import { resolveCalmHubOptions } from '../hub-commands'; +import { isNarrativeDocumentType, parseNarrativeDocument } from './narrative-document'; +import { NARRATIVE_DOCUMENT_TYPES } from '@finos/calm-shared/src/hub/calm-hub-client'; const logger: Logger = initLogger(false, 'workspace'); @@ -54,7 +56,7 @@ export function setupWorkspaceCommands(program: Command) { .argument('', 'Path to the file to add to the bundle') .option('--id ', 'Document ID to register for this file (defaults to filename without extension)') .option('--copy', 'Copy the file into the bundle instead of referencing it from its current location.') - .addOption(new Option('--type ', 'Document type').choices([...CALM_DOCUMENT_TYPES_LIST])) + .addOption(new Option('--type ', 'Document type').choices([...CALM_DOCUMENT_TYPES_LIST, ...NARRATIVE_DOCUMENT_TYPES])) .option('--namespace ', 'CalmHub namespace to associate with this file') .action(async (file: string, options: { id?: string; copy?: boolean; type?: string; namespace?: string }) => { try { @@ -66,7 +68,28 @@ export function setupWorkspaceCommands(program: Command) { const srcPath = path.resolve(file); - const type = await enforceOptionPresenceByPrompt(options.type, 'Select a document type:', CALM_DOCUMENT_TYPES_LIST); + const documentTypes = [...CALM_DOCUMENT_TYPES_LIST, ...NARRATIVE_DOCUMENT_TYPES]; + const type = await enforceOptionPresenceByPrompt(options.type, 'Select a document type:', documentTypes); + if (isNarrativeDocumentType(type)) { + if (!options.namespace?.trim()) { + throw new Error(`Narrative document '${file}' requires --namespace.`); + } + const raw = await readFile(srcPath, 'utf8'); + const narrative = parseNarrativeDocument(raw, file); + const { id: resolvedId, destPath: finalDestPath } = await addFileToBundle(bundlePath, srcPath, { + id: options.id ?? narrative.request.name, + copy: options.copy, + type, + namespace: options.namespace.trim(), + version: '1.0.0', + }); + if (options.copy) { + logger.info(`Copied ${srcPath} -> ${finalDestPath} (id: ${resolvedId})`); + } else { + logger.info(`Added reference to ${finalDestPath} (id: ${resolvedId})`); + } + return; + } if (!isValidCalmDocumentType(type)) { logger.error(`Invalid document type '${type}'. Must be one of: ${CALM_DOCUMENT_TYPES_LIST.join(', ')}`); process.exit(1); diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts new file mode 100644 index 000000000..676c27810 --- /dev/null +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { parseNarrativeDocument, parseNarrativeDocumentLocation, validateNarrativeIdentity } from './narrative-document'; + +describe('narrative document helpers', () => { + const identity = { namespace: 'finos', type: 'sad' as const, version: '1.0.0' }; + + it('uses frontmatter title and preserves CRLF Markdown', () => { + const markdown = '---\r\ntitle: Payments SAD\r\ndescription: Decisions\r\n---\r\n# Content\r\n'; + expect(parseNarrativeDocument(markdown, 'payments')).toEqual({ + request: { name: 'Payments SAD', description: 'Decisions', documentMarkdown: markdown }, markdown, + }); + }); + + it.each([ + '# No frontmatter', + '---\n---\n# Empty mapping', + '---\n- one\n---\n# Array', + '---\ntitle: 42\n---\n# Invalid title', + '---\ntitle: Good\ndescription: 42\n---\n# Invalid description', + ])('rejects invalid frontmatter', (markdown) => { + expect(() => parseNarrativeDocument(markdown, 'bad')).toThrow(/Narrative document/); + }); + + it('validates identity and matching Location', () => { + validateNarrativeIdentity(identity, false); + expect(parseNarrativeDocumentLocation('/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0', identity)).toBe(42); + expect(() => validateNarrativeIdentity({ ...identity, version: 'latest' }, false)).toThrow(/major.minor.patch/); + expect(() => parseNarrativeDocumentLocation('/api/calm/namespaces/other/documents/sad/42/versions/1.0.0', identity)).toThrow(/does not match/); + }); +}); diff --git a/cli/src/command-helpers/workspace/narrative-document.ts b/cli/src/command-helpers/workspace/narrative-document.ts new file mode 100644 index 000000000..ebc7dbebc --- /dev/null +++ b/cli/src/command-helpers/workspace/narrative-document.ts @@ -0,0 +1,87 @@ +import { NarrativeDocumentRequest, NarrativeDocumentType, NARRATIVE_DOCUMENT_TYPES } from '@finos/calm-shared/src/hub/calm-hub-client'; +import { parseYamlFrontMatterMapping } from '@finos/calm-shared/src/template/front-matter'; + +const LOCATION_PATTERN = /^\/api\/calm\/namespaces\/([^/]+)\/documents\/(knowledge|sad)\/(\d+)\/versions\/((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))$/; +const NAMESPACE_PATTERN = /^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*$/; +const SEMVER_PATTERN = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; + +export interface NarrativeDocumentIdentity { + namespace: string; + type: NarrativeDocumentType; + version: string; + calmHubDocumentId?: number; +} + +export interface ParsedNarrativeDocument { + request: NarrativeDocumentRequest; + markdown: string; +} + +export function isNarrativeDocumentType(type: string): type is NarrativeDocumentType { + return NARRATIVE_DOCUMENT_TYPES.includes(type as NarrativeDocumentType); +} + +export function parseNarrativeDocument(markdown: string, label: string): ParsedNarrativeDocument { + let frontMatter: Record | null; + try { + frontMatter = parseYamlFrontMatterMapping(markdown); + } catch { + throw new Error(`Narrative document '${label}' has malformed YAML frontmatter.`); + } + if (!frontMatter || Object.keys(frontMatter).length === 0) { + throw new Error(`Narrative document '${label}' must contain non-empty YAML mapping frontmatter.`); + } + + const title = frontMatter.title; + if (typeof title !== 'string' || !title.trim()) { + throw new Error(`Narrative document '${label}' frontmatter must contain a non-empty string title.`); + } + + const description = frontMatter.description; + if (description !== undefined && typeof description !== 'string') { + throw new Error(`Narrative document '${label}' frontmatter description must be a string.`); + } + + return { + request: { + name: title.trim(), + ...(description === undefined ? {} : { description }), + documentMarkdown: markdown, + }, + markdown, + }; +} + +export function validateNarrativeIdentity(identity: NarrativeDocumentIdentity, requireDocumentId: boolean): void { + if (!NAMESPACE_PATTERN.test(identity.namespace)) { + throw new Error('Narrative document namespace must be a non-empty valid namespace.'); + } + if (!isNarrativeDocumentType(identity.type)) { + throw new Error(`Unsupported narrative document type '${identity.type}'.`); + } + if (!SEMVER_PATTERN.test(identity.version)) { + throw new Error(`Narrative document version '${identity.version}' must be major.minor.patch.`); + } + if (requireDocumentId && (!Number.isSafeInteger(identity.calmHubDocumentId) || identity.calmHubDocumentId! <= 0)) { + throw new Error('Narrative document calmHubDocumentId must be a positive integer.'); + } +} + +export function parseNarrativeDocumentLocation(location: string, identity: NarrativeDocumentIdentity): number { + const match = LOCATION_PATTERN.exec(location); + if (!match) { + throw new Error(`Narrative document Location '${location}' has an unexpected format.`); + } + const [, namespace, type, idString, version] = match; + if (namespace !== identity.namespace || type !== identity.type || version !== identity.version) { + throw new Error(`Narrative document Location '${location}' does not match the requested identity.`); + } + const id = Number(idString); + if (!Number.isSafeInteger(id) || id <= 0) { + throw new Error(`Narrative document Location '${location}' has an invalid document id.`); + } + if (identity.calmHubDocumentId !== undefined && id !== identity.calmHubDocumentId) { + throw new Error(`Narrative document Location '${location}' does not match the stored document id.`); + } + return id; +} diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index b00c6e85b..837f0925c 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -7,11 +7,17 @@ import path from 'path'; import { existsSync } from 'fs'; const makeClient = ( - overrides: Partial> = {} + overrides: Partial> = {} ): CalmHubClient => ({ getMappedResourceVersions: vi.fn(async () => []), createMappedResourceVersion: vi.fn(async () => '/calm/namespaces/com.example/architectures/my-arch/versions/1.0.0'), getMappedResourceByVersion: vi.fn(async () => ({})), + createNarrativeDocument: vi.fn(async () => '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0'), + createNarrativeDocumentVersion: vi.fn(async () => '/api/calm/namespaces/com.example/documents/sad/42/versions/1.1.0'), + getNarrativeDocumentVersions: vi.fn(async () => []), + getNarrativeDocumentVersion: vi.fn(async () => ({ documentMarkdown: '' })), ...overrides, }) as unknown as CalmHubClient; @@ -82,6 +88,33 @@ describe('pushWorkspaceToHub', () => { expect(client.getMappedResourceVersions).not.toHaveBeenCalled(); }); + it('creates a narrative document and stores its server identity', async () => { + const markdown = '---\ntitle: Payments SAD\ndescription: Decisions\n---\n# Payments\n'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { + 'payments-sad': { path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + }); + const client = makeClient(); + + await pushWorkspaceToHub(bundlePath, client); + + expect(client.createNarrativeDocument).toHaveBeenCalledWith('com.example', 'sad', { + name: 'Payments SAD', description: 'Decisions', documentMarkdown: markdown, + }); + expect(await loadManifest(bundlePath)).toMatchObject({ + 'payments-sad': { calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0' }, + }); + }); + + it('fails the completed push when a narrative document is invalid', async () => { + await writeFile(path.join(filesPath, 'bad.md'), '# no frontmatter'); + await saveManifest(bundlePath, { + bad: { path: 'files/bad.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + }); + await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/narrative document/); + expect((await loadManifest(bundlePath)).bad.calmHubDocumentId).toBeUndefined(); + }); + it('skips entries whose file is invalid JSON', async () => { await writeFile(path.join(filesPath, 'bad.json'), 'not json {{{'); await saveManifest(bundlePath, { diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index 81ee2bf4e..8e5e647bb 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -5,6 +5,12 @@ import { CalmHubClient } from '@finos/calm-shared/src/hub/calm-hub-client'; import { DocumentMetadata, extractDocumentMetadata } from '@finos/calm-shared/src/hub/document-id-utils'; import { initLogger, Logger } from '@finos/calm-shared/src/logger'; import { canonicalEqual } from './bump'; +import { + isNarrativeDocumentType, + parseNarrativeDocument, + parseNarrativeDocumentLocation, + validateNarrativeIdentity, +} from './narrative-document'; const logger: Logger = initLogger(false, 'workspace'); @@ -34,12 +40,14 @@ export async function pushWorkspaceToHub( } const conflicts: string[] = []; + const narrativeFailures: string[] = []; for (const [id, entry] of entries) { const filePath = resolveFilePath(bundlePath, entry.path); if (!existsSync(filePath)) { logger.warn(`File not found for id '${id}': ${filePath}`); + if (isNarrativeDocumentType(entry.type)) narrativeFailures.push(`${id}: file not found`); continue; } @@ -48,6 +56,71 @@ export async function pushWorkspaceToHub( raw = await readFile(filePath, 'utf8'); } catch (e) { logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); + if (isNarrativeDocumentType(entry.type)) narrativeFailures.push(`${id}: file could not be read`); + continue; + } + + if (isNarrativeDocumentType(entry.type)) { + try { + const version = entry.version; + if (!version) throw new Error('Narrative document manifest entry has no version. Re-add the document to repair it.'); + if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { + throw new Error('Narrative document Hub identity is incomplete. Re-add the document to repair it.'); + } + const identity = { + namespace: entry.namespace ?? '', + type: entry.type, + version, + calmHubDocumentId: entry.calmHubDocumentId, + }; + const narrative = parseNarrativeDocument(raw, id); + + if (entry.calmHubDocumentId === undefined) { + validateNarrativeIdentity(identity, false); + if (version !== '1.0.0') { + throw new Error('A narrative document without calmHubDocumentId must use version 1.0.0.'); + } + const location = await client.createNarrativeDocument(identity.namespace, identity.type, narrative.request); + const documentId = parseNarrativeDocumentLocation(location, identity); + manifest[id] = { ...entry, calmHubDocumentId: documentId, calmHubId: location }; + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${location}`); + continue; + } + + validateNarrativeIdentity(identity, true); + const versions = await client.getNarrativeDocumentVersions( + identity.namespace, identity.type, identity.calmHubDocumentId! + ); + if (!versions.includes(version)) { + const location = await client.createNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId!, version, narrative.request + ); + parseNarrativeDocumentLocation(location, identity); + manifest[id] = { ...entry, calmHubId: location }; + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${location}`); + continue; + } + + if (!failIfModified) { + logger.info(`No changes for '${id}' - version ${version} already exists, skipping`); + continue; + } + const remote = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId!, version + ); + if (remote.documentMarkdown !== raw) { + logger.error(`'${id}' version ${version} already exists in CalmHub but differs on disk. Bump it before pushing.`); + conflicts.push(`${id}@${version}`); + } else { + logger.info(`No changes for '${id}' - version ${version} already exists and is unchanged, skipping`); + } + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + logger.error(`Failed to push narrative document '${id}': ${message}`); + narrativeFailures.push(`${id}: ${message}`); + } continue; } @@ -115,10 +188,19 @@ export async function pushWorkspaceToHub( } } - if (conflicts.length > 0) { + if (conflicts.length > 0 || narrativeFailures.length > 0) { + const summaries: string[] = []; + if (conflicts.length > 0) { + summaries.push( + `${conflicts.length} modified document(s) already exist in CalmHub at their declared version ` + + `(${conflicts.join(', ')}). Run \`calm workspace bump\` to create new versions for them.` + ); + } + if (narrativeFailures.length > 0) { + summaries.push(`${narrativeFailures.length} narrative document(s) failed (${narrativeFailures.join('; ')})`); + } throw new Error( - `Push failed: ${conflicts.length} modified document(s) already exist in CalmHub at their declared ` + - `version (${conflicts.join(', ')}). Run \`calm workspace bump\` to create new versions for them.` + `Push failed: ${summaries.join(' ')}` ); } } diff --git a/shared/src/hub/calm-hub-client.spec.ts b/shared/src/hub/calm-hub-client.spec.ts index c337d1bdf..53b568a48 100644 --- a/shared/src/hub/calm-hub-client.spec.ts +++ b/shared/src/hub/calm-hub-client.spec.ts @@ -15,6 +15,41 @@ describe('CalmHubClient', () => { client = new CalmHubClient({ calmHubUrl: 'http://localhost:8080' }, ax); }); + describe('narrative documents', () => { + const request = { name: 'Payments SAD', description: 'Decisions', documentMarkdown: '---\ntitle: Payments SAD\n---\n# Payments' }; + + it('creates a narrative document using the first-class endpoint', async () => { + mock.onPost('/api/calm/namespaces/finos/documents/sad').reply(201, null, { + location: '/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0', + }); + await expect(client.createNarrativeDocument('finos', 'sad', request)).resolves.toContain('/42/versions/1.0.0'); + expect(mock.history.post[0].data).toBe(JSON.stringify(request)); + }); + + it('rejects a create response without Location', async () => { + mock.onPost('/api/calm/namespaces/finos/documents/sad').reply(201, null, {}); + await expect(client.createNarrativeDocument('finos', 'sad', request)).rejects.toMatchObject({ + request: 'POST /api/calm/namespaces/finos/documents/sad', + }); + }); + + it('reads typed Markdown and rejects malformed success bodies', async () => { + const endpoint = '/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0'; + mock.onGet(endpoint).replyOnce(200, { documentMarkdown: '# Payments' }); + await expect(client.getNarrativeDocumentVersion('finos', 'sad', 42, '1.0.0')).resolves.toEqual({ documentMarkdown: '# Payments' }); + mock.onGet(endpoint).replyOnce(200, {}); + await expect(client.getNarrativeDocumentVersion('finos', 'sad', 42, '1.0.0')).rejects.toBeInstanceOf(HubClientError); + }); + + it('requires a string version array', async () => { + const endpoint = '/api/calm/namespaces/finos/documents/sad/42/versions'; + mock.onGet(endpoint).replyOnce(200, { values: ['1.0.0'] }); + await expect(client.getNarrativeDocumentVersions('finos', 'sad', 42)).resolves.toEqual(['1.0.0']); + mock.onGet(endpoint).replyOnce(200, { values: [42] }); + await expect(client.getNarrativeDocumentVersions('finos', 'sad', 42)).rejects.toBeInstanceOf(HubClientError); + }); + }); + // ── createNamespace ────────────────────────────────────────────────────── describe('createNamespace', () => { diff --git a/shared/src/hub/calm-hub-client.ts b/shared/src/hub/calm-hub-client.ts index 56490eb3a..e1c16b2ea 100644 --- a/shared/src/hub/calm-hub-client.ts +++ b/shared/src/hub/calm-hub-client.ts @@ -39,6 +39,19 @@ export interface HubControlSummary { export type ResourceChangeType = 'MAJOR' | 'MINOR' | 'PATCH'; +export type NarrativeDocumentType = 'knowledge' | 'sad'; +export const NARRATIVE_DOCUMENT_TYPES: NarrativeDocumentType[] = ['knowledge', 'sad']; + +export interface NarrativeDocumentRequest { + name: string; + description?: string; + documentMarkdown: string; +} + +export interface NarrativeDocumentVersion { + documentMarkdown: string; +} + export type ResourceType = 'patterns' | 'architectures' | 'standards' | 'interfaces'; export const RESOURCE_TYPES = ['patterns', 'architectures', 'standards', 'interfaces']; @@ -120,6 +133,79 @@ export class CalmHubClient { } } + // ── Narrative documents ───────────────────────────────────────────────── + + async createNarrativeDocument( + namespace: string, + type: NarrativeDocumentType, + request: NarrativeDocumentRequest + ): Promise { + const endpoint = `/api/calm/namespaces/${namespace}/documents/${type}`; + return this.createNarrativeDocumentAt(endpoint, request, `POST ${endpoint}`); + } + + async createNarrativeDocumentVersion( + namespace: string, + type: NarrativeDocumentType, + id: number, + version: string, + request: NarrativeDocumentRequest + ): Promise { + const endpoint = `/api/calm/namespaces/${namespace}/documents/${type}/${id}/versions/${version}`; + return this.createNarrativeDocumentAt(endpoint, request, `POST ${endpoint}`); + } + + async getNarrativeDocumentVersions(namespace: string, type: NarrativeDocumentType, id: number): Promise { + const endpoint = `/api/calm/namespaces/${namespace}/documents/${type}/${id}/versions`; + try { + const response = await this.ax.get(endpoint); + if (!response.data || typeof response.data !== 'object' || !Array.isArray(response.data.values) || + !response.data.values.every((value: unknown) => typeof value === 'string')) { + throw new HubClientError(0, 'Response does not contain a string values array', `GET ${endpoint}`); + } + return response.data.values; + } catch (err) { + throw this.wrapError(err, `GET ${endpoint}`); + } + } + + async getNarrativeDocumentVersion( + namespace: string, + type: NarrativeDocumentType, + id: number, + version: string + ): Promise { + const endpoint = `/api/calm/namespaces/${namespace}/documents/${type}/${id}/versions/${version}`; + try { + const response = await this.ax.get(endpoint); + if (!response.data || typeof response.data !== 'object' || typeof response.data.documentMarkdown !== 'string') { + throw new HubClientError(0, 'Response does not contain documentMarkdown', `GET ${endpoint}`); + } + return { documentMarkdown: response.data.documentMarkdown }; + } catch (err) { + if (err instanceof HubClientError) throw err; + throw this.wrapError(err, `GET ${endpoint}`); + } + } + + private async createNarrativeDocumentAt( + endpoint: string, + request: NarrativeDocumentRequest, + requestLabel: string + ): Promise { + try { + const response = await this.ax.post(endpoint, request); + const location = response.headers.location as string | undefined; + if (!location) { + throw new HubClientError(0, 'Response does not include Location header', requestLabel); + } + return location; + } catch (err) { + if (err instanceof HubClientError) throw err; + throw this.wrapError(err, requestLabel); + } + } + /** * Lists namespaces. * @returns Namespace summaries. diff --git a/shared/src/template/front-matter.ts b/shared/src/template/front-matter.ts index 918dde58c..b0db065af 100644 --- a/shared/src/template/front-matter.ts +++ b/shared/src/template/front-matter.ts @@ -12,6 +12,17 @@ export interface ParsedFrontMatter { urlToLocalPathMapping?: Map; } +const YAML_FRONTMATTER_PATTERN = /^---\r?\n([\s\S]*?)\r?\n---(?=\r?\n|$)/; + +/** Parses only a leading YAML mapping. It does not resolve template-specific paths. */ +export function parseYamlFrontMatterMapping(content: string): Record | null { + const match = YAML_FRONTMATTER_PATTERN.exec(content); + if (!match) return null; + const parsed = yaml.parse(match[1]); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + return parsed as Record; +} + const RESERVED_KEYS = new Set([ 'architecture', 'url-to-local-file-mapping' From 035549414910db0f4021e97d55b8a4106db4385f Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 11:50:47 +0100 Subject: [PATCH 02/28] test(cli): cover narrative workspace publish paths --- .../workspace/narrative-document.spec.ts | 17 ++++++++ .../command-helpers/workspace/push.spec.ts | 43 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index 676c27810..99ad0a308 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -27,4 +27,21 @@ describe('narrative document helpers', () => { expect(() => validateNarrativeIdentity({ ...identity, version: 'latest' }, false)).toThrow(/major.minor.patch/); expect(() => parseNarrativeDocumentLocation('/api/calm/namespaces/other/documents/sad/42/versions/1.0.0', identity)).toThrow(/does not match/); }); + + it.each([ + [{ ...identity, namespace: 'not_valid' }, false, /valid namespace/], + [{ ...identity, type: 'other' as never }, false, /Unsupported/], + [{ ...identity, version: '01.0.0' }, false, /major.minor.patch/], + [{ ...identity, calmHubDocumentId: 0 }, true, /positive integer/], + ])('rejects invalid persisted identity %#', (candidate, requireId, message) => { + expect(() => validateNarrativeIdentity(candidate, requireId)).toThrow(message); + }); + + it('rejects malformed and mismatched persisted Locations', () => { + expect(() => parseNarrativeDocumentLocation('not-a-location', identity)).toThrow(/unexpected format/); + expect(() => parseNarrativeDocumentLocation( + '/api/calm/namespaces/finos/documents/sad/43/versions/1.0.0', + { ...identity, calmHubDocumentId: 42 } + )).toThrow(/stored document id/); + }); }); diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 837f0925c..55e11e6d4 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -115,6 +115,49 @@ describe('pushWorkspaceToHub', () => { expect((await loadManifest(bundlePath)).bad.calmHubDocumentId).toBeUndefined(); }); + it('creates a later narrative version and updates its location', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments\n'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.1.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const client = makeClient({ getNarrativeDocumentVersions: vi.fn().mockResolvedValue(['1.0.0']) }); + + await pushWorkspaceToHub(bundlePath, client); + + expect(client.createNarrativeDocumentVersion).toHaveBeenCalledWith('com.example', 'sad', 42, '1.1.0', expect.objectContaining({ documentMarkdown: markdown })); + expect((await loadManifest(bundlePath)).payments.calmHubId).toContain('/1.1.0'); + }); + + it('strictly detects changed Markdown at an existing narrative version', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Changed\n'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const client = makeClient({ + getNarrativeDocumentVersions: vi.fn().mockResolvedValue(['1.0.0']), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown.replace('Changed', 'Published') }), + }); + + await expect(pushWorkspaceToHub(bundlePath, client, { failIfModified: true })).rejects.toThrow(/payments@1.0.0/); + }); + + it('fails narrative publish for missing source files and incomplete Hub identity', async () => { + await saveManifest(bundlePath, { + missing: { path: 'files/missing.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + partial: { path: 'files/partial.md', type: 'sad', namespace: 'com.example', version: '1.0.0', calmHubId: '/partial' }, + }); + await writeFile(path.join(filesPath, 'partial.md'), '---\ntitle: Partial\n---\n# Partial'); + await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/narrative document/); + }); + it('skips entries whose file is invalid JSON', async () => { await writeFile(path.join(filesPath, 'bad.json'), 'not json {{{'); await saveManifest(bundlePath, { From bce6c046588e907cb59dfc1ce6d69647d5f7bc2e Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 11:53:46 +0100 Subject: [PATCH 03/28] test(cli): cover narrative workspace version states --- .../command-helpers/workspace/bump.spec.ts | 31 +++++++++++++++++++ .../command-helpers/workspace/push.spec.ts | 20 ++++++++++++ 2 files changed, 51 insertions(+) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index 03f60ef44..c89e318ce 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -52,6 +52,37 @@ describe('bump', () => { }); describe('detectChangedResources', () => { + it('treats new, already-bumped, and unchanged narrative documents as clean', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Published\n'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + const baseEntry = { + path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }; + + await saveManifest(bundlePath, { payments: { ...baseEntry, calmHubDocumentId: undefined, calmHubId: undefined } }); + expect(await detectChangedResources(bundlePath, makeClient())).toEqual([]); + + await saveManifest(bundlePath, { payments: { ...baseEntry, version: '1.1.0' } }); + expect(await detectChangedResources(bundlePath, makeClient({ narrativeVersions: ['1.0.0'] }))).toEqual([]); + + await saveManifest(bundlePath, { payments: baseEntry }); + expect(await detectChangedResources(bundlePath, makeClient({ narrativeVersions: ['1.0.0'], narrativeMarkdown: markdown }))).toEqual([]); + }); + + it('fails narrative checks with incomplete identity or missing source', async () => { + await saveManifest(bundlePath, { + partial: { path: 'files/missing.md', type: 'sad', namespace: 'com.example', version: '1.0.0', calmHubId: '/partial' }, + }); + await expect(detectChangedResources(bundlePath, makeClient())).rejects.toThrow(/file not found/); + + await writeFile(path.join(filesPath, 'partial.md'), '---\ntitle: Partial\n---\n# Partial'); + await saveManifest(bundlePath, { + partial: { path: 'files/partial.md', type: 'sad', namespace: 'com.example', version: '1.0.0', calmHubId: '/partial' }, + }); + await expect(detectChangedResources(bundlePath, makeClient())).rejects.toThrow(/incomplete Hub identity/); + }); + it('detects and bumps changed narrative Markdown without rewriting it', async () => { const markdown = '---\ntitle: Payments SAD\n---\n# Changed\n'; await writeFile(path.join(filesPath, 'payments.md'), markdown); diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 55e11e6d4..b17e59b4c 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -149,6 +149,26 @@ describe('pushWorkspaceToHub', () => { await expect(pushWorkspaceToHub(bundlePath, client, { failIfModified: true })).rejects.toThrow(/payments@1.0.0/); }); + it('idempotently skips an existing narrative version and accepts an exact strict comparison', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments\n'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const client = makeClient({ + getNarrativeDocumentVersions: vi.fn().mockResolvedValue(['1.0.0']), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), + }); + + await pushWorkspaceToHub(bundlePath, client); + await pushWorkspaceToHub(bundlePath, client, { failIfModified: true }); + + expect(client.createNarrativeDocumentVersion).not.toHaveBeenCalled(); + }); + it('fails narrative publish for missing source files and incomplete Hub identity', async () => { await saveManifest(bundlePath, { missing: { path: 'files/missing.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, From fc425ff205160d29e6123879a1d47bca3b7c03a7 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 11:55:58 +0100 Subject: [PATCH 04/28] test(cli): cover narrative workspace error paths --- cli/src/command-helpers/workspace/bump.spec.ts | 14 ++++++++++++++ .../workspace/narrative-document.spec.ts | 7 +++++++ cli/src/command-helpers/workspace/push.spec.ts | 11 +++++++++++ 3 files changed, 32 insertions(+) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index c89e318ce..b5886d107 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -83,6 +83,20 @@ describe('bump', () => { await expect(detectChangedResources(bundlePath, makeClient())).rejects.toThrow(/incomplete Hub identity/); }); + it('fails narrative checks when Hub version retrieval fails', async () => { + await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const client = makeClient({ narrativeVersions: ['1.0.0'] }); + (client.getNarrativeDocumentVersion as ReturnType).mockRejectedValueOnce(new Error('Hub unavailable')); + + await expect(detectChangedResources(bundlePath, client)).rejects.toThrow(/Hub unavailable/); + }); + it('detects and bumps changed narrative Markdown without rewriting it', async () => { const markdown = '---\ntitle: Payments SAD\n---\n# Changed\n'; await writeFile(path.join(filesPath, 'payments.md'), markdown); diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index 99ad0a308..b62241eee 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -11,6 +11,12 @@ describe('narrative document helpers', () => { }); }); + it('publishes without an optional description and rejects malformed YAML', () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Content'; + expect(parseNarrativeDocument(markdown, 'payments').request).toEqual({ name: 'Payments SAD', documentMarkdown: markdown }); + expect(() => parseNarrativeDocument('---\ntitle: [\n---\n# Broken', 'broken')).toThrow(/malformed YAML/); + }); + it.each([ '# No frontmatter', '---\n---\n# Empty mapping', @@ -39,6 +45,7 @@ describe('narrative document helpers', () => { it('rejects malformed and mismatched persisted Locations', () => { expect(() => parseNarrativeDocumentLocation('not-a-location', identity)).toThrow(/unexpected format/); + expect(() => parseNarrativeDocumentLocation('/api/calm/namespaces/finos/documents/sad/0/versions/1.0.0', identity)).toThrow(/invalid document id/); expect(() => parseNarrativeDocumentLocation( '/api/calm/namespaces/finos/documents/sad/43/versions/1.0.0', { ...identity, calmHubDocumentId: 42 } diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index b17e59b4c..306a5e89a 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -178,6 +178,17 @@ describe('pushWorkspaceToHub', () => { await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/narrative document/); }); + it('fails narrative publish when Hub returns an unexpected Location', async () => { + await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); + await saveManifest(bundlePath, { + payments: { path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + }); + const client = makeClient({ createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected') }); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/unexpected format/); + expect((await loadManifest(bundlePath)).payments.calmHubDocumentId).toBeUndefined(); + }); + it('skips entries whose file is invalid JSON', async () => { await writeFile(path.join(filesPath, 'bad.json'), 'not json {{{'); await saveManifest(bundlePath, { From 185312143d4aa67022f1cf8571bef5329d40fd71 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 11:57:53 +0100 Subject: [PATCH 05/28] test(cli): cover narrative workspace edge cases --- cli/src/command-helpers/workspace/bump.spec.ts | 13 +++++++++++++ cli/src/command-helpers/workspace/push.spec.ts | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index b5886d107..e37faf34b 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -97,6 +97,19 @@ describe('bump', () => { await expect(detectChangedResources(bundlePath, client)).rejects.toThrow(/Hub unavailable/); }); + it('treats a document with no Hub versions as new and rejects missing manifest versions', async () => { + await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); + const entry = { + path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }; + await saveManifest(bundlePath, { payments: entry }); + expect(await detectChangedResources(bundlePath, makeClient({ narrativeVersions: [] }))).toEqual([]); + + await saveManifest(bundlePath, { payments: { ...entry, version: undefined } }); + await expect(detectChangedResources(bundlePath, makeClient())).rejects.toThrow(/no manifest version/); + }); + it('detects and bumps changed narrative Markdown without rewriting it', async () => { const markdown = '---\ntitle: Payments SAD\n---\n# Changed\n'; await writeFile(path.join(filesPath, 'payments.md'), markdown); diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 306a5e89a..9960ab296 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -189,6 +189,17 @@ describe('pushWorkspaceToHub', () => { expect((await loadManifest(bundlePath)).payments.calmHubDocumentId).toBeUndefined(); }); + it('rejects narrative manifests with no version or a non-initial unassigned version', async () => { + await writeFile(path.join(filesPath, 'missing.md'), '---\ntitle: Missing\n---\n# Missing'); + await writeFile(path.join(filesPath, 'ahead.md'), '---\ntitle: Ahead\n---\n# Ahead'); + await saveManifest(bundlePath, { + missing: { path: 'files/missing.md', type: 'sad', namespace: 'com.example' }, + ahead: { path: 'files/ahead.md', type: 'sad', namespace: 'com.example', version: '1.1.0' }, + }); + + await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/narrative document/); + }); + it('skips entries whose file is invalid JSON', async () => { await writeFile(path.join(filesPath, 'bad.json'), 'not json {{{'); await saveManifest(bundlePath, { From bd8e1f058286ecf11751bca58840ee87a6073361 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 11:59:31 +0100 Subject: [PATCH 06/28] test(cli): cover unreadable narrative sources --- cli/src/command-helpers/workspace/bump.spec.ts | 7 +++++++ cli/src/command-helpers/workspace/push.spec.ts | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index e37faf34b..9760699d4 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -110,6 +110,13 @@ describe('bump', () => { await expect(detectChangedResources(bundlePath, makeClient())).rejects.toThrow(/no manifest version/); }); + it('fails narrative checks when a tracked path cannot be read', async () => { + await saveManifest(bundlePath, { + unreadable: { path: 'files', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + }); + await expect(detectChangedResources(bundlePath, makeClient())).rejects.toThrow(/could not be read/); + }); + it('detects and bumps changed narrative Markdown without rewriting it', async () => { const markdown = '---\ntitle: Payments SAD\n---\n# Changed\n'; await writeFile(path.join(filesPath, 'payments.md'), markdown); diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 9960ab296..7cddb2ac4 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -200,6 +200,13 @@ describe('pushWorkspaceToHub', () => { await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/narrative document/); }); + it('fails narrative publish when a tracked path cannot be read', async () => { + await saveManifest(bundlePath, { + unreadable: { path: 'files', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + }); + await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/file could not be read/); + }); + it('skips entries whose file is invalid JSON', async () => { await writeFile(path.join(filesPath, 'bad.json'), 'not json {{{'); await saveManifest(bundlePath, { From 239ee7c62d0aa0a6cbffe420a7a733ca554bc45f Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 12:24:57 +0100 Subject: [PATCH 07/28] fix(cli): accept absolute document locations --- .../workspace/narrative-document.spec.ts | 1 + .../workspace/narrative-document.ts | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index b62241eee..a38a04488 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -30,6 +30,7 @@ describe('narrative document helpers', () => { it('validates identity and matching Location', () => { validateNarrativeIdentity(identity, false); expect(parseNarrativeDocumentLocation('/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0', identity)).toBe(42); + expect(parseNarrativeDocumentLocation('http://localhost:8080/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0', identity)).toBe(42); expect(() => validateNarrativeIdentity({ ...identity, version: 'latest' }, false)).toThrow(/major.minor.patch/); expect(() => parseNarrativeDocumentLocation('/api/calm/namespaces/other/documents/sad/42/versions/1.0.0', identity)).toThrow(/does not match/); }); diff --git a/cli/src/command-helpers/workspace/narrative-document.ts b/cli/src/command-helpers/workspace/narrative-document.ts index ebc7dbebc..08baeca69 100644 --- a/cli/src/command-helpers/workspace/narrative-document.ts +++ b/cli/src/command-helpers/workspace/narrative-document.ts @@ -68,7 +68,8 @@ export function validateNarrativeIdentity(identity: NarrativeDocumentIdentity, r } export function parseNarrativeDocumentLocation(location: string, identity: NarrativeDocumentIdentity): number { - const match = LOCATION_PATTERN.exec(location); + const path = extractLocationPath(location); + const match = LOCATION_PATTERN.exec(path); if (!match) { throw new Error(`Narrative document Location '${location}' has an unexpected format.`); } @@ -85,3 +86,19 @@ export function parseNarrativeDocumentLocation(location: string, identity: Narra } return id; } + +function extractLocationPath(location: string): string { + if (location.startsWith('/')) { + return location; + } + + try { + const url = new URL(location); + if (!['http:', 'https:'].includes(url.protocol) || url.search || url.hash) { + throw new Error('invalid Location URL'); + } + return url.pathname; + } catch { + throw new Error(`Narrative document Location '${location}' has an unexpected format.`); + } +} From d91631fc9f3273ac6de068d414903ff7ef029e9d Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 16:12:39 +0100 Subject: [PATCH 08/28] fix(shared): skip narrative documents in workspace loader --- .../workspace-document-loader.spec.ts | 14 ++++++++++++++ .../document-loader/workspace-document-loader.ts | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/shared/src/document-loader/workspace-document-loader.spec.ts b/shared/src/document-loader/workspace-document-loader.spec.ts index b9bfdb8e2..ffcbcfcae 100644 --- a/shared/src/document-loader/workspace-document-loader.spec.ts +++ b/shared/src/document-loader/workspace-document-loader.spec.ts @@ -161,6 +161,20 @@ describe('WorkspaceDocumentLoader', () => { // Document without an $id is stored only by its bare id. expect(mocks.schemaDirectory.storeDocument).toHaveBeenCalledWith('no-id-doc', 'schema', noIdDoc); }); + + it('ignores Markdown narrative documents', async () => { + setupBundle({ + '/ws/workspace-manifest.json': JSON.stringify({ + 'payments-sad': { path: 'files/payments-sad.md', type: 'sad' }, + }), + '/ws/files/payments-sad.md': '---\\ntitle: Payments SAD\\n---\\n# Payments\\n', + }); + const loader = new WorkspaceDocumentLoader(BUNDLE); + + await loader.initialise(mocks.schemaDirectory as unknown as SchemaDirectory); + + expect(mocks.schemaDirectory.storeDocument).not.toHaveBeenCalled(); + }); }); describe('with no usable manifest', () => { diff --git a/shared/src/document-loader/workspace-document-loader.ts b/shared/src/document-loader/workspace-document-loader.ts index 3caa4571e..c37e2e37e 100644 --- a/shared/src/document-loader/workspace-document-loader.ts +++ b/shared/src/document-loader/workspace-document-loader.ts @@ -8,6 +8,7 @@ import path from 'path'; // Mirrors MANIFEST_FILENAME in the CLI workspace bundle module. const MANIFEST_FILENAME = 'workspace-manifest.json'; +const NARRATIVE_DOCUMENT_TYPES = new Set(['knowledge', 'sad']); /** * Identity of a single tracked workspace document, used to decide whether an @@ -79,6 +80,12 @@ export class WorkspaceDocumentLoader implements DocumentLoader { const rules: WorkspaceRule[] = []; for (const [bareId, value] of Object.entries(manifest)) { // Manifest entries are `{ path, type, ... }`; tolerate the legacy plain-string form too. + const type = value && typeof value === 'object' + ? (value as { type?: unknown }).type + : undefined; + // Narrative documents are Markdown rather than CALM JSON documents. They cannot be + // schema-preloaded or resolve a CALM `$ref`, so keep them out of this JSON-only loader. + if (typeof type === 'string' && NARRATIVE_DOCUMENT_TYPES.has(type)) continue; const relPath = typeof value === 'string' ? value : (value && typeof value === 'object' && typeof (value as { path?: unknown }).path === 'string' From ce2b95d83b9e60fd1a6992f927a6b623f4546a2b Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 19:51:01 +0100 Subject: [PATCH 09/28] fix(cli): validate narrative workspace identity --- cli/README.md | 16 +++++--- .../command-helpers/workspace/bump.spec.ts | 20 ++++++++++ cli/src/command-helpers/workspace/bump.ts | 11 +++--- .../workspace/commands.spec.ts | 32 ++++++++++++++++ .../workspace/narrative-document.spec.ts | 14 ++++++- .../workspace/narrative-document.ts | 38 ++++++++++++------- .../command-helpers/workspace/push.spec.ts | 14 +++++++ cli/src/command-helpers/workspace/push.ts | 5 ++- .../workspace-document-loader.ts | 4 +- shared/src/hub/calm-hub-client.spec.ts | 31 +++++++++++++++ shared/src/hub/calm-hub-client.ts | 4 +- 11 files changed, 157 insertions(+), 32 deletions(-) diff --git a/cli/README.md b/cli/README.md index 5a55ea275..2fb1355bb 100644 --- a/cli/README.md +++ b/cli/README.md @@ -803,7 +803,7 @@ calm workspace init my-system #### `calm workspace add ` -Register a CALM document with the active workspace. By default the file is referenced at its current location on disk (no copying). Prompts interactively for document type and (manifest) name if they cannot be determined automatically. +Register a CALM JSON document or narrative Markdown document with the active workspace. By default the file is referenced at its current location on disk (no copying). Prompts interactively for document type and (manifest) name if they cannot be determined automatically. ``` calm workspace add [--id ] [--type ] [--namespace ] [--copy] @@ -812,16 +812,18 @@ calm workspace add [--id ] [--type ] [--namespace ] | Option | Description | |--------|-------------| | `--id ` | Explicit manifest registration id. Overrides automatic resolution. | -| `--type ` | Document type. If omitted, an interactive dropdown is shown. One of: `pattern`, `architecture`, `interface`, `flow`, `control`, `schema`, `timeline`, `adr`. | -| `--namespace ` | CalmHub namespace to record in the manifest. If omitted, it is derived from the document `$id`. | +| `--type ` | Document type. If omitted, an interactive dropdown is shown. One of: `pattern`, `architecture`, `interface`, `flow`, `control`, `schema`, `timeline`, `adr`, `knowledge`, `sad`. | +| `--namespace ` | CalmHub namespace to record in the manifest. It is required for narrative Markdown and otherwise derived from the document `$id` when omitted. | | `--copy` | Copy the file into the bundle's `files/` directory instead of referencing it in place. | -**Document `$id` handling.** `add` inspects the file's CalmHub `$id`: +**Narrative Markdown documents.** Use `--type knowledge` or `sad`. `add` reads YAML frontmatter. A non-empty `title` becomes the manifest name unless you supply `--id`. `--namespace` is required. The initial manifest version is `1.0.0`. Markdown has no CALM `$id` and is never rewritten. + +**JSON document `$id` handling.** For JSON mapping documents, `add` inspects the file's CalmHub `$id`: - **No `$id`** → you are prompted interactively to build one from its components (see below); the `$id` is written into the file and the document is added. - **Conformant `$id`** → left untouched; the manifest namespace is derived from it. - **Non-conformant `$id`** → left as-is; a warning is printed and the document is still tracked, but it cannot be pushed to CalmHub until the `$id` is fixed (silently rewriting it would lose data for types that don't use CalmHub URLs, e.g. `flow`, `adr`, `timeline`). -**Manifest name resolution** (when `--id` is not given): the `title` field from the JSON file, else an interactive prompt. +**Manifest name resolution** (when `--id` is not given): the `title` field from the JSON file or Markdown frontmatter, else an interactive prompt. ```shell # Interactive — prompts for type, builds the $id if needed, then the manifest name @@ -829,6 +831,9 @@ calm workspace add ./architectures/payment-service.json # Reference an already-conformant document without copying calm workspace add ./architectures/payment-service.json --type architecture + +# Register a narrative Markdown document; the frontmatter title becomes its manifest name +calm workspace add ./docs/payments-sad.md --type sad --namespace finos ``` #### `calm workspace new [type] [name] [template]` @@ -895,6 +900,7 @@ calm workspace add ./docs/payments-sad.md --type sad --namespace finos calm workspace push --calm-hub-url http://localhost:8080 calm workspace show # shows the published Hub location calm workspace check --calm-hub-url http://localhost:8080 +# Edit ./docs/payments-sad.md, then bump and publish the new version calm workspace bump --minor --calm-hub-url http://localhost:8080 calm workspace push --calm-hub-url http://localhost:8080 ``` diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index 9760699d4..51c08c9f9 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -137,6 +137,26 @@ describe('bump', () => { expect(await readFile(path.join(filesPath, 'payments.md'), 'utf8')).toBe(markdown); }); + it.each([ + ['MAJOR', '2.0.0'], + ['PATCH', '1.0.1'], + ] as const)('applies a %s bump to a changed narrative document', async (increment, version) => { + const markdown = '---\ntitle: Payments SAD\n---\n# Changed\n'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const client = makeClient({ narrativeVersions: ['1.0.0'], narrativeMarkdown: markdown.replace('Changed', 'Published') }); + + await bumpWorkspace(bundlePath, client, { increment }); + expect((await loadManifest(bundlePath)).payments.version).toBe(version); + + expect(await bumpWorkspace(bundlePath, client, { increment })).toMatchObject({ bumped: [] }); + }); + it('skips a brand-new resource with no versions in CalmHub', async () => { await write('a.json', { $id: idAt('a', '1.0.0'), title: 'A' }); await saveManifest(bundlePath, { 'a': { path: 'files/a.json', type: 'architecture' } }); diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index fde261ac9..fd57a2383 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -11,7 +11,7 @@ import { import { computeSemVerBump, sortSemVer } from '@finos/calm-shared/src/hub/semver'; import { canonicalEqual } from '@finos/calm-shared/src/hub/canonical'; import { initLogger, Logger } from '@finos/calm-shared/src/logger'; -import { NarrativeDocumentIdentity, isNarrativeDocumentType, parseNarrativeDocument, validateNarrativeIdentity } from './narrative-document'; +import { NarrativeDocumentIdentity, isNarrativeDocumentType, parseNarrativeDocument, parseNarrativeDocumentLocation, validateNarrativeIdentity } from './narrative-document'; // Re-exported for existing consumers (push.ts, tests) that import it from here. export { canonicalEqual }; @@ -47,7 +47,7 @@ interface ChangedResourceBase { export type ChangedResource = | (ChangedResourceBase & { kind: 'mapping'; metadata: DocumentMetadata }) - | (ChangedResourceBase & { kind: 'narrative'; narrativeIdentity: NarrativeDocumentIdentity }); + | (ChangedResourceBase & { kind: 'narrative' }); export interface BumpResult { bumped: Array<{ id: string; filePath: string; fromVersion: string; toVersion: string; triggeredBy?: string; increment?: ResourceChangeType }>; @@ -119,10 +119,11 @@ export async function detectChangedResources( }; parseNarrativeDocument(raw, id); if (entry.calmHubDocumentId === undefined) { - validateNarrativeIdentity(identity, false); + validateNarrativeIdentity(identity, false, id); continue; } - validateNarrativeIdentity(identity, true); + validateNarrativeIdentity(identity, true, id); + parseNarrativeDocumentLocation(entry.calmHubId, identity, false); const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId!); if (versions.length === 0 || !versions.includes(version)) continue; const remote = await client.getNarrativeDocumentVersion( @@ -131,7 +132,7 @@ export async function detectChangedResources( if (remote.documentMarkdown === raw) continue; changed.push({ id, filePath, currentVersion: version, - latestHubVersion: sortSemVer(versions)[versions.length - 1], kind: 'narrative', narrativeIdentity: identity, + latestHubVersion: sortSemVer(versions)[versions.length - 1], kind: 'narrative', }); continue; } diff --git a/cli/src/command-helpers/workspace/commands.spec.ts b/cli/src/command-helpers/workspace/commands.spec.ts index 103539e19..88969f9ff 100644 --- a/cli/src/command-helpers/workspace/commands.spec.ts +++ b/cli/src/command-helpers/workspace/commands.spec.ts @@ -224,6 +224,38 @@ describe('setupWorkspaceCommands', () => { ); }); + it('registers Markdown using its frontmatter title without rewriting it', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments\n'; + mocks.readFile.mockResolvedValueOnce(markdown); + + await program.parseAsync(['node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos']); + + expect(mocks.writeFile).not.toHaveBeenCalled(); + expect(mocks.addFileToBundle).toHaveBeenCalledWith( + '/fake/bundle', + expect.stringContaining('payments.md'), + expect.objectContaining({ id: 'Payments SAD', type: 'sad', namespace: 'finos', version: '1.0.0' }) + ); + }); + + it('requires a namespace when adding a narrative document', async () => { + await expect( + program.parseAsync(['node', 'test', 'workspace', 'add', 'payments.md', '--type', 'knowledge']) + ).rejects.toThrow(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + }); + + it('does not mutate or register Markdown with malformed frontmatter', async () => { + mocks.readFile.mockResolvedValueOnce('---\ntitle: [\n---\n# Payments\n'); + + await expect( + program.parseAsync(['node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos']) + ).rejects.toThrow(); + + expect(mocks.writeFile).not.toHaveBeenCalled(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + }); + it('should exit when no workspace bundle found', async () => { mocks.findWorkspaceManifestPath.mockReturnValueOnce(null); await expect( diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index a38a04488..64cd33f52 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -7,7 +7,7 @@ describe('narrative document helpers', () => { it('uses frontmatter title and preserves CRLF Markdown', () => { const markdown = '---\r\ntitle: Payments SAD\r\ndescription: Decisions\r\n---\r\n# Content\r\n'; expect(parseNarrativeDocument(markdown, 'payments')).toEqual({ - request: { name: 'Payments SAD', description: 'Decisions', documentMarkdown: markdown }, markdown, + request: { name: 'Payments SAD', description: 'Decisions', documentMarkdown: markdown }, }); }); @@ -37,7 +37,9 @@ describe('narrative document helpers', () => { it.each([ [{ ...identity, namespace: 'not_valid' }, false, /valid namespace/], - [{ ...identity, type: 'other' as never }, false, /Unsupported/], + [{ ...identity, namespace: 42 }, false, /valid namespace/], + [{ ...identity, type: 'other' as never }, false, /unsupported/], + [{ ...identity, version: 1 }, false, /major.minor.patch/], [{ ...identity, version: '01.0.0' }, false, /major.minor.patch/], [{ ...identity, calmHubDocumentId: 0 }, true, /positive integer/], ])('rejects invalid persisted identity %#', (candidate, requireId, message) => { @@ -51,5 +53,13 @@ describe('narrative document helpers', () => { '/api/calm/namespaces/finos/documents/sad/43/versions/1.0.0', { ...identity, calmHubDocumentId: 42 } )).toThrow(/stored document id/); + expect(() => parseNarrativeDocumentLocation(null, identity)).toThrow(/unexpected format/); + expect(() => parseNarrativeDocumentLocation( + '/api/calm/namespaces/finos/documents/sad/42/versions/01.0.0', identity, false + )).toThrow(/unexpected format/); + expect(parseNarrativeDocumentLocation( + '/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0', + { ...identity, version: '1.1.0', calmHubDocumentId: 42 }, false + )).toBe(42); }); }); diff --git a/cli/src/command-helpers/workspace/narrative-document.ts b/cli/src/command-helpers/workspace/narrative-document.ts index 08baeca69..9e62c4059 100644 --- a/cli/src/command-helpers/workspace/narrative-document.ts +++ b/cli/src/command-helpers/workspace/narrative-document.ts @@ -14,10 +14,9 @@ export interface NarrativeDocumentIdentity { export interface ParsedNarrativeDocument { request: NarrativeDocumentRequest; - markdown: string; } -export function isNarrativeDocumentType(type: string): type is NarrativeDocumentType { +export function isNarrativeDocumentType(type: unknown): type is NarrativeDocumentType { return NARRATIVE_DOCUMENT_TYPES.includes(type as NarrativeDocumentType); } @@ -48,33 +47,44 @@ export function parseNarrativeDocument(markdown: string, label: string): ParsedN ...(description === undefined ? {} : { description }), documentMarkdown: markdown, }, - markdown, }; } -export function validateNarrativeIdentity(identity: NarrativeDocumentIdentity, requireDocumentId: boolean): void { - if (!NAMESPACE_PATTERN.test(identity.namespace)) { - throw new Error('Narrative document namespace must be a non-empty valid namespace.'); +export function validateNarrativeIdentity(identity: unknown, requireDocumentId: boolean, label?: string): asserts identity is NarrativeDocumentIdentity { + const prefix = label ? `Narrative document '${label}' ` : 'Narrative document '; + if (!identity || typeof identity !== 'object') { + throw new Error(`${prefix}identity must be an object.`); } - if (!isNarrativeDocumentType(identity.type)) { - throw new Error(`Unsupported narrative document type '${identity.type}'.`); + const candidate = identity as Record; + if (typeof candidate.namespace !== 'string' || !NAMESPACE_PATTERN.test(candidate.namespace)) { + throw new Error(`${prefix}namespace must be a non-empty valid namespace.`); } - if (!SEMVER_PATTERN.test(identity.version)) { - throw new Error(`Narrative document version '${identity.version}' must be major.minor.patch.`); + if (!isNarrativeDocumentType(candidate.type)) { + throw new Error(`${prefix}has unsupported type '${String(candidate.type)}'.`); } - if (requireDocumentId && (!Number.isSafeInteger(identity.calmHubDocumentId) || identity.calmHubDocumentId! <= 0)) { - throw new Error('Narrative document calmHubDocumentId must be a positive integer.'); + if (typeof candidate.version !== 'string' || !SEMVER_PATTERN.test(candidate.version)) { + throw new Error(`${prefix}version '${String(candidate.version)}' must be major.minor.patch.`); + } + if (requireDocumentId && (!Number.isSafeInteger(candidate.calmHubDocumentId) || (candidate.calmHubDocumentId as number) <= 0)) { + throw new Error(`${prefix}calmHubDocumentId must be a positive integer.`); } } -export function parseNarrativeDocumentLocation(location: string, identity: NarrativeDocumentIdentity): number { +export function parseNarrativeDocumentLocation( + location: unknown, + identity: NarrativeDocumentIdentity, + requireIdentityVersion: boolean = true +): number { + if (typeof location !== 'string' || !location) { + throw new Error(`Narrative document Location '${String(location)}' has an unexpected format.`); + } const path = extractLocationPath(location); const match = LOCATION_PATTERN.exec(path); if (!match) { throw new Error(`Narrative document Location '${location}' has an unexpected format.`); } const [, namespace, type, idString, version] = match; - if (namespace !== identity.namespace || type !== identity.type || version !== identity.version) { + if (namespace !== identity.namespace || type !== identity.type || (requireIdentityVersion && version !== identity.version)) { throw new Error(`Narrative document Location '${location}' does not match the requested identity.`); } const id = Number(idString); diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 7cddb2ac4..9fefd21c0 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -178,6 +178,20 @@ describe('pushWorkspaceToHub', () => { await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/narrative document/); }); + it('rejects malformed persisted narrative identity before calling Hub', async () => { + await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 42 as unknown as string, version: '1.1.0', calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const client = makeClient(); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/valid namespace/); + expect(client.getNarrativeDocumentVersions).not.toHaveBeenCalled(); + }); + it('fails narrative publish when Hub returns an unexpected Location', async () => { await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); await saveManifest(bundlePath, { diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index 8e5e647bb..cff1a6b1c 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -76,7 +76,7 @@ export async function pushWorkspaceToHub( const narrative = parseNarrativeDocument(raw, id); if (entry.calmHubDocumentId === undefined) { - validateNarrativeIdentity(identity, false); + validateNarrativeIdentity(identity, false, id); if (version !== '1.0.0') { throw new Error('A narrative document without calmHubDocumentId must use version 1.0.0.'); } @@ -88,7 +88,8 @@ export async function pushWorkspaceToHub( continue; } - validateNarrativeIdentity(identity, true); + validateNarrativeIdentity(identity, true, id); + parseNarrativeDocumentLocation(entry.calmHubId, identity, false); const versions = await client.getNarrativeDocumentVersions( identity.namespace, identity.type, identity.calmHubDocumentId! ); diff --git a/shared/src/document-loader/workspace-document-loader.ts b/shared/src/document-loader/workspace-document-loader.ts index c37e2e37e..e8c7784b4 100644 --- a/shared/src/document-loader/workspace-document-loader.ts +++ b/shared/src/document-loader/workspace-document-loader.ts @@ -5,10 +5,10 @@ import { readFile } from 'fs/promises'; import { existsSync, readFileSync } from 'fs'; import { SchemaDirectory } from '../schema-directory'; import path from 'path'; +import { NARRATIVE_DOCUMENT_TYPES, type NarrativeDocumentType } from '../hub/calm-hub-client'; // Mirrors MANIFEST_FILENAME in the CLI workspace bundle module. const MANIFEST_FILENAME = 'workspace-manifest.json'; -const NARRATIVE_DOCUMENT_TYPES = new Set(['knowledge', 'sad']); /** * Identity of a single tracked workspace document, used to decide whether an @@ -85,7 +85,7 @@ export class WorkspaceDocumentLoader implements DocumentLoader { : undefined; // Narrative documents are Markdown rather than CALM JSON documents. They cannot be // schema-preloaded or resolve a CALM `$ref`, so keep them out of this JSON-only loader. - if (typeof type === 'string' && NARRATIVE_DOCUMENT_TYPES.has(type)) continue; + if (typeof type === 'string' && NARRATIVE_DOCUMENT_TYPES.includes(type as NarrativeDocumentType)) continue; const relPath = typeof value === 'string' ? value : (value && typeof value === 'object' && typeof (value as { path?: unknown }).path === 'string' diff --git a/shared/src/hub/calm-hub-client.spec.ts b/shared/src/hub/calm-hub-client.spec.ts index 53b568a48..206d7311c 100644 --- a/shared/src/hub/calm-hub-client.spec.ts +++ b/shared/src/hub/calm-hub-client.spec.ts @@ -33,6 +33,14 @@ describe('CalmHubClient', () => { }); }); + it('creates a typed later version at the version endpoint', async () => { + const endpoint = '/api/calm/namespaces/finos/documents/knowledge/42/versions/1.1.0'; + mock.onPost(endpoint).reply(201, null, { location: endpoint }); + + await expect(client.createNarrativeDocumentVersion('finos', 'knowledge', 42, '1.1.0', request)).resolves.toBe(endpoint); + expect(mock.history.post[0].data).toBe(JSON.stringify(request)); + }); + it('reads typed Markdown and rejects malformed success bodies', async () => { const endpoint = '/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0'; mock.onGet(endpoint).replyOnce(200, { documentMarkdown: '# Payments' }); @@ -48,6 +56,16 @@ describe('CalmHubClient', () => { mock.onGet(endpoint).replyOnce(200, { values: [42] }); await expect(client.getNarrativeDocumentVersions('finos', 'sad', 42)).rejects.toBeInstanceOf(HubClientError); }); + + it.each([404, 500])('wraps a %i response from a narrative endpoint', async (status) => { + const endpoint = '/api/calm/namespaces/finos/documents/sad/42/versions'; + mock.onGet(endpoint).replyOnce(status, { error: 'unavailable' }); + + await expect(client.getNarrativeDocumentVersions('finos', 'sad', 42)).rejects.toMatchObject({ + status, + request: `GET ${endpoint}`, + }); + }); }); // ── createNamespace ────────────────────────────────────────────────────── @@ -171,6 +189,19 @@ describe('CalmHubClient', () => { expect(authMock.history.get[0].headers?.Authorization).toBe('Bearer test-token'); }); + it('injects auth headers on narrative document requests', async () => { + authMock.onPost('/api/calm/namespaces/finos/documents/sad').reply(201, null, { + location: '/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0', + }); + + await authClient.createNarrativeDocument('finos', 'sad', { + name: 'Payments SAD', documentMarkdown: '# Payments', + }); + + expect(getAuthHeaders).toHaveBeenCalledOnce(); + expect(authMock.history.post[0].headers?.Authorization).toBe('Bearer test-token'); + }); + it('does not call getAuthHeaders when no auth plugin is configured', async () => { mock.onGet('/api/calm/namespaces').reply(200, { values: [] }); diff --git a/shared/src/hub/calm-hub-client.ts b/shared/src/hub/calm-hub-client.ts index e1c16b2ea..986113df4 100644 --- a/shared/src/hub/calm-hub-client.ts +++ b/shared/src/hub/calm-hub-client.ts @@ -39,8 +39,8 @@ export interface HubControlSummary { export type ResourceChangeType = 'MAJOR' | 'MINOR' | 'PATCH'; -export type NarrativeDocumentType = 'knowledge' | 'sad'; -export const NARRATIVE_DOCUMENT_TYPES: NarrativeDocumentType[] = ['knowledge', 'sad']; +export const NARRATIVE_DOCUMENT_TYPES = ['knowledge', 'sad'] as const; +export type NarrativeDocumentType = typeof NARRATIVE_DOCUMENT_TYPES[number]; export interface NarrativeDocumentRequest { name: string; From 29d3b5ea6f9d11a87533669c4ddc8d14c1d9ebcc Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sat, 22 Aug 2026 21:06:28 +0100 Subject: [PATCH 10/28] refactor(cli): derive narrative Location types --- .../command-helpers/workspace/narrative-document.spec.ts | 9 +++++++++ cli/src/command-helpers/workspace/narrative-document.ts | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index 64cd33f52..1b4494c4a 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { NARRATIVE_DOCUMENT_TYPES } from '@finos/calm-shared/src/hub/calm-hub-client'; import { parseNarrativeDocument, parseNarrativeDocumentLocation, validateNarrativeIdentity } from './narrative-document'; describe('narrative document helpers', () => { @@ -35,6 +36,14 @@ describe('narrative document helpers', () => { expect(() => parseNarrativeDocumentLocation('/api/calm/namespaces/other/documents/sad/42/versions/1.0.0', identity)).toThrow(/does not match/); }); + it.each(NARRATIVE_DOCUMENT_TYPES)('accepts the supported %s Location type', (type) => { + const narrativeIdentity = { ...identity, type }; + expect(parseNarrativeDocumentLocation( + `/api/calm/namespaces/finos/documents/${type}/42/versions/1.0.0`, + narrativeIdentity + )).toBe(42); + }); + it.each([ [{ ...identity, namespace: 'not_valid' }, false, /valid namespace/], [{ ...identity, namespace: 42 }, false, /valid namespace/], diff --git a/cli/src/command-helpers/workspace/narrative-document.ts b/cli/src/command-helpers/workspace/narrative-document.ts index 9e62c4059..3b11310bf 100644 --- a/cli/src/command-helpers/workspace/narrative-document.ts +++ b/cli/src/command-helpers/workspace/narrative-document.ts @@ -1,7 +1,10 @@ import { NarrativeDocumentRequest, NarrativeDocumentType, NARRATIVE_DOCUMENT_TYPES } from '@finos/calm-shared/src/hub/calm-hub-client'; import { parseYamlFrontMatterMapping } from '@finos/calm-shared/src/template/front-matter'; -const LOCATION_PATTERN = /^\/api\/calm\/namespaces\/([^/]+)\/documents\/(knowledge|sad)\/(\d+)\/versions\/((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))$/; +const LOCATION_PATTERN = new RegExp( + `^/api/calm/namespaces/([^/]+)/documents/(${NARRATIVE_DOCUMENT_TYPES.join('|')})/(\\d+)/versions/` + + `((?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*))$` +); const NAMESPACE_PATTERN = /^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*$/; const SEMVER_PATTERN = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; From 4e97d8e054990e88db1e56dbbad67af4645f5062 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sun, 23 Aug 2026 18:43:14 +0100 Subject: [PATCH 11/28] feat(shared): define narrative document types --- calm-models/src/types/index.spec.ts | 13 +++++++++++++ calm-models/src/types/index.ts | 8 ++++++++ 2 files changed, 21 insertions(+) create mode 100644 calm-models/src/types/index.spec.ts diff --git a/calm-models/src/types/index.spec.ts b/calm-models/src/types/index.spec.ts new file mode 100644 index 000000000..8a799cc5a --- /dev/null +++ b/calm-models/src/types/index.spec.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest'; +import { CALM_NARRATIVE_DOCUMENT_TYPES_LIST, isNarrativeDocumentType } from './index'; + +describe('isNarrativeDocumentType', () => { + it.each(CALM_NARRATIVE_DOCUMENT_TYPES_LIST)('accepts %s', (type) => { + expect(isNarrativeDocumentType(type)).toBe(true); + }); + + it('rejects unsupported values', () => { + expect(isNarrativeDocumentType('architecture')).toBe(false); + expect(isNarrativeDocumentType(1)).toBe(false); + }); +}); diff --git a/calm-models/src/types/index.ts b/calm-models/src/types/index.ts index 9881f1bbb..c3803e473 100644 --- a/calm-models/src/types/index.ts +++ b/calm-models/src/types/index.ts @@ -22,6 +22,14 @@ export const CALM_DOCUMENT_TYPES_LIST = [ export type CalmDocumentType = (typeof CALM_DOCUMENT_TYPES_LIST)[number]; +export const CALM_NARRATIVE_DOCUMENT_TYPES_LIST = ['knowledge', 'sad'] as const; + +export type NarrativeDocumentType = (typeof CALM_NARRATIVE_DOCUMENT_TYPES_LIST)[number]; + +export function isNarrativeDocumentType(input: unknown): input is NarrativeDocumentType { + return typeof input === 'string' && CALM_NARRATIVE_DOCUMENT_TYPES_LIST.includes(input as NarrativeDocumentType); +} + export function isValidCalmDocumentType(input: string): input is CalmDocumentType { return CALM_DOCUMENT_TYPES_LIST.some((type) => type === input); } From 31cfaba2753f332cd88432940da256bb2f84257c Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sun, 23 Aug 2026 18:43:42 +0100 Subject: [PATCH 12/28] refactor(shared): use canonical narrative document types --- shared/src/document-loader/workspace-document-loader.ts | 5 ++--- shared/src/hub/calm-hub-client.ts | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/shared/src/document-loader/workspace-document-loader.ts b/shared/src/document-loader/workspace-document-loader.ts index e8c7784b4..502f6377c 100644 --- a/shared/src/document-loader/workspace-document-loader.ts +++ b/shared/src/document-loader/workspace-document-loader.ts @@ -1,11 +1,10 @@ import { DocumentLoader, DocumentLoadError } from './document-loader'; -import type { CalmDocumentType } from '@finos/calm-models/types'; +import { isNarrativeDocumentType, type CalmDocumentType } from '@finos/calm-models/types'; import { initLogger, Logger } from '../logger'; import { readFile } from 'fs/promises'; import { existsSync, readFileSync } from 'fs'; import { SchemaDirectory } from '../schema-directory'; import path from 'path'; -import { NARRATIVE_DOCUMENT_TYPES, type NarrativeDocumentType } from '../hub/calm-hub-client'; // Mirrors MANIFEST_FILENAME in the CLI workspace bundle module. const MANIFEST_FILENAME = 'workspace-manifest.json'; @@ -85,7 +84,7 @@ export class WorkspaceDocumentLoader implements DocumentLoader { : undefined; // Narrative documents are Markdown rather than CALM JSON documents. They cannot be // schema-preloaded or resolve a CALM `$ref`, so keep them out of this JSON-only loader. - if (typeof type === 'string' && NARRATIVE_DOCUMENT_TYPES.includes(type as NarrativeDocumentType)) continue; + if (isNarrativeDocumentType(type)) continue; const relPath = typeof value === 'string' ? value : (value && typeof value === 'object' && typeof (value as { path?: unknown }).path === 'string' diff --git a/shared/src/hub/calm-hub-client.ts b/shared/src/hub/calm-hub-client.ts index 986113df4..36b6d06ba 100644 --- a/shared/src/hub/calm-hub-client.ts +++ b/shared/src/hub/calm-hub-client.ts @@ -1,4 +1,5 @@ import axios, { Axios } from 'axios'; +import type { NarrativeDocumentType } from '@finos/calm-models/types'; import { AuthPlugin } from '../auth/auth-plugin'; import { initLogger, Logger } from '../logger'; import { DocumentMetadata, extractDocumentMetadata, validateDocumentId } from './document-id-utils'; @@ -39,9 +40,6 @@ export interface HubControlSummary { export type ResourceChangeType = 'MAJOR' | 'MINOR' | 'PATCH'; -export const NARRATIVE_DOCUMENT_TYPES = ['knowledge', 'sad'] as const; -export type NarrativeDocumentType = typeof NARRATIVE_DOCUMENT_TYPES[number]; - export interface NarrativeDocumentRequest { name: string; description?: string; From c9b3a641f8b20fb0dee8a7f94041407a6001c1a4 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sun, 23 Aug 2026 18:43:55 +0100 Subject: [PATCH 13/28] feat(cli): recover published narrative documents --- cli/README.md | 9 +- .../command-helpers/workspace/bump.spec.ts | 18 +++ cli/src/command-helpers/workspace/bump.ts | 8 +- .../command-helpers/workspace/bundle.spec.ts | 25 ++++ cli/src/command-helpers/workspace/bundle.ts | 21 +++- .../workspace/commands.spec.ts | 114 +++++++++++++++++- cli/src/command-helpers/workspace/commands.ts | 68 +++++++++-- .../workspace/narrative-document.spec.ts | 22 +++- .../workspace/narrative-document.ts | 31 +++-- .../command-helpers/workspace/push.spec.ts | 11 ++ cli/src/command-helpers/workspace/push.ts | 11 +- 11 files changed, 308 insertions(+), 30 deletions(-) diff --git a/cli/README.md b/cli/README.md index 2fb1355bb..8cbfcfa86 100644 --- a/cli/README.md +++ b/cli/README.md @@ -806,7 +806,7 @@ calm workspace init my-system Register a CALM JSON document or narrative Markdown document with the active workspace. By default the file is referenced at its current location on disk (no copying). Prompts interactively for document type and (manifest) name if they cannot be determined automatically. ``` -calm workspace add [--id ] [--type ] [--namespace ] [--copy] +calm workspace add [--id ] [--type ] [--namespace ] [--copy] [--calm-hub-document-id --ver [--calm-hub-url ]] ``` | Option | Description | @@ -815,9 +815,13 @@ calm workspace add [--id ] [--type ] [--namespace ] | `--type ` | Document type. If omitted, an interactive dropdown is shown. One of: `pattern`, `architecture`, `interface`, `flow`, `control`, `schema`, `timeline`, `adr`, `knowledge`, `sad`. | | `--namespace ` | CalmHub namespace to record in the manifest. It is required for narrative Markdown and otherwise derived from the document `$id` when omitted. | | `--copy` | Copy the file into the bundle's `files/` directory instead of referencing it in place. | +| `--calm-hub-document-id ` and `--ver ` | Recover an existing narrative document. Both options are required together. | +| `--calm-hub-url ` | Optional CalmHub URL used only for narrative recovery. It otherwise uses the configured URL. | **Narrative Markdown documents.** Use `--type knowledge` or `sad`. `add` reads YAML frontmatter. A non-empty `title` becomes the manifest name unless you supply `--id`. `--namespace` is required. The initial manifest version is `1.0.0`. Markdown has no CALM `$id` and is never rewritten. +To restore a removed narrative document without creating a new CalmHub document, supply its verified Hub id and version. The local Markdown must exactly match the stored Hub version. + **JSON document `$id` handling.** For JSON mapping documents, `add` inspects the file's CalmHub `$id`: - **No `$id`** → you are prompted interactively to build one from its components (see below); the `$id` is written into the file and the document is added. - **Conformant `$id`** → left untouched; the manifest namespace is derived from it. @@ -834,6 +838,9 @@ calm workspace add ./architectures/payment-service.json --type architecture # Register a narrative Markdown document; the frontmatter title becomes its manifest name calm workspace add ./docs/payments-sad.md --type sad --namespace finos + +# Restore an existing narrative document +calm workspace add ./docs/payments-sad.md --type sad --namespace finos --calm-hub-document-id 42 --ver 1.2.0 ``` #### `calm workspace new [type] [name] [template]` diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index 51c08c9f9..ce17947e1 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -83,6 +83,24 @@ describe('bump', () => { await expect(detectChangedResources(bundlePath, makeClient())).rejects.toThrow(/incomplete Hub identity/); }); + it('rejects a missing namespace before Hub calls and accepts a valid namespace', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + const entry = { + path: 'files/payments.md', type: 'sad' as const, version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }; + const invalidClient = makeClient(); + await saveManifest(bundlePath, { payments: entry }); + await expect(detectChangedResources(bundlePath, invalidClient)).rejects.toThrow(/valid namespace/); + expect(invalidClient.getNarrativeDocumentVersions).not.toHaveBeenCalled(); + + const validClient = makeClient({ narrativeVersions: [] }); + await saveManifest(bundlePath, { payments: { ...entry, namespace: 'com.example' } }); + await expect(detectChangedResources(bundlePath, validClient)).resolves.toEqual([]); + expect(validClient.getNarrativeDocumentVersions).toHaveBeenCalledWith('com.example', 'sad', 42); + }); + it('fails narrative checks when Hub version retrieval fails', async () => { await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); await saveManifest(bundlePath, { diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index fd57a2383..fa53a0b0e 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -11,7 +11,8 @@ import { import { computeSemVerBump, sortSemVer } from '@finos/calm-shared/src/hub/semver'; import { canonicalEqual } from '@finos/calm-shared/src/hub/canonical'; import { initLogger, Logger } from '@finos/calm-shared/src/logger'; -import { NarrativeDocumentIdentity, isNarrativeDocumentType, parseNarrativeDocument, parseNarrativeDocumentLocation, validateNarrativeIdentity } from './narrative-document'; +import { isNarrativeDocumentType } from '@finos/calm-models/types'; +import { NarrativeDocumentIdentity, parseNarrativeDocument, validateNarrativeDocumentLocation, validateNarrativeIdentity, validateNarrativeNamespace } from './narrative-document'; // Re-exported for existing consumers (push.ts, tests) that import it from here. export { canonicalEqual }; @@ -114,8 +115,9 @@ export async function detectChangedResources( if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { throw new Error(`Narrative document '${id}' has incomplete Hub identity. Re-add the document to repair it.`); } + validateNarrativeNamespace(entry.namespace, id); const identity: NarrativeDocumentIdentity = { - namespace: entry.namespace ?? '', type: entry.type, version, calmHubDocumentId: entry.calmHubDocumentId, + namespace: entry.namespace, type: entry.type, version, calmHubDocumentId: entry.calmHubDocumentId, }; parseNarrativeDocument(raw, id); if (entry.calmHubDocumentId === undefined) { @@ -123,7 +125,7 @@ export async function detectChangedResources( continue; } validateNarrativeIdentity(identity, true, id); - parseNarrativeDocumentLocation(entry.calmHubId, identity, false); + validateNarrativeDocumentLocation(entry.calmHubId, identity, false); const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId!); if (versions.length === 0 || !versions.includes(version)) continue; const remote = await client.getNarrativeDocumentVersion( diff --git a/cli/src/command-helpers/workspace/bundle.spec.ts b/cli/src/command-helpers/workspace/bundle.spec.ts index cdfe8ce3b..b15fe1b34 100644 --- a/cli/src/command-helpers/workspace/bundle.spec.ts +++ b/cli/src/command-helpers/workspace/bundle.spec.ts @@ -203,6 +203,31 @@ describe('bundle', () => { expect(manifest['source-doc'].type).toBe('architecture'); }); + it('persists a complete narrative Hub identity with its version', async () => { + await addFileToBundle(bundlePath, srcFile, { + type: 'sad', version: '1.2.0', calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/1.2.0', + }); + + expect((await loadManifest(bundlePath))['source-doc']).toMatchObject({ + version: '1.2.0', calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/1.2.0', + }); + }); + + it('allows a version-only narrative entry for a new document', async () => { + await addFileToBundle(bundlePath, srcFile, { type: 'sad', version: '1.0.0' }); + expect((await loadManifest(bundlePath))['source-doc']).toMatchObject({ version: '1.0.0' }); + }); + + it.each([ + { type: 'sad' as const, version: '1.2.0', calmHubDocumentId: 42 }, + { type: 'sad' as const, version: '1.2.0', calmHubId: '/path' }, + { type: 'sad' as const, calmHubDocumentId: 42, calmHubId: '/path' }, + ])('rejects an incomplete narrative Hub identity', async (options) => { + await expect(addFileToBundle(bundlePath, srcFile, options)).rejects.toThrow(/Hub identity/); + }); + it('should copy file when copy option is true', async () => { const result = await addFileToBundle(bundlePath, srcFile, { copy: true }); diff --git a/cli/src/command-helpers/workspace/bundle.ts b/cli/src/command-helpers/workspace/bundle.ts index 253ee4fa2..7be71a5a4 100644 --- a/cli/src/command-helpers/workspace/bundle.ts +++ b/cli/src/command-helpers/workspace/bundle.ts @@ -3,8 +3,7 @@ import { mkdir, copyFile, readFile, writeFile } from 'fs/promises'; import { existsSync } from 'fs'; import { JSONPath } from 'jsonpath-plus'; import { printBundleTreeFromGraph } from './tree'; -import type { CalmDocumentType } from '@finos/calm-models/types'; -import type { NarrativeDocumentType } from '@finos/calm-shared/src/hub/calm-hub-client'; +import type { CalmDocumentType, NarrativeDocumentType } from '@finos/calm-models/types'; /** * Property names that can contain document references (URLs or paths) in CALM JSON. @@ -170,9 +169,24 @@ export async function determineDocumentId(srcPath: string, explicitId?: string): export async function addFileToBundle( bundlePath: string, srcPath: string, - opts?: { id?: string; destName?: string; copy?: boolean; type?: WorkspaceDocumentType; namespace?: string; version?: string } + opts?: { + id?: string; + destName?: string; + copy?: boolean; + type?: WorkspaceDocumentType; + namespace?: string; + version?: string; + calmHubDocumentId?: number; + calmHubId?: string; + } ): Promise<{ id: string; destPath: string; rel: string }> { + const hasDocumentId = opts?.calmHubDocumentId !== undefined; + const hasHubId = opts?.calmHubId !== undefined; + if (hasDocumentId !== hasHubId || ((hasDocumentId || hasHubId) && !opts?.version)) { + throw new Error('Narrative document Hub identity requires calmHubDocumentId, calmHubId, and version.'); + } + const id = await determineDocumentId(srcPath, opts?.id); let rel: string; let destPath: string; @@ -198,6 +212,7 @@ export async function addFileToBundle( type: opts?.type ?? 'unknown', ...(opts?.namespace ? { namespace: opts.namespace } : {}), ...(opts?.version ? { version: opts.version } : {}), + ...(hasDocumentId ? { calmHubDocumentId: opts!.calmHubDocumentId, calmHubId: opts!.calmHubId } : {}), }; await saveManifest(bundlePath, manifest); diff --git a/cli/src/command-helpers/workspace/commands.spec.ts b/cli/src/command-helpers/workspace/commands.spec.ts index 88969f9ff..000d6e979 100644 --- a/cli/src/command-helpers/workspace/commands.spec.ts +++ b/cli/src/command-helpers/workspace/commands.spec.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { Command } from 'commander'; +import { CALM_NARRATIVE_DOCUMENT_TYPES_LIST } from '@finos/calm-models/types'; import { setupWorkspaceCommands } from './commands'; const mocks = vi.hoisted(() => { @@ -26,7 +27,12 @@ const mocks = vi.hoisted(() => { loadCliConfig: vi.fn(async () => ({ calmHubUrl: 'https://calmhub.example.com' })), loadAuthPlugin: vi.fn(async () => ({ getAuthHeaders: vi.fn(async () => ({})) })), CalmHubClient: vi.fn().mockImplementation(function() { - return { isMockClient: true }; + return { + isMockClient: true, + getNarrativeDocumentVersion: vi.fn(async () => ({ documentMarkdown: '---\ntitle: Payments SAD\n---\n# Payments\n' })), + createNarrativeDocument: vi.fn(), + createNarrativeDocumentVersion: vi.fn(), + }; }), select: vi.fn(async () => 'architecture'), input: vi.fn(async () => 'prompted-name'), @@ -95,7 +101,6 @@ vi.mock('../../cli-config', () => ({ vi.mock('@finos/calm-shared/src/hub/calm-hub-client', () => ({ CalmHubClient: mocks.CalmHubClient, - NARRATIVE_DOCUMENT_TYPES: ['knowledge', 'sad'], })); vi.mock('./document-id-prompt', () => ({ @@ -169,6 +174,12 @@ describe('setupWorkspaceCommands', () => { const CONFORMANT_ID = 'https://calmhub.example.com/calm/namespaces/ns/architectures/my-arch/versions/1.0.0'; describe('workspace add', () => { + it('derives narrative Commander choices from the canonical list', () => { + const add = program.commands.find(command => command.name() === 'workspace')!.commands + .find(command => command.name() === 'add')!; + const typeOption = add.options.find(option => option.flags.includes('--type')) as unknown as { argChoices: string[] }; + expect(typeOption.argChoices).toEqual(expect.arrayContaining(CALM_NARRATIVE_DOCUMENT_TYPES_LIST)); + }); it('builds a $id when the file has none, writes it back, and adds with the derived namespace', async () => { // readFile mock returns JSON with title 'My Architecture' and no $id. await program.parseAsync(['node', 'test', 'workspace', 'add', 'test.json']); @@ -238,6 +249,105 @@ describe('setupWorkspaceCommands', () => { ); }); + it('recovers a verified narrative document without creating it', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments\n'; + mocks.readFile.mockResolvedValueOnce(markdown); + + await program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', + '--calm-hub-document-id', '42', '--ver', '1.2.0', '--calm-hub-url', 'https://explicit.example.com' + ]); + + expect(mocks.CalmHubClient).toHaveBeenCalledWith(expect.objectContaining({ calmHubUrl: 'https://explicit.example.com' })); + const client = mocks.CalmHubClient.mock.results[0].value; + expect(client.getNarrativeDocumentVersion).toHaveBeenCalledWith('finos', 'sad', 42, '1.2.0'); + expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + expect(client.createNarrativeDocumentVersion).not.toHaveBeenCalled(); + expect(mocks.addFileToBundle).toHaveBeenCalledWith('/fake/bundle', expect.stringContaining('payments.md'), expect.objectContaining({ + id: 'Payments SAD', type: 'sad', namespace: 'finos', version: '1.2.0', calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/1.2.0', + })); + }); + + it('uses configured CalmHub URL and authentication for recovery', async () => { + const authPlugin = { getAuthHeaders: vi.fn(async () => ({})) }; + mocks.loadCliConfig.mockResolvedValueOnce({ calmHubUrl: 'https://configured.example.com', authPluginPath: 'auth.ts' }); + mocks.loadAuthPlugin.mockResolvedValueOnce(authPlugin); + mocks.readFile.mockResolvedValueOnce('---\ntitle: Payments SAD\n---\n# Payments\n'); + + await program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', + '--calm-hub-document-id', '42', '--ver', '1.2.0' + ]); + + expect(mocks.CalmHubClient).toHaveBeenCalledWith({ calmHubUrl: 'https://configured.example.com', authPlugin }); + }); + + it('rejects recovery when neither an explicit nor configured Hub URL exists', async () => { + mocks.loadCliConfig.mockResolvedValueOnce(null); + mocks.readFile.mockResolvedValueOnce('---\ntitle: Payments SAD\n---\n# Payments\n'); + await expect(program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', + '--calm-hub-document-id', '42', '--ver', '1.2.0' + ])).rejects.toThrow(); + expect(mocks.loadCliConfig).toHaveBeenCalled(); + expect(mocks.CalmHubClient).not.toHaveBeenCalled(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + }); + + it.each([ + ['--calm-hub-document-id', '0', '--ver', '1.2.0'], + ['--calm-hub-document-id', '42', '--ver', 'invalid'], + ])('rejects invalid recovery identity values', async (idOption, id, versionOption, version) => { + await expect(program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', + idOption, id, versionOption, version + ])).rejects.toThrow(); + expect(mocks.CalmHubClient).not.toHaveBeenCalled(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + }); + + it('preserves --id as the recovery manifest key', async () => { + mocks.readFile.mockResolvedValueOnce('---\ntitle: Payments SAD\n---\n# Payments\n'); + await program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--id', 'payments-archive', '--type', 'sad', '--namespace', 'finos', + '--calm-hub-document-id', '42', '--ver', '1.2.0', '--calm-hub-url', 'https://explicit.example.com' + ]); + expect(mocks.addFileToBundle).toHaveBeenCalledWith('/fake/bundle', expect.any(String), expect.objectContaining({ id: 'payments-archive' })); + }); + + it.each([ + ['--calm-hub-document-id', '42'], + ['--ver', '1.2.0'], + ['--calm-hub-url', 'https://explicit.example.com'], + ])('rejects incomplete narrative recovery options (%s)', async (option, value) => { + await expect(program.parseAsync(['node', 'test', 'workspace', 'add', 'payments.md', option, value])).rejects.toThrow(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + expect(mocks.CalmHubClient).not.toHaveBeenCalled(); + }); + + it('rejects recovery without a narrative type or namespace before Hub calls', async () => { + await expect(program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'architecture', '--namespace', 'finos', + '--calm-hub-document-id', '42', '--ver', '1.2.0' + ])).rejects.toThrow(); + await expect(program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', + '--calm-hub-document-id', '42', '--ver', '1.2.0' + ])).rejects.toThrow(); + expect(mocks.CalmHubClient).not.toHaveBeenCalled(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + }); + + it('does not change the manifest when recovered Markdown differs', async () => { + mocks.readFile.mockResolvedValueOnce('---\ntitle: Payments SAD\n---\n# Local\n'); + await expect(program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', + '--calm-hub-document-id', '42', '--ver', '1.2.0' + ])).rejects.toThrow(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + }); + it('requires a namespace when adding a narrative document', async () => { await expect( program.parseAsync(['node', 'test', 'workspace', 'add', 'payments.md', '--type', 'knowledge']) diff --git a/cli/src/command-helpers/workspace/commands.ts b/cli/src/command-helpers/workspace/commands.ts index ed53916a8..249ec31b6 100644 --- a/cli/src/command-helpers/workspace/commands.ts +++ b/cli/src/command-helpers/workspace/commands.ts @@ -13,13 +13,12 @@ import { loadWorkspaceConfig } from './config'; import { findWorkspaceManifestPath, findGitRoot } from '../../workspace-resolver'; import { initLogger, Logger } from '@finos/calm-shared/src/logger'; import { select, input } from '@inquirer/prompts'; -import { CALM_DOCUMENT_TYPES_LIST, isValidCalmDocumentType } from '@finos/calm-models/types'; +import { CALM_DOCUMENT_TYPES_LIST, CALM_NARRATIVE_DOCUMENT_TYPES_LIST, isNarrativeDocumentType, isValidCalmDocumentType } from '@finos/calm-models/types'; import { CalmHubClient, ResourceChangeType } from '@finos/calm-shared/src/hub/calm-hub-client'; import { isConformantDocumentId, namespaceFromDocumentId } from '@finos/calm-shared/src/hub/document-id-utils'; import { loadCliConfig } from '../../cli-config'; import { resolveCalmHubOptions } from '../hub-commands'; -import { isNarrativeDocumentType, parseNarrativeDocument } from './narrative-document'; -import { NARRATIVE_DOCUMENT_TYPES } from '@finos/calm-shared/src/hub/calm-hub-client'; +import { constructNarrativeDocumentPath, parseNarrativeDocument, validateNarrativeIdentity } from './narrative-document'; const logger: Logger = initLogger(false, 'workspace'); @@ -56,9 +55,20 @@ export function setupWorkspaceCommands(program: Command) { .argument('', 'Path to the file to add to the bundle') .option('--id ', 'Document ID to register for this file (defaults to filename without extension)') .option('--copy', 'Copy the file into the bundle instead of referencing it from its current location.') - .addOption(new Option('--type ', 'Document type').choices([...CALM_DOCUMENT_TYPES_LIST, ...NARRATIVE_DOCUMENT_TYPES])) + .addOption(new Option('--type ', 'Document type').choices([...CALM_DOCUMENT_TYPES_LIST, ...CALM_NARRATIVE_DOCUMENT_TYPES_LIST])) .option('--namespace ', 'CalmHub namespace to associate with this file') - .action(async (file: string, options: { id?: string; copy?: boolean; type?: string; namespace?: string }) => { + .option('--calm-hub-document-id ', 'Existing CalmHub narrative document ID') + .option('--ver ', 'Existing CalmHub narrative document version') + .option('--calm-hub-url ', 'CalmHub URL used to verify an existing narrative document') + .action(async (file: string, options: { + id?: string; + copy?: boolean; + type?: string; + namespace?: string; + calmHubDocumentId?: string; + ver?: string; + calmHubUrl?: string; + }) => { try { const bundlePath = findWorkspaceManifestPath(process.cwd()); if (!bundlePath) { @@ -68,7 +78,52 @@ export function setupWorkspaceCommands(program: Command) { const srcPath = path.resolve(file); - const documentTypes = [...CALM_DOCUMENT_TYPES_LIST, ...NARRATIVE_DOCUMENT_TYPES]; + const hasDocumentId = options.calmHubDocumentId !== undefined; + const hasVersion = options.ver !== undefined; + const hasHubUrl = options.calmHubUrl !== undefined; + const recoveryRequested = hasDocumentId || hasVersion || hasHubUrl; + if (recoveryRequested) { + if (!hasDocumentId || !hasVersion) { + throw new Error('Narrative recovery requires both --calm-hub-document-id and --ver.'); + } + if (!options.type || !isNarrativeDocumentType(options.type)) { + throw new Error('Narrative recovery requires a narrative --type.'); + } + if (!options.namespace?.trim()) { + throw new Error(`Narrative document '${file}' recovery requires --namespace.`); + } + + const identity = { + namespace: options.namespace.trim(), + type: options.type, + version: options.ver, + calmHubDocumentId: Number(options.calmHubDocumentId), + }; + validateNarrativeIdentity(identity, true, file); + const raw = await readFile(srcPath, 'utf8'); + const narrative = parseNarrativeDocument(raw, file); + const calmHubOptions = await resolveCalmHubOptions({ calmHubUrl: options.calmHubUrl }); + const client = new CalmHubClient(calmHubOptions); + const remote = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId, identity.version + ); + if (remote.documentMarkdown !== raw) { + throw new Error(`Narrative document '${file}' does not match CalmHub version ${identity.version}.`); + } + const { id: resolvedId, destPath: finalDestPath } = await addFileToBundle(bundlePath, srcPath, { + id: options.id ?? narrative.request.name, + copy: options.copy, + type: identity.type, + namespace: identity.namespace, + version: identity.version, + calmHubDocumentId: identity.calmHubDocumentId, + calmHubId: constructNarrativeDocumentPath(identity), + }); + logger.info(`${options.copy ? 'Copied' : 'Added reference to'} ${finalDestPath} (id: ${resolvedId})`); + return; + } + + const documentTypes = [...CALM_DOCUMENT_TYPES_LIST, ...CALM_NARRATIVE_DOCUMENT_TYPES_LIST]; const type = await enforceOptionPresenceByPrompt(options.type, 'Select a document type:', documentTypes); if (isNarrativeDocumentType(type)) { if (!options.namespace?.trim()) { @@ -572,4 +627,3 @@ async function enforceOptionPresenceByPrompt(cliInput: string | undefined, promp message: prompt }); }; - diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index 1b4494c4a..a1be73d07 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest'; -import { NARRATIVE_DOCUMENT_TYPES } from '@finos/calm-shared/src/hub/calm-hub-client'; -import { parseNarrativeDocument, parseNarrativeDocumentLocation, validateNarrativeIdentity } from './narrative-document'; +import { CALM_NARRATIVE_DOCUMENT_TYPES_LIST } from '@finos/calm-models/types'; +import { + constructNarrativeDocumentPath, + parseNarrativeDocument, + parseNarrativeDocumentLocation, + validateNarrativeDocumentLocation, + validateNarrativeIdentity, +} from './narrative-document'; describe('narrative document helpers', () => { const identity = { namespace: 'finos', type: 'sad' as const, version: '1.0.0' }; @@ -36,7 +42,7 @@ describe('narrative document helpers', () => { expect(() => parseNarrativeDocumentLocation('/api/calm/namespaces/other/documents/sad/42/versions/1.0.0', identity)).toThrow(/does not match/); }); - it.each(NARRATIVE_DOCUMENT_TYPES)('accepts the supported %s Location type', (type) => { + it.each(CALM_NARRATIVE_DOCUMENT_TYPES_LIST)('accepts the supported %s Location type', (type) => { const narrativeIdentity = { ...identity, type }; expect(parseNarrativeDocumentLocation( `/api/calm/namespaces/finos/documents/${type}/42/versions/1.0.0`, @@ -71,4 +77,14 @@ describe('narrative document helpers', () => { { ...identity, version: '1.1.0', calmHubDocumentId: 42 }, false )).toBe(42); }); + + it('validates persisted Locations and constructs canonical paths', () => { + const storedIdentity = { ...identity, calmHubDocumentId: 42 }; + expect(() => validateNarrativeDocumentLocation( + '/api/calm/namespaces/other/documents/sad/42/versions/1.0.0', storedIdentity + )).toThrow(/does not match/); + expect(constructNarrativeDocumentPath(storedIdentity)).toBe( + '/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0' + ); + }); }); diff --git a/cli/src/command-helpers/workspace/narrative-document.ts b/cli/src/command-helpers/workspace/narrative-document.ts index 3b11310bf..108626a2c 100644 --- a/cli/src/command-helpers/workspace/narrative-document.ts +++ b/cli/src/command-helpers/workspace/narrative-document.ts @@ -1,9 +1,10 @@ -import { NarrativeDocumentRequest, NarrativeDocumentType, NARRATIVE_DOCUMENT_TYPES } from '@finos/calm-shared/src/hub/calm-hub-client'; +import type { NarrativeDocumentRequest } from '@finos/calm-shared/src/hub/calm-hub-client'; +import { CALM_NARRATIVE_DOCUMENT_TYPES_LIST, isNarrativeDocumentType, type NarrativeDocumentType } from '@finos/calm-models/types'; import { parseYamlFrontMatterMapping } from '@finos/calm-shared/src/template/front-matter'; const LOCATION_PATTERN = new RegExp( - `^/api/calm/namespaces/([^/]+)/documents/(${NARRATIVE_DOCUMENT_TYPES.join('|')})/(\\d+)/versions/` + - `((?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*))$` + `^/api/calm/namespaces/([^/]+)/documents/(${CALM_NARRATIVE_DOCUMENT_TYPES_LIST.join('|')})/(\\d+)/versions/` + + '((?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*))$' ); const NAMESPACE_PATTERN = /^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*$/; const SEMVER_PATTERN = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; @@ -19,10 +20,6 @@ export interface ParsedNarrativeDocument { request: NarrativeDocumentRequest; } -export function isNarrativeDocumentType(type: unknown): type is NarrativeDocumentType { - return NARRATIVE_DOCUMENT_TYPES.includes(type as NarrativeDocumentType); -} - export function parseNarrativeDocument(markdown: string, label: string): ParsedNarrativeDocument { let frontMatter: Record | null; try { @@ -73,6 +70,13 @@ export function validateNarrativeIdentity(identity: unknown, requireDocumentId: } } +export function validateNarrativeNamespace(namespace: unknown, label?: string): asserts namespace is string { + const prefix = label ? `Narrative document '${label}' ` : 'Narrative document '; + if (typeof namespace !== 'string' || !NAMESPACE_PATTERN.test(namespace)) { + throw new Error(`${prefix}namespace must be a non-empty valid namespace.`); + } +} + export function parseNarrativeDocumentLocation( location: unknown, identity: NarrativeDocumentIdentity, @@ -100,6 +104,19 @@ export function parseNarrativeDocumentLocation( return id; } +export function validateNarrativeDocumentLocation( + location: unknown, + identity: NarrativeDocumentIdentity, + requireIdentityVersion: boolean = true +): void { + parseNarrativeDocumentLocation(location, identity, requireIdentityVersion); +} + +export function constructNarrativeDocumentPath(identity: NarrativeDocumentIdentity): string { + validateNarrativeIdentity(identity, true); + return `/api/calm/namespaces/${identity.namespace}/documents/${identity.type}/${identity.calmHubDocumentId}/versions/${identity.version}`; +} + function extractLocationPath(location: string): string { if (location.startsWith('/')) { return location; diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 9fefd21c0..be44c379c 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -192,6 +192,17 @@ describe('pushWorkspaceToHub', () => { expect(client.getNarrativeDocumentVersions).not.toHaveBeenCalled(); }); + it('rejects a missing narrative namespace without Hub calls or manifest mutation', async () => { + await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); + const entry = { path: 'files/payments.md', type: 'sad' as const, version: '1.0.0' }; + await saveManifest(bundlePath, { payments: entry }); + const client = makeClient(); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/valid namespace/); + expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + expect(await loadManifest(bundlePath)).toEqual({ payments: entry }); + }); + it('fails narrative publish when Hub returns an unexpected Location', async () => { await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); await saveManifest(bundlePath, { diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index cff1a6b1c..d7dea9b50 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -5,11 +5,13 @@ import { CalmHubClient } from '@finos/calm-shared/src/hub/calm-hub-client'; import { DocumentMetadata, extractDocumentMetadata } from '@finos/calm-shared/src/hub/document-id-utils'; import { initLogger, Logger } from '@finos/calm-shared/src/logger'; import { canonicalEqual } from './bump'; +import { isNarrativeDocumentType } from '@finos/calm-models/types'; import { - isNarrativeDocumentType, parseNarrativeDocument, parseNarrativeDocumentLocation, + validateNarrativeDocumentLocation, validateNarrativeIdentity, + validateNarrativeNamespace, } from './narrative-document'; const logger: Logger = initLogger(false, 'workspace'); @@ -67,8 +69,9 @@ export async function pushWorkspaceToHub( if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { throw new Error('Narrative document Hub identity is incomplete. Re-add the document to repair it.'); } + validateNarrativeNamespace(entry.namespace, id); const identity = { - namespace: entry.namespace ?? '', + namespace: entry.namespace, type: entry.type, version, calmHubDocumentId: entry.calmHubDocumentId, @@ -89,7 +92,7 @@ export async function pushWorkspaceToHub( } validateNarrativeIdentity(identity, true, id); - parseNarrativeDocumentLocation(entry.calmHubId, identity, false); + validateNarrativeDocumentLocation(entry.calmHubId, identity, false); const versions = await client.getNarrativeDocumentVersions( identity.namespace, identity.type, identity.calmHubDocumentId! ); @@ -97,7 +100,7 @@ export async function pushWorkspaceToHub( const location = await client.createNarrativeDocumentVersion( identity.namespace, identity.type, identity.calmHubDocumentId!, version, narrative.request ); - parseNarrativeDocumentLocation(location, identity); + validateNarrativeDocumentLocation(location, identity); manifest[id] = { ...entry, calmHubId: location }; await saveManifest(bundlePath, manifest); logger.info(`Pushed '${id}' version ${version} -> ${location}`); From bab86628951342f07a8e7c3d3f13e6ac22cf0dce Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Sun, 23 Aug 2026 18:44:59 +0100 Subject: [PATCH 14/28] docs(cli): clarify narrative failure handling --- cli/src/command-helpers/workspace/bump.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index fa53a0b0e..8fbd07b85 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -110,6 +110,7 @@ export async function detectChangedResources( } if (isNarrativeDocumentType(entry.type)) { + // Bump stops on invalid narrative state because it writes local manifest versions; push can report independent failures together. const version = entry.version; if (!version) throw new Error(`Narrative document '${id}' has no manifest version.`); if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { From 552333e75c63f0cec5aef8de702536de358707e0 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Thu, 17 Sep 2026 14:41:11 +0100 Subject: [PATCH 15/28] refactor(cli): discriminate workspace manifest entries --- cli/src/command-helpers/workspace/bump.ts | 14 ++- .../command-helpers/workspace/bundle.spec.ts | 62 ++++++++++- cli/src/command-helpers/workspace/bundle.ts | 105 ++++++++++++++---- cli/src/command-helpers/workspace/push.ts | 9 +- 4 files changed, 155 insertions(+), 35 deletions(-) diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index b1f455b26..e17e5a5bc 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -1,6 +1,6 @@ import { readFile, writeFile } from 'fs/promises'; import { existsSync } from 'fs'; -import { loadManifest, resolveFilePath, saveManifest } from './bundle'; +import { isNarrativeWorkspaceManifestEntry, loadManifest, resolveFilePath, saveManifest } from './bundle'; import { buildRefRulesFromDiskIds, syncReferences, RefUpdateResult } from './ref-rewrite'; import { CalmHubClient, @@ -14,7 +14,6 @@ import { initLogger, Logger, } from '@finos/calm-shared'; -import { isNarrativeDocumentType } from '@finos/calm-models/types'; import { NarrativeDocumentIdentity, parseNarrativeDocument, validateNarrativeDocumentLocation, validateNarrativeIdentity, validateNarrativeNamespace } from './narrative-document'; // Re-exported for existing consumers (push.ts, tests) that import it from here. @@ -98,7 +97,7 @@ export async function detectChangedResources( for (const [id, entry] of Object.entries(manifest)) { const filePath = resolveFilePath(bundlePath, entry.path); if (!existsSync(filePath)) { - if (isNarrativeDocumentType(entry.type)) throw new Error(`Narrative document '${id}' file not found: ${filePath}`); + if (isNarrativeWorkspaceManifestEntry(entry)) throw new Error(`Narrative document '${id}' file not found: ${filePath}`); logger.warn(`File not found for id '${id}': ${filePath}`); continue; } @@ -107,12 +106,12 @@ export async function detectChangedResources( try { raw = await readFile(filePath, 'utf8'); } catch (e) { - if (isNarrativeDocumentType(entry.type)) throw new Error(`Narrative document '${id}' could not be read: ${e instanceof Error ? e.message : String(e)}`); + if (isNarrativeWorkspaceManifestEntry(entry)) throw new Error(`Narrative document '${id}' could not be read: ${e instanceof Error ? e.message : String(e)}`); logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); continue; } - if (isNarrativeDocumentType(entry.type)) { + if (isNarrativeWorkspaceManifestEntry(entry)) { // Bump stops on invalid narrative state because it writes local manifest versions; push can report independent failures together. const version = entry.version; if (!version) throw new Error(`Narrative document '${id}' has no manifest version.`); @@ -217,6 +216,9 @@ export async function bumpWorkspace( const manifest = await loadManifest(bundlePath); const entry = manifest[c.id]; if (!entry) throw new Error(`Narrative document '${c.id}' is no longer in the manifest.`); + if (!isNarrativeWorkspaceManifestEntry(entry)) { + throw new Error(`Narrative document '${c.id}' is no longer a narrative manifest entry.`); + } manifest[c.id] = { ...entry, version: toVersion }; await saveManifest(bundlePath, manifest); bumped.push({ id: c.id, filePath: c.filePath, fromVersion: c.currentVersion, toVersion, increment: docIncrement }); @@ -246,7 +248,7 @@ export async function bumpWorkspace( for (let depth = 0; depth < MAX_CASCADE_DEPTH; depth++) { const manifest = await loadManifest(bundlePath); const jsonManifest = Object.fromEntries( - Object.entries(manifest).filter(([, entry]) => !isNarrativeDocumentType(entry.type)) + Object.entries(manifest).filter(([, entry]) => !isNarrativeWorkspaceManifestEntry(entry)) ); const rules = await buildRefRulesFromDiskIds(jsonManifest, bundlePath); const refUpdates = await syncReferences(bundlePath, jsonManifest, rules); diff --git a/cli/src/command-helpers/workspace/bundle.spec.ts b/cli/src/command-helpers/workspace/bundle.spec.ts index b15fe1b34..2800477e6 100644 --- a/cli/src/command-helpers/workspace/bundle.spec.ts +++ b/cli/src/command-helpers/workspace/bundle.spec.ts @@ -7,8 +7,10 @@ import { buildDependencyGraph, printBundleTree, extractReferenceValue, + isNarrativeWorkspaceManifestEntry, MANIFEST_FILENAME, - REFERENCE_PROPERTIES + REFERENCE_PROPERTIES, + type WorkspaceManifestEntry, } from './bundle'; import { mkdir, writeFile, rm, readFile } from 'fs/promises'; import path from 'path'; @@ -49,6 +51,53 @@ describe('bundle', () => { }); }); + describe('WorkspaceManifestEntry', () => { + it('discriminates narrative entries by document type', () => { + const mapping: WorkspaceManifestEntry = { path: 'files/architecture.json', type: 'architecture' }; + const unpublishedNarrative: WorkspaceManifestEntry = { + path: 'files/design.md', type: 'sad', version: '1.0.0', + }; + const publishedNarrative: WorkspaceManifestEntry = { + path: 'files/design.md', type: 'sad', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/documents/sad/42/versions/1.0.0', + }; + + expect(isNarrativeWorkspaceManifestEntry(mapping)).toBe(false); + expect(isNarrativeWorkspaceManifestEntry(unpublishedNarrative)).toBe(true); + expect(isNarrativeWorkspaceManifestEntry(publishedNarrative)).toBe(true); + expect([mapping, publishedNarrative].filter(isNarrativeWorkspaceManifestEntry)[0].version).toBe('1.0.0'); + }); + + it('rejects incomplete narrative identity and narrative-owned mapping state at compile time', () => { + // @ts-expect-error Published narrative entries require calmHubId. + const narrativeWithoutHubId: WorkspaceManifestEntry = { + path: 'files/design.md', type: 'sad', version: '1.0.0', calmHubDocumentId: 42, + }; + // @ts-expect-error Published narrative entries require calmHubDocumentId. + const narrativeWithoutDocumentId: WorkspaceManifestEntry = { + path: 'files/design.md', type: 'sad', version: '1.0.0', calmHubId: '/documents/sad/42/versions/1.0.0', + }; + // @ts-expect-error Narrative entries require a manifest version. + const narrativeWithoutVersion: WorkspaceManifestEntry = { + path: 'files/design.md', type: 'sad', + }; + // @ts-expect-error Mapping entries do not own a manifest version. + const invalidMapping: WorkspaceManifestEntry = { + path: 'files/architecture.json', type: 'architecture', version: '1.0.0', + }; + // @ts-expect-error Mapping entries do not own a narrative document ID. + const invalidMappingId: WorkspaceManifestEntry = { + path: 'files/architecture.json', type: 'architecture', calmHubDocumentId: 42, + }; + + expect(narrativeWithoutHubId.type).toBe('sad'); + expect(narrativeWithoutDocumentId.type).toBe('sad'); + expect(narrativeWithoutVersion.type).toBe('sad'); + expect(invalidMapping.type).toBe('architecture'); + expect(invalidMappingId.type).toBe('architecture'); + }); + }); + describe('extractReferenceValue', () => { it('should return string value directly', () => { expect(extractReferenceValue('https://example.com/schema.json')).toBe('https://example.com/schema.json'); @@ -98,6 +147,15 @@ describe('bundle', () => { expect(manifest).toEqual(expected); }); + it('should preserve malformed persisted narrative identity for runtime validation', async () => { + const malformed = { + narrative: { path: 'files/design.md', type: 'sad', calmHubId: '/partial' }, + }; + await writeFile(path.join(bundlePath, MANIFEST_FILENAME), JSON.stringify(malformed)); + + expect(await loadManifest(bundlePath)).toEqual(malformed); + }); + it('should migrate old string-value format to new entry format', async () => { const old = { 'doc1': 'files/doc1.json', 'doc2': 'files/doc2.json' }; await writeFile(path.join(bundlePath, MANIFEST_FILENAME), JSON.stringify(old)); @@ -225,7 +283,7 @@ describe('bundle', () => { { type: 'sad' as const, version: '1.2.0', calmHubId: '/path' }, { type: 'sad' as const, calmHubDocumentId: 42, calmHubId: '/path' }, ])('rejects an incomplete narrative Hub identity', async (options) => { - await expect(addFileToBundle(bundlePath, srcFile, options)).rejects.toThrow(/Hub identity/); + await expect(addFileToBundle(bundlePath, srcFile, options as never)).rejects.toThrow(/Hub identity/); }); it('should copy file when copy option is true', async () => { diff --git a/cli/src/command-helpers/workspace/bundle.ts b/cli/src/command-helpers/workspace/bundle.ts index 26839b519..7e5a946f1 100644 --- a/cli/src/command-helpers/workspace/bundle.ts +++ b/cli/src/command-helpers/workspace/bundle.ts @@ -3,7 +3,7 @@ import { mkdir, copyFile, readFile, writeFile } from 'fs/promises'; import { existsSync } from 'fs'; import { JSONPath } from 'jsonpath-plus'; import { printBundleTreeFromGraph } from './tree'; -import type { CalmDocumentType, NarrativeDocumentType } from '@finos/calm-models/types'; +import { isNarrativeDocumentType, type CalmDocumentType, type NarrativeDocumentType } from '@finos/calm-models/types'; /** * Property names that can contain document references (URLs or paths) in CALM JSON. @@ -65,15 +65,44 @@ export function extractAllReferences(json: object): string[] { export type WorkspaceDocumentType = CalmDocumentType | NarrativeDocumentType | 'unknown'; -export type WorkspaceManifestEntry = { +export type MappingWorkspaceManifestEntry = { path: string; - type: WorkspaceDocumentType; + type: CalmDocumentType | 'unknown'; namespace?: string; calmHubId?: string; - version?: string; - calmHubDocumentId?: number; + version?: never; + calmHubDocumentId?: never; }; +type NarrativeWorkspaceManifestEntryBase = { + path: string; + type: NarrativeDocumentType; + namespace?: string; + version: string; +}; + +export type UnpublishedNarrativeWorkspaceManifestEntry = NarrativeWorkspaceManifestEntryBase & { + calmHubDocumentId?: never; + calmHubId?: never; +}; + +export type PublishedNarrativeWorkspaceManifestEntry = NarrativeWorkspaceManifestEntryBase & { + calmHubDocumentId: number; + calmHubId: string; +}; + +export type NarrativeWorkspaceManifestEntry = + | UnpublishedNarrativeWorkspaceManifestEntry + | PublishedNarrativeWorkspaceManifestEntry; + +export type WorkspaceManifestEntry = MappingWorkspaceManifestEntry | NarrativeWorkspaceManifestEntry; + +export function isNarrativeWorkspaceManifestEntry( + entry: WorkspaceManifestEntry +): entry is NarrativeWorkspaceManifestEntry { + return isNarrativeDocumentType(entry.type); +} + export type WorkspaceManifest = Record; export type DependencyGraph = { @@ -156,6 +185,36 @@ export async function determineDocumentId(srcPath: string, explicitId?: string): return path.basename(srcPath, path.extname(srcPath)); } +type AddFileToBundleCommonOptions = { + id?: string; + destName?: string; + copy?: boolean; + namespace?: string; +}; + +type AddMappingFileToBundleOptions = AddFileToBundleCommonOptions & { + type?: CalmDocumentType | 'unknown'; + version?: never; + calmHubDocumentId?: never; + calmHubId?: never; +}; + +type AddNarrativeFileToBundleOptions = AddFileToBundleCommonOptions & { + type: NarrativeDocumentType; + version: string; +} & ( + | { calmHubDocumentId?: never; calmHubId?: never } + | { calmHubDocumentId: number; calmHubId: string } +); + +type AddFileToBundleOptions = AddMappingFileToBundleOptions | AddNarrativeFileToBundleOptions; + +function isNarrativeAddFileToBundleOptions( + opts: AddFileToBundleOptions | undefined +): opts is AddNarrativeFileToBundleOptions { + return opts !== undefined && isNarrativeDocumentType(opts.type); +} + /** * Add a file into the workspace bundle and register it in the bundle manifest. * The file is copied into the bundle's 'files/' directory and the manifest is updated @@ -169,16 +228,7 @@ export async function determineDocumentId(srcPath: string, explicitId?: string): export async function addFileToBundle( bundlePath: string, srcPath: string, - opts?: { - id?: string; - destName?: string; - copy?: boolean; - type?: WorkspaceDocumentType; - namespace?: string; - version?: string; - calmHubDocumentId?: number; - calmHubId?: string; - } + opts?: AddFileToBundleOptions ): Promise<{ id: string; destPath: string; rel: string }> { const hasDocumentId = opts?.calmHubDocumentId !== undefined; @@ -208,13 +258,24 @@ export async function addFileToBundle( } const manifest = await loadManifest(bundlePath); - manifest[id] = { - path: rel, - type: opts?.type ?? 'unknown', - ...(opts?.namespace ? { namespace: opts.namespace } : {}), - ...(opts?.version ? { version: opts.version } : {}), - ...(hasDocumentId ? { calmHubDocumentId: opts!.calmHubDocumentId, calmHubId: opts!.calmHubId } : {}), - }; + if (isNarrativeAddFileToBundleOptions(opts)) { + const hubIdentity = opts.calmHubDocumentId !== undefined && opts.calmHubId !== undefined + ? { calmHubDocumentId: opts.calmHubDocumentId, calmHubId: opts.calmHubId } + : {}; + manifest[id] = { + path: rel, + type: opts.type, + ...(opts.namespace ? { namespace: opts.namespace } : {}), + version: opts.version, + ...hubIdentity, + }; + } else { + manifest[id] = { + path: rel, + type: opts?.type ?? 'unknown', + ...(opts?.namespace ? { namespace: opts.namespace } : {}), + }; + } await saveManifest(bundlePath, manifest); return { id, destPath, rel }; diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index bd7cba970..e3311dd27 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -1,9 +1,8 @@ import { readFile } from 'fs/promises'; import { existsSync } from 'fs'; -import { loadManifest, saveManifest, resolveFilePath } from './bundle'; +import { isNarrativeWorkspaceManifestEntry, loadManifest, saveManifest, resolveFilePath } from './bundle'; import { CalmHubClient, DocumentMetadata, extractDocumentMetadata, initLogger, Logger } from '@finos/calm-shared'; import { canonicalEqual } from './bump'; -import { isNarrativeDocumentType } from '@finos/calm-models/types'; import { parseNarrativeDocument, parseNarrativeDocumentLocation, @@ -47,7 +46,7 @@ export async function pushWorkspaceToHub( if (!existsSync(filePath)) { logger.warn(`File not found for id '${id}': ${filePath}`); - if (isNarrativeDocumentType(entry.type)) narrativeFailures.push(`${id}: file not found`); + if (isNarrativeWorkspaceManifestEntry(entry)) narrativeFailures.push(`${id}: file not found`); continue; } @@ -56,11 +55,11 @@ export async function pushWorkspaceToHub( raw = await readFile(filePath, 'utf8'); } catch (e) { logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); - if (isNarrativeDocumentType(entry.type)) narrativeFailures.push(`${id}: file could not be read`); + if (isNarrativeWorkspaceManifestEntry(entry)) narrativeFailures.push(`${id}: file could not be read`); continue; } - if (isNarrativeDocumentType(entry.type)) { + if (isNarrativeWorkspaceManifestEntry(entry)) { try { const version = entry.version; if (!version) throw new Error('Narrative document manifest entry has no version. Re-add the document to repair it.'); From 78b0ef8ff57d2d803229c7ab99ae19d44561ff62 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Thu, 17 Sep 2026 14:48:37 +0100 Subject: [PATCH 16/28] fix(cli): validate narrative recovery document IDs --- .../workspace/commands.spec.ts | 32 +++++++++++++------ cli/src/command-helpers/workspace/commands.ts | 10 +++++- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/cli/src/command-helpers/workspace/commands.spec.ts b/cli/src/command-helpers/workspace/commands.spec.ts index b2523db48..e9ca31d5f 100644 --- a/cli/src/command-helpers/workspace/commands.spec.ts +++ b/cli/src/command-helpers/workspace/commands.spec.ts @@ -245,23 +245,27 @@ describe('setupWorkspaceCommands', () => { ); }); - it('recovers a verified narrative document without creating it', async () => { + it.each([ + ['1', 1], + ['42', 42], + ['123456', 123456], + ])('recovers a verified narrative document with canonical ID %s', async (rawDocumentId, documentId) => { const markdown = '---\ntitle: Payments SAD\n---\n# Payments\n'; mocks.readFile.mockResolvedValueOnce(markdown); await program.parseAsync([ 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', - '--calm-hub-document-id', '42', '--ver', '1.2.0', '--calm-hub-url', 'https://explicit.example.com' + '--calm-hub-document-id', rawDocumentId, '--ver', '1.2.0', '--calm-hub-url', 'https://explicit.example.com' ]); expect(mocks.CalmHubClient).toHaveBeenCalledWith(expect.objectContaining({ calmHubUrl: 'https://explicit.example.com' })); const client = mocks.CalmHubClient.mock.results[0].value; - expect(client.getNarrativeDocumentVersion).toHaveBeenCalledWith('finos', 'sad', 42, '1.2.0'); + expect(client.getNarrativeDocumentVersion).toHaveBeenCalledWith('finos', 'sad', documentId, '1.2.0'); expect(client.createNarrativeDocument).not.toHaveBeenCalled(); expect(client.createNarrativeDocumentVersion).not.toHaveBeenCalled(); expect(mocks.addFileToBundle).toHaveBeenCalledWith('/fake/bundle', expect.stringContaining('payments.md'), expect.objectContaining({ - id: 'Payments SAD', type: 'sad', namespace: 'finos', version: '1.2.0', calmHubDocumentId: 42, - calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/1.2.0', + id: 'Payments SAD', type: 'sad', namespace: 'finos', version: '1.2.0', calmHubDocumentId: documentId, + calmHubId: `/api/calm/namespaces/finos/documents/sad/${documentId}/versions/1.2.0`, })); }); @@ -291,13 +295,21 @@ describe('setupWorkspaceCommands', () => { expect(mocks.addFileToBundle).not.toHaveBeenCalled(); }); - it.each([ - ['--calm-hub-document-id', '0', '--ver', '1.2.0'], - ['--calm-hub-document-id', '42', '--ver', 'invalid'], - ])('rejects invalid recovery identity values', async (idOption, id, versionOption, version) => { + it.each(['0', '-1', '1.5', '1e2', ' 42', '42 ', '01', '+42', '9007199254740992'])( + 'rejects non-canonical or unsafe narrative recovery ID %j', async (documentId) => { + await expect(program.parseAsync([ + 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', + '--calm-hub-document-id', documentId, '--ver', '1.2.0' + ])).rejects.toThrow(); + expect(mocks.CalmHubClient).not.toHaveBeenCalled(); + expect(mocks.addFileToBundle).not.toHaveBeenCalled(); + } + ); + + it('rejects an invalid recovery version', async () => { await expect(program.parseAsync([ 'node', 'test', 'workspace', 'add', 'payments.md', '--type', 'sad', '--namespace', 'finos', - idOption, id, versionOption, version + '--calm-hub-document-id', '42', '--ver', 'invalid' ])).rejects.toThrow(); expect(mocks.CalmHubClient).not.toHaveBeenCalled(); expect(mocks.addFileToBundle).not.toHaveBeenCalled(); diff --git a/cli/src/command-helpers/workspace/commands.ts b/cli/src/command-helpers/workspace/commands.ts index 75c1e0d09..98cb22a9e 100644 --- a/cli/src/command-helpers/workspace/commands.ts +++ b/cli/src/command-helpers/workspace/commands.ts @@ -91,11 +91,19 @@ export function setupWorkspaceCommands(program: Command) { throw new Error(`Narrative document '${file}' recovery requires --namespace.`); } + const rawDocumentId = options.calmHubDocumentId; + if (typeof rawDocumentId !== 'string' || !/^[1-9]\d*$/.test(rawDocumentId)) { + throw new Error(`Narrative document '${file}' calmHubDocumentId must be a positive integer.`); + } + const calmHubDocumentId = Number(rawDocumentId); + if (!Number.isSafeInteger(calmHubDocumentId)) { + throw new Error(`Narrative document '${file}' calmHubDocumentId must be a positive integer.`); + } const identity = { namespace: options.namespace.trim(), type: options.type, version: options.ver, - calmHubDocumentId: Number(options.calmHubDocumentId), + calmHubDocumentId, }; validateNarrativeIdentity(identity, true, file); const raw = await readFile(srcPath, 'utf8'); From 42dd30b6185e6fcf5c40fdc5518652897b2a9284 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Thu, 17 Sep 2026 15:10:49 +0100 Subject: [PATCH 17/28] fix(cli): preserve narrative identity on re-add --- .../command-helpers/workspace/bundle.spec.ts | 166 ++++++++++++++++++ cli/src/command-helpers/workspace/bundle.ts | 83 ++++++++- cli/src/command-helpers/workspace/commands.ts | 96 +++++++--- 3 files changed, 313 insertions(+), 32 deletions(-) diff --git a/cli/src/command-helpers/workspace/bundle.spec.ts b/cli/src/command-helpers/workspace/bundle.spec.ts index 2800477e6..bde495337 100644 --- a/cli/src/command-helpers/workspace/bundle.spec.ts +++ b/cli/src/command-helpers/workspace/bundle.spec.ts @@ -278,6 +278,172 @@ describe('bundle', () => { expect((await loadManifest(bundlePath))['source-doc']).toMatchObject({ version: '1.0.0' }); }); + it('keeps an unpublished narrative unpublished when re-added', async () => { + await saveManifest(bundlePath, { + 'source-doc': { + path: 'old.md', type: 'sad', namespace: 'finos', version: '1.0.0', + }, + }); + + await addFileToBundle(bundlePath, srcFile, { + type: 'sad', namespace: 'finos', version: '1.0.0', + }); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual({ + path: srcFile, type: 'sad', namespace: 'finos', version: '1.0.0', + }); + }); + + it('preserves a published narrative Hub identity when re-added normally', async () => { + await saveManifest(bundlePath, { + 'source-doc': { + path: 'old.md', type: 'sad', namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }, + }); + + await addFileToBundle(bundlePath, srcFile, { + type: 'sad', namespace: 'finos', version: '1.0.0', + }); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual({ + path: srcFile, type: 'sad', namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }); + }); + + it.each([ + [false, srcFile], + [true, 'files/source.json'], + ])('preserves a published narrative Hub identity when changing its stored path (copy: %s)', async (copy, expectedPath) => { + await saveManifest(bundlePath, { + 'source-doc': { + path: 'old.md', type: 'sad', namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }, + }); + + await addFileToBundle(bundlePath, srcFile, { + copy, type: 'sad', namespace: 'finos', version: '1.0.0', + }); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual({ + path: expectedPath, type: 'sad', namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }); + }); + + it('accepts verified recovery when the existing published identity is equivalent', async () => { + await saveManifest(bundlePath, { + 'source-doc': { + path: 'old.md', type: 'sad', namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: 'https://calmhub.example.com/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }, + }); + + await addFileToBundle(bundlePath, srcFile, { + type: 'sad', namespace: 'finos', version: '2.3.0', calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual({ + path: srcFile, type: 'sad', namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }); + }); + + it('rejects verified recovery that conflicts with an existing published identity', async () => { + const existing = { + path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }; + await saveManifest(bundlePath, { 'source-doc': existing }); + + await expect(addFileToBundle(bundlePath, srcFile, { + copy: true, type: 'sad', namespace: 'finos', version: '2.3.0', calmHubDocumentId: 43, + calmHubId: '/api/calm/namespaces/finos/documents/sad/43/versions/2.3.0', + })).rejects.toThrow(/recovery identity conflicts/); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual(existing); + expect(existsSync(path.join(filesPath, 'source.json'))).toBe(false); + }); + + it.each([ + { type: 'knowledge' as const, namespace: 'finos' }, + { type: 'sad' as const, namespace: 'other' }, + ])('rejects a normal re-add that changes published identity scope', async ({ type, namespace }) => { + const existing = { + path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }; + await saveManifest(bundlePath, { 'source-doc': existing }); + + await expect(addFileToBundle(bundlePath, srcFile, { + type, namespace, version: '1.0.0', + })).rejects.toThrow(/type or namespace conflicts/); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual(existing); + }); + + it('rejects replacing a published narrative with a mapping before copying or changing the manifest', async () => { + await saveManifest(bundlePath, { + 'source-doc': { + path: 'old.md', type: 'sad', namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }, + }); + const manifestPath = path.join(bundlePath, MANIFEST_FILENAME); + const originalManifest = await readFile(manifestPath, 'utf8'); + + await expect(addFileToBundle(bundlePath, srcFile, { + copy: true, type: 'architecture', + })).rejects.toThrow(/cannot be replaced with a non-narrative document/); + + expect(await readFile(manifestPath, 'utf8')).toBe(originalManifest); + expect(existsSync(path.join(filesPath, 'source.json'))).toBe(false); + }); + + it.each([ + ['omitted', undefined], + ['unknown', { type: 'unknown' as const }], + ])('rejects replacing a published narrative when the incoming type is %s', async (_label, options) => { + const existing = { + path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '2.3.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', + }; + await saveManifest(bundlePath, { 'source-doc': existing }); + + await expect(addFileToBundle(bundlePath, srcFile, options)).rejects.toThrow( + /cannot be replaced with a non-narrative document/ + ); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual(existing); + }); + + it('preserves normal replacement behavior for an existing mapping entry', async () => { + await saveManifest(bundlePath, { + 'source-doc': { path: 'old.json', type: 'architecture', namespace: 'finos' }, + }); + + await addFileToBundle(bundlePath, srcFile, { + type: 'pattern', namespace: 'other', + }); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual({ + path: srcFile, type: 'pattern', namespace: 'other', + }); + }); + it.each([ { type: 'sad' as const, version: '1.2.0', calmHubDocumentId: 42 }, { type: 'sad' as const, version: '1.2.0', calmHubId: '/path' }, diff --git a/cli/src/command-helpers/workspace/bundle.ts b/cli/src/command-helpers/workspace/bundle.ts index 7e5a946f1..ce6fc94f0 100644 --- a/cli/src/command-helpers/workspace/bundle.ts +++ b/cli/src/command-helpers/workspace/bundle.ts @@ -4,6 +4,7 @@ import { existsSync } from 'fs'; import { JSONPath } from 'jsonpath-plus'; import { printBundleTreeFromGraph } from './tree'; import { isNarrativeDocumentType, type CalmDocumentType, type NarrativeDocumentType } from '@finos/calm-models/types'; +import { validateNarrativeDocumentLocation } from './narrative-document'; /** * Property names that can contain document references (URLs or paths) in CALM JSON. @@ -215,6 +216,45 @@ function isNarrativeAddFileToBundleOptions( return opts !== undefined && isNarrativeDocumentType(opts.type); } +function isPublishedNarrativeWorkspaceManifestEntry( + entry: WorkspaceManifestEntry | undefined +): entry is PublishedNarrativeWorkspaceManifestEntry { + return entry !== undefined && + isNarrativeWorkspaceManifestEntry(entry) && + entry.calmHubDocumentId !== undefined && + entry.calmHubId !== undefined; +} + +function hasEquivalentPublishedNarrativeIdentity( + entry: PublishedNarrativeWorkspaceManifestEntry, + opts: AddNarrativeFileToBundleOptions & { calmHubDocumentId: number; calmHubId: string } +): boolean { + if ( + entry.namespace === undefined || + opts.namespace === undefined || + entry.namespace !== opts.namespace || + entry.type !== opts.type || + entry.version !== opts.version || + entry.calmHubDocumentId !== opts.calmHubDocumentId + ) { + return false; + } + + const identity = { + namespace: entry.namespace, + type: entry.type, + version: entry.version, + calmHubDocumentId: entry.calmHubDocumentId, + }; + try { + validateNarrativeDocumentLocation(entry.calmHubId, identity); + validateNarrativeDocumentLocation(opts.calmHubId, identity); + return true; + } catch { + return false; + } +} + /** * Add a file into the workspace bundle and register it in the bundle manifest. * The file is copied into the bundle's 'files/' directory and the manifest is updated @@ -238,6 +278,42 @@ export async function addFileToBundle( } const id = await determineDocumentId(srcPath, opts?.id); + const manifest = await loadManifest(bundlePath); + const existingEntry = manifest[id]; + let narrativeIdentity: Pick | undefined; + + if ( + isPublishedNarrativeWorkspaceManifestEntry(existingEntry) && + !isNarrativeAddFileToBundleOptions(opts) + ) { + throw new Error(`Published narrative document '${id}' cannot be replaced with a non-narrative document.`); + } + + if (isNarrativeAddFileToBundleOptions(opts)) { + if (opts.calmHubDocumentId !== undefined && opts.calmHubId !== undefined) { + if ( + isPublishedNarrativeWorkspaceManifestEntry(existingEntry) && + !hasEquivalentPublishedNarrativeIdentity(existingEntry, opts) + ) { + throw new Error(`Narrative document '${id}' recovery identity conflicts with its existing published Hub identity.`); + } + narrativeIdentity = { + version: opts.version, + calmHubDocumentId: opts.calmHubDocumentId, + calmHubId: opts.calmHubId, + }; + } else if (isPublishedNarrativeWorkspaceManifestEntry(existingEntry)) { + if (existingEntry.type !== opts.type || existingEntry.namespace !== opts.namespace) { + throw new Error(`Narrative document '${id}' type or namespace conflicts with its existing published Hub identity.`); + } + narrativeIdentity = { + version: existingEntry.version, + calmHubDocumentId: existingEntry.calmHubDocumentId, + calmHubId: existingEntry.calmHubId, + }; + } + } + let rel: string; let destPath: string; @@ -257,16 +333,15 @@ export async function addFileToBundle( rel = srcPath; } - const manifest = await loadManifest(bundlePath); if (isNarrativeAddFileToBundleOptions(opts)) { - const hubIdentity = opts.calmHubDocumentId !== undefined && opts.calmHubId !== undefined - ? { calmHubDocumentId: opts.calmHubDocumentId, calmHubId: opts.calmHubId } + const hubIdentity = narrativeIdentity + ? { calmHubDocumentId: narrativeIdentity.calmHubDocumentId, calmHubId: narrativeIdentity.calmHubId } : {}; manifest[id] = { path: rel, type: opts.type, ...(opts.namespace ? { namespace: opts.namespace } : {}), - version: opts.version, + version: narrativeIdentity?.version ?? opts.version, ...hubIdentity, }; } else { diff --git a/cli/src/command-helpers/workspace/commands.ts b/cli/src/command-helpers/workspace/commands.ts index 98cb22a9e..c178db406 100644 --- a/cli/src/command-helpers/workspace/commands.ts +++ b/cli/src/command-helpers/workspace/commands.ts @@ -16,10 +16,44 @@ import { select, input } from '@inquirer/prompts'; import { CALM_DOCUMENT_TYPES_LIST, CALM_NARRATIVE_DOCUMENT_TYPES_LIST, isNarrativeDocumentType, isValidCalmDocumentType } from '@finos/calm-models/types'; import { loadCliConfig } from '../../cli-config'; import { resolveCalmHubOptions } from '../hub-commands'; -import { constructNarrativeDocumentPath, parseNarrativeDocument, validateNarrativeIdentity } from './narrative-document'; +import { constructNarrativeDocumentPath, parseNarrativeDocument, validateNarrativeIdentity, type NarrativeDocumentIdentity } from './narrative-document'; const logger: Logger = initLogger(false, 'workspace'); +type NarrativeRegistrationOptions = { + id?: string; + copy?: boolean; + identity: NarrativeDocumentIdentity; + verify?: (documentMarkdown: string) => Promise; +}; + +async function registerNarrativeDocument( + bundlePath: string, + srcPath: string, + file: string, + options: NarrativeRegistrationOptions +): Promise<{ id: string; destPath: string; rel: string }> { + const raw = await readFile(srcPath, 'utf8'); + const narrative = parseNarrativeDocument(raw, file); + await options.verify?.(raw); + + const hubIdentity = options.identity.calmHubDocumentId === undefined + ? {} + : { + calmHubDocumentId: options.identity.calmHubDocumentId, + calmHubId: constructNarrativeDocumentPath(options.identity), + }; + + return addFileToBundle(bundlePath, srcPath, { + id: options.id ?? narrative.request.name, + copy: options.copy, + type: options.identity.type, + namespace: options.identity.namespace, + version: options.identity.version, + ...hubIdentity, + }); +} + /** * Sets up the 'workspace' command and its subcommands in the CLI. * @param program The Commander.js top-level program. @@ -106,25 +140,26 @@ export function setupWorkspaceCommands(program: Command) { calmHubDocumentId, }; validateNarrativeIdentity(identity, true, file); - const raw = await readFile(srcPath, 'utf8'); - const narrative = parseNarrativeDocument(raw, file); - const calmHubOptions = await resolveCalmHubOptions({ calmHubUrl: options.calmHubUrl }); - const client = new CalmHubClient(calmHubOptions); - const remote = await client.getNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId, identity.version + const { id: resolvedId, destPath: finalDestPath } = await registerNarrativeDocument( + bundlePath, + srcPath, + file, + { + id: options.id, + copy: options.copy, + identity, + verify: async (raw) => { + const calmHubOptions = await resolveCalmHubOptions({ calmHubUrl: options.calmHubUrl }); + const client = new CalmHubClient(calmHubOptions); + const remote = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId, identity.version + ); + if (remote.documentMarkdown !== raw) { + throw new Error(`Narrative document '${file}' does not match CalmHub version ${identity.version}.`); + } + }, + } ); - if (remote.documentMarkdown !== raw) { - throw new Error(`Narrative document '${file}' does not match CalmHub version ${identity.version}.`); - } - const { id: resolvedId, destPath: finalDestPath } = await addFileToBundle(bundlePath, srcPath, { - id: options.id ?? narrative.request.name, - copy: options.copy, - type: identity.type, - namespace: identity.namespace, - version: identity.version, - calmHubDocumentId: identity.calmHubDocumentId, - calmHubId: constructNarrativeDocumentPath(identity), - }); logger.info(`${options.copy ? 'Copied' : 'Added reference to'} ${finalDestPath} (id: ${resolvedId})`); return; } @@ -135,15 +170,20 @@ export function setupWorkspaceCommands(program: Command) { if (!options.namespace?.trim()) { throw new Error(`Narrative document '${file}' requires --namespace.`); } - const raw = await readFile(srcPath, 'utf8'); - const narrative = parseNarrativeDocument(raw, file); - const { id: resolvedId, destPath: finalDestPath } = await addFileToBundle(bundlePath, srcPath, { - id: options.id ?? narrative.request.name, - copy: options.copy, - type, - namespace: options.namespace.trim(), - version: '1.0.0', - }); + const { id: resolvedId, destPath: finalDestPath } = await registerNarrativeDocument( + bundlePath, + srcPath, + file, + { + id: options.id, + copy: options.copy, + identity: { + namespace: options.namespace.trim(), + type, + version: '1.0.0', + }, + } + ); if (options.copy) { logger.info(`Copied ${srcPath} -> ${finalDestPath} (id: ${resolvedId})`); } else { From 80332af1120f3ffa4f576f9bf487b6b2dd3e09bf Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Thu, 17 Sep 2026 16:09:17 +0100 Subject: [PATCH 18/28] fix(cli): make narrative create recovery idempotent --- .../command-helpers/workspace/bundle.spec.ts | 84 +++++ cli/src/command-helpers/workspace/bundle.ts | 50 +++ .../command-helpers/workspace/push.spec.ts | 337 +++++++++++++++++- cli/src/command-helpers/workspace/push.ts | 147 +++++++- shared/src/hub/calm-hub-client.spec.ts | 43 ++- shared/src/hub/calm-hub-client.ts | 24 +- 6 files changed, 664 insertions(+), 21 deletions(-) diff --git a/cli/src/command-helpers/workspace/bundle.spec.ts b/cli/src/command-helpers/workspace/bundle.spec.ts index bde495337..6dd1f6dbc 100644 --- a/cli/src/command-helpers/workspace/bundle.spec.ts +++ b/cli/src/command-helpers/workspace/bundle.spec.ts @@ -20,6 +20,7 @@ describe('bundle', () => { const testDir = path.join(__dirname, 'test-bundle'); const bundlePath = path.join(testDir, 'bundle'); const filesPath = path.join(bundlePath, 'files'); + const documentMarkdownSha256 = 'a'.repeat(64); beforeAll(async () => { await mkdir(testDir, { recursive: true }); @@ -61,10 +62,15 @@ describe('bundle', () => { path: 'files/design.md', type: 'sad', version: '1.0.0', calmHubDocumentId: 42, calmHubId: '/documents/sad/42/versions/1.0.0', }; + const pendingNarrative: WorkspaceManifestEntry = { + path: 'files/design.md', type: 'sad', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + }; expect(isNarrativeWorkspaceManifestEntry(mapping)).toBe(false); expect(isNarrativeWorkspaceManifestEntry(unpublishedNarrative)).toBe(true); expect(isNarrativeWorkspaceManifestEntry(publishedNarrative)).toBe(true); + expect(isNarrativeWorkspaceManifestEntry(pendingNarrative)).toBe(true); expect([mapping, publishedNarrative].filter(isNarrativeWorkspaceManifestEntry)[0].version).toBe('1.0.0'); }); @@ -89,12 +95,19 @@ describe('bundle', () => { const invalidMappingId: WorkspaceManifestEntry = { path: 'files/architecture.json', type: 'architecture', calmHubDocumentId: 42, }; + // @ts-expect-error Pending narrative entries cannot also have a published identity. + const pendingPublishedNarrative: WorkspaceManifestEntry = { + path: 'files/design.md', type: 'sad', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + calmHubDocumentId: 42, calmHubId: '/documents/sad/42/versions/1.0.0', + }; expect(narrativeWithoutHubId.type).toBe('sad'); expect(narrativeWithoutDocumentId.type).toBe('sad'); expect(narrativeWithoutVersion.type).toBe('sad'); expect(invalidMapping.type).toBe('architecture'); expect(invalidMappingId.type).toBe('architecture'); + expect(pendingPublishedNarrative.type).toBe('sad'); }); }); @@ -294,6 +307,77 @@ describe('bundle', () => { }); }); + it('rejects re-adding a narrative while create recovery is pending', async () => { + const existing = { + path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + }; + await saveManifest(bundlePath, { 'source-doc': existing }); + + await expect(addFileToBundle(bundlePath, srcFile, { + copy: true, type: 'sad', namespace: 'finos', version: '1.0.0', + })).rejects.toThrow(/pending create recovery/); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual(existing); + expect(existsSync(path.join(filesPath, 'source.json'))).toBe(false); + }); + + it('rejects replacing a pending narrative with a mapping', async () => { + const existing = { + path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + }; + await saveManifest(bundlePath, { 'source-doc': existing }); + + await expect(addFileToBundle(bundlePath, srcFile, { + copy: true, type: 'architecture', + })).rejects.toThrow(/pending create recovery/); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual(existing); + expect(existsSync(path.join(filesPath, 'source.json'))).toBe(false); + }); + + it('allows a full compatible verified identity to reconcile pending recovery', async () => { + await saveManifest(bundlePath, { + 'source-doc': { + path: 'old.md', type: 'sad', namespace: 'finos', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + }, + }); + + await addFileToBundle(bundlePath, srcFile, { + type: 'sad', namespace: 'finos', version: '1.0.0', calmHubDocumentId: 3, + calmHubId: '/api/calm/namespaces/finos/documents/sad/3/versions/1.0.0', + }); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual({ + path: srcFile, type: 'sad', namespace: 'finos', version: '1.0.0', + calmHubDocumentId: 3, + calmHubId: '/api/calm/namespaces/finos/documents/sad/3/versions/1.0.0', + }); + expect((await loadManifest(bundlePath))['source-doc']).not.toHaveProperty('createRecovery'); + }); + + it.each([ + { type: 'knowledge' as const, namespace: 'finos', version: '1.0.0' }, + { type: 'sad' as const, namespace: 'other', version: '1.0.0' }, + { type: 'sad' as const, namespace: 'finos', version: '1.1.0' }, + ])('rejects verified identity outside the pending recovery scope', async ({ type, namespace, version }) => { + const existing = { + path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + }; + await saveManifest(bundlePath, { 'source-doc': existing }); + + await expect(addFileToBundle(bundlePath, srcFile, { + copy: true, type, namespace, version, calmHubDocumentId: 3, + calmHubId: `/api/calm/namespaces/${namespace}/documents/${type}/3/versions/${version}`, + })).rejects.toThrow(/pending create recovery scope/); + + expect((await loadManifest(bundlePath))['source-doc']).toEqual(existing); + expect(existsSync(path.join(filesPath, 'source.json'))).toBe(false); + }); + it('preserves a published narrative Hub identity when re-added normally', async () => { await saveManifest(bundlePath, { 'source-doc': { diff --git a/cli/src/command-helpers/workspace/bundle.ts b/cli/src/command-helpers/workspace/bundle.ts index ce6fc94f0..9764db6dd 100644 --- a/cli/src/command-helpers/workspace/bundle.ts +++ b/cli/src/command-helpers/workspace/bundle.ts @@ -73,6 +73,7 @@ export type MappingWorkspaceManifestEntry = { calmHubId?: string; version?: never; calmHubDocumentId?: never; + createRecovery?: never; }; type NarrativeWorkspaceManifestEntryBase = { @@ -85,15 +86,29 @@ type NarrativeWorkspaceManifestEntryBase = { export type UnpublishedNarrativeWorkspaceManifestEntry = NarrativeWorkspaceManifestEntryBase & { calmHubDocumentId?: never; calmHubId?: never; + createRecovery?: never; +}; + +export type NarrativeCreateRecovery = { + documentIdsBeforeCreate: number[]; + documentMarkdownSha256: string; +}; + +export type CreateRecoveryPendingNarrativeWorkspaceManifestEntry = NarrativeWorkspaceManifestEntryBase & { + calmHubDocumentId?: never; + calmHubId?: never; + createRecovery: NarrativeCreateRecovery; }; export type PublishedNarrativeWorkspaceManifestEntry = NarrativeWorkspaceManifestEntryBase & { calmHubDocumentId: number; calmHubId: string; + createRecovery?: never; }; export type NarrativeWorkspaceManifestEntry = | UnpublishedNarrativeWorkspaceManifestEntry + | CreateRecoveryPendingNarrativeWorkspaceManifestEntry | PublishedNarrativeWorkspaceManifestEntry; export type WorkspaceManifestEntry = MappingWorkspaceManifestEntry | NarrativeWorkspaceManifestEntry; @@ -225,6 +240,14 @@ function isPublishedNarrativeWorkspaceManifestEntry( entry.calmHubId !== undefined; } +function hasNarrativeCreateRecovery( + entry: WorkspaceManifestEntry | undefined +): entry is CreateRecoveryPendingNarrativeWorkspaceManifestEntry { + return entry !== undefined && + isNarrativeWorkspaceManifestEntry(entry) && + Object.prototype.hasOwnProperty.call(entry, 'createRecovery'); +} + function hasEquivalentPublishedNarrativeIdentity( entry: PublishedNarrativeWorkspaceManifestEntry, opts: AddNarrativeFileToBundleOptions & { calmHubDocumentId: number; calmHubId: string } @@ -282,6 +305,33 @@ export async function addFileToBundle( const existingEntry = manifest[id]; let narrativeIdentity: Pick | undefined; + if (hasNarrativeCreateRecovery(existingEntry)) { + if ( + !isNarrativeAddFileToBundleOptions(opts) || + opts.calmHubDocumentId === undefined || + opts.calmHubId === undefined + ) { + throw new Error(`Narrative document '${id}' has pending create recovery and cannot be re-added until it is reconciled.`); + } + if ( + existingEntry.type !== opts.type || + existingEntry.namespace !== opts.namespace || + existingEntry.version !== opts.version + ) { + throw new Error(`Narrative document '${id}' recovery identity conflicts with its pending create recovery scope.`); + } + try { + validateNarrativeDocumentLocation(opts.calmHubId, { + namespace: opts.namespace ?? '', + type: opts.type, + version: opts.version, + calmHubDocumentId: opts.calmHubDocumentId, + }); + } catch { + throw new Error(`Narrative document '${id}' recovery identity conflicts with its pending create recovery scope.`); + } + } + if ( isPublishedNarrativeWorkspaceManifestEntry(existingEntry) && !isNarrativeAddFileToBundleOptions(opts) diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 28634539c..f5adbe05a 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -5,17 +5,20 @@ import { CalmHubClient, HubClientError } from '@finos/calm-shared'; import { mkdir, writeFile, rm } from 'fs/promises'; import path from 'path'; import { existsSync } from 'fs'; +import { createHash } from 'node:crypto'; const makeClient = ( overrides: Partial> = {} + 'createNarrativeDocument' | 'createNarrativeDocumentVersion' | 'getNarrativeDocumentIds' | + 'getNarrativeDocumentVersions' | 'getNarrativeDocumentVersion'>> = {} ): CalmHubClient => ({ getMappedResourceVersions: vi.fn(async () => []), createMappedResourceVersion: vi.fn(async () => '/calm/namespaces/com.example/architectures/my-arch/versions/1.0.0'), getMappedResourceByVersion: vi.fn(async () => ({})), createNarrativeDocument: vi.fn(async () => '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0'), createNarrativeDocumentVersion: vi.fn(async () => '/api/calm/namespaces/com.example/documents/sad/42/versions/1.1.0'), + getNarrativeDocumentIds: vi.fn(async () => []), getNarrativeDocumentVersions: vi.fn(async () => []), getNarrativeDocumentVersion: vi.fn(async () => ({ documentMarkdown: '' })), ...overrides, @@ -34,6 +37,7 @@ vi.mock('@finos/calm-shared', async (importOriginal) => ({ const BASE = 'https://hub.example.com'; const mappingId = (resource: string, version = '1.0.0', type = 'architectures', ns = 'com.example') => `${BASE}/calm/namespaces/${ns}/${type}/${resource}/versions/${version}`; +const sha256 = (value: string) => createHash('sha256').update(value, 'utf8').digest('hex'); describe('pushWorkspaceToHub', () => { const testDir = path.join(__dirname, 'test-push'); @@ -59,6 +63,13 @@ describe('pushWorkspaceToHub', () => { await mkdir(filesPath, { recursive: true }); }); + async function writeFreshNarrative(markdown: string): Promise { + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { + payments: { path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + }); + } + it('resolves file path when entry.path is absolute', async () => { const absoluteFilePath = path.join(filesPath, 'doc-a.json'); await writeFile(absoluteFilePath, JSON.stringify(docA)); @@ -105,6 +116,319 @@ describe('pushWorkspaceToHub', () => { expect(await loadManifest(bundlePath)).toMatchObject({ 'payments-sad': { calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0' }, }); + expect((await loadManifest(bundlePath))['payments-sad']).not.toHaveProperty('createRecovery'); + expect(client.getNarrativeDocumentIds).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + }); + + it('preserves an absolute valid Location without recovery', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const location = 'https://hub.example.com/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0'; + const client = makeClient({ createNarrativeDocument: vi.fn().mockResolvedValue(location) }); + + await pushWorkspaceToHub(bundlePath, client); + + expect((await loadManifest(bundlePath)).payments).toMatchObject({ calmHubDocumentId: 42, calmHubId: location }); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); + expect(client.getNarrativeDocumentIds).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); + }); + + it('preserves independent identities for multiple narratives', async () => { + const payments = '---\ntitle: Payments SAD\n---\n# Payments'; + const orders = '---\ntitle: Orders Knowledge\n---\n# Orders'; + await writeFile(path.join(filesPath, 'payments.md'), payments); + await writeFile(path.join(filesPath, 'orders.md'), orders); + await saveManifest(bundlePath, { + payments: { path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + orders: { path: 'files/orders.md', type: 'knowledge', namespace: 'com.example', version: '1.0.0' }, + }); + const createNarrativeDocument = vi.fn().mockImplementation(async (_namespace, type) => + `/api/calm/namespaces/com.example/documents/${type}/${type === 'sad' ? 3 : 4}/versions/1.0.0` + ); + const client = makeClient({ createNarrativeDocument }); + + await pushWorkspaceToHub(bundlePath, client); + + expect(await loadManifest(bundlePath)).toMatchObject({ + payments: { calmHubDocumentId: 3 }, + orders: { calmHubDocumentId: 4 }, + }); + expect(createNarrativeDocument).toHaveBeenCalledTimes(2); + expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); + }); + + it('recovers a created narrative document from authoritative Hub state', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const getNarrativeDocumentIds = vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2, 3]); + const client = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + getNarrativeDocumentIds, + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), + }); + + await pushWorkspaceToHub(bundlePath, client); + + expect((await loadManifest(bundlePath)).payments).toMatchObject({ + calmHubDocumentId: 3, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', + }); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); + expect(client.getNarrativeDocumentVersion).toHaveBeenCalledWith('com.example', 'sad', 3, '1.0.0'); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + }); + + it('recovers only the matching document when concurrent documents appear', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const client = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2, 3, 4]), + getNarrativeDocumentVersion: vi.fn().mockImplementation(async (_namespace, _type, documentId) => ({ + documentMarkdown: documentId === 4 ? markdown : '---\ntitle: Other\n---\n# Other', + })), + }); + + await pushWorkspaceToHub(bundlePath, client); + + expect((await loadManifest(bundlePath)).payments).toMatchObject({ calmHubDocumentId: 4 }); + expect(client.getNarrativeDocumentVersion).toHaveBeenCalledTimes(2); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + }); + + it('uses the same recovery path when a confirmed create has no Location', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const client = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue(undefined), + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2, 3]), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), + }); + + await pushWorkspaceToHub(bundlePath, client); + + expect((await loadManifest(bundlePath)).payments).toMatchObject({ + calmHubDocumentId: 3, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', + }); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + }); + + it('persists a recovery fence when the post-create list is stale', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const client = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2]), + }); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/no new document has matching Markdown/); + + expect((await loadManifest(bundlePath)).payments).toEqual({ + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + }); + + it('fails ambiguous recovery when multiple new documents match', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const client = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2, 3, 4]), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), + }); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/multiple new documents have matching Markdown/); + + expect((await loadManifest(bundlePath)).payments).toMatchObject({ + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('calmHubDocumentId'); + expect(client.getNarrativeDocumentVersion).toHaveBeenCalledTimes(2); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + }); + + it('recovers a pending create on retry without another POST', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const firstClient = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2]), + }); + await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow(/no new document/); + + const retryClient = makeClient({ + getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3]), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), + }); + await pushWorkspaceToHub(bundlePath, retryClient); + + expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments).toEqual({ + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 3, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', + }); + }); + + it('does not recover against changed local Markdown', async () => { + const markdownA = '---\ntitle: Payments SAD\n---\n# Payload A'; + const markdownB = '---\ntitle: Payments SAD\n---\n# Payload B'; + await writeFreshNarrative(markdownA); + const firstClient = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2]), + }); + await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow(/no new document/); + const pending = (await loadManifest(bundlePath)).payments; + + await writeFile(path.join(filesPath, 'payments.md'), markdownB); + const retryClient = makeClient({ + getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3, 4]), + getNarrativeDocumentVersion: vi.fn().mockImplementation(async (_namespace, _type, documentId) => ({ + documentMarkdown: documentId === 3 ? markdownA : markdownB, + })), + }); + + await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/Markdown changed while create recovery is pending/); + + expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); + expect(retryClient.getNarrativeDocumentIds).not.toHaveBeenCalled(); + expect(retryClient.getNarrativeDocumentVersion).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments).toEqual(pending); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('calmHubDocumentId'); + }); + + it('keeps a pending recovery fence when retry still has no match', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const pending = { + path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }; + await saveManifest(bundlePath, { payments: pending }); + const client = makeClient({ + getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3]), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: '# Different' }), + }); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/no new document has matching Markdown/); + + expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments).toEqual(pending); + }); + + it('keeps a pending recovery fence when retry remains ambiguous', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const pending = { + path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }; + await saveManifest(bundlePath, { payments: pending }); + const client = makeClient({ + getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3, 4]), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), + }); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/multiple new documents have matching Markdown/); + + expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments).toEqual(pending); + }); + + it('keeps a recovery fence after a candidate GET error and never POSTs on retry', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const firstClient = makeClient({ + createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockResolvedValueOnce([1, 2, 3]), + getNarrativeDocumentVersion: vi.fn().mockRejectedValue(new Error('temporarily unavailable')), + }); + await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow(/temporarily unavailable/); + expect((await loadManifest(bundlePath)).payments).toMatchObject({ + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }); + + const retryClient = makeClient({ + getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3]), + getNarrativeDocumentVersion: vi.fn().mockRejectedValue(new Error('still unavailable')), + }); + await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/still unavailable/); + + expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments).toMatchObject({ + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }); + }); + + it.each([ + null, + {}, + { documentIdsBeforeCreate: '1,2', documentMarkdownSha256: 'a'.repeat(64) }, + { documentIdsBeforeCreate: [0], documentMarkdownSha256: 'a'.repeat(64) }, + { documentIdsBeforeCreate: [1.5], documentMarkdownSha256: 'a'.repeat(64) }, + { documentIdsBeforeCreate: [Number.MAX_SAFE_INTEGER + 1], documentMarkdownSha256: 'a'.repeat(64) }, + { documentIdsBeforeCreate: [1, 2] }, + { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'not-a-digest' }, + { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'A'.repeat(64) }, + { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'a'.repeat(63) }, + ])('rejects malformed persisted create recovery without posting: %j', async (createRecovery) => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', createRecovery, + } as never, + }); + const client = makeClient(); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/createRecovery/); + + expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); + }); + + it('does not run success recovery after a genuine create failure', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const getNarrativeDocumentIds = vi.fn().mockResolvedValue([1, 2]); + const client = makeClient({ + createNarrativeDocument: vi.fn().mockRejectedValue( + new HubClientError(500, 'unavailable', 'POST /api/calm/namespaces/com.example/documents/sad') + ), + getNarrativeDocumentIds, + }); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/Hub error 500/); + + expect(getNarrativeDocumentIds).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('calmHubDocumentId'); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); }); it('fails the completed push when a narrative document is invalid', async () => { @@ -204,17 +528,6 @@ describe('pushWorkspaceToHub', () => { expect(await loadManifest(bundlePath)).toEqual({ payments: entry }); }); - it('fails narrative publish when Hub returns an unexpected Location', async () => { - await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); - await saveManifest(bundlePath, { - payments: { path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, - }); - const client = makeClient({ createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected') }); - - await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/unexpected format/); - expect((await loadManifest(bundlePath)).payments.calmHubDocumentId).toBeUndefined(); - }); - it('rejects narrative manifests with no version or a non-initial unassigned version', async () => { await writeFile(path.join(filesPath, 'missing.md'), '---\ntitle: Missing\n---\n# Missing'); await writeFile(path.join(filesPath, 'ahead.md'), '---\ntitle: Ahead\n---\n# Ahead'); diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index e3311dd27..930bd7955 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -1,11 +1,28 @@ import { readFile } from 'fs/promises'; import { existsSync } from 'fs'; -import { isNarrativeWorkspaceManifestEntry, loadManifest, saveManifest, resolveFilePath } from './bundle'; -import { CalmHubClient, DocumentMetadata, extractDocumentMetadata, initLogger, Logger } from '@finos/calm-shared'; +import { createHash } from 'node:crypto'; +import { + isNarrativeWorkspaceManifestEntry, + loadManifest, + saveManifest, + resolveFilePath, + type NarrativeCreateRecovery, + type NarrativeWorkspaceManifestEntry, + type PublishedNarrativeWorkspaceManifestEntry, +} from './bundle'; +import { + CalmHubClient, + DocumentMetadata, + extractDocumentMetadata, + initLogger, + Logger, +} from '@finos/calm-shared'; import { canonicalEqual } from './bump'; import { + constructNarrativeDocumentPath, parseNarrativeDocument, parseNarrativeDocumentLocation, + type NarrativeDocumentIdentity, validateNarrativeDocumentLocation, validateNarrativeIdentity, validateNarrativeNamespace, @@ -66,6 +83,7 @@ export async function pushWorkspaceToHub( if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { throw new Error('Narrative document Hub identity is incomplete. Re-add the document to repair it.'); } + const createRecovery = getCreateRecovery(entry); validateNarrativeNamespace(entry.namespace, id); const identity = { namespace: entry.namespace, @@ -80,11 +98,57 @@ export async function pushWorkspaceToHub( if (version !== '1.0.0') { throw new Error('A narrative document without calmHubDocumentId must use version 1.0.0.'); } + + if (createRecovery !== undefined) { + if (sha256(narrative.request.documentMarkdown) !== createRecovery.documentMarkdownSha256) { + throw new Error('Narrative document Markdown changed while create recovery is pending. Restore the submitted content or reconcile it explicitly.'); + } + const recovered = await recoverCreatedNarrativeDocument( + client, identity, createRecovery.documentIdsBeforeCreate, narrative.request.documentMarkdown + ); + manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); + await saveManifest(bundlePath, manifest); + logger.info(`Recovered '${id}' version ${version} -> ${recovered.location}`); + continue; + } + + const documentIdsBeforeCreate = await client.getNarrativeDocumentIds(identity.namespace, identity.type); const location = await client.createNarrativeDocument(identity.namespace, identity.type, narrative.request); - const documentId = parseNarrativeDocumentLocation(location, identity); - manifest[id] = { ...entry, calmHubDocumentId: documentId, calmHubId: location }; + + let documentId: number | undefined; + if (location !== undefined) { + try { + documentId = parseNarrativeDocumentLocation(location, identity); + } catch { + // The POST succeeded, but the Location cannot establish the document identity. + } + } + + if (documentId !== undefined && location !== undefined) { + manifest[id] = publishNarrativeEntry(entry, documentId, location); + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${location}`); + continue; + } + + manifest[id] = { + path: entry.path, + type: entry.type, + ...(entry.namespace === undefined ? {} : { namespace: entry.namespace }), + version, + createRecovery: { + documentIdsBeforeCreate, + documentMarkdownSha256: sha256(narrative.request.documentMarkdown), + }, + }; await saveManifest(bundlePath, manifest); - logger.info(`Pushed '${id}' version ${version} -> ${location}`); + + const recovered = await recoverCreatedNarrativeDocument( + client, identity, documentIdsBeforeCreate, narrative.request.documentMarkdown + ); + manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${recovered.location}`); continue; } @@ -205,3 +269,76 @@ export async function pushWorkspaceToHub( ); } } + +function getCreateRecovery(entry: NarrativeWorkspaceManifestEntry): NarrativeCreateRecovery | undefined { + const createRecovery = (entry as unknown as Record).createRecovery; + if (createRecovery === undefined) return undefined; + + if (entry.calmHubDocumentId !== undefined || entry.calmHubId !== undefined) { + throw new Error('Narrative document cannot be both published and pending create recovery.'); + } + if (!createRecovery || typeof createRecovery !== 'object' || Array.isArray(createRecovery)) { + throw new Error('Narrative document createRecovery must contain a valid documentIdsBeforeCreate array.'); + } + const documentIdsBeforeCreate = (createRecovery as Record).documentIdsBeforeCreate; + if (!Array.isArray(documentIdsBeforeCreate) || + !documentIdsBeforeCreate.every((documentId: unknown) => Number.isSafeInteger(documentId) && (documentId as number) > 0)) { + throw new Error('Narrative document createRecovery documentIdsBeforeCreate must contain only positive safe integers.'); + } + const documentMarkdownSha256 = (createRecovery as Record).documentMarkdownSha256; + if (typeof documentMarkdownSha256 !== 'string' || !/^[0-9a-f]{64}$/.test(documentMarkdownSha256)) { + throw new Error('Narrative document createRecovery documentMarkdownSha256 must be a lowercase SHA-256 hex digest.'); + } + return { documentIdsBeforeCreate, documentMarkdownSha256 }; +} + +function sha256(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function publishNarrativeEntry( + entry: NarrativeWorkspaceManifestEntry, + documentId: number, + location: string +): PublishedNarrativeWorkspaceManifestEntry { + return { + path: entry.path, + type: entry.type, + ...(entry.namespace === undefined ? {} : { namespace: entry.namespace }), + version: entry.version, + calmHubDocumentId: documentId, + calmHubId: location, + }; +} + +async function recoverCreatedNarrativeDocument( + client: CalmHubClient, + identity: NarrativeDocumentIdentity, + documentIdsBeforeCreate: number[], + documentMarkdown: string +): Promise<{ documentId: number; location: string }> { + const documentIdsAfterCreate = await client.getNarrativeDocumentIds(identity.namespace, identity.type); + const existingIds = new Set(documentIdsBeforeCreate); + const candidateIds = [...new Set(documentIdsAfterCreate.filter((documentId) => !existingIds.has(documentId)))]; + const matchingIds: number[] = []; + + for (const candidateId of candidateIds) { + const candidate = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, candidateId, '1.0.0' + ); + if (candidate.documentMarkdown === documentMarkdown) { + matchingIds.push(candidateId); + } + } + + if (matchingIds.length === 0) { + throw new Error('Could not recover the created narrative document: no new document has matching Markdown.'); + } + if (matchingIds.length > 1) { + throw new Error('Could not recover the created narrative document: multiple new documents have matching Markdown.'); + } + + const documentId = matchingIds[0]; + const location = constructNarrativeDocumentPath({ ...identity, calmHubDocumentId: documentId }); + return { documentId, location }; +} diff --git a/shared/src/hub/calm-hub-client.spec.ts b/shared/src/hub/calm-hub-client.spec.ts index 206d7311c..f45d54395 100644 --- a/shared/src/hub/calm-hub-client.spec.ts +++ b/shared/src/hub/calm-hub-client.spec.ts @@ -26,13 +26,43 @@ describe('CalmHubClient', () => { expect(mock.history.post[0].data).toBe(JSON.stringify(request)); }); - it('rejects a create response without Location', async () => { + it('returns undefined after a confirmed create response without Location', async () => { mock.onPost('/api/calm/namespaces/finos/documents/sad').reply(201, null, {}); + await expect(client.createNarrativeDocument('finos', 'sad', request)).resolves.toBeUndefined(); + }); + + it('still rejects a genuine narrative create failure', async () => { + const endpoint = '/api/calm/namespaces/finos/documents/sad'; + mock.onPost(endpoint).reply(500, { error: 'unavailable' }); + await expect(client.createNarrativeDocument('finos', 'sad', request)).rejects.toMatchObject({ - request: 'POST /api/calm/namespaces/finos/documents/sad', + status: 500, + request: `POST ${endpoint}`, }); }); + it('lists narrative document IDs from the type-scoped endpoint', async () => { + const endpoint = '/api/calm/namespaces/finos/documents/sad'; + mock.onGet(endpoint).reply(200, { values: [1, 2, 42] }); + + await expect(client.getNarrativeDocumentIds('finos', 'sad')).resolves.toEqual([1, 2, 42]); + expect(mock.history.get[0].url).toBe(endpoint); + }); + + it.each([ + {}, + { values: '1' }, + { values: [0] }, + { values: [-1] }, + { values: [1.5] }, + { values: [Number.MAX_SAFE_INTEGER + 1] }, + { values: ['1'] }, + ])('rejects a malformed narrative document ID list: %j', async (body) => { + mock.onGet('/api/calm/namespaces/finos/documents/sad').reply(200, body); + + await expect(client.getNarrativeDocumentIds('finos', 'sad')).rejects.toBeInstanceOf(HubClientError); + }); + it('creates a typed later version at the version endpoint', async () => { const endpoint = '/api/calm/namespaces/finos/documents/knowledge/42/versions/1.1.0'; mock.onPost(endpoint).reply(201, null, { location: endpoint }); @@ -202,6 +232,15 @@ describe('CalmHubClient', () => { expect(authMock.history.post[0].headers?.Authorization).toBe('Bearer test-token'); }); + it('injects auth headers when listing narrative document IDs', async () => { + authMock.onGet('/api/calm/namespaces/finos/documents/sad').reply(200, { values: [] }); + + await authClient.getNarrativeDocumentIds('finos', 'sad'); + + expect(getAuthHeaders).toHaveBeenCalledOnce(); + expect(authMock.history.get[0].headers?.Authorization).toBe('Bearer test-token'); + }); + it('does not call getAuthHeaders when no auth plugin is configured', async () => { mock.onGet('/api/calm/namespaces').reply(200, { values: [] }); diff --git a/shared/src/hub/calm-hub-client.ts b/shared/src/hub/calm-hub-client.ts index 5e5ea717f..c4e6821dc 100644 --- a/shared/src/hub/calm-hub-client.ts +++ b/shared/src/hub/calm-hub-client.ts @@ -133,9 +133,14 @@ export class CalmHubClient { namespace: string, type: NarrativeDocumentType, request: NarrativeDocumentRequest - ): Promise { + ): Promise { const endpoint = `/api/calm/namespaces/${namespace}/documents/${type}`; - return this.createNarrativeDocumentAt(endpoint, request, `POST ${endpoint}`); + try { + const response = await this.ax.post(endpoint, request); + return response.headers.location as string | undefined; + } catch (err) { + throw this.wrapError(err, `POST ${endpoint}`); + } } async createNarrativeDocumentVersion( @@ -149,6 +154,21 @@ export class CalmHubClient { return this.createNarrativeDocumentAt(endpoint, request, `POST ${endpoint}`); } + async getNarrativeDocumentIds(namespace: string, type: NarrativeDocumentType): Promise { + const endpoint = `/api/calm/namespaces/${namespace}/documents/${type}`; + try { + const response = await this.ax.get(endpoint); + if (!response.data || typeof response.data !== 'object' || !Array.isArray(response.data.values) || + !response.data.values.every((value: unknown) => Number.isSafeInteger(value) && (value as number) > 0)) { + throw new HubClientError(0, 'Response does not contain a positive integer values array', `GET ${endpoint}`); + } + return response.data.values; + } catch (err) { + if (err instanceof HubClientError) throw err; + throw this.wrapError(err, `GET ${endpoint}`); + } + } + async getNarrativeDocumentVersions(namespace: string, type: NarrativeDocumentType, id: number): Promise { const endpoint = `/api/calm/namespaces/${namespace}/documents/${type}/${id}/versions`; try { From d19843d1c6da30d557d7ac35bd0263f828321d3e Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Thu, 17 Sep 2026 16:30:17 +0100 Subject: [PATCH 19/28] refactor(cli): centralize narrative entry validation --- .../command-helpers/workspace/bump.spec.ts | 19 ++++++ cli/src/command-helpers/workspace/bump.ts | 23 ++----- .../workspace/narrative-document.spec.ts | 66 ++++++++++++++++++ .../workspace/narrative-document.ts | 67 +++++++++++++++++++ .../command-helpers/workspace/push.spec.ts | 6 +- cli/src/command-helpers/workspace/push.ts | 35 +++------- 6 files changed, 171 insertions(+), 45 deletions(-) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index 996c40324..6c1bc6d8b 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -71,6 +71,25 @@ describe('bump', () => { expect(await detectChangedResources(bundlePath, makeClient({ narrativeVersions: ['1.0.0'], narrativeMarkdown: markdown }))).toEqual([]); }); + it('treats pending create recovery as unassigned without mutating the fence', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Published\n'; + const entry = { + path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', + createRecovery: { + documentIdsBeforeCreate: [1, 2], + documentMarkdownSha256: 'a'.repeat(64), + }, + }; + await writeFile(path.join(filesPath, 'payments.md'), markdown); + await saveManifest(bundlePath, { payments: entry }); + const client = makeClient(); + + expect(await detectChangedResources(bundlePath, client)).toEqual([]); + + expect(client.getNarrativeDocumentVersions).not.toHaveBeenCalled(); + expect(await loadManifest(bundlePath)).toEqual({ payments: entry }); + }); + it('fails narrative checks with incomplete identity or missing source', async () => { await saveManifest(bundlePath, { partial: { path: 'files/missing.md', type: 'sad', namespace: 'com.example', version: '1.0.0', calmHubId: '/partial' }, diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index e17e5a5bc..4d9374b87 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -14,7 +14,7 @@ import { initLogger, Logger, } from '@finos/calm-shared'; -import { NarrativeDocumentIdentity, parseNarrativeDocument, validateNarrativeDocumentLocation, validateNarrativeIdentity, validateNarrativeNamespace } from './narrative-document'; +import { resolveNarrativeEntry, validateNarrativeDocumentLocation } from './narrative-document'; // Re-exported for existing consumers (push.ts, tests) that import it from here. export { canonicalEqual }; @@ -113,26 +113,13 @@ export async function detectChangedResources( if (isNarrativeWorkspaceManifestEntry(entry)) { // Bump stops on invalid narrative state because it writes local manifest versions; push can report independent failures together. - const version = entry.version; - if (!version) throw new Error(`Narrative document '${id}' has no manifest version.`); - if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { - throw new Error(`Narrative document '${id}' has incomplete Hub identity. Re-add the document to repair it.`); - } - validateNarrativeNamespace(entry.namespace, id); - const identity: NarrativeDocumentIdentity = { - namespace: entry.namespace, type: entry.type, version, calmHubDocumentId: entry.calmHubDocumentId, - }; - parseNarrativeDocument(raw, id); - if (entry.calmHubDocumentId === undefined) { - validateNarrativeIdentity(identity, false, id); - continue; - } - validateNarrativeIdentity(identity, true, id); + const { version, identity, hubIdentityAssigned } = resolveNarrativeEntry(id, entry, raw); + if (!hubIdentityAssigned) continue; validateNarrativeDocumentLocation(entry.calmHubId, identity, false); - const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId!); + const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId); if (versions.length === 0 || !versions.includes(version)) continue; const remote = await client.getNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId!, version + identity.namespace, identity.type, identity.calmHubDocumentId, version ); if (remote.documentMarkdown === raw) continue; changed.push({ diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index a1be73d07..fb2cde7ae 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -4,12 +4,78 @@ import { constructNarrativeDocumentPath, parseNarrativeDocument, parseNarrativeDocumentLocation, + resolveNarrativeEntry, validateNarrativeDocumentLocation, validateNarrativeIdentity, } from './narrative-document'; describe('narrative document helpers', () => { const identity = { namespace: 'finos', type: 'sad' as const, version: '1.0.0' }; + const markdown = '---\ntitle: Payments SAD\ndescription: Decisions\n---\n# Content\n'; + + describe('resolveNarrativeEntry', () => { + it('resolves a valid unpublished narrative', () => { + const resolved = resolveNarrativeEntry('payments', identity, markdown); + + expect(resolved).toMatchObject({ + version: '1.0.0', + hubIdentityAssigned: false, + identity, + narrative: { + request: { name: 'Payments SAD', description: 'Decisions', documentMarkdown: markdown }, + }, + }); + expect(resolved.identity.calmHubDocumentId).toBeUndefined(); + }); + + it('resolves a valid published narrative', () => { + const resolved = resolveNarrativeEntry('payments', { + ...identity, + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/1.0.0', + }, markdown); + + expect(resolved.hubIdentityAssigned).toBe(true); + expect(resolved.identity.calmHubDocumentId).toBe(42); + expect(resolved.narrative.request.documentMarkdown).toBe(markdown); + }); + + it('rejects a missing manifest version', () => { + expect(() => resolveNarrativeEntry('payments', { + type: 'sad', namespace: 'finos', + }, markdown)).toThrow(/Narrative document 'payments' has no manifest version\./); + }); + + it.each([ + ['document ID only', { ...identity, calmHubDocumentId: 42 }], + ['Location only', { ...identity, calmHubId: '/stored-location' }], + ])('rejects incomplete Hub identity with %s', (_case, entry) => { + expect(() => resolveNarrativeEntry('payments', entry, markdown)).toThrow( + /Narrative document 'payments' has incomplete Hub identity\. Re-add the document to repair it\./ + ); + }); + + it.each([ + [{ ...identity, namespace: 'not_valid' }, /valid namespace/], + [{ ...identity, type: 'unsupported' }, /unsupported type/], + [{ ...identity, version: 'latest' }, /major.minor.patch/], + [{ ...identity, calmHubDocumentId: 0, calmHubId: '/stored-location' }, /positive integer/], + ])('rejects invalid identity data %#', (entry, message) => { + expect(() => resolveNarrativeEntry('payments', entry, markdown)).toThrow(message); + }); + + it('parses Markdown before validating the constructed identity', () => { + expect(() => resolveNarrativeEntry( + 'payments', { ...identity, version: 'latest' }, '# No frontmatter' + )).toThrow(/must contain non-empty YAML mapping frontmatter/); + }); + + it('rejects malformed narrative Markdown', () => { + expect(() => resolveNarrativeEntry('payments', identity, '# No frontmatter')).toThrow( + /must contain non-empty YAML mapping frontmatter/ + ); + }); + }); it('uses frontmatter title and preserves CRLF Markdown', () => { const markdown = '---\r\ntitle: Payments SAD\r\ndescription: Decisions\r\n---\r\n# Content\r\n'; diff --git a/cli/src/command-helpers/workspace/narrative-document.ts b/cli/src/command-helpers/workspace/narrative-document.ts index d0394d06f..5afd3dbfe 100644 --- a/cli/src/command-helpers/workspace/narrative-document.ts +++ b/cli/src/command-helpers/workspace/narrative-document.ts @@ -19,6 +19,73 @@ export interface ParsedNarrativeDocument { request: NarrativeDocumentRequest; } +export interface NarrativeEntryInput { + type: unknown; + namespace?: unknown; + version?: unknown; + calmHubDocumentId?: unknown; + calmHubId?: unknown; +} + +export type ResolvedNarrativeEntry = { + version: string; + narrative: ParsedNarrativeDocument; +} & ( + | { + hubIdentityAssigned: false; + identity: NarrativeDocumentIdentity & { calmHubDocumentId?: undefined }; + } + | { + hubIdentityAssigned: true; + identity: NarrativeDocumentIdentity & { calmHubDocumentId: number }; + } +); + +export function resolveNarrativeEntry( + id: string, + entry: NarrativeEntryInput, + raw: string +): ResolvedNarrativeEntry { + if (!entry.version) { + throw new Error(`Narrative document '${id}' has no manifest version.`); + } + + const hubIdentityAssigned = entry.calmHubDocumentId !== undefined; + if (hubIdentityAssigned !== (entry.calmHubId !== undefined)) { + throw new Error(`Narrative document '${id}' has incomplete Hub identity. Re-add the document to repair it.`); + } + + const namespace = entry.namespace; + validateNarrativeNamespace(namespace, id); + const identity = { + namespace, + type: entry.type, + version: entry.version, + ...(hubIdentityAssigned ? { calmHubDocumentId: entry.calmHubDocumentId } : {}), + }; + const narrative = parseNarrativeDocument(raw, id); + validateNarrativeIdentity(identity, hubIdentityAssigned, id); + + if (hubIdentityAssigned) { + return { + version: identity.version, + identity: identity as NarrativeDocumentIdentity & { calmHubDocumentId: number }, + narrative, + hubIdentityAssigned: true, + }; + } + return { + version: identity.version, + identity: { + namespace: identity.namespace, + type: identity.type, + version: identity.version, + }, + narrative, + hubIdentityAssigned: false, + }; +} + export function parseNarrativeDocument(markdown: string, label: string): ParsedNarrativeDocument { let frontMatter: Record | null; try { diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index f5adbe05a..fb6aa041b 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -395,8 +395,8 @@ describe('pushWorkspaceToHub', () => { { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'not-a-digest' }, { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'A'.repeat(64) }, { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'a'.repeat(63) }, - ])('rejects malformed persisted create recovery without posting: %j', async (createRecovery) => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + ])('rejects malformed persisted create recovery before parsing or posting: %j', async (createRecovery) => { + const markdown = '# no frontmatter'; await writeFreshNarrative(markdown); await saveManifest(bundlePath, { payments: { @@ -500,7 +500,7 @@ describe('pushWorkspaceToHub', () => { partial: { path: 'files/partial.md', type: 'sad', namespace: 'com.example', version: '1.0.0', calmHubId: '/partial' }, }); await writeFile(path.join(filesPath, 'partial.md'), '---\ntitle: Partial\n---\n# Partial'); - await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/narrative document/); + await expect(pushWorkspaceToHub(bundlePath, makeClient())).rejects.toThrow(/incomplete Hub identity/); }); it('rejects malformed persisted narrative identity before calling Hub', async () => { diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index 930bd7955..146240ff8 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -20,12 +20,10 @@ import { import { canonicalEqual } from './bump'; import { constructNarrativeDocumentPath, - parseNarrativeDocument, parseNarrativeDocumentLocation, + resolveNarrativeEntry, type NarrativeDocumentIdentity, validateNarrativeDocumentLocation, - validateNarrativeIdentity, - validateNarrativeNamespace, } from './narrative-document'; const logger: Logger = initLogger(false, 'workspace'); @@ -78,23 +76,12 @@ export async function pushWorkspaceToHub( if (isNarrativeWorkspaceManifestEntry(entry)) { try { - const version = entry.version; - if (!version) throw new Error('Narrative document manifest entry has no version. Re-add the document to repair it.'); - if ((entry.calmHubId === undefined) !== (entry.calmHubDocumentId === undefined)) { - throw new Error('Narrative document Hub identity is incomplete. Re-add the document to repair it.'); - } const createRecovery = getCreateRecovery(entry); - validateNarrativeNamespace(entry.namespace, id); - const identity = { - namespace: entry.namespace, - type: entry.type, - version, - calmHubDocumentId: entry.calmHubDocumentId, - }; - const narrative = parseNarrativeDocument(raw, id); - - if (entry.calmHubDocumentId === undefined) { - validateNarrativeIdentity(identity, false, id); + const resolved = resolveNarrativeEntry(id, entry, raw); + const { version, narrative } = resolved; + + if (!resolved.hubIdentityAssigned) { + const { identity } = resolved; if (version !== '1.0.0') { throw new Error('A narrative document without calmHubDocumentId must use version 1.0.0.'); } @@ -152,17 +139,17 @@ export async function pushWorkspaceToHub( continue; } - validateNarrativeIdentity(identity, true, id); + const { identity } = resolved; validateNarrativeDocumentLocation(entry.calmHubId, identity, false); const versions = await client.getNarrativeDocumentVersions( - identity.namespace, identity.type, identity.calmHubDocumentId! + identity.namespace, identity.type, identity.calmHubDocumentId ); if (!versions.includes(version)) { const location = await client.createNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId!, version, narrative.request + identity.namespace, identity.type, identity.calmHubDocumentId, version, narrative.request ); validateNarrativeDocumentLocation(location, identity); - manifest[id] = { ...entry, calmHubId: location }; + manifest[id] = publishNarrativeEntry(entry, identity.calmHubDocumentId, location); await saveManifest(bundlePath, manifest); logger.info(`Pushed '${id}' version ${version} -> ${location}`); continue; @@ -173,7 +160,7 @@ export async function pushWorkspaceToHub( continue; } const remote = await client.getNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId!, version + identity.namespace, identity.type, identity.calmHubDocumentId, version ); if (remote.documentMarkdown !== raw) { logger.error(`'${id}' version ${version} already exists in CalmHub but differs on disk. Bump it before pushing.`); From 615d99d9384aaae1815e8d08f3d9df6fc46b3c3a Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Fri, 18 Sep 2026 08:45:46 +0100 Subject: [PATCH 20/28] fix(cli): fail workspace push on mapping errors --- .../command-helpers/workspace/push.spec.ts | 85 ++++++++++++++++--- cli/src/command-helpers/workspace/push.ts | 18 +++- 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index fb6aa041b..5a596e7da 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -653,11 +653,14 @@ describe('pushWorkspaceToHub', () => { .mockResolvedValueOnce(mappingId('doc-b')), }); - await expect(pushWorkspaceToHub(bundlePath, client)).resolves.not.toThrow(); + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow( + /mapping document\(s\) failed \(doc-a: create failed\)/ + ); expect(client.createMappedResourceVersion).toHaveBeenCalledTimes(2); + expect((await loadManifest(bundlePath))['doc-b'].calmHubId).toBe(mappingId('doc-b')); }); - it('logs error and continues when fetching existing versions fails', async () => { + it('fails after processing later entries when fetching existing versions fails', async () => { await writeFile(path.join(filesPath, 'doc-a.json'), JSON.stringify(docA)); await writeFile(path.join(filesPath, 'doc-b.json'), JSON.stringify(docB)); await saveManifest(bundlePath, { @@ -668,14 +671,71 @@ describe('pushWorkspaceToHub', () => { getMappedResourceVersions: vi.fn() .mockRejectedValueOnce(new HubClientError(500, 'Internal Server Error', 'GET ...')) .mockResolvedValueOnce([]), + createMappedResourceVersion: vi.fn().mockResolvedValue(mappingId('doc-b')), }); - await expect(pushWorkspaceToHub(bundlePath, client)).resolves.not.toThrow(); + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/doc-a: .*Internal Server Error/); expect(client.createMappedResourceVersion).toHaveBeenCalledTimes(1); expect(client.createMappedResourceVersion).toHaveBeenCalledWith( expect.objectContaining({ mapping: 'doc-b' }), JSON.stringify(docB) ); + expect((await loadManifest(bundlePath))['doc-b'].calmHubId).toBe(mappingId('doc-b')); + }); + + it('reports multiple mapping failures without losing document details', async () => { + await writeFile(path.join(filesPath, 'doc-a.json'), JSON.stringify(docA)); + await writeFile(path.join(filesPath, 'doc-b.json'), JSON.stringify(docB)); + await saveManifest(bundlePath, { + 'doc-a': { path: 'files/doc-a.json', type: 'architecture', namespace: 'com.example' }, + 'doc-b': { path: 'files/doc-b.json', type: 'architecture', namespace: 'com.example' }, + }); + const client = makeClient({ + createMappedResourceVersion: vi.fn() + .mockRejectedValueOnce(new Error('first create failed')) + .mockRejectedValueOnce(new Error('second create failed')), + }); + + const push = pushWorkspaceToHub(bundlePath, client); + await expect(push).rejects.toThrow(/doc-a: first create failed/); + await expect(push).rejects.toThrow(/doc-b: second create failed/); + expect(client.createMappedResourceVersion).toHaveBeenCalledTimes(2); + }); + + it('reports mapping and narrative failures together', async () => { + await writeFile(path.join(filesPath, 'doc-a.json'), JSON.stringify(docA)); + await writeFile(path.join(filesPath, 'bad.md'), '# no frontmatter'); + await saveManifest(bundlePath, { + 'doc-a': { path: 'files/doc-a.json', type: 'architecture', namespace: 'com.example' }, + bad: { path: 'files/bad.md', type: 'sad', namespace: 'com.example', version: '1.0.0' }, + }); + const client = makeClient({ + createMappedResourceVersion: vi.fn().mockRejectedValue(new Error('mapping unavailable')), + }); + + const push = pushWorkspaceToHub(bundlePath, client); + await expect(push).rejects.toThrow(/mapping document\(s\) failed \(doc-a: mapping unavailable\)/); + await expect(push).rejects.toThrow(/narrative document\(s\) failed \(bad:/); + }); + + it('reports mapping failures and modified-version conflicts together', async () => { + const changedDocB = { ...docB, extra: 'edited' }; + await writeFile(path.join(filesPath, 'doc-a.json'), JSON.stringify(docA)); + await writeFile(path.join(filesPath, 'doc-b.json'), JSON.stringify(changedDocB)); + await saveManifest(bundlePath, { + 'doc-a': { path: 'files/doc-a.json', type: 'architecture', namespace: 'com.example' }, + 'doc-b': { path: 'files/doc-b.json', type: 'architecture', namespace: 'com.example' }, + }); + const client = makeClient({ + getMappedResourceVersions: vi.fn(async (_namespace: string, resource: string) => + resource === 'doc-b' ? ['1.0.0'] : []), + getMappedResourceByVersion: vi.fn().mockResolvedValue(docB), + createMappedResourceVersion: vi.fn().mockRejectedValue(new Error('create unavailable')), + }); + + const push = pushWorkspaceToHub(bundlePath, client, { failIfModified: true }); + await expect(push).rejects.toThrow(/doc-b@1\.0\.0/); + await expect(push).rejects.toThrow(/mapping document\(s\) failed \(doc-a: create unavailable\)/); }); describe('failIfModified (strict merge-time push)', () => { @@ -709,21 +769,26 @@ describe('pushWorkspaceToHub', () => { expect(client.createMappedResourceVersion).not.toHaveBeenCalled(); }); - it('skips and does not fail when fetching the published version to compare fails', async () => { + it('fails after processing later entries when fetching the published version to compare fails', async () => { await writeFile(path.join(filesPath, 'doc-a.json'), JSON.stringify({ ...docA, extra: 'edited' })); + await writeFile(path.join(filesPath, 'doc-b.json'), JSON.stringify(docB)); await saveManifest(bundlePath, { - 'doc-a': { path: 'files/doc-a.json', type: 'architecture', namespace: 'com.example' } + 'doc-a': { path: 'files/doc-a.json', type: 'architecture', namespace: 'com.example' }, + 'doc-b': { path: 'files/doc-b.json', type: 'architecture', namespace: 'com.example' }, }); const client = makeClient({ - getMappedResourceVersions: vi.fn().mockResolvedValue(['1.0.0']), + getMappedResourceVersions: vi.fn(async (_namespace: string, resource: string) => + resource === 'doc-a' ? ['1.0.0'] : []), getMappedResourceByVersion: vi.fn().mockRejectedValue(new Error('boom')), + createMappedResourceVersion: vi.fn().mockResolvedValue(mappingId('doc-b')), }); - await expect(pushWorkspaceToHub(bundlePath, client, { failIfModified: true })).resolves.not.toThrow(); - expect(client.createMappedResourceVersion).not.toHaveBeenCalled(); + await expect(pushWorkspaceToHub(bundlePath, client, { failIfModified: true })).rejects.toThrow(/doc-a: boom/); + expect(client.createMappedResourceVersion).toHaveBeenCalledOnce(); + expect((await loadManifest(bundlePath))['doc-b'].calmHubId).toBe(mappingId('doc-b')); }); - it('skips when the compare fetch rejects with a non-Error value', async () => { + it('reports a non-Error comparison failure', async () => { await writeFile(path.join(filesPath, 'doc-a.json'), JSON.stringify({ ...docA, extra: 'edited' })); await saveManifest(bundlePath, { 'doc-a': { path: 'files/doc-a.json', type: 'architecture', namespace: 'com.example' } @@ -733,7 +798,7 @@ describe('pushWorkspaceToHub', () => { getMappedResourceByVersion: vi.fn().mockRejectedValue('boom-string'), }); - await expect(pushWorkspaceToHub(bundlePath, client, { failIfModified: true })).resolves.not.toThrow(); + await expect(pushWorkspaceToHub(bundlePath, client, { failIfModified: true })).rejects.toThrow(/doc-a: boom-string/); expect(client.createMappedResourceVersion).not.toHaveBeenCalled(); }); diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index 146240ff8..e45b4b9de 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -54,6 +54,7 @@ export async function pushWorkspaceToHub( } const conflicts: string[] = []; + const mappingFailures: string[] = []; const narrativeFailures: string[] = []; for (const [id, entry] of entries) { @@ -201,7 +202,9 @@ export async function pushWorkspaceToHub( try { existingVersions = await client.getMappedResourceVersions(namespace, mappingId, resourceType); } catch (e) { - logger.error(`Failed to fetch existing versions for '${id}' from CalmHub: ${e instanceof Error ? e.message : String(e)}`); + const message = e instanceof Error ? e.message : String(e); + logger.error(`Failed to fetch existing versions for '${id}' from CalmHub: ${message}`); + mappingFailures.push(`${id}: ${message}`); continue; } @@ -217,7 +220,9 @@ export async function pushWorkspaceToHub( try { remote = await client.getMappedResourceByVersion(namespace, mappingId, version, resourceType); } catch (e) { - logger.error(`Failed to fetch '${id}' @ ${version} from CalmHub to compare: ${e instanceof Error ? e.message : String(e)}`); + const message = e instanceof Error ? e.message : String(e); + logger.error(`Failed to fetch '${id}' @ ${version} from CalmHub to compare: ${message}`); + mappingFailures.push(`${id}: ${message}`); continue; } @@ -236,11 +241,13 @@ export async function pushWorkspaceToHub( await saveManifest(bundlePath, manifest); logger.info(`Pushed '${id}' version ${version} -> ${calmHubId}`); } catch (e) { - logger.error(`Failed to push '${id}': ${e instanceof Error ? e.message : String(e)}`); + const message = e instanceof Error ? e.message : String(e); + logger.error(`Failed to push '${id}': ${message}`); + mappingFailures.push(`${id}: ${message}`); } } - if (conflicts.length > 0 || narrativeFailures.length > 0) { + if (conflicts.length > 0 || mappingFailures.length > 0 || narrativeFailures.length > 0) { const summaries: string[] = []; if (conflicts.length > 0) { summaries.push( @@ -248,6 +255,9 @@ export async function pushWorkspaceToHub( `(${conflicts.join(', ')}). Run \`calm workspace bump\` to create new versions for them.` ); } + if (mappingFailures.length > 0) { + summaries.push(`${mappingFailures.length} mapping document(s) failed (${mappingFailures.join('; ')})`); + } if (narrativeFailures.length > 0) { summaries.push(`${narrativeFailures.length} narrative document(s) failed (${narrativeFailures.join('; ')})`); } From 7c5860eebd7eece4e15e67b5aef16b783f60b454 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Fri, 18 Sep 2026 09:23:15 +0100 Subject: [PATCH 21/28] refactor(cli): centralize workspace document dispatch --- cli/src/command-helpers/workspace/bump.ts | 166 +++++--- cli/src/command-helpers/workspace/bundle.ts | 9 +- cli/src/command-helpers/workspace/commands.ts | 232 ++++++----- .../workspace/document-kind.spec.ts | 83 ++++ .../workspace/document-kind.ts | 129 ++++++ cli/src/command-helpers/workspace/push.ts | 379 ++++++++++-------- .../workspace/ref-rewrite.spec.ts | 27 ++ .../command-helpers/workspace/ref-rewrite.ts | 21 +- 8 files changed, 708 insertions(+), 338 deletions(-) create mode 100644 cli/src/command-helpers/workspace/document-kind.spec.ts create mode 100644 cli/src/command-helpers/workspace/document-kind.ts diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index 4d9374b87..b5bd290e4 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -1,6 +1,12 @@ import { readFile, writeFile } from 'fs/promises'; import { existsSync } from 'fs'; -import { isNarrativeWorkspaceManifestEntry, loadManifest, resolveFilePath, saveManifest } from './bundle'; +import { + loadManifest, + resolveFilePath, + saveManifest, + type MappingWorkspaceManifestEntry, + type NarrativeWorkspaceManifestEntry, +} from './bundle'; import { buildRefRulesFromDiskIds, syncReferences, RefUpdateResult } from './ref-rewrite'; import { CalmHubClient, @@ -15,6 +21,11 @@ import { Logger, } from '@finos/calm-shared'; import { resolveNarrativeEntry, validateNarrativeDocumentLocation } from './narrative-document'; +import { + dispatchWorkspaceManifestEntry, + resolveWorkspaceManifestEntry, + type WorkspaceManifestEntryOperations, +} from './document-kind'; // Re-exported for existing consumers (push.ts, tests) that import it from here. export { canonicalEqual }; @@ -69,6 +80,18 @@ export interface BumpOptions { getCascadeIncrement?: (docId: string, triggeredBy: string, defaultIncrement: ResourceChangeType) => Promise; } +interface DetectChangedEntryContext { + client: CalmHubClient; + filePath: string; + id: string; + raw: string; +} + +const DETECT_CHANGED_ENTRY_OPERATIONS = { + mapping: detectChangedMappingEntry, + narrative: detectChangedNarrativeEntry, +} satisfies WorkspaceManifestEntryOperations, [DetectChangedEntryContext]>; + /** Returns the highest-priority increment from a list (MAJOR > MINOR > PATCH). */ export function maxIncrement(increments: ResourceChangeType[]): ResourceChangeType { if (increments.includes('MAJOR')) return 'MAJOR'; @@ -95,9 +118,10 @@ export async function detectChangedResources( const changed: ChangedResource[] = []; for (const [id, entry] of Object.entries(manifest)) { + const document = resolveWorkspaceManifestEntry(entry); const filePath = resolveFilePath(bundlePath, entry.path); if (!existsSync(filePath)) { - if (isNarrativeWorkspaceManifestEntry(entry)) throw new Error(`Narrative document '${id}' file not found: ${filePath}`); + if (document.handler.unreadableFile === 'fail') throw new Error(`Narrative document '${id}' file not found: ${filePath}`); logger.warn(`File not found for id '${id}': ${filePath}`); continue; } @@ -106,73 +130,89 @@ export async function detectChangedResources( try { raw = await readFile(filePath, 'utf8'); } catch (e) { - if (isNarrativeWorkspaceManifestEntry(entry)) throw new Error(`Narrative document '${id}' could not be read: ${e instanceof Error ? e.message : String(e)}`); + if (document.handler.unreadableFile === 'fail') throw new Error(`Narrative document '${id}' could not be read: ${e instanceof Error ? e.message : String(e)}`); logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); continue; } - if (isNarrativeWorkspaceManifestEntry(entry)) { - // Bump stops on invalid narrative state because it writes local manifest versions; push can report independent failures together. - const { version, identity, hubIdentityAssigned } = resolveNarrativeEntry(id, entry, raw); - if (!hubIdentityAssigned) continue; - validateNarrativeDocumentLocation(entry.calmHubId, identity, false); - const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId); - if (versions.length === 0 || !versions.includes(version)) continue; - const remote = await client.getNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId, version - ); - if (remote.documentMarkdown === raw) continue; - changed.push({ - id, filePath, currentVersion: version, - latestHubVersion: sortSemVer(versions)[versions.length - 1], kind: 'narrative', - }); - continue; - } + const changedResource = await dispatchWorkspaceManifestEntry( + document, + DETECT_CHANGED_ENTRY_OPERATIONS, + { client, filePath, id, raw } + ); + if (changedResource) changed.push(changedResource); + } - let metadata: DocumentMetadata; - try { - metadata = extractDocumentMetadata(raw); - } catch (e) { - logger.warn(`Skipping '${id}': not mappable to CalmHub (${e instanceof Error ? e.message : String(e)})`); - continue; - } - if (!metadata.namespace) { - logger.warn(`Skipping '${id}': document $id has no namespace.`); - continue; - } + return changed; +} - let versions: string[]; - try { - versions = await client.getMappedResourceVersions(metadata.namespace, metadata.mapping, metadata.type); - } catch (e) { - logger.error(`Failed to fetch versions for '${id}': ${e instanceof Error ? e.message : String(e)}`); - continue; - } +async function detectChangedNarrativeEntry( + entry: NarrativeWorkspaceManifestEntry, + context: DetectChangedEntryContext +): Promise { + const { client, filePath, id, raw } = context; + // Bump stops on invalid narrative state because it writes local manifest versions; push can report independent failures together. + const { version, identity, hubIdentityAssigned } = resolveNarrativeEntry(id, entry, raw); + if (!hubIdentityAssigned) return undefined; + validateNarrativeDocumentLocation(entry.calmHubId, identity, false); + const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId); + if (versions.length === 0 || !versions.includes(version)) return undefined; + const remote = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId, version + ); + if (remote.documentMarkdown === raw) return undefined; + return { + id, filePath, currentVersion: version, + latestHubVersion: sortSemVer(versions)[versions.length - 1], kind: 'narrative', + }; +} - if (versions.length === 0) continue; // new resource — nothing to bump - if (!versions.includes(metadata.version)) continue; // already ahead — already bumped +async function detectChangedMappingEntry( + _entry: MappingWorkspaceManifestEntry, + context: DetectChangedEntryContext +): Promise { + const { client, filePath, id, raw } = context; + let metadata: DocumentMetadata; + try { + metadata = extractDocumentMetadata(raw); + } catch (e) { + logger.warn(`Skipping '${id}': not mappable to CalmHub (${e instanceof Error ? e.message : String(e)})`); + return undefined; + } + if (!metadata.namespace) { + logger.warn(`Skipping '${id}': document $id has no namespace.`); + return undefined; + } - let remote: object; - try { - remote = await client.getMappedResourceByVersion(metadata.namespace, metadata.mapping, metadata.version, metadata.type); - } catch (e) { - logger.error(`Failed to fetch '${id}' @ ${metadata.version} from CalmHub: ${e instanceof Error ? e.message : String(e)}`); - continue; - } + let versions: string[]; + try { + versions = await client.getMappedResourceVersions(metadata.namespace, metadata.mapping, metadata.type); + } catch (e) { + logger.error(`Failed to fetch versions for '${id}': ${e instanceof Error ? e.message : String(e)}`); + return undefined; + } - if (canonicalEqual(JSON.parse(raw), remote)) continue; // unchanged + if (versions.length === 0) return undefined; // new resource — nothing to bump + if (!versions.includes(metadata.version)) return undefined; // already ahead — already bumped - changed.push({ - id, - filePath, - metadata, - currentVersion: metadata.version, - latestHubVersion: sortSemVer(versions)[versions.length - 1], - kind: 'mapping', - }); + let remote: object; + try { + remote = await client.getMappedResourceByVersion(metadata.namespace, metadata.mapping, metadata.version, metadata.type); + } catch (e) { + logger.error(`Failed to fetch '${id}' @ ${metadata.version} from CalmHub: ${e instanceof Error ? e.message : String(e)}`); + return undefined; } - return changed; + if (canonicalEqual(JSON.parse(raw), remote)) return undefined; + + return { + id, + filePath, + metadata, + currentVersion: metadata.version, + latestHubVersion: sortSemVer(versions)[versions.length - 1], + kind: 'mapping', + }; } /** @@ -203,10 +243,11 @@ export async function bumpWorkspace( const manifest = await loadManifest(bundlePath); const entry = manifest[c.id]; if (!entry) throw new Error(`Narrative document '${c.id}' is no longer in the manifest.`); - if (!isNarrativeWorkspaceManifestEntry(entry)) { + const document = resolveWorkspaceManifestEntry(entry); + if (document.kind !== 'narrative') { throw new Error(`Narrative document '${c.id}' is no longer a narrative manifest entry.`); } - manifest[c.id] = { ...entry, version: toVersion }; + manifest[c.id] = { ...document.entry, version: toVersion }; await saveManifest(bundlePath, manifest); bumped.push({ id: c.id, filePath: c.filePath, fromVersion: c.currentVersion, toVersion, increment: docIncrement }); appliedIncrements.set(c.id, docIncrement); @@ -234,11 +275,8 @@ export async function bumpWorkspace( for (let depth = 0; depth < MAX_CASCADE_DEPTH; depth++) { const manifest = await loadManifest(bundlePath); - const jsonManifest = Object.fromEntries( - Object.entries(manifest).filter(([, entry]) => !isNarrativeWorkspaceManifestEntry(entry)) - ); - const rules = await buildRefRulesFromDiskIds(jsonManifest, bundlePath); - const refUpdates = await syncReferences(bundlePath, jsonManifest, rules); + const rules = await buildRefRulesFromDiskIds(manifest, bundlePath); + const refUpdates = await syncReferences(bundlePath, manifest, rules); allRefUpdates.push(...refUpdates); const cascadeCandidates = refUpdates.filter(r => r.changeCount > 0 && !bumpedIds.has(r.docId)); diff --git a/cli/src/command-helpers/workspace/bundle.ts b/cli/src/command-helpers/workspace/bundle.ts index 9764db6dd..cd04e1df1 100644 --- a/cli/src/command-helpers/workspace/bundle.ts +++ b/cli/src/command-helpers/workspace/bundle.ts @@ -5,6 +5,9 @@ import { JSONPath } from 'jsonpath-plus'; import { printBundleTreeFromGraph } from './tree'; import { isNarrativeDocumentType, type CalmDocumentType, type NarrativeDocumentType } from '@finos/calm-models/types'; import { validateNarrativeDocumentLocation } from './narrative-document'; +import { isNarrativeWorkspaceManifestEntry } from './document-kind'; + +export { isNarrativeWorkspaceManifestEntry } from './document-kind'; /** * Property names that can contain document references (URLs or paths) in CALM JSON. @@ -113,12 +116,6 @@ export type NarrativeWorkspaceManifestEntry = export type WorkspaceManifestEntry = MappingWorkspaceManifestEntry | NarrativeWorkspaceManifestEntry; -export function isNarrativeWorkspaceManifestEntry( - entry: WorkspaceManifestEntry -): entry is NarrativeWorkspaceManifestEntry { - return isNarrativeDocumentType(entry.type); -} - export type WorkspaceManifest = Record; export type DependencyGraph = { diff --git a/cli/src/command-helpers/workspace/commands.ts b/cli/src/command-helpers/workspace/commands.ts index c178db406..d12add356 100644 --- a/cli/src/command-helpers/workspace/commands.ts +++ b/cli/src/command-helpers/workspace/commands.ts @@ -13,10 +13,20 @@ import { loadWorkspaceConfig } from './config'; import { findWorkspaceManifestPath, findGitRoot } from '../../workspace-resolver'; import { initLogger, Logger, CalmHubClient, ResourceChangeType, isConformantDocumentId, namespaceFromDocumentId } from '@finos/calm-shared'; import { select, input } from '@inquirer/prompts'; -import { CALM_DOCUMENT_TYPES_LIST, CALM_NARRATIVE_DOCUMENT_TYPES_LIST, isNarrativeDocumentType, isValidCalmDocumentType } from '@finos/calm-models/types'; +import { + CALM_DOCUMENT_TYPES_LIST, + CALM_NARRATIVE_DOCUMENT_TYPES_LIST, + isValidCalmDocumentType, + type CalmDocumentType, +} from '@finos/calm-models/types'; import { loadCliConfig } from '../../cli-config'; import { resolveCalmHubOptions } from '../hub-commands'; import { constructNarrativeDocumentPath, parseNarrativeDocument, validateNarrativeIdentity, type NarrativeDocumentIdentity } from './narrative-document'; +import { + dispatchWorkspaceDocumentType, + resolveWorkspaceDocumentType, + type WorkspaceDocumentTypeOperations, +} from './document-kind'; const logger: Logger = initLogger(false, 'workspace'); @@ -27,6 +37,23 @@ type NarrativeRegistrationOptions = { verify?: (documentMarkdown: string) => Promise; }; +type WorkspaceAddOptions = { + id?: string; + copy?: boolean; + type?: string; + namespace?: string; + calmHubDocumentId?: string; + ver?: string; + calmHubUrl?: string; +}; + +interface AddDocumentContext { + bundlePath: string; + file: string; + options: WorkspaceAddOptions; + srcPath: string; +} + async function registerNarrativeDocument( bundlePath: string, srcPath: string, @@ -54,6 +81,104 @@ async function registerNarrativeDocument( }); } +const ADD_DOCUMENT_OPERATIONS = { + mapping: addMappingDocument, + narrative: addNarrativeDocument, +} satisfies WorkspaceDocumentTypeOperations, [AddDocumentContext]>; + +async function addNarrativeDocument( + type: NarrativeDocumentIdentity['type'], + context: AddDocumentContext +): Promise { + const { bundlePath, file, options, srcPath } = context; + if (!options.namespace?.trim()) { + throw new Error(`Narrative document '${file}' requires --namespace.`); + } + const { id: resolvedId, destPath: finalDestPath } = await registerNarrativeDocument( + bundlePath, + srcPath, + file, + { + id: options.id, + copy: options.copy, + identity: { + namespace: options.namespace.trim(), + type, + version: '1.0.0', + }, + } + ); + if (options.copy) { + logger.info(`Copied ${srcPath} -> ${finalDestPath} (id: ${resolvedId})`); + } else { + logger.info(`Added reference to ${finalDestPath} (id: ${resolvedId})`); + } +} + +async function addMappingDocument( + type: CalmDocumentType | 'unknown', + context: AddDocumentContext +): Promise { + const { bundlePath, options, srcPath } = context; + if (!isValidCalmDocumentType(type)) { + throw new Error(`Invalid document type '${type}'. Must be one of: ${CALM_DOCUMENT_TYPES_LIST.join(', ')}`); + } + + // Parse the file once; we manage its $id only when it is valid JSON. + let fileJson: Record | undefined; + try { + fileJson = JSON.parse(await readFile(srcPath, 'utf8')); + } catch (_) { + fileJson = undefined; + } + + const baseUrlDefault = (await loadCliConfig())?.calmHubUrl; + const existingId = fileJson && typeof fileJson['$id'] === 'string' ? (fileJson['$id'] as string) : undefined; + let builtNamespace: string | undefined; + let effectiveId = existingId; + + if (fileJson) { + if (!existingId) { + // No $id present: build one interactively and write it into the file. + const built = await promptForDocumentId({ baseUrlDefault }); + fileJson['$id'] = built.id; + await writeFile(srcPath, JSON.stringify(fileJson, null, 2), 'utf8'); + logger.info(`Set document $id to ${built.id}`); + builtNamespace = built.namespace; + effectiveId = built.id; + } else if (!isConformantDocumentId(existingId)) { + // Non-conformant $id: warn but still add — push will skip non-pushable types anyway. + // Silently rewriting would be data loss for types that don't use CalmHub URLs (flow, adr, timeline, etc.). + logger.warn(`Document $id '${existingId}' is not a conformant CalmHub id. The document will be tracked but cannot be pushed to CalmHub.`); + } + } + + const namespace = options.namespace + ?? builtNamespace + ?? (effectiveId ? namespaceFromDocumentId(effectiveId) : undefined); + + let id = options.id; + if (!id && fileJson && typeof fileJson['title'] === 'string' && (fileJson['title'] as string).trim()) { + id = (fileJson['title'] as string).trim(); + } + if (!id) { + id = await input({ message: 'Enter a name for this document:' }); + } + + const { id: resolvedId, destPath: finalDestPath } = await addFileToBundle(bundlePath, srcPath, { + id, + copy: options.copy, + type, + namespace: namespace?.trim() + }); + + if (options.copy) { + logger.info(`Copied ${srcPath} -> ${finalDestPath} (id: ${resolvedId})`); + } else { + logger.info(`Added reference to ${finalDestPath} (id: ${resolvedId})`); + } +} + /** * Sets up the 'workspace' command and its subcommands in the CLI. * @param program The Commander.js top-level program. @@ -92,15 +217,7 @@ export function setupWorkspaceCommands(program: Command) { .option('--calm-hub-document-id ', 'Existing CalmHub narrative document ID') .option('--ver ', 'Existing CalmHub narrative document version') .option('--calm-hub-url ', 'CalmHub URL used to verify an existing narrative document') - .action(async (file: string, options: { - id?: string; - copy?: boolean; - type?: string; - namespace?: string; - calmHubDocumentId?: string; - ver?: string; - calmHubUrl?: string; - }) => { + .action(async (file: string, options: WorkspaceAddOptions) => { try { const bundlePath = findWorkspaceManifestPath(process.cwd()); if (!bundlePath) { @@ -118,7 +235,11 @@ export function setupWorkspaceCommands(program: Command) { if (!hasDocumentId || !hasVersion) { throw new Error('Narrative recovery requires both --calm-hub-document-id and --ver.'); } - if (!options.type || !isNarrativeDocumentType(options.type)) { + if (!options.type) { + throw new Error('Narrative recovery requires a narrative --type.'); + } + const recoveredType = resolveWorkspaceDocumentType(options.type); + if (recoveredType?.kind !== 'narrative') { throw new Error('Narrative recovery requires a narrative --type.'); } if (!options.namespace?.trim()) { @@ -135,7 +256,7 @@ export function setupWorkspaceCommands(program: Command) { } const identity = { namespace: options.namespace.trim(), - type: options.type, + type: recoveredType.type, version: options.ver, calmHubDocumentId, }; @@ -166,89 +287,16 @@ export function setupWorkspaceCommands(program: Command) { const documentTypes = [...CALM_DOCUMENT_TYPES_LIST, ...CALM_NARRATIVE_DOCUMENT_TYPES_LIST]; const type = await enforceOptionPresenceByPrompt(options.type, 'Select a document type:', documentTypes); - if (isNarrativeDocumentType(type)) { - if (!options.namespace?.trim()) { - throw new Error(`Narrative document '${file}' requires --namespace.`); - } - const { id: resolvedId, destPath: finalDestPath } = await registerNarrativeDocument( - bundlePath, - srcPath, - file, - { - id: options.id, - copy: options.copy, - identity: { - namespace: options.namespace.trim(), - type, - version: '1.0.0', - }, - } - ); - if (options.copy) { - logger.info(`Copied ${srcPath} -> ${finalDestPath} (id: ${resolvedId})`); - } else { - logger.info(`Added reference to ${finalDestPath} (id: ${resolvedId})`); - } - return; - } - if (!isValidCalmDocumentType(type)) { + const resolvedType = resolveWorkspaceDocumentType(type); + if (!resolvedType) { logger.error(`Invalid document type '${type}'. Must be one of: ${CALM_DOCUMENT_TYPES_LIST.join(', ')}`); process.exit(1); } - - // Parse the file once; we manage its $id only when it is valid JSON. - let fileJson: Record | undefined; - try { - fileJson = JSON.parse(await readFile(srcPath, 'utf8')); - } catch (_) { - fileJson = undefined; - } - - const baseUrlDefault = (await loadCliConfig())?.calmHubUrl; - const existingId = fileJson && typeof fileJson['$id'] === 'string' ? (fileJson['$id'] as string) : undefined; - let builtNamespace: string | undefined; - let effectiveId = existingId; - - if (fileJson) { - if (!existingId) { - // No $id present: build one interactively and write it into the file. - const built = await promptForDocumentId({ baseUrlDefault }); - fileJson['$id'] = built.id; - await writeFile(srcPath, JSON.stringify(fileJson, null, 2), 'utf8'); - logger.info(`Set document $id to ${built.id}`); - builtNamespace = built.namespace; - effectiveId = built.id; - } else if (!isConformantDocumentId(existingId)) { - // Non-conformant $id: warn but still add — push will skip non-pushable types anyway. - // Silently rewriting would be data loss for types that don't use CalmHub URLs (flow, adr, timeline, etc.). - logger.warn(`Document $id '${existingId}' is not a conformant CalmHub id. The document will be tracked but cannot be pushed to CalmHub.`); - } - } - - const namespace = options.namespace - ?? builtNamespace - ?? (effectiveId ? namespaceFromDocumentId(effectiveId) : undefined); - - let id = options.id; - if (!id && fileJson && typeof fileJson['title'] === 'string' && (fileJson['title'] as string).trim()) { - id = (fileJson['title'] as string).trim(); - } - if (!id) { - id = await input({ message: 'Enter a name for this document:' }); - } - - const { id: resolvedId, destPath: finalDestPath } = await addFileToBundle(bundlePath, srcPath, { - id, - copy: options.copy, - type, - namespace: namespace?.trim() - }); - - if (options.copy) { - logger.info(`Copied ${srcPath} -> ${finalDestPath} (id: ${resolvedId})`); - } else { - logger.info(`Added reference to ${finalDestPath} (id: ${resolvedId})`); - } + await dispatchWorkspaceDocumentType( + resolvedType, + ADD_DOCUMENT_OPERATIONS, + { bundlePath, file, options, srcPath } + ); } catch (err) { logger.error('Failed to add file to workspace bundle: ' + (err instanceof Error ? err.message : String(err))); process.exit(1); diff --git a/cli/src/command-helpers/workspace/document-kind.spec.ts b/cli/src/command-helpers/workspace/document-kind.spec.ts new file mode 100644 index 000000000..25b5f09fb --- /dev/null +++ b/cli/src/command-helpers/workspace/document-kind.spec.ts @@ -0,0 +1,83 @@ +import { + dispatchWorkspaceManifestEntry, + getJsonReferenceWorkspaceManifest, + resolveWorkspaceDocumentType, + resolveWorkspaceManifestEntry, + WORKSPACE_DOCUMENT_HANDLERS, + type WorkspaceManifestEntryOperations, +} from './document-kind'; +import type { WorkspaceManifest, WorkspaceManifestEntry } from './bundle'; + +describe('workspace document handlers', () => { + it('resolves mapping and narrative entries through the central handler record', () => { + const mapping: WorkspaceManifestEntry = { path: 'architecture.json', type: 'architecture' }; + const narrative: WorkspaceManifestEntry = { + path: 'decision.md', + type: 'sad', + namespace: 'example', + version: '1.0.0', + }; + + expect(resolveWorkspaceManifestEntry(mapping)).toEqual({ + kind: 'mapping', + handler: WORKSPACE_DOCUMENT_HANDLERS.mapping, + entry: mapping, + }); + expect(resolveWorkspaceManifestEntry(narrative)).toEqual({ + kind: 'narrative', + handler: WORKSPACE_DOCUMENT_HANDLERS.narrative, + entry: narrative, + }); + expect(resolveWorkspaceDocumentType('architecture')?.handler).toBe(WORKSPACE_DOCUMENT_HANDLERS.mapping); + expect(resolveWorkspaceDocumentType('sad')?.handler).toBe(WORKSPACE_DOCUMENT_HANDLERS.narrative); + }); + + it('intentionally retains mapping behavior for the legacy unknown type', () => { + const unknown: WorkspaceManifestEntry = { path: 'legacy.json', type: 'unknown' }; + + expect(resolveWorkspaceManifestEntry(unknown)).toEqual({ + kind: 'mapping', + handler: WORKSPACE_DOCUMENT_HANDLERS.mapping, + entry: unknown, + }); + expect(resolveWorkspaceDocumentType('unknown')?.handler).toBe(WORKSPACE_DOCUMENT_HANDLERS.mapping); + }); + + it('does not assign unsupported future types to the mapping strategy', () => { + expect(resolveWorkspaceDocumentType('future-document-kind')).toBeUndefined(); + }); + + it('selects only handlers that support JSON reference processing', () => { + const manifest: WorkspaceManifest = { + architecture: { path: 'architecture.json', type: 'architecture' }, + decision: { + path: 'decision.md', + type: 'sad', + namespace: 'example', + version: '1.0.0', + }, + }; + + expect(getJsonReferenceWorkspaceManifest(manifest)).toEqual({ + architecture: manifest.architecture, + }); + }); + + it('dispatches every registered kind through an exhaustive operation table', () => { + const operations = { + mapping: entry => `mapping:${entry.type}`, + narrative: entry => `narrative:${entry.type}`, + } satisfies WorkspaceManifestEntryOperations; + + const mapping = resolveWorkspaceManifestEntry({ path: 'architecture.json', type: 'architecture' }); + const narrative = resolveWorkspaceManifestEntry({ + path: 'decision.md', + type: 'sad', + namespace: 'example', + version: '1.0.0', + }); + + expect(dispatchWorkspaceManifestEntry(mapping, operations)).toBe('mapping:architecture'); + expect(dispatchWorkspaceManifestEntry(narrative, operations)).toBe('narrative:sad'); + }); +}); diff --git a/cli/src/command-helpers/workspace/document-kind.ts b/cli/src/command-helpers/workspace/document-kind.ts new file mode 100644 index 000000000..a79342fff --- /dev/null +++ b/cli/src/command-helpers/workspace/document-kind.ts @@ -0,0 +1,129 @@ +import { + isNarrativeDocumentType, + isValidCalmDocumentType, + type CalmDocumentType, + type NarrativeDocumentType, +} from '@finos/calm-models/types'; +import type { + MappingWorkspaceManifestEntry, + NarrativeWorkspaceManifestEntry, + WorkspaceManifest, + WorkspaceManifestEntry, +} from './bundle'; + +export const WORKSPACE_DOCUMENT_HANDLERS = { + mapping: { + kind: 'mapping', + format: 'json', + unreadableFile: 'warn', + supportsJsonReferences: true, + }, + narrative: { + kind: 'narrative', + format: 'markdown', + unreadableFile: 'fail', + supportsJsonReferences: false, + }, +} as const; + +export type WorkspaceDocumentHandler = + typeof WORKSPACE_DOCUMENT_HANDLERS[keyof typeof WORKSPACE_DOCUMENT_HANDLERS]; +export type WorkspaceDocumentKind = keyof typeof WORKSPACE_DOCUMENT_HANDLERS; + +export type ResolvedWorkspaceDocumentType = + | { kind: 'mapping'; handler: typeof WORKSPACE_DOCUMENT_HANDLERS.mapping; type: CalmDocumentType | 'unknown' } + | { kind: 'narrative'; handler: typeof WORKSPACE_DOCUMENT_HANDLERS.narrative; type: NarrativeDocumentType }; + +export type ResolvedWorkspaceManifestEntry = + | { kind: 'mapping'; handler: typeof WORKSPACE_DOCUMENT_HANDLERS.mapping; entry: MappingWorkspaceManifestEntry } + | { kind: 'narrative'; handler: typeof WORKSPACE_DOCUMENT_HANDLERS.narrative; entry: NarrativeWorkspaceManifestEntry }; + +export type WorkspaceDocumentTypeOperations = { + [K in WorkspaceDocumentKind]: ( + type: Extract['type'], + ...args: TArgs + ) => TResult; +}; + +export type WorkspaceManifestEntryOperations = { + [K in WorkspaceDocumentKind]: ( + entry: Extract['entry'], + ...args: TArgs + ) => TResult; +}; + +/** Resolve only explicitly supported workspace types plus the legacy literal `unknown`. */ +export function resolveWorkspaceDocumentType(type: unknown): ResolvedWorkspaceDocumentType | undefined { + if (typeof type !== 'string') return undefined; + if (isValidCalmDocumentType(type) || type === 'unknown') { + return { kind: 'mapping', handler: WORKSPACE_DOCUMENT_HANDLERS.mapping, type }; + } + if (isNarrativeDocumentType(type)) { + return { kind: 'narrative', handler: WORKSPACE_DOCUMENT_HANDLERS.narrative, type }; + } + return undefined; +} + +export function isNarrativeWorkspaceManifestEntry( + entry: WorkspaceManifestEntry +): entry is NarrativeWorkspaceManifestEntry { + return resolveWorkspaceDocumentType(entry.type)?.kind === 'narrative'; +} + +export function resolveWorkspaceManifestEntry( + entry: WorkspaceManifestEntry +): ResolvedWorkspaceManifestEntry { + if (isNarrativeWorkspaceManifestEntry(entry)) { + return { kind: 'narrative', handler: WORKSPACE_DOCUMENT_HANDLERS.narrative, entry }; + } + if (resolveWorkspaceDocumentType(entry.type)?.kind !== 'mapping') { + throw new Error(`Unsupported workspace document type '${String(entry.type)}'.`); + } + return { kind: 'mapping', handler: WORKSPACE_DOCUMENT_HANDLERS.mapping, entry }; +} + +export function dispatchWorkspaceDocumentType( + document: ResolvedWorkspaceDocumentType, + operations: WorkspaceDocumentTypeOperations, + ...args: TArgs +): TResult { + switch (document.kind) { + case 'mapping': + return operations.mapping(document.type, ...args); + case 'narrative': + return operations.narrative(document.type, ...args); + default: + return assertNever(document); + } +} + +export function dispatchWorkspaceManifestEntry( + document: ResolvedWorkspaceManifestEntry, + operations: WorkspaceManifestEntryOperations, + ...args: TArgs +): TResult { + switch (document.kind) { + case 'mapping': + return operations.mapping(document.entry, ...args); + case 'narrative': + return operations.narrative(document.entry, ...args); + default: + return assertNever(document); + } +} + +/** Keep non-JSON workspace documents out of mapping reference reads and rewrites. */ +export function getJsonReferenceWorkspaceManifest(manifest: WorkspaceManifest): WorkspaceManifest { + const result: WorkspaceManifest = {}; + for (const [id, entry] of Object.entries(manifest)) { + const resolved = resolveWorkspaceManifestEntry(entry); + if (resolved.handler.supportsJsonReferences) { + result[id] = resolved.entry; + } + } + return result; +} + +function assertNever(value: never): never { + throw new Error(`Unsupported workspace document kind '${String(value)}'.`); +} diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index e45b4b9de..23ea2f8a7 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -2,13 +2,14 @@ import { readFile } from 'fs/promises'; import { existsSync } from 'fs'; import { createHash } from 'node:crypto'; import { - isNarrativeWorkspaceManifestEntry, loadManifest, saveManifest, resolveFilePath, type NarrativeCreateRecovery, type NarrativeWorkspaceManifestEntry, type PublishedNarrativeWorkspaceManifestEntry, + type MappingWorkspaceManifestEntry, + type WorkspaceManifest, } from './bundle'; import { CalmHubClient, @@ -25,6 +26,11 @@ import { type NarrativeDocumentIdentity, validateNarrativeDocumentLocation, } from './narrative-document'; +import { + dispatchWorkspaceManifestEntry, + resolveWorkspaceManifestEntry, + type WorkspaceManifestEntryOperations, +} from './document-kind'; const logger: Logger = initLogger(false, 'workspace'); @@ -39,6 +45,23 @@ export interface PushOptions { failIfModified?: boolean; } +interface PushEntryContext { + bundlePath: string; + client: CalmHubClient; + conflicts: string[]; + failIfModified: boolean; + id: string; + manifest: WorkspaceManifest; + mappingFailures: string[]; + narrativeFailures: string[]; + raw: string; +} + +const PUSH_ENTRY_OPERATIONS = { + mapping: pushMappingEntry, + narrative: pushNarrativeEntry, +} satisfies WorkspaceManifestEntryOperations, [PushEntryContext]>; + export async function pushWorkspaceToHub( bundlePath: string, client: CalmHubClient, @@ -58,11 +81,12 @@ export async function pushWorkspaceToHub( const narrativeFailures: string[] = []; for (const [id, entry] of entries) { + const document = resolveWorkspaceManifestEntry(entry); const filePath = resolveFilePath(bundlePath, entry.path); if (!existsSync(filePath)) { logger.warn(`File not found for id '${id}': ${filePath}`); - if (isNarrativeWorkspaceManifestEntry(entry)) narrativeFailures.push(`${id}: file not found`); + if (document.handler.unreadableFile === 'fail') narrativeFailures.push(`${id}: file not found`); continue; } @@ -71,199 +95,220 @@ export async function pushWorkspaceToHub( raw = await readFile(filePath, 'utf8'); } catch (e) { logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); - if (isNarrativeWorkspaceManifestEntry(entry)) narrativeFailures.push(`${id}: file could not be read`); + if (document.handler.unreadableFile === 'fail') narrativeFailures.push(`${id}: file could not be read`); continue; } - if (isNarrativeWorkspaceManifestEntry(entry)) { - try { - const createRecovery = getCreateRecovery(entry); - const resolved = resolveNarrativeEntry(id, entry, raw); - const { version, narrative } = resolved; - - if (!resolved.hubIdentityAssigned) { - const { identity } = resolved; - if (version !== '1.0.0') { - throw new Error('A narrative document without calmHubDocumentId must use version 1.0.0.'); - } - - if (createRecovery !== undefined) { - if (sha256(narrative.request.documentMarkdown) !== createRecovery.documentMarkdownSha256) { - throw new Error('Narrative document Markdown changed while create recovery is pending. Restore the submitted content or reconcile it explicitly.'); - } - const recovered = await recoverCreatedNarrativeDocument( - client, identity, createRecovery.documentIdsBeforeCreate, narrative.request.documentMarkdown - ); - manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); - await saveManifest(bundlePath, manifest); - logger.info(`Recovered '${id}' version ${version} -> ${recovered.location}`); - continue; - } - - const documentIdsBeforeCreate = await client.getNarrativeDocumentIds(identity.namespace, identity.type); - const location = await client.createNarrativeDocument(identity.namespace, identity.type, narrative.request); - - let documentId: number | undefined; - if (location !== undefined) { - try { - documentId = parseNarrativeDocumentLocation(location, identity); - } catch { - // The POST succeeded, but the Location cannot establish the document identity. - } - } - - if (documentId !== undefined && location !== undefined) { - manifest[id] = publishNarrativeEntry(entry, documentId, location); - await saveManifest(bundlePath, manifest); - logger.info(`Pushed '${id}' version ${version} -> ${location}`); - continue; - } - - manifest[id] = { - path: entry.path, - type: entry.type, - ...(entry.namespace === undefined ? {} : { namespace: entry.namespace }), - version, - createRecovery: { - documentIdsBeforeCreate, - documentMarkdownSha256: sha256(narrative.request.documentMarkdown), - }, - }; - await saveManifest(bundlePath, manifest); - - const recovered = await recoverCreatedNarrativeDocument( - client, identity, documentIdsBeforeCreate, narrative.request.documentMarkdown - ); - manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); - await saveManifest(bundlePath, manifest); - logger.info(`Pushed '${id}' version ${version} -> ${recovered.location}`); - continue; - } + await dispatchWorkspaceManifestEntry(document, PUSH_ENTRY_OPERATIONS, { + bundlePath, + client, + conflicts, + failIfModified, + id, + manifest, + mappingFailures, + narrativeFailures, + raw, + }); + } - const { identity } = resolved; - validateNarrativeDocumentLocation(entry.calmHubId, identity, false); - const versions = await client.getNarrativeDocumentVersions( - identity.namespace, identity.type, identity.calmHubDocumentId - ); - if (!versions.includes(version)) { - const location = await client.createNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId, version, narrative.request - ); - validateNarrativeDocumentLocation(location, identity); - manifest[id] = publishNarrativeEntry(entry, identity.calmHubDocumentId, location); - await saveManifest(bundlePath, manifest); - logger.info(`Pushed '${id}' version ${version} -> ${location}`); - continue; - } + if (conflicts.length > 0 || mappingFailures.length > 0 || narrativeFailures.length > 0) { + const summaries: string[] = []; + if (conflicts.length > 0) { + summaries.push( + `${conflicts.length} modified document(s) already exist in CalmHub at their declared version ` + + `(${conflicts.join(', ')}). Run \`calm workspace bump\` to create new versions for them.` + ); + } + if (mappingFailures.length > 0) { + summaries.push(`${mappingFailures.length} mapping document(s) failed (${mappingFailures.join('; ')})`); + } + if (narrativeFailures.length > 0) { + summaries.push(`${narrativeFailures.length} narrative document(s) failed (${narrativeFailures.join('; ')})`); + } + throw new Error( + `Push failed: ${summaries.join(' ')}` + ); + } +} + +async function pushNarrativeEntry( + entry: NarrativeWorkspaceManifestEntry, + context: PushEntryContext +): Promise { + const { bundlePath, client, conflicts, failIfModified, id, manifest, narrativeFailures, raw } = context; + try { + const createRecovery = getCreateRecovery(entry); + const resolved = resolveNarrativeEntry(id, entry, raw); + const { version, narrative } = resolved; + + if (!resolved.hubIdentityAssigned) { + const { identity } = resolved; + if (version !== '1.0.0') { + throw new Error('A narrative document without calmHubDocumentId must use version 1.0.0.'); + } - if (!failIfModified) { - logger.info(`No changes for '${id}' - version ${version} already exists, skipping`); - continue; + if (createRecovery !== undefined) { + if (sha256(narrative.request.documentMarkdown) !== createRecovery.documentMarkdownSha256) { + throw new Error('Narrative document Markdown changed while create recovery is pending. Restore the submitted content or reconcile it explicitly.'); } - const remote = await client.getNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId, version + const recovered = await recoverCreatedNarrativeDocument( + client, identity, createRecovery.documentIdsBeforeCreate, narrative.request.documentMarkdown ); - if (remote.documentMarkdown !== raw) { - logger.error(`'${id}' version ${version} already exists in CalmHub but differs on disk. Bump it before pushing.`); - conflicts.push(`${id}@${version}`); - } else { - logger.info(`No changes for '${id}' - version ${version} already exists and is unchanged, skipping`); + manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); + await saveManifest(bundlePath, manifest); + logger.info(`Recovered '${id}' version ${version} -> ${recovered.location}`); + return; + } + + const documentIdsBeforeCreate = await client.getNarrativeDocumentIds(identity.namespace, identity.type); + const location = await client.createNarrativeDocument(identity.namespace, identity.type, narrative.request); + + let documentId: number | undefined; + if (location !== undefined) { + try { + documentId = parseNarrativeDocumentLocation(location, identity); + } catch { + // The POST succeeded, but the Location cannot establish the document identity. } - } catch (e) { - const message = e instanceof Error ? e.message : String(e); - logger.error(`Failed to push narrative document '${id}': ${message}`); - narrativeFailures.push(`${id}: ${message}`); } - continue; - } - // The mapping API addresses resources by (namespace, type, mappingId, version), - // all encoded in the document's $id. Documents without a well-formed mapping $id - // (or whose type has no ResourceType, e.g. flow/adr) cannot be pushed and are skipped. - let metadata: DocumentMetadata; - try { - metadata = extractDocumentMetadata(raw); - } catch (e) { - logger.warn( - `Skipping '${id}': not mappable to CalmHub. Documents must have a '$id' of the form ` + - '$BASE_URL/calm/namespaces/$NAMESPACE/$TYPE/$MAPPING_ID/versions/$VERSION ' + - `(${e instanceof Error ? e.message : String(e)})` + if (documentId !== undefined && location !== undefined) { + manifest[id] = publishNarrativeEntry(entry, documentId, location); + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${location}`); + return; + } + + manifest[id] = { + path: entry.path, + type: entry.type, + ...(entry.namespace === undefined ? {} : { namespace: entry.namespace }), + version, + createRecovery: { + documentIdsBeforeCreate, + documentMarkdownSha256: sha256(narrative.request.documentMarkdown), + }, + }; + await saveManifest(bundlePath, manifest); + + const recovered = await recoverCreatedNarrativeDocument( + client, identity, documentIdsBeforeCreate, narrative.request.documentMarkdown ); - continue; + manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${recovered.location}`); + return; } - const { namespace, type: resourceType, mapping: mappingId, version } = metadata; - if (!namespace) { - logger.warn(`Skipping '${id}': document $id has no namespace.`); - continue; + const { identity } = resolved; + validateNarrativeDocumentLocation(entry.calmHubId, identity, false); + const versions = await client.getNarrativeDocumentVersions( + identity.namespace, identity.type, identity.calmHubDocumentId + ); + if (!versions.includes(version)) { + const location = await client.createNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId, version, narrative.request + ); + validateNarrativeDocumentLocation(location, identity); + manifest[id] = publishNarrativeEntry(entry, identity.calmHubDocumentId, location); + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${location}`); + return; } - let existingVersions: string[]; - try { - existingVersions = await client.getMappedResourceVersions(namespace, mappingId, resourceType); - } catch (e) { - const message = e instanceof Error ? e.message : String(e); - logger.error(`Failed to fetch existing versions for '${id}' from CalmHub: ${message}`); - mappingFailures.push(`${id}: ${message}`); - continue; + if (!failIfModified) { + logger.info(`No changes for '${id}' - version ${version} already exists, skipping`); + return; + } + const remote = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId, version + ); + if (remote.documentMarkdown !== raw) { + logger.error(`'${id}' version ${version} already exists in CalmHub but differs on disk. Bump it before pushing.`); + conflicts.push(`${id}@${version}`); + } else { + logger.info(`No changes for '${id}' - version ${version} already exists and is unchanged, skipping`); } + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + logger.error(`Failed to push narrative document '${id}': ${message}`); + narrativeFailures.push(`${id}: ${message}`); + } +} - if (existingVersions.includes(version)) { - if (!failIfModified) { - logger.info(`No changes for '${id}' - version ${version} already exists, skipping`); - continue; - } +async function pushMappingEntry( + entry: MappingWorkspaceManifestEntry, + context: PushEntryContext +): Promise { + const { bundlePath, client, conflicts, failIfModified, id, manifest, mappingFailures, raw } = context; + // The mapping API addresses resources by (namespace, type, mappingId, version), + // all encoded in the document's $id. Documents without a well-formed mapping $id + // (or whose type has no ResourceType, e.g. flow/adr) cannot be pushed and are skipped. + let metadata: DocumentMetadata; + try { + metadata = extractDocumentMetadata(raw); + } catch (e) { + logger.warn( + `Skipping '${id}': not mappable to CalmHub. Documents must have a '$id' of the form ` + + '$BASE_URL/calm/namespaces/$NAMESPACE/$TYPE/$MAPPING_ID/versions/$VERSION ' + + `(${e instanceof Error ? e.message : String(e)})` + ); + return; + } - // Strict mode: an existing version is only a conflict if the on-disk content differs - // from what is already published. Unchanged content is still skipped. - let remote: object; - try { - remote = await client.getMappedResourceByVersion(namespace, mappingId, version, resourceType); - } catch (e) { - const message = e instanceof Error ? e.message : String(e); - logger.error(`Failed to fetch '${id}' @ ${version} from CalmHub to compare: ${message}`); - mappingFailures.push(`${id}: ${message}`); - continue; - } + const { namespace, type: resourceType, mapping: mappingId, version } = metadata; + if (!namespace) { + logger.warn(`Skipping '${id}': document $id has no namespace.`); + return; + } - if (canonicalEqual(JSON.parse(raw), remote)) { - logger.info(`No changes for '${id}' - version ${version} already exists and is unchanged, skipping`); - } else { - logger.error(`'${id}' version ${version} already exists in CalmHub but differs on disk. Bump it before pushing.`); - conflicts.push(`${id}@${version}`); - } - continue; + let existingVersions: string[]; + try { + existingVersions = await client.getMappedResourceVersions(namespace, mappingId, resourceType); + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + logger.error(`Failed to fetch existing versions for '${id}' from CalmHub: ${message}`); + mappingFailures.push(`${id}: ${message}`); + return; + } + + if (existingVersions.includes(version)) { + if (!failIfModified) { + logger.info(`No changes for '${id}' - version ${version} already exists, skipping`); + return; } + // Strict mode: an existing version is only a conflict if the on-disk content differs + // from what is already published. Unchanged content is still skipped. + let remote: object; try { - const calmHubId = await client.createMappedResourceVersion(metadata, raw); - manifest[id] = { ...entry, calmHubId }; - await saveManifest(bundlePath, manifest); - logger.info(`Pushed '${id}' version ${version} -> ${calmHubId}`); + remote = await client.getMappedResourceByVersion(namespace, mappingId, version, resourceType); } catch (e) { const message = e instanceof Error ? e.message : String(e); - logger.error(`Failed to push '${id}': ${message}`); + logger.error(`Failed to fetch '${id}' @ ${version} from CalmHub to compare: ${message}`); mappingFailures.push(`${id}: ${message}`); + return; } - } - if (conflicts.length > 0 || mappingFailures.length > 0 || narrativeFailures.length > 0) { - const summaries: string[] = []; - if (conflicts.length > 0) { - summaries.push( - `${conflicts.length} modified document(s) already exist in CalmHub at their declared version ` + - `(${conflicts.join(', ')}). Run \`calm workspace bump\` to create new versions for them.` - ); - } - if (mappingFailures.length > 0) { - summaries.push(`${mappingFailures.length} mapping document(s) failed (${mappingFailures.join('; ')})`); - } - if (narrativeFailures.length > 0) { - summaries.push(`${narrativeFailures.length} narrative document(s) failed (${narrativeFailures.join('; ')})`); + if (canonicalEqual(JSON.parse(raw), remote)) { + logger.info(`No changes for '${id}' - version ${version} already exists and is unchanged, skipping`); + } else { + logger.error(`'${id}' version ${version} already exists in CalmHub but differs on disk. Bump it before pushing.`); + conflicts.push(`${id}@${version}`); } - throw new Error( - `Push failed: ${summaries.join(' ')}` - ); + return; + } + + try { + const calmHubId = await client.createMappedResourceVersion(metadata, raw); + manifest[id] = { ...entry, calmHubId }; + await saveManifest(bundlePath, manifest); + logger.info(`Pushed '${id}' version ${version} -> ${calmHubId}`); + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + logger.error(`Failed to push '${id}': ${message}`); + mappingFailures.push(`${id}: ${message}`); } } diff --git a/cli/src/command-helpers/workspace/ref-rewrite.spec.ts b/cli/src/command-helpers/workspace/ref-rewrite.spec.ts index da1b6b4ce..2ebc447d3 100644 --- a/cli/src/command-helpers/workspace/ref-rewrite.spec.ts +++ b/cli/src/command-helpers/workspace/ref-rewrite.spec.ts @@ -8,6 +8,7 @@ import { } from './ref-rewrite'; import { mkdir, writeFile, rm, readFile } from 'fs/promises'; import path from 'path'; +import type { WorkspaceManifest } from './bundle'; // eslint-disable-next-line @typescript-eslint/no-explicit-any const loadJson = async (p: string): Promise => JSON.parse(await readFile(p, 'utf8')); @@ -152,6 +153,32 @@ describe('ref-rewrite orchestrators', () => { expect(b.$schema.const).toBe(idAt('a', '1.1.0')); }); + it('excludes narrative entries from JSON reference rules and rewrites', async () => { + await write('a.json', { $id: idAt('a', '1.1.0'), title: 'A' }); + const narrativePath = path.join(bundlePath, 'files', 'decision.md'); + const narrativeContent = JSON.stringify({ + $id: idAt('decision', '1.0.0'), + $ref: idAt('a', '1.0.0'), + }); + await writeFile(narrativePath, narrativeContent, 'utf8'); + const manifest: WorkspaceManifest = { + 'a': { path: 'files/a.json', type: 'architecture' }, + 'decision': { + path: 'files/decision.md', + type: 'sad', + namespace: 'com.example', + version: '1.0.0', + }, + }; + + const rules = await buildRefRulesFromDiskIds(manifest, bundlePath); + const results = await syncReferences(bundlePath, manifest, rules); + + expect(rules.map(rule => rule.bareId)).toEqual(['a']); + expect(results.map(result => result.docId)).toEqual(['a']); + expect(await readFile(narrativePath, 'utf8')).toBe(narrativeContent); + }); + it('skips missing files and unparseable JSON when building rules and syncing', async () => { await write('a.json', { $id: idAt('a', '1.1.0'), title: 'A' }); await writeFile(path.join(bundlePath, 'files', 'bad.json'), 'not json {{{', 'utf8'); diff --git a/cli/src/command-helpers/workspace/ref-rewrite.ts b/cli/src/command-helpers/workspace/ref-rewrite.ts index eef4122b3..e249c1554 100644 --- a/cli/src/command-helpers/workspace/ref-rewrite.ts +++ b/cli/src/command-helpers/workspace/ref-rewrite.ts @@ -2,6 +2,7 @@ import { readFile, writeFile } from 'fs/promises'; import { existsSync } from 'fs'; import { REFERENCE_PROPERTIES, WorkspaceManifest, resolveFilePath } from './bundle'; import { initLogger, Logger } from '@finos/calm-shared'; +import { getJsonReferenceWorkspaceManifest } from './document-kind'; const logger: Logger = initLogger(false, 'workspace'); @@ -157,17 +158,18 @@ export function replaceRefsInObject( } /** - * Build rewrite rules from the *current on-disk* `$id` of every tracked document. Any reference to - * a tracked document (bare id, stale versioned path, or full URL) is mapped to that document's - * current `$id`. Documents without a usable `$id` are skipped as targets (they can still contain - * references that get rewritten). + * Build rewrite rules from the *current on-disk* `$id` of each tracked JSON document. Any reference + * to a tracked JSON document (bare id, stale versioned path, or full URL) is mapped to that + * document's current `$id`. Documents without a usable `$id` are skipped as targets (they can still + * contain references that get rewritten). Non-JSON handlers are excluded by document-kind policy. */ export async function buildRefRulesFromDiskIds( manifest: WorkspaceManifest, bundlePath: string ): Promise { const rules: RefRule[] = []; - for (const [id, entry] of Object.entries(manifest)) { + const jsonManifest = getJsonReferenceWorkspaceManifest(manifest); + for (const [id, entry] of Object.entries(jsonManifest)) { const filePath = resolveFilePath(bundlePath, entry.path); if (!existsSync(filePath)) continue; try { @@ -189,9 +191,9 @@ export async function buildRefRulesFromDiskIds( } /** - * Rewrite references across all tracked documents according to the given rules, writing back any - * file that changed. Idempotent: a second run finds references already at their target and is a - * no-op. + * Rewrite references across tracked JSON documents according to the given rules, writing back any + * file that changed. Non-JSON handlers are excluded by document-kind policy. Idempotent: a second + * run finds references already at their target and is a no-op. */ export async function syncReferences( bundlePath: string, @@ -200,7 +202,8 @@ export async function syncReferences( ): Promise { const results: RefUpdateResult[] = []; - for (const [id, entry] of Object.entries(manifest)) { + const jsonManifest = getJsonReferenceWorkspaceManifest(manifest); + for (const [id, entry] of Object.entries(jsonManifest)) { const filePath = resolveFilePath(bundlePath, entry.path); if (!existsSync(filePath)) { logger.warn(`File not found for '${id}': ${filePath}`); From aba75824d7c6fd16ce43e1ce479187d4e3204a81 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Fri, 18 Sep 2026 09:38:56 +0100 Subject: [PATCH 22/28] perf(cli): batch narrative bump manifest updates --- .../command-helpers/workspace/bump.spec.ts | 128 ++++++++++++++++++ cli/src/command-helpers/workspace/bump.ts | 28 ++-- 2 files changed, 147 insertions(+), 9 deletions(-) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index 6c1bc6d8b..48fcef3cb 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -10,6 +10,15 @@ vi.mock('@finos/calm-shared', async (importOriginal) => ({ initLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }), })); +vi.mock('./bundle', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + loadManifest: vi.fn(actual.loadManifest), + saveManifest: vi.fn(actual.saveManifest), + }; +}); + const BASE = 'https://hub.example.com'; const idAt = (resource: string, version: string, type = 'architectures', ns = 'com.example') => `${BASE}/calm/namespaces/${ns}/${type}/${resource}/versions/${version}`; @@ -288,6 +297,125 @@ describe('bump', () => { }); describe('bumpWorkspace', () => { + it('loads and saves the manifest once for a batch of narrative updates', async () => { + const entry = (name: string, documentId: number) => ({ + path: `files/${name}.md`, type: 'sad' as const, namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: documentId, + calmHubId: `/api/calm/namespaces/com.example/documents/sad/${documentId}/versions/1.0.0`, + }); + await writeFile(path.join(filesPath, 'payments.md'), '# Payments'); + await writeFile(path.join(filesPath, 'orders.md'), '# Orders'); + await saveManifest(bundlePath, { payments: entry('payments', 42), orders: entry('orders', 43) }); + vi.mocked(loadManifest).mockClear(); + vi.mocked(saveManifest).mockClear(); + + const result = await bumpWorkspace(bundlePath, makeClient(), { + increment: 'MINOR', + preDetectedChanges: [ + { id: 'payments', kind: 'narrative', filePath: path.join(filesPath, 'payments.md'), currentVersion: '1.0.0', latestHubVersion: '1.0.0' }, + { id: 'orders', kind: 'narrative', filePath: path.join(filesPath, 'orders.md'), currentVersion: '1.0.0', latestHubVersion: '1.0.0' }, + ], + }); + + expect(result.bumped.map(({ id, toVersion }) => ({ id, toVersion }))).toEqual([ + { id: 'payments', toVersion: '1.1.0' }, + { id: 'orders', toVersion: '1.1.0' }, + ]); + expect(saveManifest).toHaveBeenCalledTimes(1); + expect(saveManifest).toHaveBeenCalledWith(bundlePath, expect.objectContaining({ + payments: expect.objectContaining({ version: '1.1.0' }), + orders: expect.objectContaining({ version: '1.1.0' }), + })); + const saveCallOrder = vi.mocked(saveManifest).mock.invocationCallOrder[0]; + const updatePhaseLoads = vi.mocked(loadManifest).mock.invocationCallOrder.filter(order => order < saveCallOrder); + expect(updatePhaseLoads).toHaveLength(1); + }); + + it('does not save a partially validated narrative batch', async () => { + await writeFile(path.join(filesPath, 'payments.md'), '# Payments'); + await saveManifest(bundlePath, { + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + architecture: { path: 'files/a.json', type: 'architecture' }, + }); + vi.mocked(loadManifest).mockClear(); + vi.mocked(saveManifest).mockClear(); + + await expect(bumpWorkspace(bundlePath, makeClient(), { + increment: 'MINOR', + preDetectedChanges: [ + { id: 'payments', kind: 'narrative', filePath: path.join(filesPath, 'payments.md'), currentVersion: '1.0.0', latestHubVersion: '1.0.0' }, + { id: 'architecture', kind: 'narrative', filePath: path.join(filesPath, 'a.json'), currentVersion: '1.0.0', latestHubVersion: '1.0.0' }, + ], + })).rejects.toThrow(/no longer a narrative manifest entry/); + + expect(loadManifest).toHaveBeenCalledTimes(1); + expect(saveManifest).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments.version).toBe('1.0.0'); + }); + + it('does not save the manifest update phase for mapping-only changes', async () => { + await write('a.json', { $id: idAt('a', '1.0.0'), title: 'A', extra: 'edited' }); + await saveManifest(bundlePath, { a: { path: 'files/a.json', type: 'architecture' } }); + vi.mocked(saveManifest).mockClear(); + + const result = await bumpWorkspace(bundlePath, makeClient(), { + increment: 'MINOR', + preDetectedChanges: [{ + id: 'a', kind: 'mapping', filePath: path.join(filesPath, 'a.json'), + currentVersion: '1.0.0', latestHubVersion: '1.0.0', + metadata: { + rawDocumentId: idAt('a', '1.0.0'), baseUrl: BASE, name: 'A', + namespace: 'com.example', type: 'architectures', mapping: 'a', version: '1.0.0', + }, + }], + }); + + expect(saveManifest).not.toHaveBeenCalled(); + expect(result.bumped).toEqual([ + expect.objectContaining({ id: 'a', fromVersion: '1.0.0', toVersion: '1.1.0' }), + ]); + expect((await read('a.json')).$id).toBe(idAt('a', '1.1.0')); + }); + + it('preserves result order and updates both document kinds in a mixed batch', async () => { + await write('a.json', { $id: idAt('a', '1.0.0'), title: 'A', extra: 'edited' }); + await writeFile(path.join(filesPath, 'payments.md'), '# Payments'); + await saveManifest(bundlePath, { + a: { path: 'files/a.json', type: 'architecture' }, + payments: { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + + const result = await bumpWorkspace(bundlePath, makeClient(), { + increment: 'MINOR', + preDetectedChanges: [ + { + id: 'a', kind: 'mapping', filePath: path.join(filesPath, 'a.json'), + currentVersion: '1.0.0', latestHubVersion: '1.0.0', + metadata: { + rawDocumentId: idAt('a', '1.0.0'), baseUrl: BASE, name: 'A', + namespace: 'com.example', type: 'architectures', mapping: 'a', version: '1.0.0', + }, + }, + { id: 'payments', kind: 'narrative', filePath: path.join(filesPath, 'payments.md'), currentVersion: '1.0.0', latestHubVersion: '1.0.0' }, + ], + }); + + expect(result.bumped.map(({ id, toVersion }) => ({ id, toVersion }))).toEqual([ + { id: 'a', toVersion: '1.1.0' }, + { id: 'payments', toVersion: '1.1.0' }, + ]); + expect((await read('a.json')).$id).toBe(idAt('a', '1.1.0')); + expect((await loadManifest(bundlePath)).payments.version).toBe('1.1.0'); + }); + it('bumps a changed doc by one MINOR increment relative to the latest hub version', async () => { await write('a.json', { $id: idAt('a', '1.0.0'), title: 'A', extra: 'edited' }); await saveManifest(bundlePath, { 'a': { path: 'files/a.json', type: 'architecture' } }); diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index b5bd290e4..2ebc71bf3 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -236,19 +236,28 @@ export async function bumpWorkspace( // Tracks the actual increment used for each bumped doc, so cascade passes can inherit it. const appliedIncrements = new Map(); + const narrativeChanges = changed.filter((change): change is Extract => + change.kind === 'narrative' + ); + const narrativeManifest = narrativeChanges.length > 0 ? await loadManifest(bundlePath) : undefined; + for (const change of narrativeChanges) { + const entry = narrativeManifest?.[change.id]; + if (!entry) throw new Error(`Narrative document '${change.id}' is no longer in the manifest.`); + const document = resolveWorkspaceManifestEntry(entry); + if (document.kind !== 'narrative') { + throw new Error(`Narrative document '${change.id}' is no longer a narrative manifest entry.`); + } + const increment = options.perDocIncrements?.get(change.id) ?? options.increment; + narrativeManifest[change.id] = { + ...document.entry, + version: computeSemVerBump(change.latestHubVersion, increment), + }; + } + for (const c of changed) { const docIncrement = options.perDocIncrements?.get(c.id) ?? options.increment; const toVersion = computeSemVerBump(c.latestHubVersion, docIncrement); if (c.kind === 'narrative') { - const manifest = await loadManifest(bundlePath); - const entry = manifest[c.id]; - if (!entry) throw new Error(`Narrative document '${c.id}' is no longer in the manifest.`); - const document = resolveWorkspaceManifestEntry(entry); - if (document.kind !== 'narrative') { - throw new Error(`Narrative document '${c.id}' is no longer a narrative manifest entry.`); - } - manifest[c.id] = { ...document.entry, version: toVersion }; - await saveManifest(bundlePath, manifest); bumped.push({ id: c.id, filePath: c.filePath, fromVersion: c.currentVersion, toVersion, increment: docIncrement }); appliedIncrements.set(c.id, docIncrement); bumpedIds.add(c.id); @@ -263,6 +272,7 @@ export async function bumpWorkspace( bumpedIds.add(c.id); logger.info(`Bumped '${c.id}' ${c.currentVersion} -> ${toVersion}`); } + if (narrativeManifest) await saveManifest(bundlePath, narrativeManifest); // Cascade: sync refs, then bump any document that was modified by the sync but not yet bumped. // Repeat until nothing new gets changed (fixed-point). Terminates because each iteration adds From e4ee2fb96acdf94d6116b3d6d7de76520de1aeff Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Fri, 18 Sep 2026 10:08:46 +0100 Subject: [PATCH 23/28] perf(cli): parallelize workspace Hub checks --- .../command-helpers/workspace/bump.spec.ts | 163 ++++++++++++++++++ cli/src/command-helpers/workspace/bump.ts | 146 +++++++++------- 2 files changed, 247 insertions(+), 62 deletions(-) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index 48fcef3cb..d75fe7602 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -36,6 +36,16 @@ const makeClient = (opts: ClientOpts = {}): CalmHubClient => ({ getNarrativeDocumentVersion: vi.fn(async () => ({ documentMarkdown: opts.narrativeMarkdown ?? '' })), }) as unknown as CalmHubClient; +const deferred = () => { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + describe('bump', () => { const bundlePath = path.join(__dirname, 'test-bump', 'bundle'); const filesPath = path.join(bundlePath, 'files'); @@ -164,6 +174,159 @@ describe('bump', () => { await expect(detectChangedResources(bundlePath, client)).rejects.toThrow(/Hub unavailable/); }); + it('checks narrative documents concurrently and returns changes in manifest order', async () => { + await writeFile(path.join(filesPath, 'first.md'), '---\ntitle: First\n---\n# First changed'); + await writeFile(path.join(filesPath, 'second.md'), '---\ntitle: Second\n---\n# Second changed'); + await saveManifest(bundlePath, { + first: { + path: 'files/first.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 41, calmHubId: '/api/calm/namespaces/com.example/documents/sad/41/versions/1.0.0', + }, + second: { + path: 'files/second.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const firstEntered = deferred(); + const secondEntered = deferred(); + const releaseFirst = deferred(); + const releaseSecond = deferred(); + const secondCompleted = deferred(); + const client = makeClient(); + vi.mocked(client.getNarrativeDocumentVersions).mockImplementation(async (_namespace, _type, documentId) => { + if (documentId === 41) { + firstEntered.resolve(); + await releaseFirst.promise; + } else { + secondEntered.resolve(); + await releaseSecond.promise; + } + return ['1.0.0']; + }); + vi.mocked(client.getNarrativeDocumentVersion).mockImplementation(async (_namespace, _type, documentId) => { + if (documentId === 42) secondCompleted.resolve(); + return { documentMarkdown: '# Published' }; + }); + + const detection = detectChangedResources(bundlePath, client); + await Promise.all([firstEntered.promise, secondEntered.promise]); + releaseSecond.resolve(); + await secondCompleted.promise; + releaseFirst.resolve(); + + await expect(detection).resolves.toMatchObject([{ id: 'first' }, { id: 'second' }]); + }); + + it('checks mapping documents concurrently', async () => { + await write('first.json', { $id: idAt('first', '1.0.0'), title: 'First changed' }); + await write('second.json', { $id: idAt('second', '1.0.0'), title: 'Second changed' }); + await saveManifest(bundlePath, { + first: { path: 'files/first.json', type: 'architecture' }, + second: { path: 'files/second.json', type: 'architecture' }, + }); + const firstEntered = deferred(); + const secondEntered = deferred(); + const releaseFirst = deferred(); + const releaseSecond = deferred(); + const secondCompleted = deferred(); + const client = makeClient(); + vi.mocked(client.getMappedResourceVersions).mockImplementation(async (_namespace, mappingId) => { + if (mappingId === 'first') { + firstEntered.resolve(); + await releaseFirst.promise; + } else { + secondEntered.resolve(); + await releaseSecond.promise; + } + return ['1.0.0']; + }); + vi.mocked(client.getMappedResourceByVersion).mockImplementation(async (_namespace, mappingId) => { + if (mappingId === 'second') secondCompleted.resolve(); + return { $id: idAt(mappingId, '1.0.0'), title: 'Published' }; + }); + + const detection = detectChangedResources(bundlePath, client); + await Promise.all([firstEntered.promise, secondEntered.promise]); + releaseSecond.resolve(); + await secondCompleted.promise; + releaseFirst.resolve(); + + await expect(detection).resolves.toMatchObject([{ id: 'first' }, { id: 'second' }]); + }); + + it('overlaps mapping and narrative Hub checks', async () => { + await write('architecture.json', { $id: idAt('architecture', '1.0.0'), title: 'Changed' }); + await writeFile(path.join(filesPath, 'sad.md'), '---\ntitle: SAD\n---\n# Changed'); + await saveManifest(bundlePath, { + architecture: { path: 'files/architecture.json', type: 'architecture' }, + sad: { + path: 'files/sad.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const mappingEntered = deferred(); + const narrativeEntered = deferred(); + const releaseMapping = deferred(); + const releaseNarrative = deferred(); + const client = makeClient(); + vi.mocked(client.getMappedResourceVersions).mockImplementation(async () => { + mappingEntered.resolve(); + await releaseMapping.promise; + return ['1.0.0']; + }); + vi.mocked(client.getMappedResourceByVersion).mockResolvedValue({ + $id: idAt('architecture', '1.0.0'), title: 'Published', + }); + vi.mocked(client.getNarrativeDocumentVersions).mockImplementation(async () => { + narrativeEntered.resolve(); + await releaseNarrative.promise; + return ['1.0.0']; + }); + vi.mocked(client.getNarrativeDocumentVersion).mockResolvedValue({ documentMarkdown: '# Published' }); + + const detection = detectChangedResources(bundlePath, client); + await Promise.all([mappingEntered.promise, narrativeEntered.promise]); + releaseNarrative.resolve(); + releaseMapping.resolve(); + + await expect(detection).resolves.toMatchObject([{ id: 'architecture' }, { id: 'sad' }]); + }); + + it('reports the first narrative Hub failure in manifest order', async () => { + await writeFile(path.join(filesPath, 'first.md'), '---\ntitle: First\n---\n# First'); + await writeFile(path.join(filesPath, 'second.md'), '---\ntitle: Second\n---\n# Second'); + await saveManifest(bundlePath, { + first: { + path: 'files/first.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 41, calmHubId: '/api/calm/namespaces/com.example/documents/sad/41/versions/1.0.0', + }, + second: { + path: 'files/second.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }, + }); + const firstEntered = deferred(); + const secondEntered = deferred(); + const failFirst = deferred(); + const failSecond = deferred(); + const client = makeClient(); + vi.mocked(client.getNarrativeDocumentVersions).mockImplementation(async (_namespace, _type, documentId) => { + if (documentId === 41) { + firstEntered.resolve(); + return failFirst.promise; + } + secondEntered.resolve(); + return failSecond.promise; + }); + + const detection = detectChangedResources(bundlePath, client); + await Promise.all([firstEntered.promise, secondEntered.promise]); + failSecond.reject(new Error('second failure')); + failFirst.reject(new Error('first failure')); + + await expect(detection).rejects.toThrow('first failure'); + }); + it('treats a document with no Hub versions as new and rejects missing manifest versions', async () => { await writeFile(path.join(filesPath, 'payments.md'), '---\ntitle: Payments SAD\n---\n# Payments'); const entry = { diff --git a/cli/src/command-helpers/workspace/bump.ts b/cli/src/command-helpers/workspace/bump.ts index 2ebc71bf3..f28378921 100644 --- a/cli/src/command-helpers/workspace/bump.ts +++ b/cli/src/command-helpers/workspace/bump.ts @@ -87,10 +87,12 @@ interface DetectChangedEntryContext { raw: string; } +type DetectChangedEntryCheck = () => Promise; + const DETECT_CHANGED_ENTRY_OPERATIONS = { - mapping: detectChangedMappingEntry, - narrative: detectChangedNarrativeEntry, -} satisfies WorkspaceManifestEntryOperations, [DetectChangedEntryContext]>; + mapping: prepareChangedMappingEntry, + narrative: prepareChangedNarrativeEntry, +} satisfies WorkspaceManifestEntryOperations; /** Returns the highest-priority increment from a list (MAJOR > MINOR > PATCH). */ export function maxIncrement(increments: ResourceChangeType[]): ResourceChangeType { @@ -115,62 +117,79 @@ export async function detectChangedResources( client: CalmHubClient ): Promise { const manifest = await loadManifest(bundlePath); - const changed: ChangedResource[] = []; + const checks: DetectChangedEntryCheck[] = []; + let preparationFailure: { reason: unknown } | undefined; for (const [id, entry] of Object.entries(manifest)) { - const document = resolveWorkspaceManifestEntry(entry); - const filePath = resolveFilePath(bundlePath, entry.path); - if (!existsSync(filePath)) { - if (document.handler.unreadableFile === 'fail') throw new Error(`Narrative document '${id}' file not found: ${filePath}`); - logger.warn(`File not found for id '${id}': ${filePath}`); - continue; - } - - let raw: string; try { - raw = await readFile(filePath, 'utf8'); - } catch (e) { - if (document.handler.unreadableFile === 'fail') throw new Error(`Narrative document '${id}' could not be read: ${e instanceof Error ? e.message : String(e)}`); - logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); - continue; + const document = resolveWorkspaceManifestEntry(entry); + const filePath = resolveFilePath(bundlePath, entry.path); + if (!existsSync(filePath)) { + if (document.handler.unreadableFile === 'fail') throw new Error(`Narrative document '${id}' file not found: ${filePath}`); + logger.warn(`File not found for id '${id}': ${filePath}`); + continue; + } + + let raw: string; + try { + raw = await readFile(filePath, 'utf8'); + } catch (e) { + if (document.handler.unreadableFile === 'fail') throw new Error(`Narrative document '${id}' could not be read: ${e instanceof Error ? e.message : String(e)}`); + logger.warn(`Failed to read file for id '${id}': ${e instanceof Error ? e.message : String(e)}`); + continue; + } + + const check = dispatchWorkspaceManifestEntry( + document, + DETECT_CHANGED_ENTRY_OPERATIONS, + { client, filePath, id, raw } + ); + if (check) checks.push(check); + } catch (reason) { + preparationFailure = { reason }; + break; } + } - const changedResource = await dispatchWorkspaceManifestEntry( - document, - DETECT_CHANGED_ENTRY_OPERATIONS, - { client, filePath, id, raw } - ); - if (changedResource) changed.push(changedResource); + // Run only entries before the first local failure, then inspect results in manifest order. + const results = await Promise.allSettled(checks.map(check => check())); + const changed: ChangedResource[] = []; + for (const result of results) { + if (result.status === 'rejected') throw result.reason; + if (result.value) changed.push(result.value); } + if (preparationFailure) throw preparationFailure.reason; return changed; } -async function detectChangedNarrativeEntry( +function prepareChangedNarrativeEntry( entry: NarrativeWorkspaceManifestEntry, context: DetectChangedEntryContext -): Promise { +): DetectChangedEntryCheck | undefined { const { client, filePath, id, raw } = context; // Bump stops on invalid narrative state because it writes local manifest versions; push can report independent failures together. const { version, identity, hubIdentityAssigned } = resolveNarrativeEntry(id, entry, raw); if (!hubIdentityAssigned) return undefined; validateNarrativeDocumentLocation(entry.calmHubId, identity, false); - const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId); - if (versions.length === 0 || !versions.includes(version)) return undefined; - const remote = await client.getNarrativeDocumentVersion( - identity.namespace, identity.type, identity.calmHubDocumentId, version - ); - if (remote.documentMarkdown === raw) return undefined; - return { - id, filePath, currentVersion: version, - latestHubVersion: sortSemVer(versions)[versions.length - 1], kind: 'narrative', + return async () => { + const versions = await client.getNarrativeDocumentVersions(identity.namespace, identity.type, identity.calmHubDocumentId); + if (versions.length === 0 || !versions.includes(version)) return undefined; + const remote = await client.getNarrativeDocumentVersion( + identity.namespace, identity.type, identity.calmHubDocumentId, version + ); + if (remote.documentMarkdown === raw) return undefined; + return { + id, filePath, currentVersion: version, + latestHubVersion: sortSemVer(versions)[versions.length - 1], kind: 'narrative', + }; }; } -async function detectChangedMappingEntry( +function prepareChangedMappingEntry( _entry: MappingWorkspaceManifestEntry, context: DetectChangedEntryContext -): Promise { +): DetectChangedEntryCheck | undefined { const { client, filePath, id, raw } = context; let metadata: DocumentMetadata; try { @@ -179,39 +198,42 @@ async function detectChangedMappingEntry( logger.warn(`Skipping '${id}': not mappable to CalmHub (${e instanceof Error ? e.message : String(e)})`); return undefined; } - if (!metadata.namespace) { + const namespace = metadata.namespace; + if (!namespace) { logger.warn(`Skipping '${id}': document $id has no namespace.`); return undefined; } - let versions: string[]; - try { - versions = await client.getMappedResourceVersions(metadata.namespace, metadata.mapping, metadata.type); - } catch (e) { - logger.error(`Failed to fetch versions for '${id}': ${e instanceof Error ? e.message : String(e)}`); - return undefined; - } + return async () => { + let versions: string[]; + try { + versions = await client.getMappedResourceVersions(namespace, metadata.mapping, metadata.type); + } catch (e) { + logger.error(`Failed to fetch versions for '${id}': ${e instanceof Error ? e.message : String(e)}`); + return undefined; + } - if (versions.length === 0) return undefined; // new resource — nothing to bump - if (!versions.includes(metadata.version)) return undefined; // already ahead — already bumped + if (versions.length === 0) return undefined; // new resource — nothing to bump + if (!versions.includes(metadata.version)) return undefined; // already ahead — already bumped - let remote: object; - try { - remote = await client.getMappedResourceByVersion(metadata.namespace, metadata.mapping, metadata.version, metadata.type); - } catch (e) { - logger.error(`Failed to fetch '${id}' @ ${metadata.version} from CalmHub: ${e instanceof Error ? e.message : String(e)}`); - return undefined; - } + let remote: object; + try { + remote = await client.getMappedResourceByVersion(namespace, metadata.mapping, metadata.version, metadata.type); + } catch (e) { + logger.error(`Failed to fetch '${id}' @ ${metadata.version} from CalmHub: ${e instanceof Error ? e.message : String(e)}`); + return undefined; + } - if (canonicalEqual(JSON.parse(raw), remote)) return undefined; + if (canonicalEqual(JSON.parse(raw), remote)) return undefined; - return { - id, - filePath, - metadata, - currentVersion: metadata.version, - latestHubVersion: sortSemVer(versions)[versions.length - 1], - kind: 'mapping', + return { + id, + filePath, + metadata, + currentVersion: metadata.version, + latestHubVersion: sortSemVer(versions)[versions.length - 1], + kind: 'mapping', + }; }; } From b30cba4bc224bb2d53a99947f8d36b23f6bbaa11 Mon Sep 17 00:00:00 2001 From: 101Steeps Date: Fri, 18 Sep 2026 12:30:27 +0100 Subject: [PATCH 24/28] test(cli): align bundle path expectations with main --- cli/src/command-helpers/workspace/bundle.spec.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cli/src/command-helpers/workspace/bundle.spec.ts b/cli/src/command-helpers/workspace/bundle.spec.ts index a10f518cd..2987fde6d 100644 --- a/cli/src/command-helpers/workspace/bundle.spec.ts +++ b/cli/src/command-helpers/workspace/bundle.spec.ts @@ -251,6 +251,7 @@ describe('bundle', () => { describe('addFileToBundle', () => { const srcFile = path.join(testDir, 'source.json'); + const referencedSrcPath = path.relative(bundlePath, srcFile).split(path.sep).join('/'); beforeEach(async () => { await writeFile(srcFile, JSON.stringify({ '$id': 'source-doc', data: 'test' })); @@ -304,7 +305,7 @@ describe('bundle', () => { }); expect((await loadManifest(bundlePath))['source-doc']).toEqual({ - path: srcFile, type: 'sad', namespace: 'finos', version: '1.0.0', + path: referencedSrcPath, type: 'sad', namespace: 'finos', version: '1.0.0', }); }); @@ -352,7 +353,7 @@ describe('bundle', () => { }); expect((await loadManifest(bundlePath))['source-doc']).toEqual({ - path: srcFile, type: 'sad', namespace: 'finos', version: '1.0.0', + path: referencedSrcPath, type: 'sad', namespace: 'finos', version: '1.0.0', calmHubDocumentId: 3, calmHubId: '/api/calm/namespaces/finos/documents/sad/3/versions/1.0.0', }); @@ -393,14 +394,14 @@ describe('bundle', () => { }); expect((await loadManifest(bundlePath))['source-doc']).toEqual({ - path: srcFile, type: 'sad', namespace: 'finos', version: '2.3.0', + path: referencedSrcPath, type: 'sad', namespace: 'finos', version: '2.3.0', calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', }); }); it.each([ - [false, srcFile], + [false, referencedSrcPath], [true, 'files/source.json'], ])('preserves a published narrative Hub identity when changing its stored path (copy: %s)', async (copy, expectedPath) => { await saveManifest(bundlePath, { @@ -437,7 +438,7 @@ describe('bundle', () => { }); expect((await loadManifest(bundlePath))['source-doc']).toEqual({ - path: srcFile, type: 'sad', namespace: 'finos', version: '2.3.0', + path: referencedSrcPath, type: 'sad', namespace: 'finos', version: '2.3.0', calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/finos/documents/sad/42/versions/2.3.0', }); @@ -525,7 +526,7 @@ describe('bundle', () => { }); expect((await loadManifest(bundlePath))['source-doc']).toEqual({ - path: srcFile, type: 'pattern', namespace: 'other', + path: referencedSrcPath, type: 'pattern', namespace: 'other', }); }); From 0eff4503a0d2d218e0fe82f8bd3aea17239ded4d Mon Sep 17 00:00:00 2001 From: Matthew Steeples Date: Fri, 18 Sep 2026 19:08:51 +0100 Subject: [PATCH 25/28] fix(cli): persist narrative recovery before create --- .../command-helpers/workspace/push.spec.ts | 128 +++++++++++++++++- cli/src/command-helpers/workspace/push.ts | 24 ++-- 2 files changed, 134 insertions(+), 18 deletions(-) diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 5a596e7da..552a45d5b 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -1,6 +1,7 @@ -import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; import { pushWorkspaceToHub } from './push'; import { loadManifest, saveManifest } from './bundle'; +import * as bundle from './bundle'; import { CalmHubClient, HubClientError } from '@finos/calm-shared'; import { mkdir, writeFile, rm } from 'fs/promises'; import path from 'path'; @@ -40,7 +41,7 @@ const mappingId = (resource: string, version = '1.0.0', type = 'architectures', const sha256 = (value: string) => createHash('sha256').update(value, 'utf8').digest('hex'); describe('pushWorkspaceToHub', () => { - const testDir = path.join(__dirname, 'test-push'); + const testDir = path.resolve(__dirname, '../../../../sandbox/test-push'); const bundlePath = path.join(testDir, 'bundle'); const filesPath = path.join(bundlePath, 'files'); @@ -63,6 +64,10 @@ describe('pushWorkspaceToHub', () => { await mkdir(filesPath, { recursive: true }); }); + afterEach(() => { + vi.restoreAllMocks(); + }); + async function writeFreshNarrative(markdown: string): Promise { await writeFile(path.join(filesPath, 'payments.md'), markdown); await saveManifest(bundlePath, { @@ -122,6 +127,115 @@ describe('pushWorkspaceToHub', () => { expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); }); + it('saves recovery state before invoking narrative create', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const persistManifest = saveManifest; + let notifySaveStarted!: () => void; + const saveStarted = new Promise((resolve) => { notifySaveStarted = resolve; }); + let allowSave!: () => void; + const saveAllowed = new Promise((resolve) => { allowSave = resolve; }); + const save = vi.spyOn(bundle, 'saveManifest').mockImplementationOnce(async (bundlePath, manifest) => { + notifySaveStarted(); + await saveAllowed; + await persistManifest(bundlePath, manifest); + }); + const location = '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0'; + const client = makeClient({ + getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2]), + createNarrativeDocument: vi.fn(async () => { + expect((await loadManifest(bundlePath)).payments).toEqual({ + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }); + return location; + }), + }); + + const push = pushWorkspaceToHub(bundlePath, client); + await saveStarted; + try { + expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + } finally { + allowSave(); + } + await push; + + expect(save.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(client.createNarrativeDocument).mock.invocationCallOrder[0]); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + expect((await loadManifest(bundlePath)).payments).toMatchObject({ calmHubDocumentId: 42, calmHubId: location }); + expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); + }); + + it('does not POST when saving pre-create recovery state fails', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const originalManifest = await loadManifest(bundlePath); + const save = vi.spyOn(bundle, 'saveManifest').mockRejectedValueOnce(new Error('manifest write failed')); + const client = makeClient({ getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2]) }); + + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/manifest write failed/); + + expect(save).toHaveBeenCalledOnce(); + expect(save).toHaveBeenCalledWith(bundlePath, { + payments: { + ...originalManifest.payments, + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }, + }); + expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + expect(await loadManifest(bundlePath)).toEqual(originalManifest); + }); + + it.each([new Error('connection reset'), 'transport failure'])( + 'recovers server-side creation after POST throws %s without another POST', async (failure) => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const documents = new Map([[1, '# Existing'], [2, '# Other']]); + const firstClient = makeClient({ + getNarrativeDocumentIds: vi.fn(async () => [...documents.keys()]), + createNarrativeDocument: vi.fn(async (_namespace, _type, request) => { + documents.set(3, request.documentMarkdown); + throw failure; + }), + }); + + await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow( + failure instanceof Error ? failure.message : failure + ); + + const pending = { + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + }; + expect((await loadManifest(bundlePath)).payments).toEqual(pending); + expect(firstClient.createNarrativeDocument).toHaveBeenCalledOnce(); + expect(firstClient.getNarrativeDocumentIds).toHaveBeenCalledOnce(); + + const retryClient = makeClient({ + getNarrativeDocumentIds: vi.fn() + .mockResolvedValueOnce([1, 2]) + .mockImplementation(async () => [...documents.keys()]), + getNarrativeDocumentVersion: vi.fn(async (_namespace, _type, documentId) => ({ + documentMarkdown: documents.get(documentId)!, + })), + }); + await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/no new document has matching Markdown/); + expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments).toEqual(pending); + + await pushWorkspaceToHub(bundlePath, retryClient); + + expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); + expect(retryClient.getNarrativeDocumentVersion).toHaveBeenCalledWith('com.example', 'sad', 3, '1.0.0'); + expect((await loadManifest(bundlePath)).payments).toEqual({ + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + calmHubDocumentId: 3, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', + }); + } + ); + it('preserves an absolute valid Location without recovery', async () => { const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; await writeFreshNarrative(markdown); @@ -224,11 +338,11 @@ describe('pushWorkspaceToHub', () => { expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); }); - it('persists a recovery fence when the post-create list is stale', async () => { + it.each(['/unexpected', undefined])('retains recovery state for Location %s when the post-create list is stale', async (location) => { const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; await writeFreshNarrative(markdown); const client = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), + createNarrativeDocument: vi.fn().mockResolvedValue(location), getNarrativeDocumentIds: vi.fn() .mockResolvedValueOnce([1, 2]) .mockResolvedValueOnce([1, 2]), @@ -411,7 +525,7 @@ describe('pushWorkspaceToHub', () => { expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); }); - it('does not run success recovery after a genuine create failure', async () => { + it('retains recovery state without immediate recovery after a Hub create error', async () => { const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; await writeFreshNarrative(markdown); const getNarrativeDocumentIds = vi.fn().mockResolvedValue([1, 2]); @@ -428,7 +542,9 @@ describe('pushWorkspaceToHub', () => { expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('calmHubDocumentId'); - expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); + expect((await loadManifest(bundlePath)).payments).toHaveProperty('createRecovery', { + documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown), + }); }); it('fails the completed push when a narrative document is invalid', async () => { diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index 23ea2f8a7..56f865aa6 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -162,6 +162,18 @@ async function pushNarrativeEntry( } const documentIdsBeforeCreate = await client.getNarrativeDocumentIds(identity.namespace, identity.type); + manifest[id] = { + path: entry.path, + type: entry.type, + ...(entry.namespace === undefined ? {} : { namespace: entry.namespace }), + version, + createRecovery: { + documentIdsBeforeCreate, + documentMarkdownSha256: sha256(narrative.request.documentMarkdown), + }, + }; + // Persist recovery before POST because a transport failure can hide a successful create. + await saveManifest(bundlePath, manifest); const location = await client.createNarrativeDocument(identity.namespace, identity.type, narrative.request); let documentId: number | undefined; @@ -180,18 +192,6 @@ async function pushNarrativeEntry( return; } - manifest[id] = { - path: entry.path, - type: entry.type, - ...(entry.namespace === undefined ? {} : { namespace: entry.namespace }), - version, - createRecovery: { - documentIdsBeforeCreate, - documentMarkdownSha256: sha256(narrative.request.documentMarkdown), - }, - }; - await saveManifest(bundlePath, manifest); - const recovered = await recoverCreatedNarrativeDocument( client, identity, documentIdsBeforeCreate, narrative.request.documentMarkdown ); From 20e36685e46cea2fcd2b5bcb4992003b6daa6e86 Mon Sep 17 00:00:00 2001 From: Matthew Steeples Date: Fri, 18 Sep 2026 20:00:55 +0100 Subject: [PATCH 26/28] refactor(shared): centralize workspace document classification --- .../workspace/document-kind.spec.ts | 12 ++++++ .../workspace/document-kind.ts | 30 ++++++++----- .../workspace-document-kind.spec.ts | 30 +++++++++++++ .../workspace-document-kind.ts | 37 ++++++++++++++++ .../workspace-document-loader.spec.ts | 29 +++++++++---- .../workspace-document-loader.ts | 42 +++++++++++++++---- shared/src/index.ts | 4 ++ 7 files changed, 159 insertions(+), 25 deletions(-) create mode 100644 shared/src/document-loader/workspace-document-kind.spec.ts create mode 100644 shared/src/document-loader/workspace-document-kind.ts diff --git a/cli/src/command-helpers/workspace/document-kind.spec.ts b/cli/src/command-helpers/workspace/document-kind.spec.ts index 25b5f09fb..558b4530c 100644 --- a/cli/src/command-helpers/workspace/document-kind.spec.ts +++ b/cli/src/command-helpers/workspace/document-kind.spec.ts @@ -7,6 +7,10 @@ import { type WorkspaceManifestEntryOperations, } from './document-kind'; import type { WorkspaceManifest, WorkspaceManifestEntry } from './bundle'; +import { + CALM_DOCUMENT_TYPES_LIST, + CALM_NARRATIVE_DOCUMENT_TYPES_LIST, +} from '@finos/calm-models/types'; describe('workspace document handlers', () => { it('resolves mapping and narrative entries through the central handler record', () => { @@ -47,6 +51,14 @@ describe('workspace document handlers', () => { expect(resolveWorkspaceDocumentType('future-document-kind')).toBeUndefined(); }); + it('keeps the handler table exhaustive for every classified document kind', () => { + const resolvedKinds = [...CALM_DOCUMENT_TYPES_LIST, ...CALM_NARRATIVE_DOCUMENT_TYPES_LIST] + .map(type => resolveWorkspaceDocumentType(type)?.kind); + + expect(new Set(resolvedKinds)).toEqual(new Set(Object.keys(WORKSPACE_DOCUMENT_HANDLERS))); + expect(resolvedKinds).not.toContain(undefined); + }); + it('selects only handlers that support JSON reference processing', () => { const manifest: WorkspaceManifest = { architecture: { path: 'architecture.json', type: 'architecture' }, diff --git a/cli/src/command-helpers/workspace/document-kind.ts b/cli/src/command-helpers/workspace/document-kind.ts index a79342fff..c724a0391 100644 --- a/cli/src/command-helpers/workspace/document-kind.ts +++ b/cli/src/command-helpers/workspace/document-kind.ts @@ -1,9 +1,11 @@ import { - isNarrativeDocumentType, - isValidCalmDocumentType, type CalmDocumentType, type NarrativeDocumentType, } from '@finos/calm-models/types'; +import { + classifyWorkspaceDocumentType, + type WorkspaceDocumentKind, +} from '@finos/calm-shared'; import type { MappingWorkspaceManifestEntry, NarrativeWorkspaceManifestEntry, @@ -24,11 +26,15 @@ export const WORKSPACE_DOCUMENT_HANDLERS = { unreadableFile: 'fail', supportsJsonReferences: false, }, -} as const; +} as const satisfies Record; export type WorkspaceDocumentHandler = typeof WORKSPACE_DOCUMENT_HANDLERS[keyof typeof WORKSPACE_DOCUMENT_HANDLERS]; -export type WorkspaceDocumentKind = keyof typeof WORKSPACE_DOCUMENT_HANDLERS; export type ResolvedWorkspaceDocumentType = | { kind: 'mapping'; handler: typeof WORKSPACE_DOCUMENT_HANDLERS.mapping; type: CalmDocumentType | 'unknown' } @@ -54,14 +60,16 @@ export type WorkspaceManifestEntryOperations { + it.each(CALM_DOCUMENT_TYPES_LIST)('assigns mapping type %s to the JSON loader policy', (type) => { + expect(classifyWorkspaceDocumentType(type)).toEqual({ kind: 'mapping', type }); + expect(getWorkspaceDocumentLoadPolicy(type)).toBe('json'); + }); + + it.each(CALM_NARRATIVE_DOCUMENT_TYPES_LIST)('assigns narrative type %s to the non-JSON loader policy', (type) => { + expect(classifyWorkspaceDocumentType(type)).toEqual({ kind: 'narrative', type }); + expect(getWorkspaceDocumentLoadPolicy(type)).toBe('non-json'); + }); + + it('retains the JSON loader policy for the legacy unknown type', () => { + expect(classifyWorkspaceDocumentType('unknown')).toEqual({ kind: 'mapping', type: 'unknown' }); + expect(getWorkspaceDocumentLoadPolicy('unknown')).toBe('json'); + }); + + it('does not assign a loader policy to an unsupported future type', () => { + expect(classifyWorkspaceDocumentType('future-document-kind')).toBeUndefined(); + expect(getWorkspaceDocumentLoadPolicy('future-document-kind')).toBeUndefined(); + }); +}); diff --git a/shared/src/document-loader/workspace-document-kind.ts b/shared/src/document-loader/workspace-document-kind.ts new file mode 100644 index 000000000..b099f1982 --- /dev/null +++ b/shared/src/document-loader/workspace-document-kind.ts @@ -0,0 +1,37 @@ +import { + isNarrativeDocumentType, + isValidCalmDocumentType, + type CalmDocumentType, + type NarrativeDocumentType, +} from '@finos/calm-models/types'; + +export type WorkspaceDocumentKind = 'mapping' | 'narrative'; + +export type ClassifiedWorkspaceDocumentType = + | { kind: 'mapping'; type: CalmDocumentType | 'unknown' } + | { kind: 'narrative'; type: NarrativeDocumentType }; + +export const WORKSPACE_DOCUMENT_LOAD_POLICIES = { + mapping: 'json', + narrative: 'non-json', +} as const satisfies Record; + +export type WorkspaceDocumentLoadPolicy = + typeof WORKSPACE_DOCUMENT_LOAD_POLICIES[WorkspaceDocumentKind]; + +/** Classify only supported workspace document types and the legacy `unknown` type. */ +export function classifyWorkspaceDocumentType(type: unknown): ClassifiedWorkspaceDocumentType | undefined { + if (typeof type !== 'string') return undefined; + if (isValidCalmDocumentType(type) || type === 'unknown') { + return { kind: 'mapping', type }; + } + if (isNarrativeDocumentType(type)) { + return { kind: 'narrative', type }; + } + return undefined; +} + +export function getWorkspaceDocumentLoadPolicy(type: unknown): WorkspaceDocumentLoadPolicy | undefined { + const document = classifyWorkspaceDocumentType(type); + return document === undefined ? undefined : WORKSPACE_DOCUMENT_LOAD_POLICIES[document.kind]; +} diff --git a/shared/src/document-loader/workspace-document-loader.spec.ts b/shared/src/document-loader/workspace-document-loader.spec.ts index ffcbcfcae..41bf93543 100644 --- a/shared/src/document-loader/workspace-document-loader.spec.ts +++ b/shared/src/document-loader/workspace-document-loader.spec.ts @@ -25,19 +25,19 @@ const BUNDLE = '/ws'; const PATTERN_ID = 'https://hub.example.com/calm/namespaces/ws/patterns/workshop/versions/1.0.0'; const patternDoc = { '$id': PATTERN_ID, title: 'Workshop Pattern' }; // A document whose $id is a host-less CalmHub path, so a full URL can match it by path. -const STANDARD_ID = '/calm/namespaces/ws/standards/security/versions/1.0.0'; -const standardDoc = { '$id': STANDARD_ID, title: 'Security Standard' }; +const SCHEMA_ID = '/calm/namespaces/ws/schemas/security/versions/1.0.0'; +const schemaDoc = { '$id': SCHEMA_ID, title: 'Security Schema' }; const noIdDoc = { title: 'No Id Document' }; function setupBundle(extra: Record = {}) { vol.fromJSON({ '/ws/workspace-manifest.json': JSON.stringify({ 'workshop-pattern': { path: 'files/workshop-pattern.json', type: 'pattern' }, - 'security-standard': { path: 'files/security-standard.json', type: 'standard' }, + 'security-schema': { path: 'files/security-schema.json', type: 'schema' }, 'no-id-doc': { path: 'files/no-id.json', type: 'architecture' }, }), '/ws/files/workshop-pattern.json': JSON.stringify(patternDoc), - '/ws/files/security-standard.json': JSON.stringify(standardDoc), + '/ws/files/security-schema.json': JSON.stringify(schemaDoc), '/ws/files/no-id.json': JSON.stringify(noIdDoc), ...extra, }); @@ -76,7 +76,7 @@ describe('WorkspaceDocumentLoader', () => { // A host-less $id should not match a ref from an arbitrary host — the ref may // point to a completely different service on the same CalmHub path. const loader = new WorkspaceDocumentLoader(BUNDLE); - const url = 'https://any-host.example.com/calm/namespaces/ws/standards/security/versions/3.0.0'; + const url = 'https://any-host.example.com/calm/namespaces/ws/schemas/security/versions/3.0.0'; expect(loader.resolvePath(url)).toBeUndefined(); }); @@ -96,8 +96,8 @@ describe('WorkspaceDocumentLoader', () => { it('resolves an unversioned path ref that equals the $id base path', () => { // A $ref with no /versions/ segment should still resolve locally. const loader = new WorkspaceDocumentLoader(BUNDLE); - const unversioned = '/calm/namespaces/ws/standards/security'; - expect(loader.resolvePath(unversioned)).toBe('/ws/files/security-standard.json'); + const unversioned = '/calm/namespaces/ws/schemas/security'; + expect(loader.resolvePath(unversioned)).toBe('/ws/files/security-schema.json'); }); it('ignores a #/... fragment when matching', () => { @@ -175,6 +175,21 @@ describe('WorkspaceDocumentLoader', () => { expect(mocks.schemaDirectory.storeDocument).not.toHaveBeenCalled(); }); + + it('ignores unsupported future document types', async () => { + setupBundle({ + '/ws/workspace-manifest.json': JSON.stringify({ + future: { path: 'files/future.json', type: 'future-document-kind' }, + }), + '/ws/files/future.json': JSON.stringify({ '$id': 'future' }), + }); + const loader = new WorkspaceDocumentLoader(BUNDLE); + + await loader.initialise(mocks.schemaDirectory as unknown as SchemaDirectory); + + expect(loader.resolvePath('future')).toBeUndefined(); + expect(mocks.schemaDirectory.storeDocument).not.toHaveBeenCalled(); + }); }); describe('with no usable manifest', () => { diff --git a/shared/src/document-loader/workspace-document-loader.ts b/shared/src/document-loader/workspace-document-loader.ts index 502f6377c..63acf2ab4 100644 --- a/shared/src/document-loader/workspace-document-loader.ts +++ b/shared/src/document-loader/workspace-document-loader.ts @@ -1,10 +1,15 @@ import { DocumentLoader, DocumentLoadError } from './document-loader'; -import { isNarrativeDocumentType, type CalmDocumentType } from '@finos/calm-models/types'; +import type { CalmDocumentType } from '@finos/calm-models/types'; import { initLogger, Logger } from '../logger'; import { readFile } from 'fs/promises'; import { existsSync, readFileSync } from 'fs'; import { SchemaDirectory } from '../schema-directory'; import path from 'path'; +import { + getWorkspaceDocumentLoadPolicy, + WORKSPACE_DOCUMENT_LOAD_POLICIES, + type WorkspaceDocumentLoadPolicy, +} from './workspace-document-kind'; // Mirrors MANIFEST_FILENAME in the CLI workspace bundle module. const MANIFEST_FILENAME = 'workspace-manifest.json'; @@ -79,12 +84,6 @@ export class WorkspaceDocumentLoader implements DocumentLoader { const rules: WorkspaceRule[] = []; for (const [bareId, value] of Object.entries(manifest)) { // Manifest entries are `{ path, type, ... }`; tolerate the legacy plain-string form too. - const type = value && typeof value === 'object' - ? (value as { type?: unknown }).type - : undefined; - // Narrative documents are Markdown rather than CALM JSON documents. They cannot be - // schema-preloaded or resolve a CALM `$ref`, so keep them out of this JSON-only loader. - if (isNarrativeDocumentType(type)) continue; const relPath = typeof value === 'string' ? value : (value && typeof value === 'object' && typeof (value as { path?: unknown }).path === 'string' @@ -92,6 +91,20 @@ export class WorkspaceDocumentLoader implements DocumentLoader { : undefined); if (!relPath) continue; + const type = value && typeof value === 'object' + ? (value as { type?: unknown }).type + : undefined; + const loadPolicy = typeof value === 'string' + ? WORKSPACE_DOCUMENT_LOAD_POLICIES.mapping + : getWorkspaceDocumentLoadPolicy(type); + if (loadPolicy === undefined) { + this.logger.warn(`Ignoring '${bareId}' with unsupported workspace document type '${String(type)}'.`); + continue; + } + if (!usesJsonLoader(loadPolicy)) { + continue; + } + const localPath = path.isAbsolute(relPath) ? relPath : path.resolve(this.bundlePath, relPath); const rule: WorkspaceRule = { bareId, localPath }; @@ -212,3 +225,18 @@ export class WorkspaceDocumentLoader implements DocumentLoader { return this.resolveRule(reference); } } + +function usesJsonLoader(policy: WorkspaceDocumentLoadPolicy): boolean { + switch (policy) { + case 'json': + return true; + case 'non-json': + return false; + default: + return assertNever(policy); + } +} + +function assertNever(value: never): never { + throw new Error(`Unsupported workspace document load policy '${String(value)}'.`); +} diff --git a/shared/src/index.ts b/shared/src/index.ts index 8600f3b09..1ee2c7c9b 100644 --- a/shared/src/index.ts +++ b/shared/src/index.ts @@ -70,6 +70,10 @@ export { DocumentLoader, DocumentLoaderOptions, DocumentLoadError, assertJsonObj export { buildDocumentLoader } from './document-loader/node-document-loader.js'; export { FileSystemDocumentLoader } from './document-loader/file-system-document-loader.js'; export { WorkspaceDocumentLoader } from './document-loader/workspace-document-loader.js'; +export { + classifyWorkspaceDocumentType, + type WorkspaceDocumentKind, +} from './document-loader/workspace-document-kind.js'; export * from './document-loader/loading-helpers.js'; export { hasArchitectureExtension, From 470294620196ee963e06e501bd891e42ba2768d3 Mon Sep 17 00:00:00 2001 From: Matthew Steeples Date: Fri, 18 Sep 2026 21:17:48 +0100 Subject: [PATCH 27/28] chore(cli): remove unrelated review noise --- cli/src/command-helpers/workspace/commands.spec.ts | 3 +-- cli/src/command-helpers/workspace/commands.ts | 1 + cli/src/command-helpers/workspace/config.spec.ts | 2 +- cli/src/command-helpers/workspace/document-kind.ts | 9 --------- .../command-helpers/workspace/narrative-document.spec.ts | 6 ------ cli/src/command-helpers/workspace/push.spec.ts | 3 +-- 6 files changed, 4 insertions(+), 20 deletions(-) diff --git a/cli/src/command-helpers/workspace/commands.spec.ts b/cli/src/command-helpers/workspace/commands.spec.ts index 3c9d8ace5..153e4b531 100644 --- a/cli/src/command-helpers/workspace/commands.spec.ts +++ b/cli/src/command-helpers/workspace/commands.spec.ts @@ -1,6 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { Command } from 'commander'; -import path from 'path'; import { CALM_NARRATIVE_DOCUMENT_TYPES_LIST } from '@finos/calm-models/types'; import { setupWorkspaceCommands } from './commands'; @@ -155,7 +154,7 @@ describe('setupWorkspaceCommands', () => { it('should call ensureWorkspaceBundle with custom dir', async () => { await program.parseAsync(['node', 'test', 'workspace', 'init', 'my-ws', '--dir', '/custom/dir']); expect(mocks.ensureWorkspaceBundle).toHaveBeenCalledWith( - path.resolve('/custom/dir'), + '/custom/dir', 'my-ws' ); }); diff --git a/cli/src/command-helpers/workspace/commands.ts b/cli/src/command-helpers/workspace/commands.ts index 41288ab4f..74c0de9f4 100644 --- a/cli/src/command-helpers/workspace/commands.ts +++ b/cli/src/command-helpers/workspace/commands.ts @@ -703,3 +703,4 @@ async function enforceOptionPresenceByPrompt(cliInput: string | undefined, promp message: prompt }); }; + diff --git a/cli/src/command-helpers/workspace/config.spec.ts b/cli/src/command-helpers/workspace/config.spec.ts index 2ffc6956e..bdbad59fa 100644 --- a/cli/src/command-helpers/workspace/config.spec.ts +++ b/cli/src/command-helpers/workspace/config.spec.ts @@ -24,7 +24,7 @@ describe('workspace config', () => { writeFile(getWorkspaceConfigPath(gitRoot), content, 'utf8'); it('getWorkspaceConfigPath points at .calm-workspace/config.json', () => { - expect(getWorkspaceConfigPath('/repo')).toBe(path.join('/repo', '.calm-workspace', 'config.json')); + expect(getWorkspaceConfigPath('/repo')).toBe('/repo/.calm-workspace/config.json'); }); it('returns defaults when the config file is absent', async () => { diff --git a/cli/src/command-helpers/workspace/document-kind.ts b/cli/src/command-helpers/workspace/document-kind.ts index c724a0391..53f5de282 100644 --- a/cli/src/command-helpers/workspace/document-kind.ts +++ b/cli/src/command-helpers/workspace/document-kind.ts @@ -15,27 +15,18 @@ import type { export const WORKSPACE_DOCUMENT_HANDLERS = { mapping: { - kind: 'mapping', - format: 'json', unreadableFile: 'warn', supportsJsonReferences: true, }, narrative: { - kind: 'narrative', - format: 'markdown', unreadableFile: 'fail', supportsJsonReferences: false, }, } as const satisfies Record; -export type WorkspaceDocumentHandler = - typeof WORKSPACE_DOCUMENT_HANDLERS[keyof typeof WORKSPACE_DOCUMENT_HANDLERS]; - export type ResolvedWorkspaceDocumentType = | { kind: 'mapping'; handler: typeof WORKSPACE_DOCUMENT_HANDLERS.mapping; type: CalmDocumentType | 'unknown' } | { kind: 'narrative'; handler: typeof WORKSPACE_DOCUMENT_HANDLERS.narrative; type: NarrativeDocumentType }; diff --git a/cli/src/command-helpers/workspace/narrative-document.spec.ts b/cli/src/command-helpers/workspace/narrative-document.spec.ts index fb2cde7ae..03a731a02 100644 --- a/cli/src/command-helpers/workspace/narrative-document.spec.ts +++ b/cli/src/command-helpers/workspace/narrative-document.spec.ts @@ -64,12 +64,6 @@ describe('narrative document helpers', () => { expect(() => resolveNarrativeEntry('payments', entry, markdown)).toThrow(message); }); - it('parses Markdown before validating the constructed identity', () => { - expect(() => resolveNarrativeEntry( - 'payments', { ...identity, version: 'latest' }, '# No frontmatter' - )).toThrow(/must contain non-empty YAML mapping frontmatter/); - }); - it('rejects malformed narrative Markdown', () => { expect(() => resolveNarrativeEntry('payments', identity, '# No frontmatter')).toThrow( /must contain non-empty YAML mapping frontmatter/ diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index 552a45d5b..d37a477c8 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -135,7 +135,7 @@ describe('pushWorkspaceToHub', () => { const saveStarted = new Promise((resolve) => { notifySaveStarted = resolve; }); let allowSave!: () => void; const saveAllowed = new Promise((resolve) => { allowSave = resolve; }); - const save = vi.spyOn(bundle, 'saveManifest').mockImplementationOnce(async (bundlePath, manifest) => { + vi.spyOn(bundle, 'saveManifest').mockImplementationOnce(async (bundlePath, manifest) => { notifySaveStarted(); await saveAllowed; await persistManifest(bundlePath, manifest); @@ -161,7 +161,6 @@ describe('pushWorkspaceToHub', () => { } await push; - expect(save.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(client.createNarrativeDocument).mock.invocationCallOrder[0]); expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); expect((await loadManifest(bundlePath)).payments).toMatchObject({ calmHubDocumentId: 42, calmHubId: location }); expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); From 474c07e644b584c1f02fac3fcbce242334206a6f Mon Sep 17 00:00:00 2001 From: Matthew Steeples Date: Fri, 18 Sep 2026 21:43:22 +0100 Subject: [PATCH 28/28] fix(cli): make narrative create recovery explicit --- .../command-helpers/workspace/bump.spec.ts | 5 +- .../command-helpers/workspace/bundle.spec.ts | 13 +- cli/src/command-helpers/workspace/bundle.ts | 3 +- .../command-helpers/workspace/push.spec.ts | 358 ++++-------------- cli/src/command-helpers/workspace/push.ts | 122 ++---- 5 files changed, 128 insertions(+), 373 deletions(-) diff --git a/cli/src/command-helpers/workspace/bump.spec.ts b/cli/src/command-helpers/workspace/bump.spec.ts index d75fe7602..69cec9e53 100644 --- a/cli/src/command-helpers/workspace/bump.spec.ts +++ b/cli/src/command-helpers/workspace/bump.spec.ts @@ -94,10 +94,7 @@ describe('bump', () => { const markdown = '---\ntitle: Payments SAD\n---\n# Published\n'; const entry = { path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', - createRecovery: { - documentIdsBeforeCreate: [1, 2], - documentMarkdownSha256: 'a'.repeat(64), - }, + createRecovery: { pending: true as const }, }; await writeFile(path.join(filesPath, 'payments.md'), markdown); await saveManifest(bundlePath, { payments: entry }); diff --git a/cli/src/command-helpers/workspace/bundle.spec.ts b/cli/src/command-helpers/workspace/bundle.spec.ts index 2987fde6d..8670a0883 100644 --- a/cli/src/command-helpers/workspace/bundle.spec.ts +++ b/cli/src/command-helpers/workspace/bundle.spec.ts @@ -21,7 +21,6 @@ describe('bundle', () => { const testDir = path.join(__dirname, 'test-bundle'); const bundlePath = path.join(testDir, 'bundle'); const filesPath = path.join(bundlePath, 'files'); - const documentMarkdownSha256 = 'a'.repeat(64); beforeAll(async () => { await mkdir(testDir, { recursive: true }); @@ -65,7 +64,7 @@ describe('bundle', () => { }; const pendingNarrative: WorkspaceManifestEntry = { path: 'files/design.md', type: 'sad', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + createRecovery: { pending: true }, }; expect(isNarrativeWorkspaceManifestEntry(mapping)).toBe(false); @@ -99,7 +98,7 @@ describe('bundle', () => { // @ts-expect-error Pending narrative entries cannot also have a published identity. const pendingPublishedNarrative: WorkspaceManifestEntry = { path: 'files/design.md', type: 'sad', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + createRecovery: { pending: true }, calmHubDocumentId: 42, calmHubId: '/documents/sad/42/versions/1.0.0', }; @@ -312,7 +311,7 @@ describe('bundle', () => { it('rejects re-adding a narrative while create recovery is pending', async () => { const existing = { path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + createRecovery: { pending: true as const }, }; await saveManifest(bundlePath, { 'source-doc': existing }); @@ -327,7 +326,7 @@ describe('bundle', () => { it('rejects replacing a pending narrative with a mapping', async () => { const existing = { path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + createRecovery: { pending: true as const }, }; await saveManifest(bundlePath, { 'source-doc': existing }); @@ -343,7 +342,7 @@ describe('bundle', () => { await saveManifest(bundlePath, { 'source-doc': { path: 'old.md', type: 'sad', namespace: 'finos', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + createRecovery: { pending: true }, }, }); @@ -367,7 +366,7 @@ describe('bundle', () => { ])('rejects verified identity outside the pending recovery scope', async ({ type, namespace, version }) => { const existing = { path: 'old.md', type: 'sad' as const, namespace: 'finos', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256 }, + createRecovery: { pending: true as const }, }; await saveManifest(bundlePath, { 'source-doc': existing }); diff --git a/cli/src/command-helpers/workspace/bundle.ts b/cli/src/command-helpers/workspace/bundle.ts index 7677deaff..c4dcc4ade 100644 --- a/cli/src/command-helpers/workspace/bundle.ts +++ b/cli/src/command-helpers/workspace/bundle.ts @@ -94,8 +94,7 @@ export type UnpublishedNarrativeWorkspaceManifestEntry = NarrativeWorkspaceManif }; export type NarrativeCreateRecovery = { - documentIdsBeforeCreate: number[]; - documentMarkdownSha256: string; + pending: true; }; export type CreateRecoveryPendingNarrativeWorkspaceManifestEntry = NarrativeWorkspaceManifestEntryBase & { diff --git a/cli/src/command-helpers/workspace/push.spec.ts b/cli/src/command-helpers/workspace/push.spec.ts index d37a477c8..a6de7002c 100644 --- a/cli/src/command-helpers/workspace/push.spec.ts +++ b/cli/src/command-helpers/workspace/push.spec.ts @@ -6,7 +6,6 @@ import { CalmHubClient, HubClientError } from '@finos/calm-shared'; import { mkdir, writeFile, rm } from 'fs/promises'; import path from 'path'; import { existsSync } from 'fs'; -import { createHash } from 'node:crypto'; const makeClient = ( overrides: Partial ({ const BASE = 'https://hub.example.com'; const mappingId = (resource: string, version = '1.0.0', type = 'architectures', ns = 'com.example') => `${BASE}/calm/namespaces/${ns}/${type}/${resource}/versions/${version}`; -const sha256 = (value: string) => createHash('sha256').update(value, 'utf8').digest('hex'); describe('pushWorkspaceToHub', () => { const testDir = path.resolve(__dirname, '../../../../sandbox/test-push'); @@ -122,7 +120,7 @@ describe('pushWorkspaceToHub', () => { 'payments-sad': { calmHubDocumentId: 42, calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0' }, }); expect((await loadManifest(bundlePath))['payments-sad']).not.toHaveProperty('createRecovery'); - expect(client.getNarrativeDocumentIds).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); }); @@ -142,11 +140,10 @@ describe('pushWorkspaceToHub', () => { }); const location = '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0'; const client = makeClient({ - getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2]), createNarrativeDocument: vi.fn(async () => { expect((await loadManifest(bundlePath)).payments).toEqual({ path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + createRecovery: { pending: true }, }); return location; }), @@ -171,7 +168,7 @@ describe('pushWorkspaceToHub', () => { await writeFreshNarrative(markdown); const originalManifest = await loadManifest(bundlePath); const save = vi.spyOn(bundle, 'saveManifest').mockRejectedValueOnce(new Error('manifest write failed')); - const client = makeClient({ getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2]) }); + const client = makeClient(); await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/manifest write failed/); @@ -179,61 +176,67 @@ describe('pushWorkspaceToHub', () => { expect(save).toHaveBeenCalledWith(bundlePath, { payments: { ...originalManifest.payments, - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + createRecovery: { pending: true }, }, }); expect(client.createNarrativeDocument).not.toHaveBeenCalled(); expect(await loadManifest(bundlePath)).toEqual(originalManifest); }); - it.each([new Error('connection reset'), 'transport failure'])( - 'recovers server-side creation after POST throws %s without another POST', async (failure) => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const documents = new Map([[1, '# Existing'], [2, '# Other']]); - const firstClient = makeClient({ - getNarrativeDocumentIds: vi.fn(async () => [...documents.keys()]), - createNarrativeDocument: vi.fn(async (_namespace, _type, request) => { - documents.set(3, request.documentMarkdown); - throw failure; - }), - }); + it('keeps a status 0 create pending and never adopts another matching document on retry', async () => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const firstClient = makeClient({ + createNarrativeDocument: vi.fn().mockRejectedValue( + new HubClientError(0, 'connection reset', 'POST /api/calm/namespaces/com.example/documents/sad') + ), + }); - await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow( - failure instanceof Error ? failure.message : failure - ); + await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow(/Hub error 0/); - const pending = { - path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, - }; - expect((await loadManifest(bundlePath)).payments).toEqual(pending); - expect(firstClient.createNarrativeDocument).toHaveBeenCalledOnce(); - expect(firstClient.getNarrativeDocumentIds).toHaveBeenCalledOnce(); - - const retryClient = makeClient({ - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockImplementation(async () => [...documents.keys()]), - getNarrativeDocumentVersion: vi.fn(async (_namespace, _type, documentId) => ({ - documentMarkdown: documents.get(documentId)!, - })), - }); - await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/no new document has matching Markdown/); - expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); - expect((await loadManifest(bundlePath)).payments).toEqual(pending); + const pending = { + path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', + createRecovery: { pending: true as const }, + }; + expect((await loadManifest(bundlePath)).payments).toEqual(pending); + expect(firstClient.createNarrativeDocument).toHaveBeenCalledOnce(); - await pushWorkspaceToHub(bundlePath, retryClient); + const retryClient = makeClient({ + getNarrativeDocumentIds: vi.fn().mockResolvedValue([77]), + getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), + }); + await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/Explicit reconciliation is required/); - expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); - expect(retryClient.getNarrativeDocumentVersion).toHaveBeenCalledWith('com.example', 'sad', 3, '1.0.0'); - expect((await loadManifest(bundlePath)).payments).toEqual({ - path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', - calmHubDocumentId: 3, - calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', - }); - } - ); + expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); + expect(retryClient.getNarrativeDocumentIds).not.toHaveBeenCalled(); + expect(retryClient.getNarrativeDocumentVersion).not.toHaveBeenCalled(); + expect((await loadManifest(bundlePath)).payments).toEqual(pending); + }); + + it.each([400, 401, 403, 404])('restores unpublished state after definite create rejection %i and permits retry', async (status) => { + const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; + await writeFreshNarrative(markdown); + const unpublished = (await loadManifest(bundlePath)).payments; + const rejectedClient = makeClient({ + createNarrativeDocument: vi.fn().mockRejectedValue( + new HubClientError(status, 'rejected', 'POST /api/calm/namespaces/com.example/documents/sad') + ), + }); + + await expect(pushWorkspaceToHub(bundlePath, rejectedClient)).rejects.toThrow(`Hub error ${status}`); + + expect((await loadManifest(bundlePath)).payments).toEqual(unpublished); + expect(rejectedClient.createNarrativeDocument).toHaveBeenCalledOnce(); + + const retryClient = makeClient(); + await pushWorkspaceToHub(bundlePath, retryClient); + + expect(retryClient.createNarrativeDocument).toHaveBeenCalledOnce(); + expect((await loadManifest(bundlePath)).payments).toMatchObject({ + calmHubDocumentId: 42, + calmHubId: '/api/calm/namespaces/com.example/documents/sad/42/versions/1.0.0', + }); + }); it('preserves an absolute valid Location without recovery', async () => { const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; @@ -245,7 +248,7 @@ describe('pushWorkspaceToHub', () => { expect((await loadManifest(bundlePath)).payments).toMatchObject({ calmHubDocumentId: 42, calmHubId: location }); expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); - expect(client.getNarrativeDocumentIds).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); }); @@ -273,241 +276,39 @@ describe('pushWorkspaceToHub', () => { expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); }); - it('recovers a created narrative document from authoritative Hub state', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const getNarrativeDocumentIds = vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2, 3]); - const client = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), - getNarrativeDocumentIds, - getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), - }); - - await pushWorkspaceToHub(bundlePath, client); - - expect((await loadManifest(bundlePath)).payments).toMatchObject({ - calmHubDocumentId: 3, - calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', - }); - expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); - expect(client.getNarrativeDocumentVersion).toHaveBeenCalledWith('com.example', 'sad', 3, '1.0.0'); - expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); - }); - - it('recovers only the matching document when concurrent documents appear', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const client = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2, 3, 4]), - getNarrativeDocumentVersion: vi.fn().mockImplementation(async (_namespace, _type, documentId) => ({ - documentMarkdown: documentId === 4 ? markdown : '---\ntitle: Other\n---\n# Other', - })), - }); - - await pushWorkspaceToHub(bundlePath, client); - - expect((await loadManifest(bundlePath)).payments).toMatchObject({ calmHubDocumentId: 4 }); - expect(client.getNarrativeDocumentVersion).toHaveBeenCalledTimes(2); - expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); - }); - - it('uses the same recovery path when a confirmed create has no Location', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const client = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue(undefined), - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2, 3]), - getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), - }); - - await pushWorkspaceToHub(bundlePath, client); - - expect((await loadManifest(bundlePath)).payments).toMatchObject({ - calmHubDocumentId: 3, - calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', - }); - expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('createRecovery'); - expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); - }); - - it.each(['/unexpected', undefined])('retains recovery state for Location %s when the post-create list is stale', async (location) => { + it.each(['/unexpected', undefined])('retains pending state for unusable Location %s without automatic recovery', async (location) => { const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; await writeFreshNarrative(markdown); const client = makeClient({ createNarrativeDocument: vi.fn().mockResolvedValue(location), - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2]), - }); - - await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/no new document has matching Markdown/); - - expect((await loadManifest(bundlePath)).payments).toEqual({ - path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, - }); - expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); - }); - - it('fails ambiguous recovery when multiple new documents match', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const client = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2, 3, 4]), - getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), - }); - - await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/multiple new documents have matching Markdown/); - - expect((await loadManifest(bundlePath)).payments).toMatchObject({ - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, - }); - expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('calmHubDocumentId'); - expect(client.getNarrativeDocumentVersion).toHaveBeenCalledTimes(2); - expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); - }); - - it('recovers a pending create on retry without another POST', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const firstClient = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2]), - }); - await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow(/no new document/); - - const retryClient = makeClient({ - getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3]), + getNarrativeDocumentIds: vi.fn().mockResolvedValue([77]), getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), }); - await pushWorkspaceToHub(bundlePath, retryClient); - - expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); - expect((await loadManifest(bundlePath)).payments).toEqual({ - path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', - calmHubDocumentId: 3, - calmHubId: '/api/calm/namespaces/com.example/documents/sad/3/versions/1.0.0', - }); - }); - - it('does not recover against changed local Markdown', async () => { - const markdownA = '---\ntitle: Payments SAD\n---\n# Payload A'; - const markdownB = '---\ntitle: Payments SAD\n---\n# Payload B'; - await writeFreshNarrative(markdownA); - const firstClient = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2]), - }); - await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow(/no new document/); - const pending = (await loadManifest(bundlePath)).payments; - - await writeFile(path.join(filesPath, 'payments.md'), markdownB); - const retryClient = makeClient({ - getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3, 4]), - getNarrativeDocumentVersion: vi.fn().mockImplementation(async (_namespace, _type, documentId) => ({ - documentMarkdown: documentId === 3 ? markdownA : markdownB, - })), - }); - - await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/Markdown changed while create recovery is pending/); - expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); - expect(retryClient.getNarrativeDocumentIds).not.toHaveBeenCalled(); - expect(retryClient.getNarrativeDocumentVersion).not.toHaveBeenCalled(); - expect((await loadManifest(bundlePath)).payments).toEqual(pending); - expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('calmHubDocumentId'); - }); + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/Explicit reconciliation is required/); - it('keeps a pending recovery fence when retry still has no match', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); const pending = { - path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, + path: 'files/payments.md', type: 'sad', namespace: 'com.example', version: '1.0.0', + createRecovery: { pending: true }, }; - await saveManifest(bundlePath, { payments: pending }); - const client = makeClient({ - getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3]), - getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: '# Different' }), - }); - - await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/no new document has matching Markdown/); - - expect(client.createNarrativeDocument).not.toHaveBeenCalled(); expect((await loadManifest(bundlePath)).payments).toEqual(pending); - }); - - it('keeps a pending recovery fence when retry remains ambiguous', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const pending = { - path: 'files/payments.md', type: 'sad' as const, namespace: 'com.example', version: '1.0.0', - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, - }; - await saveManifest(bundlePath, { payments: pending }); - const client = makeClient({ - getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3, 4]), - getNarrativeDocumentVersion: vi.fn().mockResolvedValue({ documentMarkdown: markdown }), - }); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); + expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); - await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/multiple new documents have matching Markdown/); + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/--calm-hub-document-id /); - expect(client.createNarrativeDocument).not.toHaveBeenCalled(); + expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); + expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); expect((await loadManifest(bundlePath)).payments).toEqual(pending); }); - it('keeps a recovery fence after a candidate GET error and never POSTs on retry', async () => { - const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; - await writeFreshNarrative(markdown); - const firstClient = makeClient({ - createNarrativeDocument: vi.fn().mockResolvedValue('/unexpected'), - getNarrativeDocumentIds: vi.fn() - .mockResolvedValueOnce([1, 2]) - .mockResolvedValueOnce([1, 2, 3]), - getNarrativeDocumentVersion: vi.fn().mockRejectedValue(new Error('temporarily unavailable')), - }); - await expect(pushWorkspaceToHub(bundlePath, firstClient)).rejects.toThrow(/temporarily unavailable/); - expect((await loadManifest(bundlePath)).payments).toMatchObject({ - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, - }); - - const retryClient = makeClient({ - getNarrativeDocumentIds: vi.fn().mockResolvedValue([1, 2, 3]), - getNarrativeDocumentVersion: vi.fn().mockRejectedValue(new Error('still unavailable')), - }); - await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/still unavailable/); - - expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); - expect((await loadManifest(bundlePath)).payments).toMatchObject({ - createRecovery: { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown) }, - }); - }); - it.each([ null, {}, - { documentIdsBeforeCreate: '1,2', documentMarkdownSha256: 'a'.repeat(64) }, - { documentIdsBeforeCreate: [0], documentMarkdownSha256: 'a'.repeat(64) }, - { documentIdsBeforeCreate: [1.5], documentMarkdownSha256: 'a'.repeat(64) }, - { documentIdsBeforeCreate: [Number.MAX_SAFE_INTEGER + 1], documentMarkdownSha256: 'a'.repeat(64) }, - { documentIdsBeforeCreate: [1, 2] }, - { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'not-a-digest' }, - { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'A'.repeat(64) }, - { documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: 'a'.repeat(63) }, + { pending: false }, + { pending: true, extra: true }, ])('rejects malformed persisted create recovery before parsing or posting: %j', async (createRecovery) => { const markdown = '# no frontmatter'; await writeFreshNarrative(markdown); @@ -524,26 +325,29 @@ describe('pushWorkspaceToHub', () => { expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); }); - it('retains recovery state without immediate recovery after a Hub create error', async () => { + it.each([500, 413])('retains pending state after ambiguous Hub status %i and does not POST on retry', async (status) => { const markdown = '---\ntitle: Payments SAD\n---\n# Payments'; await writeFreshNarrative(markdown); - const getNarrativeDocumentIds = vi.fn().mockResolvedValue([1, 2]); const client = makeClient({ createNarrativeDocument: vi.fn().mockRejectedValue( - new HubClientError(500, 'unavailable', 'POST /api/calm/namespaces/com.example/documents/sad') + new HubClientError(status, 'unavailable', 'POST /api/calm/namespaces/com.example/documents/sad') ), - getNarrativeDocumentIds, }); - await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(/Hub error 500/); + await expect(pushWorkspaceToHub(bundlePath, client)).rejects.toThrow(`Hub error ${status}`); - expect(getNarrativeDocumentIds).toHaveBeenCalledOnce(); + expect(client.getNarrativeDocumentIds).not.toHaveBeenCalled(); expect(client.getNarrativeDocumentVersion).not.toHaveBeenCalled(); expect(client.createNarrativeDocument).toHaveBeenCalledOnce(); expect((await loadManifest(bundlePath)).payments).not.toHaveProperty('calmHubDocumentId'); - expect((await loadManifest(bundlePath)).payments).toHaveProperty('createRecovery', { - documentIdsBeforeCreate: [1, 2], documentMarkdownSha256: sha256(markdown), - }); + expect((await loadManifest(bundlePath)).payments).toHaveProperty('createRecovery', { pending: true }); + + const retryClient = makeClient(); + await expect(pushWorkspaceToHub(bundlePath, retryClient)).rejects.toThrow(/Explicit reconciliation is required/); + + expect(retryClient.createNarrativeDocument).not.toHaveBeenCalled(); + expect(retryClient.getNarrativeDocumentIds).not.toHaveBeenCalled(); + expect(retryClient.getNarrativeDocumentVersion).not.toHaveBeenCalled(); }); it('fails the completed push when a narrative document is invalid', async () => { diff --git a/cli/src/command-helpers/workspace/push.ts b/cli/src/command-helpers/workspace/push.ts index 56f865aa6..85c41f0cd 100644 --- a/cli/src/command-helpers/workspace/push.ts +++ b/cli/src/command-helpers/workspace/push.ts @@ -1,6 +1,5 @@ import { readFile } from 'fs/promises'; import { existsSync } from 'fs'; -import { createHash } from 'node:crypto'; import { loadManifest, saveManifest, @@ -14,16 +13,15 @@ import { import { CalmHubClient, DocumentMetadata, + HubClientError, extractDocumentMetadata, initLogger, Logger, } from '@finos/calm-shared'; import { canonicalEqual } from './bump'; import { - constructNarrativeDocumentPath, parseNarrativeDocumentLocation, resolveNarrativeEntry, - type NarrativeDocumentIdentity, validateNarrativeDocumentLocation, } from './narrative-document'; import { @@ -33,6 +31,7 @@ import { } from './document-kind'; const logger: Logger = initLogger(false, 'workspace'); +const DEFINITE_CREATE_REJECTION_STATUSES = new Set([400, 401, 403, 404]); export interface PushOptions { /** @@ -139,6 +138,9 @@ async function pushNarrativeEntry( const { bundlePath, client, conflicts, failIfModified, id, manifest, narrativeFailures, raw } = context; try { const createRecovery = getCreateRecovery(entry); + if (createRecovery !== undefined) { + throw new Error(createReconciliationMessage(id, entry)); + } const resolved = resolveNarrativeEntry(id, entry, raw); const { version, narrative } = resolved; @@ -148,56 +150,40 @@ async function pushNarrativeEntry( throw new Error('A narrative document without calmHubDocumentId must use version 1.0.0.'); } - if (createRecovery !== undefined) { - if (sha256(narrative.request.documentMarkdown) !== createRecovery.documentMarkdownSha256) { - throw new Error('Narrative document Markdown changed while create recovery is pending. Restore the submitted content or reconcile it explicitly.'); - } - const recovered = await recoverCreatedNarrativeDocument( - client, identity, createRecovery.documentIdsBeforeCreate, narrative.request.documentMarkdown - ); - manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); - await saveManifest(bundlePath, manifest); - logger.info(`Recovered '${id}' version ${version} -> ${recovered.location}`); - return; - } - - const documentIdsBeforeCreate = await client.getNarrativeDocumentIds(identity.namespace, identity.type); manifest[id] = { path: entry.path, type: entry.type, ...(entry.namespace === undefined ? {} : { namespace: entry.namespace }), version, - createRecovery: { - documentIdsBeforeCreate, - documentMarkdownSha256: sha256(narrative.request.documentMarkdown), - }, + createRecovery: { pending: true }, }; // Persist recovery before POST because a transport failure can hide a successful create. await saveManifest(bundlePath, manifest); - const location = await client.createNarrativeDocument(identity.namespace, identity.type, narrative.request); - - let documentId: number | undefined; - if (location !== undefined) { - try { - documentId = parseNarrativeDocumentLocation(location, identity); - } catch { - // The POST succeeded, but the Location cannot establish the document identity. + let location: string | undefined; + try { + location = await client.createNarrativeDocument(identity.namespace, identity.type, narrative.request); + } catch (e) { + if (isDefiniteCreateRejection(e)) { + manifest[id] = entry; + await saveManifest(bundlePath, manifest); } + throw e; } - if (documentId !== undefined && location !== undefined) { - manifest[id] = publishNarrativeEntry(entry, documentId, location); - await saveManifest(bundlePath, manifest); - logger.info(`Pushed '${id}' version ${version} -> ${location}`); - return; + if (location === undefined) { + throw new Error(createReconciliationMessage(id, entry)); } - const recovered = await recoverCreatedNarrativeDocument( - client, identity, documentIdsBeforeCreate, narrative.request.documentMarkdown - ); - manifest[id] = publishNarrativeEntry(entry, recovered.documentId, recovered.location); + let documentId: number; + try { + documentId = parseNarrativeDocumentLocation(location, identity); + } catch { + throw new Error(createReconciliationMessage(id, entry)); + } + + manifest[id] = publishNarrativeEntry(entry, documentId, location); await saveManifest(bundlePath, manifest); - logger.info(`Pushed '${id}' version ${version} -> ${recovered.location}`); + logger.info(`Pushed '${id}' version ${version} -> ${location}`); return; } @@ -320,22 +306,24 @@ function getCreateRecovery(entry: NarrativeWorkspaceManifestEntry): NarrativeCre throw new Error('Narrative document cannot be both published and pending create recovery.'); } if (!createRecovery || typeof createRecovery !== 'object' || Array.isArray(createRecovery)) { - throw new Error('Narrative document createRecovery must contain a valid documentIdsBeforeCreate array.'); - } - const documentIdsBeforeCreate = (createRecovery as Record).documentIdsBeforeCreate; - if (!Array.isArray(documentIdsBeforeCreate) || - !documentIdsBeforeCreate.every((documentId: unknown) => Number.isSafeInteger(documentId) && (documentId as number) > 0)) { - throw new Error('Narrative document createRecovery documentIdsBeforeCreate must contain only positive safe integers.'); + throw new Error('Narrative document createRecovery must be a pending-create marker.'); } - const documentMarkdownSha256 = (createRecovery as Record).documentMarkdownSha256; - if (typeof documentMarkdownSha256 !== 'string' || !/^[0-9a-f]{64}$/.test(documentMarkdownSha256)) { - throw new Error('Narrative document createRecovery documentMarkdownSha256 must be a lowercase SHA-256 hex digest.'); + if ((createRecovery as Record).pending !== true || Object.keys(createRecovery).length !== 1) { + throw new Error('Narrative document createRecovery must contain only pending: true.'); } - return { documentIdsBeforeCreate, documentMarkdownSha256 }; + return { pending: true }; +} + +function isDefiniteCreateRejection(error: unknown): boolean { + return error instanceof HubClientError && DEFINITE_CREATE_REJECTION_STATUSES.has(error.status); } -function sha256(value: string): string { - return createHash('sha256').update(value, 'utf8').digest('hex'); +function createReconciliationMessage(id: string, entry: NarrativeWorkspaceManifestEntry): string { + return `Narrative document '${id}' has a pending create with no authoritative CalmHub identity. ` + + 'Explicit reconciliation is required. Confirm the CalmHub document ID, then run ' + + `\`calm workspace add --id ${id} --type ${entry.type} --namespace ${entry.namespace ?? ''} ` + + `--calm-hub-document-id --ver ${entry.version}\` ` + + '(add `--calm-hub-url ` if it is not configured).'; } function publishNarrativeEntry( @@ -352,35 +340,3 @@ function publishNarrativeEntry( calmHubId: location, }; } - -async function recoverCreatedNarrativeDocument( - client: CalmHubClient, - identity: NarrativeDocumentIdentity, - documentIdsBeforeCreate: number[], - documentMarkdown: string -): Promise<{ documentId: number; location: string }> { - const documentIdsAfterCreate = await client.getNarrativeDocumentIds(identity.namespace, identity.type); - const existingIds = new Set(documentIdsBeforeCreate); - const candidateIds = [...new Set(documentIdsAfterCreate.filter((documentId) => !existingIds.has(documentId)))]; - const matchingIds: number[] = []; - - for (const candidateId of candidateIds) { - const candidate = await client.getNarrativeDocumentVersion( - identity.namespace, identity.type, candidateId, '1.0.0' - ); - if (candidate.documentMarkdown === documentMarkdown) { - matchingIds.push(candidateId); - } - } - - if (matchingIds.length === 0) { - throw new Error('Could not recover the created narrative document: no new document has matching Markdown.'); - } - if (matchingIds.length > 1) { - throw new Error('Could not recover the created narrative document: multiple new documents have matching Markdown.'); - } - - const documentId = matchingIds[0]; - const location = constructNarrativeDocumentPath({ ...identity, calmHubDocumentId: documentId }); - return { documentId, location }; -}