diff --git a/README.md b/README.md index 5e02cb7..35bb4aa 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Madrigal -A config-driven, pluggable knowledge compiler. Madrigal transforms structured knowledge (markdown files with frontmatter) into multiple output formats — JSON bundles, AI skill files, rule sets, and more. +A config-driven, pluggable knowledge compiler. Madrigal transforms structured knowledge (markdown files with frontmatter) into normalized, schema-aware output formats - JSON bundles, AI skill files, rule sets, and more. -Inspired by [Style Dictionary](https://amzn.github.io/style-dictionary/), Madrigal applies the same "define once, compile everywhere" philosophy to design knowledge, coding guidelines, and organizational rules. +Inspired by [Style Dictionary](https://amzn.github.io/style-dictionary/), Madrigal applies the same "define once, compile everywhere" philosophy to knowledge bases, research repositories, coding guidelines, design systems, and organizational rules. Consumers own the domain vocabulary; Madrigal owns reliable parsing, normalization, linting, and compilation. ## Quick Start @@ -16,37 +16,50 @@ Create a `madrigal.config.yaml`: sources: - "knowledge/**/*.md" -domains: - accessibility: - description: "Accessibility guidelines" - -brands: - acme: - include: - - global +schema: + preserveUnknownFrontmatter: true + id: + field: id + strategy: path + kind: + field: type + default: record + byPath: + "knowledge/studies/**": study + title: + field: title + relationships: + wikilinks: true + +kinds: + study: + required: [title, methodology, research_period, source_url] + theme: {} + +vocabularies: + durability: + values: [evergreen, timebound] platforms: - skill-file: - format: skill-md json-export: format: json-bundle ``` -Create a knowledge file at `knowledge/contrast.md`: +Create a knowledge file at `knowledge/studies/checkout-trust.md`: ```markdown --- -title: Color Contrast Requirements -domain: accessibility -severity: error +title: Checkout Trust Study +type: study +methodology: Interviews +research_period: Q1 +source_url: https://example.com/research +durability: evergreen tags: - - a11y - - wcag + - onboarding --- -All text must meet WCAG 2.1 AA contrast requirements: -- Normal text: minimum 4.5:1 contrast ratio -- Large text: minimum 3:1 contrast ratio +Participants connected onboarding clarity to [[Trust Signals]]. ``` Build programmatically: @@ -66,19 +79,31 @@ for (const output of result.results) { ### Knowledge Units -The atomic unit. Each `.md` file with frontmatter becomes a `KnowledgeUnit` with an id, title, body, domain, severity, tags, and provenance tracking. +The atomic unit. Each `.md` file with frontmatter becomes a normalized `KnowledgeUnit` with an id, title, body, kind, tags, source path, raw `frontmatter`, normalized `attributes`, extracted `relationships`, and provenance tracking. Optional rule-oriented fields such as `domain`, `brand`, `system`, and `enforcement` are preserved when present. + +### Schema + +Consumer-owned mapping rules that tell Madrigal how to read a knowledge base: where IDs, titles, kinds, and relationships come from, how path-based fallback should work, and whether unknown frontmatter should be preserved. + +### Kinds + +Consumer-defined structural types (for example `study`, `theme`, `competitor`, or `rule`). Kinds can declare required fields for linting and strict validation. + +### Vocabularies + +Consumer-owned controlled terms and aliases. Madrigal can canonicalize configured fields in `attributes` while preserving the raw parsed value in `frontmatter`. ### Domains -Logical groupings of knowledge (e.g., `accessibility`, `typography`, `layout`). Defined in config and validated at load time. +Optional logical groupings of knowledge (e.g., `accessibility`, `typography`, `layout`). Defined in config and validated at load time when present. ### Brands Organizational units that can inherit from each other. A brand can `include` other brands/groups, and brand-specific knowledge overrides globals with the same id. -### Severity +### Enforcement -Five levels: `error` > `warning` > `info` > `context` > `deprecated`. Severity controls enforcement behavior and output filtering. +Optional rule-oriented strength: `must`, `should`, `may`, `context`, or `deprecated`. Existing `severity` frontmatter is still accepted as a legacy alias for rule-focused workflows. ### Formats @@ -102,6 +127,36 @@ Named build targets in config. Each platform specifies a format and optional gro sources: - "knowledge/**/*.md" +# Schema mapping and normalization +schema: + preserveUnknownFrontmatter: true + id: + field: id # Explicit frontmatter ID field + strategy: path # path | filename + kind: + field: type # Frontmatter field used as the unit kind + default: record + byPath: + "knowledge/studies/**": study + title: + field: title + relationships: + wikilinks: true # Extract [[Target]] and [[Target|Label]] + +# Kind definitions and required fields +kinds: + : + required: + - title + +# Controlled vocabularies and aliases +vocabularies: + : + values: + - canonical-value + aliases: + old-value: canonical-value + # Domain definitions domains: : @@ -127,18 +182,17 @@ platforms: ```markdown --- -title: Rule Title # Required (or id) -id: custom-id # Optional, generated from filename if omitted -domain: accessibility # Optional, defaults to 'default' -severity: error # Optional: error|warning|info|context|deprecated -brand: acme # Optional, omit for global rules -system: web # Optional -tags: # Optional - - a11y - - wcag +title: Record Title # Field name is configurable +id: custom-id # Optional, generated from path by default +type: study # Field name is configurable +source_url: https://example.com/source +durability: evergreen # Consumer-owned attribute +enforcement: should # Optional for rule-oriented records +tags: + - research --- -Markdown body content here. +Markdown body content here, including optional [[Wiki Links]]. ``` ## Plugin System diff --git a/src/__tests__/bm25.test.ts b/src/__tests__/bm25.test.ts index b17fe5b..5e64820 100644 --- a/src/__tests__/bm25.test.ts +++ b/src/__tests__/bm25.test.ts @@ -14,7 +14,9 @@ function makeUnit( kind: 'rule', tags: [], enforcement: 'should' as Enforcement, + frontmatter: {}, attributes: {}, + relationships: [], provenance: { origin: 'human-authored', confidence: 1.0 }, ...overrides, }; diff --git a/src/__tests__/formats.test.ts b/src/__tests__/formats.test.ts new file mode 100644 index 0000000..1a771d8 --- /dev/null +++ b/src/__tests__/formats.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest'; +import { aiRulesMdFormat } from '../formats/ai-rules-md.js'; +import { jsonBundleFormat } from '../formats/json-bundle.js'; +import type { FormatOptions } from '../formats/registry.js'; +import { skillMdFormat } from '../formats/skill-md.js'; +import type { KnowledgeUnit } from '../schema/index.js'; + +const options: FormatOptions = { + platform: { format: 'json-bundle' }, + config: { + sources: [], + schema: { + preserveUnknownFrontmatter: true, + id: { field: 'id', strategy: 'path' }, + kind: { field: 'kind', default: 'record', byPath: {} }, + title: { field: 'title' }, + relationships: { wikilinks: true }, + }, + vocabularies: {}, + domains: {}, + kinds: {}, + brands: {}, + platforms: {}, + }, +}; + +function makeUnit(overrides: Partial = {}): KnowledgeUnit { + return { + id: 'record-one', + title: 'Record One', + body: 'Readable body.', + kind: 'record', + tags: ['research'], + sourcePath: 'context/record-one.md', + frontmatter: { title: 'Record One', custom: 'kept' }, + attributes: { custom: 'kept' }, + relationships: [ + { + type: 'wikilink', + target: 'Record Two', + targetId: 'record-two', + resolved: true, + }, + ], + provenance: { origin: 'human-authored', confidence: 1.0 }, + ...overrides, + }; +} + +describe('built-in formats with neutral units', () => { + it('json-bundle emits the full normalized record', async () => { + const output = await jsonBundleFormat.compile([makeUnit()], options); + const bundle = JSON.parse(output) as { units: KnowledgeUnit[] }; + + expect(bundle.units[0]).toMatchObject({ + id: 'record-one', + kind: 'record', + frontmatter: { custom: 'kept' }, + attributes: { custom: 'kept' }, + relationships: [ + { + target: 'Record Two', + targetId: 'record-two', + resolved: true, + }, + ], + }); + }); + + it('rule-oriented markdown formats tolerate missing enforcement', async () => { + const units = [makeUnit({ enforcement: undefined })]; + + const skillOutput = await skillMdFormat.compile(units, options); + const rulesOutput = await aiRulesMdFormat.compile(units, options); + + expect(skillOutput).toBeTruthy(); + expect(rulesOutput).toContain('Additional Context'); + }); +}); diff --git a/src/__tests__/loader.test.ts b/src/__tests__/loader.test.ts new file mode 100644 index 0000000..1f6b3a2 --- /dev/null +++ b/src/__tests__/loader.test.ts @@ -0,0 +1,383 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { MadrigalConfig } from '../config.js'; +import { loadKnowledgeSync } from '../loader.js'; +import { build } from '../pipeline.js'; + +function makeConfig( + overrides: { + sources?: string[]; + schema?: Partial; + kinds?: MadrigalConfig['kinds']; + vocabularies?: MadrigalConfig['vocabularies']; + platforms?: MadrigalConfig['platforms']; + } = {}, +): MadrigalConfig { + const schema: MadrigalConfig['schema'] = { + preserveUnknownFrontmatter: true, + id: { field: 'id', strategy: 'path' }, + kind: { field: 'type', default: 'record', byPath: {} }, + title: { field: 'title' }, + relationships: { wikilinks: true }, + }; + + return { + sources: overrides.sources || ['context/**/*.md'], + schema: { + ...schema, + ...overrides.schema, + id: { ...schema.id, ...overrides.schema?.id }, + kind: { ...schema.kind, ...overrides.schema?.kind }, + title: { ...schema.title, ...overrides.schema?.title }, + relationships: { + ...schema.relationships, + ...overrides.schema?.relationships, + }, + }, + vocabularies: overrides.vocabularies || {}, + domains: {}, + kinds: overrides.kinds || { + record: {}, + study: { + required: ['title', 'methodology', 'research_period', 'source_url'], + }, + theme: {}, + competitor: { + required: ['title', 'competitor_id', 'threat_level'], + }, + }, + brands: {}, + platforms: overrides.platforms || {}, + }; +} + +describe('loadKnowledge schema compiler behavior', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), 'madrigal-loader-')); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + function write(relativePath: string, content: string): void { + const filePath = join(tempDir, relativePath); + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, content, 'utf-8'); + } + + it('preserves frontmatter, normalizes attributes, maps kinds, and resolves wikilinks', () => { + write( + 'context/themes/trust.md', + `--- +title: Trust +type: theme +aliases: + - Confidence +--- +Signals that help people feel safe continuing. +`, + ); + write( + 'context/studies/checkout-trust.md', + `--- +title: Checkout Trust Study +type: study +brand: Old Co +methodology: Interviews +research_period: Q1 +source_url: https://example.com/research +durability: Evergreen +--- +Participants connected onboarding clarity to [[Trust]] and [[Trust|trusted flows]]. +`, + ); + + const result = loadKnowledgeSync({ + sources: ['context/**/*.md'], + config: makeConfig({ + vocabularies: { + brand: { values: ['Acme'], aliases: { 'Old Co': 'Acme' } }, + durability: { values: ['Evergreen', 'Timebound'] }, + }, + }), + baseDir: tempDir, + }); + + expect(result.errors).toEqual([]); + expect(result.warnings).toEqual([]); + + const study = result.units.find((unit) => unit.kind === 'study'); + const theme = result.units.find((unit) => unit.kind === 'theme'); + + expect(study?.id).toBe('context-studies-checkout-trust'); + expect(study?.frontmatter.brand).toBe('Old Co'); + expect(study?.frontmatter.durability).toBe('Evergreen'); + expect(study?.attributes.brand).toBe('Acme'); + expect(study?.attributes.methodology).toBe('Interviews'); + expect(study?.relationships).toEqual([ + { + type: 'wikilink', + target: 'Trust', + resolved: true, + targetId: theme?.id, + }, + { + type: 'wikilink', + target: 'Trust', + label: 'trusted flows', + resolved: true, + targetId: theme?.id, + }, + ]); + }); + + it('uses path-based kind fallback when the configured field is absent', () => { + write( + 'context/studies/path-mapped.md', + `--- +title: Path Mapped Study +methodology: Survey +research_period: Q2 +source_url: https://example.com/survey +--- +Body. +`, + ); + + const result = loadKnowledgeSync({ + sources: ['context/**/*.md'], + config: makeConfig({ + schema: { + kind: { + field: 'type', + default: 'record', + byPath: { 'context/studies/**': 'study' }, + }, + }, + }), + baseDir: tempDir, + }); + + expect(result.errors).toEqual([]); + expect(result.units[0].kind).toBe('study'); + }); + + it('generates path-safe IDs for index files and duplicate filenames', () => { + write( + 'context/studies/_index.md', + `--- +title: Study Overview +type: record +--- +Overview. +`, + ); + write( + 'context/studies/shared.md', + `--- +title: Shared Name +type: record +--- +Study body. +`, + ); + write( + 'context/themes/shared.md', + `--- +title: Shared Name +type: record +--- +Theme body. +`, + ); + + const result = loadKnowledgeSync({ + sources: ['context/**/*.md'], + config: makeConfig(), + baseDir: tempDir, + }); + + expect(result.errors).toEqual([]); + expect(result.units.map((unit) => unit.id).sort()).toEqual([ + 'context-studies', + 'context-studies-shared', + 'context-themes-shared', + ]); + }); + + it('reports missing kind-required fields as strict lint errors', () => { + write( + 'context/studies/missing-required.md', + `--- +title: Missing Source Study +type: study +methodology: Interviews +research_period: Q3 +--- +Body. +`, + ); + + const result = loadKnowledgeSync({ + sources: ['context/**/*.md'], + config: makeConfig(), + baseDir: tempDir, + }); + + expect(result.errors).toEqual([]); + expect(result.warnings).toContainEqual( + expect.objectContaining({ + code: 'missing-required-field', + severity: 'error', + field: 'source_url', + unitId: 'context-studies-missing-required', + }), + ); + }); + + it('reports invalid vocabulary values without mutating raw frontmatter', () => { + write( + 'context/records/vendor.md', + `--- +title: Vendor Record +type: record +brand: Unknown Brand +--- +Body. +`, + ); + + const result = loadKnowledgeSync({ + sources: ['context/**/*.md'], + config: makeConfig({ + vocabularies: { + brand: { values: ['Acme'] }, + }, + }), + baseDir: tempDir, + }); + + expect(result.units[0].frontmatter.brand).toBe('Unknown Brand'); + expect(result.units[0].attributes.brand).toBe('Unknown Brand'); + expect(result.warnings).toContainEqual( + expect.objectContaining({ + code: 'invalid-vocabulary', + severity: 'error', + field: 'brand', + }), + ); + }); + + it('excludes conflicting duplicate IDs and reports all source paths', () => { + write( + 'context/a.md', + `--- +id: shared +title: First +type: record +--- +First. +`, + ); + write( + 'context/b.md', + `--- +id: shared +title: Second +type: record +--- +Second. +`, + ); + + const result = loadKnowledgeSync({ + sources: ['context/**/*.md'], + config: makeConfig(), + baseDir: tempDir, + }); + + expect(result.units).toEqual([]); + expect(result.errors).toContainEqual( + expect.objectContaining({ + code: 'duplicate-id', + filePath: 'shared', + message: expect.stringContaining('context/a.md, context/b.md'), + }), + ); + }); + + it('build continues with valid records while excluding duplicate conflicts', async () => { + write( + 'madrigal.config.yaml', + `sources: + - "context/**/*.md" +schema: + id: + field: id + strategy: path + kind: + field: type + default: record + title: + field: title + relationships: + wikilinks: true +kinds: + record: {} +platforms: + bundle: + format: json-bundle +`, + ); + write( + 'context/valid.md', + `--- +title: Valid Record +type: record +--- +Valid. +`, + ); + write( + 'context/duplicate-a.md', + `--- +id: duplicate +title: Duplicate A +type: record +--- +A. +`, + ); + write( + 'context/duplicate-b.md', + `--- +id: duplicate +title: Duplicate B +type: record +--- +B. +`, + ); + + const result = await build({ + configPath: join(tempDir, 'madrigal.config.yaml'), + baseDir: tempDir, + }); + + expect(result.success).toBe(true); + expect(result.totalUnits).toBe(1); + expect(result.loadErrors).toContainEqual( + expect.objectContaining({ code: 'duplicate-id' }), + ); + + const bundle = JSON.parse(result.results[0].output) as { + units: Array<{ id: string }>; + }; + expect(bundle.units.map((unit) => unit.id)).toEqual(['context-valid']); + }); +}); diff --git a/src/__tests__/pipeline.test.ts b/src/__tests__/pipeline.test.ts index 1560aa6..e3e204a 100644 --- a/src/__tests__/pipeline.test.ts +++ b/src/__tests__/pipeline.test.ts @@ -12,7 +12,9 @@ function makeUnit( kind: 'rule', tags: [], enforcement: 'may', + frontmatter: {}, attributes: {}, + relationships: [], provenance: { origin: 'human-authored', confidence: 1.0 }, ...overrides, }; diff --git a/src/__tests__/propose.test.ts b/src/__tests__/propose.test.ts index 595ecfb..a10c755 100644 --- a/src/__tests__/propose.test.ts +++ b/src/__tests__/propose.test.ts @@ -11,7 +11,9 @@ function makeUnit( kind: 'rule', tags: [], enforcement: 'may', + frontmatter: {}, attributes: {}, + relationships: [], provenance: { origin: 'human-authored', confidence: 1.0 }, ...overrides, }; diff --git a/src/__tests__/resolver.test.ts b/src/__tests__/resolver.test.ts index 5d71bb2..564c30c 100644 --- a/src/__tests__/resolver.test.ts +++ b/src/__tests__/resolver.test.ts @@ -16,7 +16,9 @@ function makeUnit( kind: 'rule', tags: [], enforcement: 'may', + frontmatter: {}, attributes: {}, + relationships: [], provenance: { origin: 'human-authored', confidence: 1.0 }, ...overrides, }; @@ -24,6 +26,14 @@ function makeUnit( const baseConfig: MadrigalConfig = { sources: ['**/*.md'], + schema: { + preserveUnknownFrontmatter: true, + id: { field: 'id', strategy: 'path' }, + kind: { field: 'kind', default: 'rule', byPath: {} }, + title: { field: 'title' }, + relationships: { wikilinks: false }, + }, + vocabularies: {}, domains: { default: { description: 'Default' } }, kinds: { rule: { description: 'Rule' } }, brands: { diff --git a/src/cli.ts b/src/cli.ts index a5e36da..5d7cce7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -216,11 +216,16 @@ async function runValidate() { if (loadResult.warnings.length > 0) { console.warn(`\n${loadResult.warnings.length} load warning(s):`); for (const w of loadResult.warnings) - console.warn(` ${w.filePath}: ${w.message}`); + console.warn( + ` ${w.filePath}: ${w.severity === 'error' ? '[error] ' : ''}${w.message}`, + ); } + const lintErrors = loadResult.warnings.filter((w) => w.severity === 'error'); const hasErrors = - validation.errors.length > 0 || loadResult.errors.length > 0; + validation.errors.length > 0 || + loadResult.errors.length > 0 || + lintErrors.length > 0; if (hasErrors) { console.error('\nValidation failed.'); process.exit(1); diff --git a/src/compliance/checker.ts b/src/compliance/checker.ts index 40b5257..70dfdf7 100644 --- a/src/compliance/checker.ts +++ b/src/compliance/checker.ts @@ -1,5 +1,6 @@ import type { SearchAdapter } from '../adapters/search.js'; import type { MadrigalConfig } from '../config.js'; +import { effectiveEnforcement } from '../enforcement.js'; import { resolveForBrand } from '../resolver.js'; import type { KnowledgeUnit } from '../schema/index.js'; import type { ComplianceResult, ComplianceViolation } from './index.js'; @@ -72,6 +73,7 @@ export async function checkCompliance( const info: ComplianceViolation[] = []; for (const result of scored) { + const enforcement = effectiveEnforcement(result.unit.enforcement); const violation: ComplianceViolation = { knowledgeUnit: result.unit, matchResult: { @@ -79,10 +81,10 @@ export async function checkCompliance( matched: true, confidence: result.score, }, - message: `${result.unit.title} [${result.unit.enforcement.toUpperCase()}]`, + message: `${result.unit.title} [${enforcement.toUpperCase()}]`, }; - switch (result.unit.enforcement) { + switch (enforcement) { case 'must': violations.push(violation); break; diff --git a/src/compliance/report.ts b/src/compliance/report.ts index fbd4c46..077bd5c 100644 --- a/src/compliance/report.ts +++ b/src/compliance/report.ts @@ -1,3 +1,4 @@ +import { type Enforcement, effectiveEnforcement } from '../enforcement.js'; import type { ComplianceResult, ComplianceViolation, @@ -84,7 +85,8 @@ function formatViolationMd( options: ReportOptions, ): string { const score = v.matchResult.confidence.toFixed(2); - let line = `- **${v.knowledgeUnit.title}** [${v.knowledgeUnit.enforcement.toUpperCase()}] — relevance: ${score}`; + const enforcement = effectiveEnforcement(v.knowledgeUnit.enforcement); + let line = `- **${v.knowledgeUnit.title}** [${enforcement.toUpperCase()}] — relevance: ${score}`; if (options.includeSuggestions) { // Include a brief excerpt from the rule body as guidance @@ -114,7 +116,7 @@ function violationToJson(v: ComplianceViolation) { return { unitId: v.knowledgeUnit.id, title: v.knowledgeUnit.title, - enforcement: v.knowledgeUnit.enforcement, + enforcement: effectiveEnforcement(v.knowledgeUnit.enforcement), confidence: v.matchResult.confidence, message: v.message, tags: v.knowledgeUnit.tags, @@ -147,7 +149,7 @@ function formatSarif(result: ComplianceResult): string { }, properties: { tags: v.knowledgeUnit.tags, - enforcement: v.knowledgeUnit.enforcement, + enforcement: effectiveEnforcement(v.knowledgeUnit.enforcement), }, })); @@ -182,9 +184,9 @@ function formatSarif(result: ComplianceResult): string { } function enforcementToSarifLevel( - enforcement: string, + enforcement: Enforcement | undefined, ): 'error' | 'warning' | 'note' { - switch (enforcement) { + switch (effectiveEnforcement(enforcement)) { case 'must': return 'error'; case 'should': diff --git a/src/config.ts b/src/config.ts index 5db6a5f..7296d3e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -15,7 +15,57 @@ export interface DomainConfig { */ export interface KindConfig { /** Human-readable description of the kind */ - description: string; + description?: string; + /** Consumer-defined required frontmatter fields for this kind */ + required?: string[]; +} + +export type IdStrategy = 'path' | 'filename'; + +export interface IdSchemaConfig { + /** Frontmatter field to use for explicit IDs */ + field?: string; + /** Generated ID strategy when no explicit ID exists */ + strategy?: IdStrategy; +} + +export interface KindSchemaConfig { + /** Frontmatter field to use as the kind */ + field?: string; + /** Default kind when no field/path rule matches */ + default?: string; + /** Glob patterns mapped to kind names */ + byPath?: Record; +} + +export interface TitleSchemaConfig { + /** Frontmatter field to use as the title */ + field?: string; +} + +export interface RelationshipSchemaConfig { + /** Extract Obsidian-style [[Wiki Links]] from markdown bodies */ + wikilinks?: boolean; +} + +export interface SchemaConfig { + /** Preserve non-core frontmatter fields in attributes */ + preserveUnknownFrontmatter?: boolean; + /** ID mapping and generation rules */ + id?: IdSchemaConfig; + /** Kind mapping rules */ + kind?: KindSchemaConfig; + /** Title mapping rules */ + title?: TitleSchemaConfig; + /** Relationship extraction rules */ + relationships?: RelationshipSchemaConfig; +} + +export interface VocabularyConfig { + /** Allowed canonical values */ + values?: string[]; + /** Raw value to canonical value mapping */ + aliases?: Record; } /** @@ -47,6 +97,10 @@ export interface PlatformConfig { export interface MadrigalConfig { /** Glob patterns for knowledge source files */ sources: string[]; + /** Consumer-owned schema mapping and linting config */ + schema: SchemaConfig; + /** Consumer-owned controlled vocabularies */ + vocabularies: Record; /** Domain definitions (key is domain name) */ domains: Record; /** Kind definitions (key is kind name, structural types of knowledge) */ @@ -146,6 +200,9 @@ function normalizeConfig(raw: unknown, _baseDir: string): MadrigalConfig { return { sources: config.sources as string[], + schema: normalizeSchemaConfig(config.schema), + vocabularies: + (config.vocabularies as Record) || {}, domains: (config.domains as Record) || {}, kinds: (config.kinds as Record) || {}, brands: (config.brands as Record) || {}, @@ -153,6 +210,32 @@ function normalizeConfig(raw: unknown, _baseDir: string): MadrigalConfig { }; } +function normalizeSchemaConfig(raw: unknown): SchemaConfig { + const schema = (raw && typeof raw === 'object' ? raw : {}) as SchemaConfig; + + return { + preserveUnknownFrontmatter: + schema.preserveUnknownFrontmatter !== undefined + ? schema.preserveUnknownFrontmatter + : true, + id: { + field: schema.id?.field || 'id', + strategy: schema.id?.strategy || 'path', + }, + kind: { + field: schema.kind?.field || 'kind', + default: schema.kind?.default || 'rule', + byPath: schema.kind?.byPath || {}, + }, + title: { + field: schema.title?.field || 'title', + }, + relationships: { + wikilinks: schema.relationships?.wikilinks || false, + }, + }; +} + /** * Validate a Madrigal configuration. * @@ -185,6 +268,34 @@ export function validateConfig( } } + const idStrategy = config.schema.id?.strategy; + if (idStrategy && !['path', 'filename'].includes(idStrategy)) { + errors.push({ + path: 'schema.id.strategy', + message: 'ID strategy must be "path" or "filename"', + }); + } + + for (const [field, vocabulary] of Object.entries(config.vocabularies)) { + const values = new Set(vocabulary.values || []); + if (vocabulary.aliases) { + for (const [alias, canonical] of Object.entries(vocabulary.aliases)) { + if (alias === canonical) { + warnings.push({ + path: `vocabularies.${field}.aliases.${alias}`, + message: 'Alias maps to itself', + }); + } + if (values.size > 0 && !values.has(canonical)) { + errors.push({ + path: `vocabularies.${field}.aliases.${alias}`, + message: `Alias maps to unknown canonical value "${canonical}"`, + }); + } + } + } + } + // Validate brands const brandNames = new Set(Object.keys(config.brands)); brandNames.add('global'); // 'global' is always valid as an include target diff --git a/src/dev/server.ts b/src/dev/server.ts index 28dc5f3..4be8547 100644 --- a/src/dev/server.ts +++ b/src/dev/server.ts @@ -13,7 +13,10 @@ import { cors } from 'hono/cors'; import { getState, loadState } from './state.js'; import { resolveForBrand } from '../resolver.js'; import { build, buildPlatformByName } from '../pipeline.js'; -import { ENFORCEMENT_ORDER } from '../enforcement.js'; +import { + effectiveEnforcement, + enforcementRank, +} from '../enforcement.js'; import type { KnowledgeUnit } from '../schema/index.js'; import { generateTopology, createOpenAIProvider, createVoyageProvider, createProviderFromEnv, cosineSimilarity } from '../topology/index.js'; import type { TopologyData, EmbeddingProvider } from '../topology/index.js'; @@ -59,8 +62,10 @@ export function createApp(baseDir: string): Hono { const byBrand: Record = {}; for (const u of units) { - byDomain[u.domain] = (byDomain[u.domain] || 0) + 1; - byEnforcement[u.enforcement] = (byEnforcement[u.enforcement] || 0) + 1; + const domain = u.domain || 'default'; + const enforcement = effectiveEnforcement(u.enforcement); + byDomain[domain] = (byDomain[domain] || 0) + 1; + byEnforcement[enforcement] = (byEnforcement[enforcement] || 0) + 1; byKind[u.kind] = (byKind[u.kind] || 0) + 1; const brand = u.brand || 'global'; byBrand[brand] = (byBrand[brand] || 0) + 1; @@ -98,7 +103,10 @@ export function createApp(baseDir: string): Hono { let results = scored; if (kind) results = results.filter((r) => r.unit.kind === kind); - if (enforcement) results = results.filter((r) => r.unit.enforcement === enforcement); + if (enforcement) + results = results.filter( + (r) => effectiveEnforcement(r.unit.enforcement) === enforcement, + ); const paged = results.slice(offset, offset + limit); return c.json({ @@ -111,12 +119,13 @@ export function createApp(baseDir: string): Hono { if (domain) filtered = filtered.filter((u) => u.domain === domain); if (brand) filtered = filtered.filter((u) => !u.brand || u.brand === brand); if (kind) filtered = filtered.filter((u) => u.kind === kind); - if (enforcement) filtered = filtered.filter((u) => u.enforcement === enforcement); + if (enforcement) + filtered = filtered.filter( + (u) => effectiveEnforcement(u.enforcement) === enforcement, + ); - filtered.sort( - (a, b) => - (ENFORCEMENT_ORDER[a.enforcement] ?? 99) - - (ENFORCEMENT_ORDER[b.enforcement] ?? 99), + filtered.sort((a, b) => + enforcementRank(a.enforcement) - enforcementRank(b.enforcement), ); return c.json({ @@ -142,7 +151,7 @@ export function createApp(baseDir: string): Hono { const resolvedUnit = resolved.find((u) => u.id === id); if (resolvedUnit) { brandResolutions[brandName] = { - enforcement: resolvedUnit.enforcement, + enforcement: effectiveEnforcement(resolvedUnit.enforcement), overridden: resolvedUnit.enforcement !== unit.enforcement, }; } @@ -212,7 +221,7 @@ export function createApp(baseDir: string): Hono { const original = units.find((u) => u.id === ru.id); return { ...ru, - _baseEnforcement: original?.enforcement ?? ru.enforcement, + _baseEnforcement: effectiveEnforcement(original?.enforcement ?? ru.enforcement), _overridden: original ? original.enforcement !== ru.enforcement : false, }; }); @@ -240,13 +249,15 @@ export function createApp(baseDir: string): Hono { const domainMap = new Map }>(); for (const u of units) { - let entry = domainMap.get(u.domain); + const domain = u.domain || 'default'; + const enforcement = effectiveEnforcement(u.enforcement); + let entry = domainMap.get(domain); if (!entry) { entry = { count: 0, byEnforcement: {} }; - domainMap.set(u.domain, entry); + domainMap.set(domain, entry); } entry.count++; - entry.byEnforcement[u.enforcement] = (entry.byEnforcement[u.enforcement] || 0) + 1; + entry.byEnforcement[enforcement] = (entry.byEnforcement[enforcement] || 0) + 1; } // Include config domains with zero units (coverage gaps) @@ -661,7 +672,7 @@ export function createApp(baseDir: string): Hono { } else if (sortBy === 'title') { cmp = a.title.localeCompare(b.title); } else if (sortBy === 'domain') { - cmp = a.domain.localeCompare(b.domain); + cmp = (a.domain || 'default').localeCompare(b.domain || 'default'); } else if (sortBy === 'origin') { cmp = a.provenance.origin.localeCompare(b.provenance.origin); } diff --git a/src/enforcement.ts b/src/enforcement.ts index 1b2c332..92687b2 100644 --- a/src/enforcement.ts +++ b/src/enforcement.ts @@ -17,14 +17,27 @@ export const ENFORCEMENT_ORDER: Record = { deprecated: 4, }; -export function compareEnforcement(a: Enforcement, b: Enforcement): number { - return ENFORCEMENT_ORDER[a] - ENFORCEMENT_ORDER[b]; +export function compareEnforcement( + a: Enforcement | undefined, + b: Enforcement | undefined, +): number { + return enforcementRank(a) - enforcementRank(b); } -export function isEnforceable(enforcement: Enforcement): boolean { +export function isEnforceable(enforcement: Enforcement | undefined): boolean { return enforcement === 'must' || enforcement === 'should'; } +export function effectiveEnforcement( + enforcement: Enforcement | undefined, +): Enforcement { + return enforcement || 'may'; +} + +export function enforcementRank(enforcement: Enforcement | undefined): number { + return ENFORCEMENT_ORDER[effectiveEnforcement(enforcement)]; +} + /** * Parse a string into an Enforcement value. * Returns null if the string is not a valid enforcement level. diff --git a/src/formats/ai-rules-md.ts b/src/formats/ai-rules-md.ts index d1768ee..3895232 100644 --- a/src/formats/ai-rules-md.ts +++ b/src/formats/ai-rules-md.ts @@ -1,4 +1,8 @@ -import { compareEnforcement, isEnforceable } from '../enforcement.js'; +import { + compareEnforcement, + effectiveEnforcement, + isEnforceable, +} from '../enforcement.js'; import type { KnowledgeUnit } from '../schema/index.js'; import type { Format, FormatOptions } from './registry.js'; @@ -40,8 +44,12 @@ export const aiRulesMdFormat: Format = { ); // Group by enforcement - const musts = enforceableUnits.filter((u) => u.enforcement === 'must'); - const shoulds = enforceableUnits.filter((u) => u.enforcement === 'should'); + const musts = enforceableUnits.filter( + (u) => effectiveEnforcement(u.enforcement) === 'must', + ); + const shoulds = enforceableUnits.filter( + (u) => effectiveEnforcement(u.enforcement) === 'should', + ); if (musts.length > 0) { lines.push('## MUST Follow'); @@ -69,7 +77,9 @@ export const aiRulesMdFormat: Format = { // Add context rules in a collapsed section const contextUnits = units.filter( - (u) => u.enforcement === 'context' || u.enforcement === 'may', + (u) => + effectiveEnforcement(u.enforcement) === 'context' || + effectiveEnforcement(u.enforcement) === 'may', ); if (contextUnits.length > 0) { lines.push('## Additional Context'); diff --git a/src/formats/json-bundle.ts b/src/formats/json-bundle.ts index 1b412b7..b962563 100644 --- a/src/formats/json-bundle.ts +++ b/src/formats/json-bundle.ts @@ -38,20 +38,21 @@ export const jsonBundleFormat: Format = { unitCount: units.length, }, units: units.map((unit) => ({ - ...unit, // Ensure consistent field ordering in output id: unit.id, title: unit.title, body: unit.body, - domain: unit.domain, kind: unit.kind, - system: unit.system, - brand: unit.brand, tags: unit.tags, - enforcement: unit.enforcement, + sourcePath: unit.sourcePath, + frontmatter: unit.frontmatter, attributes: unit.attributes, + relationships: unit.relationships, provenance: unit.provenance, - sourcePath: unit.sourcePath, + domain: unit.domain, + system: unit.system, + brand: unit.brand, + enforcement: unit.enforcement, })), }; diff --git a/src/formats/mesh-domain.ts b/src/formats/mesh-domain.ts index 23b7d2d..4f3d133 100644 --- a/src/formats/mesh-domain.ts +++ b/src/formats/mesh-domain.ts @@ -1,3 +1,4 @@ +import { effectiveEnforcement } from '../enforcement.js'; import type { KnowledgeUnit } from '../schema/index.js'; import type { Format, FormatOptions } from './registry.js'; @@ -46,12 +47,14 @@ export const meshDomainFormat: Format = { id: unit.id, title: unit.title, content: unit.body, - domain: unit.domain, - enforcement: unit.enforcement, + domain: unit.domain || 'default', + enforcement: effectiveEnforcement(unit.enforcement), tags: unit.tags, metadata: { kind: unit.kind, + frontmatter: unit.frontmatter, attributes: unit.attributes, + relationships: unit.relationships, brand: unit.brand, system: unit.system, sourcePath: unit.sourcePath, diff --git a/src/formats/skill-md.ts b/src/formats/skill-md.ts index 378ff4f..5b8c6d1 100644 --- a/src/formats/skill-md.ts +++ b/src/formats/skill-md.ts @@ -1,4 +1,4 @@ -import { compareEnforcement } from '../enforcement.js'; +import { compareEnforcement, effectiveEnforcement } from '../enforcement.js'; import type { KnowledgeUnit } from '../schema/index.js'; import type { Format, FormatOptions } from './registry.js'; @@ -100,8 +100,10 @@ function formatUnit(unit: KnowledgeUnit): string { /** * Get an enforcement badge. */ -function getEnforcementBadge(enforcement: string): string { - switch (enforcement) { +function getEnforcementBadge( + enforcement: KnowledgeUnit['enforcement'], +): string { + switch (effectiveEnforcement(enforcement)) { case 'must': return '[MUST]'; case 'should': diff --git a/src/index.ts b/src/index.ts index 59f4409..5040a58 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,12 +22,18 @@ export { formatReport } from './compliance/report.js'; export type { BrandConfig, DomainConfig, + IdSchemaConfig, KindConfig, + KindSchemaConfig, MadrigalConfig, PlatformConfig, + RelationshipSchemaConfig, + SchemaConfig, + TitleSchemaConfig, ValidationError, ValidationResult, ValidationWarning, + VocabularyConfig, } from './config.js'; export { getBrandNames, @@ -42,6 +48,8 @@ export type { Enforcement } from './enforcement.js'; export { compareEnforcement, ENFORCEMENT_ORDER, + effectiveEnforcement, + enforcementRank, isEnforceable, parseEnforcement, } from './enforcement.js'; @@ -130,6 +138,7 @@ export type { CreateKnowledgeUnit, Domain, KnowledgeFrontmatter, + KnowledgeRelationship, KnowledgeUnit, KnowledgeUnitWithEmbedding, UpdateKnowledgeUnit, diff --git a/src/loader.ts b/src/loader.ts index f2fbf19..22231de 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -3,11 +3,15 @@ import { basename, extname, relative } from 'node:path'; import fg from 'fast-glob'; import matter from 'gray-matter'; import { parse as parseYaml } from 'yaml'; -import type { MadrigalConfig } from './config.js'; +import type { MadrigalConfig, VocabularyConfig } from './config.js'; import type { Enforcement } from './enforcement.js'; import { parseEnforcement } from './enforcement.js'; import { createFileProvenance } from './provenance.js'; -import type { KnowledgeFrontmatter, KnowledgeUnit } from './schema/index.js'; +import type { + KnowledgeFrontmatter, + KnowledgeRelationship, + KnowledgeUnit, +} from './schema/index.js'; /** * Options for loading knowledge units. @@ -25,8 +29,10 @@ export interface LoadOptions { * Error that occurred while loading a file. */ export interface LoadError { - /** Path to the file */ + /** Path to the file or conflicting generated ID */ filePath: string; + /** Machine-readable error category */ + code?: string; /** Error message */ message: string; /** Original error if available */ @@ -34,15 +40,21 @@ export interface LoadError { } /** - * Warning about a loaded file. + * Warning or lint violation about a loaded file. */ export interface LoadWarning { /** Path to the file */ filePath: string; + /** Machine-readable warning category */ + code?: string; + /** Severity. Validate treats "error" as failing; build warns and continues. */ + severity?: 'warning' | 'error'; /** Warning message */ message: string; /** Field that caused the warning */ field?: string; + /** Knowledge unit ID, when available */ + unitId?: string; } /** @@ -51,25 +63,41 @@ export interface LoadWarning { export interface LoadResult { /** Successfully loaded knowledge units */ units: KnowledgeUnit[]; - /** Files that failed to parse */ + /** Files that failed to parse or invalid duplicate records that were excluded */ errors: LoadError[]; - /** Files with validation warnings */ + /** Files with validation warnings or lint violations */ warnings: LoadWarning[]; } +type ParsedUnit = KnowledgeUnit & { + _relationshipTargets?: Array<{ + relationship: KnowledgeRelationship; + targetKey: string; + }>; +}; + +const CORE_FRONTMATTER_KEYS = new Set([ + 'id', + 'title', + 'domain', + 'kind', + 'type', + 'system', + 'brand', + 'tags', + 'enforcement', + 'severity', + 'attributes', + 'provenance', + 'body', + 'entries', +]); + /** - * Load knowledge units from markdown files. - * - * @param options - Load options including source patterns and config - * @returns Loaded units with any errors and warnings + * Load knowledge units from markdown and YAML files. */ export async function loadKnowledge(options: LoadOptions): Promise { const { sources, config, baseDir = process.cwd() } = options; - const units: KnowledgeUnit[] = []; - const errors: LoadError[] = []; - const warnings: LoadWarning[] = []; - - // Find all matching files const files = await fg(sources, { cwd: baseDir, absolute: true, @@ -77,466 +105,645 @@ export async function loadKnowledge(options: LoadOptions): Promise { ignore: ['**/node_modules/**', '**/dist/**'], }); - const domainNames = new Set(Object.keys(config.domains)); - const kindNames = new Set(Object.keys(config.kinds)); - const brandNames = new Set(Object.keys(config.brands)); - brandNames.add('global'); // 'global' is always valid + return loadFiles(files, config, baseDir); +} + +/** + * Synchronous version of loadKnowledge for simple use cases. + */ +export function loadKnowledgeSync(options: LoadOptions): LoadResult { + const { sources, config, baseDir = process.cwd() } = options; + const files = fg.sync(sources, { + cwd: baseDir, + absolute: true, + onlyFiles: true, + ignore: ['**/node_modules/**', '**/dist/**'], + }); + + return loadFiles(files, config, baseDir); +} + +function loadFiles( + files: string[], + config: MadrigalConfig, + baseDir: string, +): LoadResult { + const units: ParsedUnit[] = []; + const errors: LoadError[] = []; + const warnings: LoadWarning[] = []; for (const filePath of files) { try { const ext = extname(filePath).toLowerCase(); if (ext === '.yaml' || ext === '.yml') { - const parsed = parseKnowledgeYamlFile( - filePath, - baseDir, - domainNames, - kindNames, - brandNames, - warnings, + units.push( + ...parseKnowledgeYamlFile(filePath, baseDir, config, warnings), ); - units.push(...parsed); } else { - const unit = parseKnowledgeFile( + const unit = parseKnowledgeMarkdownFile( filePath, baseDir, - domainNames, - kindNames, - brandNames, + config, warnings, ); - if (unit) { - units.push(unit); - } + if (unit) units.push(unit); } } catch (err) { errors.push({ filePath, + code: 'parse-error', message: err instanceof Error ? err.message : String(err), error: err instanceof Error ? err : undefined, }); } } - // Detect duplicate IDs — silently taking the first would lose units in the BM25 index - const idSeen = new Map(); // id → first unit title - for (const unit of units) { - const first = idSeen.get(unit.id); - if (first) { - warnings.push({ - filePath: unit.id, - message: `Duplicate ID "${unit.id}" (also used by "${first}"). Add explicit id: frontmatter to disambiguate. The second unit will be silently dropped by the search index.`, - }); - } else { - idSeen.set(unit.id, unit.title); - } - } + const deduped = excludeDuplicateIds(units, errors); + resolveRelationships(deduped, config, warnings); - return { units, errors, warnings }; + return { + units: deduped.map(stripInternalFields), + errors, + warnings, + }; } -/** - * Parse a single knowledge markdown file. - */ -function parseKnowledgeFile( +function parseKnowledgeMarkdownFile( filePath: string, baseDir: string, - domainNames: Set, - kindNames: Set, - brandNames: Set, + config: MadrigalConfig, warnings: LoadWarning[], -): KnowledgeUnit | null { +): ParsedUnit | null { const content = readFileSync(filePath, 'utf-8'); const { data, content: body } = matter(content); - const frontmatter = data as KnowledgeFrontmatter; + const frontmatter = { ...(data as KnowledgeFrontmatter) }; + + return buildUnit({ + filePath, + baseDir, + config, + frontmatter, + body: body.trim(), + warnings, + }); +} - // Validate required fields - if (!frontmatter.title && !frontmatter.id) { +function parseKnowledgeYamlFile( + filePath: string, + baseDir: string, + config: MadrigalConfig, + warnings: LoadWarning[], +): ParsedUnit[] { + const content = readFileSync(filePath, 'utf-8'); + const parsed = parseYaml(content) as Record; + + if (!parsed || typeof parsed !== 'object') { warnings.push({ filePath, - message: 'Missing title and id in frontmatter; using filename', + code: 'yaml-non-object', + severity: 'error', + message: 'YAML file parsed to null or non-object', }); + return []; } - // Generate ID from filename if not provided. - // Include brand prefix so cross-brand files with the same filename stay unique. - const id = - frontmatter.id || generateIdFromFilename(filePath, frontmatter.brand); - const title = frontmatter.title || id; + const entries = parsed.entries as Array> | undefined; + if (!Array.isArray(entries)) { + return [ + buildUnit({ + filePath, + baseDir, + config, + frontmatter: { ...parsed }, + body: typeof parsed.body === 'string' ? parsed.body.trim() : '', + warnings, + }), + ]; + } - // Validate domain - const domain = frontmatter.domain || 'default'; - if (!domainNames.has(domain) && domainNames.size > 0) { - warnings.push({ + return entries.map((entry, index) => { + const frontmatter = { ...parsed, ...entry }; + delete frontmatter.entries; + const body = + typeof frontmatter.body === 'string' + ? frontmatter.body.trim() + : stringifyStructuredBody(entry); + + return buildUnit({ filePath, - field: 'domain', - message: `Unknown domain "${domain}". Known domains: ${Array.from(domainNames).join(', ')}`, + baseDir, + config, + frontmatter, + body, + warnings, + entryIndex: index, }); - } + }); +} - // Validate brand - if (frontmatter.brand && !brandNames.has(frontmatter.brand)) { +function buildUnit(options: { + filePath: string; + baseDir: string; + config: MadrigalConfig; + frontmatter: Record; + body: string; + warnings: LoadWarning[]; + entryIndex?: number; +}): ParsedUnit { + const { filePath, baseDir, config, frontmatter, body, warnings, entryIndex } = + options; + const sourcePath = normalizePath(relative(baseDir, filePath)); + const schema = config.schema; + const explicitIdField = schema.id?.field || 'id'; + const titleField = schema.title?.field || 'title'; + + const rawExplicitId = frontmatter[explicitIdField]; + const baseId = + typeof rawExplicitId === 'string' && rawExplicitId.trim() + ? rawExplicitId.trim() + : generateId(filePath, baseDir, config); + const id = entryIndex === undefined ? baseId : `${baseId}--${entryIndex}`; + + const rawTitle = frontmatter[titleField]; + const title = + typeof rawTitle === 'string' && rawTitle.trim() ? rawTitle.trim() : id; + if (!rawTitle && !rawExplicitId) { warnings.push({ filePath, - field: 'brand', - message: `Unknown brand "${frontmatter.brand}". Known brands: ${Array.from(brandNames).join(', ')}`, + code: 'missing-title', + severity: 'warning', + message: + 'Missing title and id in frontmatter; using generated ID as title', + unitId: id, }); } - // Parse kind (default: 'rule') - const kind = frontmatter.kind || 'rule'; - if (frontmatter.kind && kindNames.size > 0 && !kindNames.has(kind)) { - warnings.push({ - filePath, - field: 'kind', - message: `Unknown kind "${kind}". Known kinds: ${Array.from(kindNames).join(', ')}`, - }); + const kind = resolveKind(sourcePath, frontmatter, config); + const tags = normalizeStringArray(frontmatter.tags); + const attributes = buildAttributes(frontmatter, config); + + applyVocabularies(attributes, config.vocabularies, { + filePath, + unitId: id, + warnings, + }); + + const domain = getString(attributes.domain); + const brand = getString(attributes.brand); + const system = getString(attributes.system); + const enforcement = parseUnitEnforcement(frontmatter, warnings, filePath, id); + + if (config.domains && Object.keys(config.domains).length > 0 && domain) { + if (!config.domains[domain]) { + warnings.push({ + filePath, + code: 'unknown-domain', + severity: 'error', + field: 'domain', + unitId: id, + message: `Unknown domain "${domain}". Known domains: ${Object.keys(config.domains).join(', ')}`, + }); + } } - // Parse enforcement (with backward compat for 'severity' field) - let enforcement: Enforcement = 'may'; - const rawEnforcement = frontmatter.enforcement || frontmatter.severity; - if (rawEnforcement) { - const parsed = parseEnforcement(rawEnforcement); - if (parsed) { - enforcement = parsed; - } else { + if (config.kinds && Object.keys(config.kinds).length > 0 && kind) { + if (!config.kinds[kind]) { warnings.push({ filePath, - field: 'enforcement', - message: `Invalid enforcement "${rawEnforcement}". Using "may".`, + code: 'unknown-kind', + severity: 'error', + field: 'kind', + unitId: id, + message: `Unknown kind "${kind}". Known kinds: ${Object.keys(config.kinds).join(', ')}`, }); } } - // Parse attributes - const attributes = - ((frontmatter as Record).attributes as Record< - string, - unknown - >) || {}; + if (config.brands && Object.keys(config.brands).length > 0 && brand) { + const brandNames = new Set([...Object.keys(config.brands), 'global']); + if (!brandNames.has(brand)) { + warnings.push({ + filePath, + code: 'unknown-brand', + severity: 'error', + field: 'brand', + unitId: id, + message: `Unknown brand "${brand}". Known brands: ${Array.from(brandNames).join(', ')}`, + }); + } + } - // Build provenance const provenance = frontmatter.provenance ? { ...createFileProvenance(), - ...frontmatter.provenance, + ...(frontmatter.provenance as Record), } : createFileProvenance(); - return { + const relationships = config.schema.relationships?.wikilinks + ? extractWikiLinks(body) + : []; + + const unit: ParsedUnit = { id, title, - body: body.trim(), - domain, + body, kind, - system: frontmatter.system, - brand: frontmatter.brand, - tags: frontmatter.tags || [], - enforcement, + tags, + sourcePath, + frontmatter, attributes, + relationships, provenance, - sourcePath: relative(baseDir, filePath), + domain, + system, + brand, + enforcement, + _relationshipTargets: relationships.map((relationship) => ({ + relationship, + targetKey: relationship.target, + })), }; -} -/** - * Generate a slug ID from a filename. - * When a brand is provided, prefixes the slug with `{brand}-` so that - * cross-brand files with the same filename (e.g. voice-principles.md in both - * `shared/` and `tidal/`) produce unique IDs. - */ -function generateIdFromFilename(filePath: string, brand?: string): string { - const ext = extname(filePath); - const name = basename(filePath, ext) - .toLowerCase() - .replace(/[^a-z0-9]+/g, '-') - .replace(/^-|-$/g, ''); - return brand ? `${brand}-${name}` : name; + validateRequiredFields(unit, config, warnings, filePath); + + return unit; } -/** - * Known metadata keys in YAML knowledge files. - * Everything else goes into attributes. - */ -const KNOWN_YAML_KEYS = new Set([ - 'id', - 'title', - 'domain', - 'kind', - 'system', - 'brand', - 'tags', - 'enforcement', - 'severity', - 'provenance', - 'body', - 'entries', -]); +function buildAttributes( + frontmatter: Record, + config: MadrigalConfig, +): Record { + const explicitAttrs = + frontmatter.attributes && typeof frontmatter.attributes === 'object' + ? (frontmatter.attributes as Record) + : {}; + const attributes: Record = { ...explicitAttrs }; + + if (config.schema.preserveUnknownFrontmatter !== false) { + const extraCoreKeys = new Set(); + extraCoreKeys.add(config.schema.id?.field || 'id'); + extraCoreKeys.add(config.schema.kind?.field || 'kind'); + extraCoreKeys.add(config.schema.title?.field || 'title'); + + for (const [key, value] of Object.entries(frontmatter)) { + if ( + !CORE_FRONTMATTER_KEYS.has(key) && + !extraCoreKeys.has(key) && + key !== 'attributes' + ) { + attributes[key] = value; + } + } + } -/** - * Parse a YAML knowledge file. - * - * Supports two modes: - * - Single-unit: top-level YAML becomes one KnowledgeUnit - * - Multi-unit: when an `entries` array key exists, each entry becomes its own unit, - * inheriting top-level metadata as defaults - */ -function parseKnowledgeYamlFile( - filePath: string, - baseDir: string, - domainNames: Set, - kindNames: Set, - brandNames: Set, - warnings: LoadWarning[], -): KnowledgeUnit[] { - const content = readFileSync(filePath, 'utf-8'); - const parsed = parseYaml(content) as Record; + for (const key of ['domain', 'brand', 'system']) { + if (frontmatter[key] !== undefined) attributes[key] = frontmatter[key]; + } - if (!parsed || typeof parsed !== 'object') { - warnings.push({ - filePath, - message: 'YAML file parsed to null or non-object', - }); - return []; + return attributes; +} + +function resolveKind( + sourcePath: string, + frontmatter: Record, + config: MadrigalConfig, +): string { + const kindField = config.schema.kind?.field || 'kind'; + const fieldValue = frontmatter[kindField]; + if (typeof fieldValue === 'string' && fieldValue.trim()) { + return fieldValue.trim(); } - const entries = parsed.entries as Array> | undefined; - if (Array.isArray(entries)) { - // Multi-unit mode: each entry becomes a unit, inheriting top-level defaults - return entries.map((entry, index) => - buildYamlUnit( - filePath, - baseDir, - parsed, - entry, - index, - domainNames, - kindNames, - brandNames, - warnings, - ), - ); + for (const [pattern, kind] of Object.entries( + config.schema.kind?.byPath || {}, + )) { + if (matchesGlob(sourcePath, pattern)) return kind; } - // Single-unit mode - return [ - buildYamlUnit( - filePath, - baseDir, - parsed, - undefined, - 0, - domainNames, - kindNames, - brandNames, - warnings, - ), - ]; + return config.schema.kind?.default || 'rule'; } -/** - * Build a KnowledgeUnit from YAML data, with optional entry override. - */ -function buildYamlUnit( - filePath: string, - baseDir: string, - topLevel: Record, - entry: Record | undefined, - index: number, - domainNames: Set, - kindNames: Set, - brandNames: Set, +function parseUnitEnforcement( + frontmatter: Record, warnings: LoadWarning[], -): KnowledgeUnit { - // Merge top-level defaults with entry overrides - const merged = entry ? { ...topLevel, ...entry } : { ...topLevel }; - - // Extract standard fields - const topLevelBrand = topLevel.brand ? String(topLevel.brand) : undefined; - const parentId = String( - topLevel.id || generateIdFromFilename(filePath, topLevelBrand), - ); - const id = entry ? String(entry.id || `${parentId}--${index}`) : parentId; - const title = String(merged.title || id); - const domain = String(merged.domain || 'default'); - const kind = String(merged.kind || 'rule'); - const system = merged.system ? String(merged.system) : undefined; - const brand = merged.brand ? String(merged.brand) : undefined; - const tags = Array.isArray(merged.tags) - ? (merged.tags as unknown[]).map(String) - : []; + filePath: string, + unitId: string, +): Enforcement | undefined { + const raw = frontmatter.enforcement || frontmatter.severity; + if (!raw) return undefined; + + const parsed = parseEnforcement(String(raw)); + if (parsed) return parsed; + + warnings.push({ + filePath, + code: 'invalid-enforcement', + severity: 'error', + field: 'enforcement', + unitId, + message: `Invalid enforcement "${String(raw)}".`, + }); + return undefined; +} - // Validate domain - if (domainNames.size > 0 && !domainNames.has(domain)) { - warnings.push({ - filePath, - field: 'domain', - message: `Unknown domain "${domain}" in YAML unit "${id}"`, - }); +function applyVocabularies( + attributes: Record, + vocabularies: Record, + context: { + filePath: string; + unitId: string; + warnings: LoadWarning[]; + }, +): void { + for (const [field, vocabulary] of Object.entries(vocabularies)) { + if (attributes[field] === undefined) continue; + attributes[field] = normalizeVocabularyValue( + attributes[field], + field, + vocabulary, + context, + ); } +} - // Validate kind - if (kindNames.size > 0 && !kindNames.has(kind)) { - warnings.push({ - filePath, - field: 'kind', - message: `Unknown kind "${kind}" in YAML unit "${id}"`, - }); +function normalizeVocabularyValue( + value: unknown, + field: string, + vocabulary: VocabularyConfig, + context: { + filePath: string; + unitId: string; + warnings: LoadWarning[]; + }, +): unknown { + if (Array.isArray(value)) { + return value.map((item) => + normalizeVocabularyValue(item, field, vocabulary, context), + ); } - // Validate brand - if (brand && !brandNames.has(brand)) { - warnings.push({ - filePath, - field: 'brand', - message: `Unknown brand "${brand}" in YAML unit "${id}"`, + if (typeof value !== 'string') return value; + + const canonical = vocabulary.aliases?.[value] || value; + const allowed = new Set(vocabulary.values || []); + if (allowed.size > 0 && !allowed.has(canonical)) { + context.warnings.push({ + filePath: context.filePath, + code: 'invalid-vocabulary', + severity: 'error', + field, + unitId: context.unitId, + message: `Invalid ${field} value "${value}". Allowed values: ${Array.from(allowed).join(', ')}`, }); } - // Parse enforcement - let enforcement: Enforcement = 'may'; - const rawEnforcement = (merged.enforcement || merged.severity) as - | string - | undefined; - if (rawEnforcement) { - const parsed = parseEnforcement(String(rawEnforcement)); - if (parsed) { - enforcement = parsed; - } else { + return canonical; +} + +function validateRequiredFields( + unit: KnowledgeUnit, + config: MadrigalConfig, + warnings: LoadWarning[], + filePath: string, +): void { + const required = config.kinds[unit.kind]?.required || []; + for (const field of required) { + const value = getUnitFieldValue(unit, field); + if (isMissing(value)) { warnings.push({ filePath, - field: 'enforcement', - message: `Invalid enforcement "${rawEnforcement}" in YAML unit "${id}". Using "may".`, + code: 'missing-required-field', + severity: 'error', + field, + unitId: unit.id, + message: `Missing required field "${field}" for kind "${unit.kind}".`, }); } } +} - // Collect attributes: everything that's not a known metadata key - const attributes: Record = {}; - const mergedAttrs = (merged.attributes || {}) as Record; - // Explicit attributes field takes priority - Object.assign(attributes, mergedAttrs); - // Also collect any unknown top-level keys from the entry as attributes - if (entry) { - for (const [key, value] of Object.entries(entry)) { - if (!KNOWN_YAML_KEYS.has(key) && key !== 'attributes') { - attributes[key] = value; +function getUnitFieldValue(unit: KnowledgeUnit, field: string): unknown { + if (field === 'id') return unit.id; + if (field === 'title') return unit.title; + if (field === 'kind') return unit.kind; + if (field === 'body') return unit.body; + if (field === 'tags') return unit.tags; + if (field === 'domain') return unit.domain ?? unit.attributes.domain; + if (field === 'brand') return unit.brand ?? unit.attributes.brand; + if (field === 'system') return unit.system ?? unit.attributes.system; + if (field === 'enforcement') return unit.enforcement; + if (unit.attributes[field] !== undefined) return unit.attributes[field]; + return unit.frontmatter[field]; +} + +function isMissing(value: unknown): boolean { + if (value === undefined || value === null) return true; + if (typeof value === 'string') return value.trim().length === 0; + if (Array.isArray(value)) return value.length === 0; + return false; +} + +function excludeDuplicateIds( + units: ParsedUnit[], + errors: LoadError[], +): ParsedUnit[] { + const byId = new Map(); + for (const unit of units) { + const existing = byId.get(unit.id) || []; + existing.push(unit); + byId.set(unit.id, existing); + } + + const duplicateIds = new Set(); + for (const [id, records] of byId.entries()) { + if (records.length <= 1) continue; + duplicateIds.add(id); + errors.push({ + filePath: id, + code: 'duplicate-id', + message: `Duplicate ID "${id}" across: ${records.map((u) => u.sourcePath).join(', ')}. Conflicting records were excluded.`, + }); + } + + return units.filter((unit) => !duplicateIds.has(unit.id)); +} + +function resolveRelationships( + units: ParsedUnit[], + config: MadrigalConfig, + warnings: LoadWarning[], +): void { + if (!config.schema.relationships?.wikilinks) return; + + const index = buildRelationshipIndex(units, config); + + for (const unit of units) { + for (const pending of unit._relationshipTargets || []) { + const targetId = index.get(normalizeLookupKey(pending.targetKey)); + if (targetId) { + pending.relationship.targetId = targetId; + pending.relationship.resolved = true; + } else { + pending.relationship.resolved = false; + warnings.push({ + filePath: unit.sourcePath || unit.id, + code: 'unresolved-relationship', + severity: 'error', + unitId: unit.id, + message: `Unresolved wikilink target "${pending.targetKey}".`, + }); } } } +} - // Build provenance - const rawProvenance = merged.provenance as - | Partial - | undefined; - const provenance = rawProvenance - ? { ...createFileProvenance(), ...rawProvenance } - : createFileProvenance(); - - // Body: use explicit body field, or stringify the entry data for structured entries - let body = ''; - if (typeof merged.body === 'string') { - body = merged.body.trim(); - } else if (entry) { - // For structured entries without explicit body, create a readable representation - const bodyData = { ...entry }; - for (const key of [ - 'id', - 'title', - 'domain', - 'kind', - 'system', - 'brand', - 'tags', - 'enforcement', - 'severity', - 'provenance', - 'body', - ]) { - delete bodyData[key]; +function buildRelationshipIndex( + units: KnowledgeUnit[], + config: MadrigalConfig, +): Map { + const index = new Map(); + const aliasToCanonical = new Map(); + for (const vocabulary of Object.values(config.vocabularies)) { + for (const [alias, canonical] of Object.entries(vocabulary.aliases || {})) { + aliasToCanonical.set(normalizeLookupKey(alias), canonical); } - if (Object.keys(bodyData).length > 0) { - body = Object.entries(bodyData) - .map( - ([k, v]) => - `**${k}:** ${Array.isArray(v) ? v.join(', ') : String(v)}`, - ) - .join('\n\n'); + } + + for (const unit of units) { + addLookup(index, unit.id, unit.id); + addLookup(index, unit.title, unit.id); + addLookup(index, slug(unit.title), unit.id); + if (unit.sourcePath) { + const withoutExt = unit.sourcePath.replace(/\.[^.]+$/, ''); + addLookup(index, withoutExt, unit.id); + addLookup(index, basename(withoutExt), unit.id); + addLookup(index, slug(basename(withoutExt)), unit.id); } } - return { - id, - title, - body, - domain, - kind, - system, - brand, - tags, - enforcement, - attributes, - provenance, - sourcePath: relative(baseDir, filePath), - }; + for (const [aliasKey, canonical] of aliasToCanonical.entries()) { + const targetId = + index.get(normalizeLookupKey(canonical)) || + index.get(normalizeLookupKey(slug(canonical))); + if (targetId && !index.has(aliasKey)) index.set(aliasKey, targetId); + } + + return index; } -/** - * Synchronous version of loadKnowledge for simple use cases. - */ -export function loadKnowledgeSync(options: LoadOptions): LoadResult { - const { sources, config, baseDir = process.cwd() } = options; - const units: KnowledgeUnit[] = []; - const errors: LoadError[] = []; - const warnings: LoadWarning[] = []; +function addLookup(index: Map, key: string, id: string): void { + const normalized = normalizeLookupKey(key); + if (normalized && !index.has(normalized)) index.set(normalized, id); +} - // Find all matching files (sync) - const files = fg.sync(sources, { - cwd: baseDir, - absolute: true, - onlyFiles: true, - ignore: ['**/node_modules/**', '**/dist/**'], - }); +function extractWikiLinks(body: string): KnowledgeRelationship[] { + const relationships: KnowledgeRelationship[] = []; + const seen = new Set(); + const regex = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g; + + for (const match of body.matchAll(regex)) { + const target = match[1].trim(); + const label = match[2]?.trim(); + const key = `${target}|${label || ''}`; + if (seen.has(key)) continue; + seen.add(key); + relationships.push({ + type: 'wikilink', + target, + label, + resolved: false, + }); + } - const domainNames = new Set(Object.keys(config.domains)); - const kindNames = new Set(Object.keys(config.kinds)); - const brandNames = new Set(Object.keys(config.brands)); - brandNames.add('global'); + return relationships; +} - for (const filePath of files) { - try { - const ext = extname(filePath).toLowerCase(); - if (ext === '.yaml' || ext === '.yml') { - const parsed = parseKnowledgeYamlFile( - filePath, - baseDir, - domainNames, - kindNames, - brandNames, - warnings, - ); - units.push(...parsed); - } else { - const unit = parseKnowledgeFile( - filePath, - baseDir, - domainNames, - kindNames, - brandNames, - warnings, - ); - if (unit) { - units.push(unit); - } - } - } catch (err) { - errors.push({ - filePath, - message: err instanceof Error ? err.message : String(err), - error: err instanceof Error ? err : undefined, - }); +function stripInternalFields(unit: ParsedUnit): KnowledgeUnit { + const { _relationshipTargets, ...clean } = unit; + return clean; +} + +function generateId( + filePath: string, + baseDir: string, + config: MadrigalConfig, +): string { + const strategy = config.schema.id?.strategy || 'path'; + if (strategy === 'filename') { + return slug(basename(filePath, extname(filePath))); + } + + const sourcePath = normalizePath(relative(baseDir, filePath)); + const withoutExtension = sourcePath.replace(/\.[^.]+$/, ''); + const segments = withoutExtension.split('/'); + const last = segments[segments.length - 1]; + if (last === 'index' || last === '_index') { + segments.pop(); + } + + return slug(segments.join('/')); +} + +function stringifyStructuredBody(entry: Record): string { + const bodyData = { ...entry }; + for (const key of CORE_FRONTMATTER_KEYS) delete bodyData[key]; + + return Object.entries(bodyData) + .map(([k, v]) => `**${k}:** ${Array.isArray(v) ? v.join(', ') : String(v)}`) + .join('\n\n'); +} + +function normalizeStringArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.map(String); +} + +function getString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + +function normalizePath(path: string): string { + return path.replace(/\\/g, '/'); +} + +function normalizeLookupKey(value: string): string { + return value.trim().toLowerCase(); +} + +function slug(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); +} + +function matchesGlob(path: string, pattern: string): boolean { + const normalizedPath = normalizePath(path); + const normalizedPattern = normalizePath(pattern); + const regex = new RegExp(`^${globToRegex(normalizedPattern)}$`); + return regex.test(normalizedPath); +} + +function globToRegex(pattern: string): string { + let output = ''; + for (let i = 0; i < pattern.length; i++) { + const char = pattern[i]; + const next = pattern[i + 1]; + if (char === '*' && next === '*') { + output += '.*'; + i++; + } else if (char === '*') { + output += '[^/]*'; + } else if (char === '?') { + output += '[^/]'; + } else { + output += escapeRegex(char); } } + return output; +} - return { units, errors, warnings }; +function escapeRegex(char: string): string { + return /[\\^$+?.()|[\]{}]/.test(char) ? `\\${char}` : char; } diff --git a/src/schema/index.ts b/src/schema/index.ts index 7d0d01c..1aa215d 100644 --- a/src/schema/index.ts +++ b/src/schema/index.ts @@ -8,6 +8,7 @@ export type { CreateKnowledgeUnit, Domain, KnowledgeFrontmatter, + KnowledgeRelationship, KnowledgeUnit, KnowledgeUnitWithEmbedding, UpdateKnowledgeUnit, diff --git a/src/schema/knowledge-unit.ts b/src/schema/knowledge-unit.ts index dde0266..fd1bc53 100644 --- a/src/schema/knowledge-unit.ts +++ b/src/schema/knowledge-unit.ts @@ -8,13 +8,32 @@ import type { Provenance } from '../provenance.js'; */ export type Domain = string; +/** + * Relationship extracted from source content or frontmatter. + * Consumers define relationship semantics; Madrigal preserves and resolves them. + */ +export interface KnowledgeRelationship { + /** Relationship kind, e.g. wikilink */ + type: string; + /** Raw target as written in the source */ + target: string; + /** Optional display label, e.g. the label in [[Target|Label]] */ + label?: string; + /** Resolved target knowledge unit ID when Madrigal can resolve it */ + targetId?: string; + /** Whether this relationship could be resolved to a loaded unit */ + resolved: boolean; +} + /** * Frontmatter fields expected in knowledge markdown files. */ export interface KnowledgeFrontmatter { + [key: string]: unknown; id?: string; title?: string; domain?: string; + type?: string; kind?: string; system?: string; brand?: string; @@ -27,11 +46,12 @@ export interface KnowledgeFrontmatter { } /** - * A KnowledgeUnit is the atomic unit of design knowledge. - * It represents a single rule, guideline, pattern, or insight. + * A KnowledgeUnit is Madrigal's normalized representation of a source record. + * Consumer configs define domain terms; Madrigal preserves raw metadata and + * normalizes the fields needed for compilation, linting, and retrieval. */ export interface KnowledgeUnit { - /** Unique identifier (UUID) */ + /** Unique identifier */ id: string; /** Human-readable title */ @@ -40,32 +60,38 @@ export interface KnowledgeUnit { /** Markdown content describing the knowledge */ body: string; - /** Knowledge domain this unit belongs to */ - domain: Domain; - - /** Structural type of knowledge (e.g., 'rule', 'glossary', 'rubric', 'template') */ + /** Structural type of knowledge (consumer-defined, e.g. study, theme, rule) */ kind: string; - /** Design system this applies to (e.g., 'market', 'arcade', 'wave') */ - system?: string; - - /** Brand this applies to, null for global rules */ - brand?: string; - /** Searchable tags for categorization */ tags: string[]; - /** Enforcement level */ - enforcement: Enforcement; + /** Relative path to the source file */ + sourcePath?: string; + + /** Raw parsed frontmatter, preserved exactly as parsed from the source */ + frontmatter: Record; - /** Open metadata for domain-specific attributes (surfaces, audiences, etc.) */ + /** Open metadata for domain-specific attributes after normalization */ attributes: Record; + /** Extracted relationships, such as wiki links */ + relationships: KnowledgeRelationship[]; + /** Origin and approval tracking */ provenance: Provenance; - /** Relative path to the source file */ - sourcePath?: string; + /** Knowledge domain this unit belongs to, when configured/present */ + domain?: Domain; + + /** Design system this applies to (e.g., 'market', 'arcade', 'wave') */ + system?: string; + + /** Brand this applies to, null for global rules */ + brand?: string; + + /** Enforcement level for rule-oriented consumers */ + enforcement?: Enforcement; /** ISO 8601 timestamp of creation */ createdAt?: string; @@ -81,13 +107,15 @@ export interface KnowledgeUnit { export interface CreateKnowledgeUnit { title: string; body: string; - domain: Domain; + domain?: Domain; kind?: string; system?: string; brand?: string; tags: string[]; - enforcement: Enforcement; + enforcement?: Enforcement; + frontmatter?: Record; attributes?: Record; + relationships?: KnowledgeRelationship[]; provenance: Provenance; } @@ -104,7 +132,9 @@ export interface UpdateKnowledgeUnit { brand?: string; tags?: string[]; enforcement?: Enforcement; + frontmatter?: Record; attributes?: Record; + relationships?: KnowledgeRelationship[]; provenance?: Provenance; } diff --git a/src/search/adapter.ts b/src/search/adapter.ts index b4914c0..7731bc1 100644 --- a/src/search/adapter.ts +++ b/src/search/adapter.ts @@ -4,7 +4,7 @@ import type { SearchAdapter, SemanticSearchOptions, } from '../adapters/search.js'; -import { ENFORCEMENT_ORDER } from '../enforcement.js'; +import { effectiveEnforcement, enforcementRank } from '../enforcement.js'; import type { KnowledgeUnit } from '../schema/index.js'; import { BM25Index } from './bm25.js'; @@ -45,8 +45,7 @@ export class BM25SearchAdapter implements SearchAdapter { // Sort by enforcement: must first results.sort( (a, b) => - (ENFORCEMENT_ORDER[a.enforcement] ?? 99) - - (ENFORCEMENT_ORDER[b.enforcement] ?? 99), + enforcementRank(a.enforcement) - enforcementRank(b.enforcement), ); } @@ -83,17 +82,18 @@ export class BM25SearchAdapter implements SearchAdapter { } if (options?.minEnforcement) { - const minOrder = ENFORCEMENT_ORDER[options.minEnforcement] ?? 99; + const minOrder = enforcementRank(options.minEnforcement); results = results.filter( - (r) => (ENFORCEMENT_ORDER[r.unit.enforcement] ?? 99) <= minOrder, + (r) => enforcementRank(r.unit.enforcement) <= minOrder, ); } // Apply enforcement boost: enforceable rules score slightly higher results = results.map((r) => { let boosted = r.score; - if (r.unit.enforcement === 'must') boosted *= 1.2; - else if (r.unit.enforcement === 'should') boosted *= 1.1; + const enforcement = effectiveEnforcement(r.unit.enforcement); + if (enforcement === 'must') boosted *= 1.2; + else if (enforcement === 'should') boosted *= 1.1; return { unit: r.unit, score: boosted }; }); @@ -144,7 +144,9 @@ export class BM25SearchAdapter implements SearchAdapter { if (filter.enforcement && filter.enforcement.length > 0) { const allowed = new Set(filter.enforcement); - results = results.filter((u) => allowed.has(u.enforcement)); + results = results.filter((u) => + allowed.has(effectiveEnforcement(u.enforcement)), + ); } if (filter.kind) { diff --git a/src/serve/mcp-server.ts b/src/serve/mcp-server.ts index 0b39211..5e7d6cc 100644 --- a/src/serve/mcp-server.ts +++ b/src/serve/mcp-server.ts @@ -11,7 +11,11 @@ import { existsSync, readFileSync } from 'node:fs'; import type { MadrigalConfig } from '../config.js'; import { loadConfig } from '../config.js'; -import { ENFORCEMENT_ORDER, type Enforcement } from '../enforcement.js'; +import { + type Enforcement, + effectiveEnforcement, + enforcementRank, +} from '../enforcement.js'; import { loadKnowledge } from '../loader.js'; import type { KnowledgeUnit } from '../schema/index.js'; import { BM25SearchAdapter } from '../search/adapter.js'; @@ -174,13 +178,14 @@ export async function serveMcp(options: ServeOptions = {}): Promise { const lines = results.map((u) => { const tags = u.tags.length ? ` (${u.tags.slice(0, 3).join(', ')})` : ''; + const enforcement = effectiveEnforcement(u.enforcement); // Include a short excerpt from the body (first 150 chars) for context const excerpt = u.body .replace(/^#+\s+/gm, '') .replace(/\n+/g, ' ') .trim() .slice(0, 150); - return `- [${u.enforcement.toUpperCase()}] **${u.id}**${tags}\n ${u.title}${excerpt ? `\n _${excerpt}…_` : ''}`; + return `- [${enforcement.toUpperCase()}] **${u.id}**${tags}\n ${u.title}${excerpt ? `\n _${excerpt}…_` : ''}`; }); return { @@ -216,9 +221,9 @@ export async function serveMcp(options: ServeOptions = {}): Promise { const meta = [ `**ID:** ${unit.id}`, - `**Domain:** ${unit.domain}`, + unit.domain ? `**Domain:** ${unit.domain}` : null, `**Kind:** ${unit.kind}`, - `**Enforcement:** ${unit.enforcement}`, + `**Enforcement:** ${effectiveEnforcement(unit.enforcement)}`, `**Tags:** ${unit.tags.join(', ')}`, unit.brand ? `**Brand:** ${unit.brand}` : null, unit.system ? `**System:** ${unit.system}` : null, @@ -258,18 +263,19 @@ export async function serveMcp(options: ServeOptions = {}): Promise { (u) => isGlobal(u.brand) || u.brand === brand, ); if (enforcement) - filtered = filtered.filter((u) => u.enforcement === enforcement); + filtered = filtered.filter( + (u) => effectiveEnforcement(u.enforcement) === enforcement, + ); filtered.sort( (a, b) => - (ENFORCEMENT_ORDER[a.enforcement] ?? 99) - - (ENFORCEMENT_ORDER[b.enforcement] ?? 99), + enforcementRank(a.enforcement) - enforcementRank(b.enforcement), ); const text = filtered .map( (u) => - `- **${u.id}** [${u.enforcement.toUpperCase()}]: ${u.title} (${u.tags.join(', ')})`, + `- **${u.id}** [${effectiveEnforcement(u.enforcement).toUpperCase()}]: ${u.title} (${u.tags.join(', ')})`, ) .join('\n'); @@ -312,22 +318,15 @@ export async function serveMcp(options: ServeOptions = {}): Promise { } // Sort by enforcement (must first), then title - const order: Record = { - must: 0, - should: 1, - may: 2, - context: 3, - deprecated: 4, - }; brandUnits.sort( (a, b) => - (order[a.enforcement] ?? 9) - (order[b.enforcement] ?? 9) || + enforcementRank(a.enforcement) - enforcementRank(b.enforcement) || a.title.localeCompare(b.title), ); const lines = brandUnits.map( (u) => - `- [${u.enforcement.toUpperCase()}] **${u.id}** — ${u.title}${u.tags.length ? ` (${u.tags.slice(0, 3).join(', ')})` : ''}`, + `- [${effectiveEnforcement(u.enforcement).toUpperCase()}] **${u.id}** — ${u.title}${u.tags.length ? ` (${u.tags.slice(0, 3).join(', ')})` : ''}`, ); const text = `${brandUnits.length} knowledge unit(s) for brand "${brand}":\n\n${lines.join('\n')}\n\nUse get_knowledge_unit(id) to read the full content of any unit.`; @@ -373,11 +372,12 @@ export async function serveMcp(options: ServeOptions = {}): Promise { const rulesText = scored .map((r) => { const brandTag = r.unit.brand ? ` [${r.unit.brand}]` : ''; + const enforcement = effectiveEnforcement(r.unit.enforcement); const excerpt = r.unit.body.length > EXCERPT_CHARS ? `${r.unit.body.slice(0, EXCERPT_CHARS)}\n[…truncated]` : r.unit.body; - return `### ${r.unit.title} [${r.unit.enforcement.toUpperCase()}]${brandTag}\n**ID:** ${r.unit.id}\n\n${excerpt}`; + return `### ${r.unit.title} [${enforcement.toUpperCase()}]${brandTag}\n**ID:** ${r.unit.id}\n\n${excerpt}`; }) .join('\n\n---\n\n'); diff --git a/src/topology/generate.ts b/src/topology/generate.ts index 94e8782..1c311e2 100644 --- a/src/topology/generate.ts +++ b/src/topology/generate.ts @@ -1,3 +1,4 @@ +import { effectiveEnforcement } from '../enforcement.js'; import type { KnowledgeUnit } from '../schema/index.js'; import { fallbackClusterNames, @@ -115,9 +116,9 @@ export async function generateTopology( const nodes: TopologyNode[] = units.map((unit, i) => ({ id: unit.id, title: unit.title, - domain: unit.domain, + domain: unit.domain || 'default', kind: unit.kind, - enforcement: unit.enforcement, + enforcement: effectiveEnforcement(unit.enforcement), brand: unit.brand, tags: unit.tags, excerpt: unit.body.slice(0, 200).replace(/\n/g, ' ').trim(), diff --git a/src/topology/labeler.ts b/src/topology/labeler.ts index 850e1a0..78e6ebd 100644 --- a/src/topology/labeler.ts +++ b/src/topology/labeler.ts @@ -162,7 +162,7 @@ export function fallbackClusterNames( for (let c = 0; c < nClusters; c++) { const domains = units .filter((_, i) => labels[i] === c) - .map((u) => u.domain); + .map((u) => u.domain || 'default'); if (domains.length > 0) { const counts = new Map(); for (const d of domains) counts.set(d, (counts.get(d) ?? 0) + 1);