diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb32c5f2..fe31b923a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Added Oxlint lint fence. - Document metadata and metadata filtering. Markdown documents can opt into typed metadata through a namespaced frontmatter block (`qmd.metadata` with strings, numbers, booleans, or flat homogeneous arrays), and every search surface — CLI `search`/`vsearch`/`query` via `--filter `, the SDK's `filter` option on `search()`/`searchLex()`/`searchVector()`, the MCP `query` tool, and HTTP `POST /query` and `/search` — accepts one shared recursive filter AST discriminated by `operator`: `and`/`or`/`not` logical groups, `eq`/`ne`/`gt`/`gte`/`lt`/`lte` comparisons, `in`/`nin`/`all` membership, and `exists` presence. Every returned result satisfies the filter (applied before RRF fusion and reranking); like collection filtering, highly selective filters remain best-effort for top-K completeness. Frontmatter stays ordinary searchable content — no chunking, embedding, snippet, or line-number changes — and documents without `qmd.metadata` behave exactly as before. JSON/SDK/MCP/HTTP results now include each document's indexed metadata, and `qmd status` reports how many documents still need metadata extraction (a normal `qmd update` backfills existing indexes). +- Metadata discovery. Filtering is only useful when the caller knows what to filter on, so every surface now reports the metadata keys, types, and value counts already in the index. The CLI adds `qmd collection metadata [name...]` with `--key ` and `--value ` to select a region of the key/value space (including reverse lookup: which keys hold a value), `--filter ` to count only documents matching a filter (same AST as search), and `-n`/`--all`, `--sort count|value`, `--min-count ` to shape the per-key value window. `qmd collection list` names each collection's top keys, `qmd collection show` details them with a value preview, and `qmd status` summarizes coverage. The SDK adds `listMetadata(options)`, the MCP server adds a `metadata` tool and lists key names and types per collection in `status`, and the HTTP server adds `POST /metadata`. Counts are documents, not values. Truncation always reports the remainder. Numbers report min, median, and max. Keys whose documents disagree on type (metadata is validated per document, so this can happen within one collection as well as across collections) are split per type with their own document counts rather than resolved. Discovery applies the same extraction gate and scope as filtering, so every value it reports is one an `eq` filter can match. Reads the existing metadata tables, so no re-indexing is needed. ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index ee952e4a0..34df82bbe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,7 @@ qmd collection add . --name # Create/index collection qmd collection list # List all collections with details qmd collection remove # Remove a collection by name qmd collection rename # Rename a collection +qmd collection metadata [name...] # Discover metadata keys, types, and value counts (--key, --value, --filter) qmd init # Create a project-local .qmd index qmd ls [collection[/path]] # List collections or files in a collection qmd context add [path] "text" # Add context for path (defaults to current dir) @@ -47,9 +48,15 @@ qmd collection remove mynotes # Rename a collection qmd collection rename mynotes my-notes -# Show collection details +# Show collection details, including the top metadata keys qmd collection show mynotes +# Discover metadata keys and values to filter on +qmd collection metadata mynotes +qmd collection metadata mynotes --key topics +qmd collection metadata mynotes --value docs-team +qmd collection metadata mynotes --key topics --filter '{"key":"status","operator":"eq","value":"published"}' + # Set or clear the pre-update hook (runs before re-indexing on `qmd update`) qmd collection update-cmd mynotes 'git pull --ff-only' qmd collection update-cmd mynotes # clear diff --git a/README.md b/README.md index 76b822b20..e7ae187d1 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ runs in a container and a liveness probe connects from a non-loopback address. The HTTP server exposes two endpoints: - `POST /mcp` — MCP Streamable HTTP (JSON responses, stateless) - `POST /query` (alias `/search`) — structured search without the MCP protocol. Accepts the same optional `filter` object as the `query` tool (invalid filters return `400`); see [Metadata Filtering](#metadata-filtering) +- `POST /metadata` — metadata discovery without the MCP protocol. Same body as the `metadata` tool (invalid filters return `400`); see [Metadata Discovery](#metadata-discovery) - `GET /health` — liveness check with uptime @@ -198,6 +199,13 @@ Point any MCP client at `http://localhost:8181/mcp` to connect. | `get` | `maxLines` | number | Limit returned lines | | `get` | `lineNumbers` | boolean | Prefix lines with numbers (default **true**) | | `multi_get` | `pattern` | string | Glob pattern or comma-separated list | +| `metadata` | `collections` | string[] | Restrict discovery to collection names (default: the collections `query` searches) | +| `metadata` | `key` | string | Glob over metadata key names (`topics` exact, `mem-*` a family) | +| `metadata` | `value` | string | Glob over values in text form (reverse lookup: which keys hold this value) | +| `metadata` | `filter` | object | Count only documents matching this filter (same AST as `query`) | +| `metadata` | `limit` | number | Values shown per key (default 10). `remaining` reports the rest | +| `metadata` | `sort` | string | `count` (default) or `value` | +| `metadata` | `minCount` | number | Hide values held by fewer documents (default 1) | | `multi_get` | `maxBytes` | number | Skip files larger than N (default 10240) | | `multi_get` | `maxLines` | number | Limit lines per file | | `multi_get` | `lineNumbers` | boolean | Prefix lines with numbers (default **true**) | @@ -644,9 +652,14 @@ qmd collection rename myproject my-project qmd ls notes qmd ls notes/subfolder -# Show collection details (path, glob mask, include status, context count) +# Show collection details (path, glob mask, include status, context count, top metadata keys) qmd collection show notes +# Discover metadata keys, types, and value counts (see Metadata Discovery) +qmd collection metadata notes +qmd collection metadata notes --key topics +qmd collection metadata notes --key topics --filter '{"key":"status","operator":"eq","value":"published"}' + # Include or exclude a collection from default (unscoped) queries qmd collection include notes qmd collection exclude notes @@ -985,6 +998,140 @@ Guarantees and limits: JSON output (`--format json`), the SDK, MCP structured results, and the HTTP endpoints include each result's indexed metadata. +### Metadata Discovery + +Filtering is only useful if you know what to filter on. Discovery reports the metadata keys, types, and value counts already in the index, turning "what dimensions exist" into a well-shaped filter in a few steps. It reads the same tables filtering reads: no re-indexing, and every value it reports is one an `eq` filter can match. + +The mental model has two halves. `--key` and `--value` are glob patterns (picomatch, as `multi-get` uses) that select **where to look** in the key/value space. `--filter` selects **which documents are counted**. They compose: + +| `--key` | `--value` | Question answered | +|---------|-----------|-------------------| +| | | Which keys exist, with a window of values each | +| `topics` | | Everything about one key | +| `mem-*` | | Which keys look like this | +| | `docs-team` | Which keys hold this value | +| | `2025-*` | Which keys hold values shaped like this | +| `topics` | `type*` | Values of one key matching a pattern | + +Start wide and narrow: + +```sh +# Which keys does this collection use? (also shown by `qmd collection show notes`) +qmd collection metadata notes + +# Everything about one key: coverage, distinct count, top values +qmd collection metadata notes --key topics + +# Reverse lookup: which keys hold this value +qmd collection metadata notes --value docs-team + +# What remains after a filter, before committing to it in a query +qmd collection metadata notes --key topics --filter '{"key":"status","operator":"eq","value":"published"}' + +# Then search with the filter you just validated +qmd query "dependency injection" -c notes --filter '{ + "operator": "and", + "operands": [ + { "key": "status", "operator": "eq", "value": "published" }, + { "key": "topics", "operator": "all", "value": ["typescript"] } + ] +}' +``` + +The drill-down prints one block per key, in coverage order: + +```sh +qmd collection metadata notes +``` + +``` +topics string[] 388 of 480 documents 1,204 distinct + typescript 140 + sqlite 92 + search 77 + architecture 61 + sqlite-vec 44 + mcp 39 + embeddings 35 + agents 31 + cli 28 + testing 26 +1,194 more values, use -n or --all + +priority number 205 of 480 documents 5 distinct + min 1 median 3 max 5 + 1 (12) 2 (40) 3 (88) 4 (50) 5 (15) + +reviewed boolean 480 of 480 documents + true 61 false 419 +``` + +A reverse lookup answers "where does this value live" by returning every key that holds it, here a scalar key and an array key: + +```sh +qmd collection metadata notes --value docs-team +``` + +``` +owner string 212 of 480 documents 1 distinct + docs-team 212 + +reviewers string[] 97 of 480 documents 1 distinct + docs-team 97 +``` + +Options: `-n `/`--all` size the value window (default 10), `--sort count|value` orders it (count descending by default, value ascending for ranges and dates), and `--min-count ` drops the long tail. Omitting the collection name covers the default collections, exactly as an unscoped search does. + +Rules that matter when reading the output: + +- **Counts are documents, not values.** A document with `topics: [a, b]` contributes one to each. Coverage is "documents declaring this key". +- **Truncation is never silent.** Every capped list ends with the remainder and the flag that removes the cap. Structured results carry it as `remaining`. +- **Numbers report min, median, and max**, plus the enumerated values when they fit, which is enough to write a sound `gt`/`lt` threshold in one call. +- **Discovery sees exactly what filtering sees.** Same extraction gate, same active-document rule, same collection scope. Documents still pending extraction are reported on stderr and excluded until `qmd update` runs. +- **Type conflicts are reported, not resolved.** Metadata is validated one document at a time. Nothing requires two documents to agree on a key's type, whether they sit in the same collection or in different ones, so `priority: 3` in one file and `priority: high` in another both index. Discovery splits such a key by type and gives each type its own document count, which tells you how much of the corpus a typed filter would reach. Within one collection: + +```sh +qmd collection metadata work --key priority +``` + +``` +priority number | string 1,222 of 1,620 documents + number 18 docs min 1 median 2 max 3 + string 1,204 docs high (700), medium (380), low (124) +``` + +Across collections, each type also names where it comes from: + +```sh +qmd collection metadata --key priority +``` + +``` +priority number | string 1,427 of 2,100 documents + number 223 docs min 1 median 3 max 5 notes, work + string 1,204 docs high (700), medium (380), low (124) work +``` + +`qmd collection list` names each collection's top keys, `qmd collection show ` details the top five with a value preview, and `qmd status` summarizes how many keys and files carry metadata. + +The same discovery is available on every surface with the same options (`collection`, `key`, `value`, `filter`, `limit`, `sort`, `minCount`) and the same result shape: + +```typescript +// SDK: one flat result, keys in coverage order, each split per type +const discovery = await store.listMetadata({ collection: "notes", key: "topics", limit: 5 }) +discovery.documents // active documents in scope (the denominator) +discovery.keys[0].types[0] // { type, multiValued, documents, distinctValues, values, remaining, range?, collections } + +// Narrowed by a filter: how many documents pass, and what is left to filter on +const published = await store.listMetadata({ + collection: "notes", + filter: { key: "status", operator: "eq", value: "published" }, +}) +published.filteredDocuments +``` + +The MCP `metadata` tool takes the same options with `collections` spelled as on `query`, returns the CLI shape as text and the result as `structuredContent`, and the MCP `status` tool lists each collection's key names and types so an agent's first call reveals that metadata exists. `POST /metadata` accepts the same body as the tool and returns the same result (`400` on an invalid filter). + ### Output Format Default output is colorized CLI format (respects `NO_COLOR` env). diff --git a/skills/qmd/SKILL.md b/skills/qmd/SKILL.md index 7693d4273..79b8ef970 100644 --- a/skills/qmd/SKILL.md +++ b/skills/qmd/SKILL.md @@ -210,6 +210,21 @@ qmd query "dependency injection" --filter '{"operator":"and","operands":[{"key": Nodes are discriminated by `operator`: groups `and`/`or` take `operands`, `not` takes one `operand`, and conditions take `key` + `value` with operators `eq`/`ne`/`gt`/`gte`/`lt`/`lte` (comparison), `in`/`nin`/`all` (membership), or `exists` (presence). Matching is typed and exact; missing keys do not match `ne`/`nin` (add an `exists: false` branch in an `or` group to include them). The MCP `query` tool accepts the same AST as a `filter` object. JSON output includes each result's `metadata`. +## Discover metadata before filtering + +Do not guess keys or values. `qmd collection show ` lists the top keys with types and a value preview, and `qmd collection metadata` drills in. `--key` and `--value` are globs that pick where to look, and `--filter` picks which documents are counted: + +```bash +qmd collection metadata notes # every key, ten values each +qmd collection metadata notes --key topics # one key: coverage, distinct count, top values +qmd collection metadata notes --value docs-team # reverse lookup: which keys hold this value +qmd collection metadata notes --key topics --filter '{"key":"status","operator":"eq","value":"published"}' +``` + +Read the header first: `topics string[] 388 of 480 documents 1,204 distinct` tells you coverage and cardinality before you commit to a filter, and with `--filter` it tells you how many documents pass. Counts are documents, not values. A `N more values, use -n or --all` footer means the list was cut. Raise `-n` rather than assuming the rest. Numbers print `min`, `median`, and `max` so you can write a `gt`/`lt` threshold in one call. A key shown as `number | string` means documents disagree on type. Metadata is validated per document, so this happens within a single collection as readily as across collections. Each type reports its own document count. Filter by the type that covers the documents you want. Every value shown can be matched with `eq` under the same collection scope. + +Over MCP, call the `metadata` tool (same options, `collections` as an array) and read `remaining` and `range` from the structured result. The `status` tool lists each collection's key names and types, so check it first. + ## MCP Tool: `query` When using the MCP server, prefer structured searches: diff --git a/src/cli/qmd.ts b/src/cli/qmd.ts index c5caa2668..53009149d 100644 --- a/src/cli/qmd.ts +++ b/src/cli/qmd.ts @@ -86,7 +86,15 @@ import { type ReindexResult, type ChunkStrategy, } from "../store.js"; -import { syncDocumentMetadata, countDocumentsPendingMetadata } from "../metadata-store.js"; +import { + syncDocumentMetadata, + countDocumentsPendingMetadata, + countDocumentsWithMetadata, + listMetadata, + listMetadataKeys, + type ListMetadataOptions, +} from "../metadata-store.js"; +import { formatMetadataKeySummaries, formatMetadataOverview } from "../metadata-format.js"; import type { DocumentMetadata } from "../metadata.js"; import { parseMetadataFilter, type MetadataFilter } from "../metadata-filter.js"; import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_MODEL_CACHE_DIR, resolveEmbedModel, resolveGenerateModel, resolveRerankModel, resolveModels, inspectGgufFile, isDarwinMetalMitigationActive } from "../llm.js"; @@ -574,6 +582,10 @@ async function showStatus(): Promise { if (needsEmbedding > 0) { console.log(` ${c.yellow}Pending: ${needsEmbedding} need embedding${c.reset} (run 'qmd embed')`); } + const metadataKeys = listMetadataKeys(db); + if (metadataKeys.length > 0) { + console.log(` Metadata: ${metadataKeys.length} keys across ${countDocumentsWithMetadata(db)} files (explore with 'qmd collection metadata')`); + } const pendingMetadata = countDocumentsPendingMetadata(db); if (pendingMetadata > 0) { console.log(` ${c.yellow}Metadata: ${pendingMetadata} need extraction${c.reset} (run 'qmd update'; excluded from --filter searches)`); @@ -1811,6 +1823,12 @@ function collectionList(): void { console.log(` ${c.dim}Ignore:${c.reset} ${yamlColl.ignore.join(', ')}`); } console.log(` ${c.dim}Files:${c.reset} ${coll.active_count}`); + const metadataKeys = listMetadataKeys(db, [coll.name]); + if (metadataKeys.length > 0) { + const shownKeys = metadataKeys.slice(0, COLLECTION_LIST_METADATA_KEYS).map(overview => overview.key); + const hiddenKeys = metadataKeys.length - shownKeys.length; + console.log(` ${c.dim}Metadata:${c.reset} ${shownKeys.join(', ')}${hiddenKeys > 0 ? `, +${hiddenKeys} more` : ''}`); + } console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`); console.log(); } @@ -1818,6 +1836,27 @@ function collectionList(): void { closeDb(); } +/** Key names shown on `collection list`, and keys detailed on `collection show`. */ +const COLLECTION_LIST_METADATA_KEYS = 5; + +// The Metadata section of `collection show`: top keys by coverage with a +// value preview, and a pointer at the drill-down for the rest. +function collectionShowMetadata(name: string): void { + const db = getDb(); + const result = listMetadata(db, { collection: name, limit: 3 }); + const documentsWithMetadata = countDocumentsWithMetadata(db, [name]); + const pendingMetadata = countDocumentsPendingMetadata(db, [name]); + closeDb(); + + console.log(formatMetadataOverview(result, { + documentsWithMetadata, + pendingMetadata, + keyLimit: COLLECTION_LIST_METADATA_KEYS, + drillDownHint: `qmd collection metadata ${name}`, + colors: c, + })); +} + /** Canonical --mask, with --glob as the alias OpenClaw and others already pass (#536). */ function collectionGlobFromCli(values: { mask?: unknown; glob?: unknown }): string { const mask = typeof values.mask === "string" && values.mask.length > 0 ? values.mask : undefined; @@ -1916,6 +1955,80 @@ function collectionRename(oldName: string, newName: string): void { console.log(` Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`); } +// Metadata discovery drill-down. Collection names are already validated; +// an empty list means the default collections resolved to nothing, which +// the store reads as "every collection" exactly as search does. +function collectionMetadata(collectionNames: string[], options: ListMetadataOptions): void { + const db = getDb(); + + if (listCollections(db).length === 0) { + console.log("No collections found. Run 'qmd collection add .' to create one."); + closeDb(); + return; + } + + // Discovery always applies the extraction gate, filter or not. + warnPendingMetadata(db); + + const result = listMetadata(db, { ...options, collection: collectionSearchFilter(collectionNames) }); + closeDb(); + + if (result.keys.length === 0) { + const selection = options.key || options.value || options.filter; + console.log(selection + ? `${c.dim}No metadata matches. Run 'qmd collection metadata' without patterns to see every key.${c.reset}` + : `${c.dim}No metadata found. Add qmd.metadata frontmatter and run 'qmd update'.${c.reset}`); + return; + } + + console.log(formatMetadataKeySummaries(result, { + showCollections: collectionNames.length !== 1, + limitHint: "-n or --all", + colors: c, + })); +} + +// Parse the discovery-specific flags; exits with usage on a bad value. +function parseCliMetadataOptions(values: Record): ListMetadataOptions { + const options: ListMetadataOptions = { + filter: parseCliMetadataFilter(values.filter), + }; + + if (typeof values.key === "string" && values.key.length > 0) options.key = values.key; + if (typeof values.value === "string" && values.value.length > 0) options.value = values.value; + + if (values.all) { + options.limit = Infinity; + } else if (values.n !== undefined) { + options.limit = parsePositiveInteger(values.n, "-n"); + } + + if (values["min-count"] !== undefined) { + options.minCount = parsePositiveInteger(values["min-count"], "--min-count"); + } + + if (values.sort !== undefined) { + if (values.sort !== "count" && values.sort !== "value") { + console.error(`Invalid --sort value: ${String(values.sort)}`); + console.error("Valid: count, value"); + process.exit(1); + } + options.sort = values.sort; + } + + return options; +} + +function parsePositiveInteger(raw: unknown, flag: string): number { + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 1) { + console.error(`Invalid ${flag} value: ${String(raw)}`); + console.error(`${flag} must be a positive integer`); + process.exit(1); + } + return parsed; +} + async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, collectionName?: string, suppressEmbedNotice: boolean = false, ignorePatterns?: string[]): Promise { const db = getDb(); const resolvedPwd = pwd || getPwd(); @@ -3112,7 +3225,12 @@ function parseCLI() { json: { type: "boolean" }, explain: { type: "boolean" }, collection: { type: "string", short: "c", multiple: true }, // Filter by collection(s) - filter: { type: "string" }, // Metadata filter (JSON AST) for search/vsearch/query + filter: { type: "string" }, // Metadata filter (JSON AST) for search/vsearch/query/collection metadata + // Metadata discovery options (collection metadata) + key: { type: "string" }, // picomatch pattern over metadata keys + value: { type: "string" }, // picomatch pattern over metadata values + sort: { type: "string" }, // count (default) | value + "min-count": { type: "string" }, // drop values held by fewer documents // Collection options name: { type: "string" }, // collection name mask: { type: "string" }, // glob pattern @@ -3652,6 +3770,7 @@ function showHelp(): void { console.log(""); console.log("Collections & context:"); console.log(" qmd collection add/list/remove/rename/show - Manage indexed folders"); + console.log(" qmd collection metadata [name] [--key K] - Discover metadata keys and values to filter on"); console.log(" qmd context add/list/rm - Attach human-written summaries"); console.log(" qmd ls [collection[/path]] - Inspect indexed files"); console.log(""); @@ -4644,6 +4763,16 @@ if (isMain) { const ctxCount = Object.keys(col.context).length; console.log(` Contexts: ${ctxCount}`); } + collectionShowMetadata(name); + break; + } + + case "metadata": { + // Positional names are optional; omitted means the default + // collections, as an unscoped search does. + const rawNames = cli.args.length > 1 ? cli.args.slice(1) : undefined; + const collectionNames = resolveCollectionFilter(rawNames, true); + collectionMetadata(collectionNames, parseCliMetadataOptions(cli.values)); break; } @@ -4657,6 +4786,11 @@ if (isMain) { console.log(" remove Remove a collection"); console.log(" rename Rename a collection"); console.log(" show Show collection details"); + console.log(" metadata [name...] Discover metadata keys, types, and value counts"); + console.log(" --key Select keys (glob), --value selects values"); + console.log(" --filter Count only documents matching a metadata filter"); + console.log(" -n | --all Values shown per key (default 10)"); + console.log(" --sort count|value Value order (default count), --min-count drops the tail"); console.log(" update-cmd [cmd] Set pre-update command (e.g., 'git pull')"); console.log(" include Include in default queries"); console.log(" exclude Exclude from default queries"); @@ -4666,6 +4800,8 @@ if (isMain) { console.log(" qmd collection add ~/notes --name notes --mask 'a.md,journals/*.md'"); console.log(" qmd collection update-cmd brain 'git pull'"); console.log(" qmd collection exclude archive"); + console.log(" qmd collection metadata notes --key topics"); + console.log(" qmd collection metadata notes --key topics --filter '{\"key\":\"status\",\"operator\":\"eq\",\"value\":\"published\"}'"); process.exit(0); } diff --git a/src/index.ts b/src/index.ts index 223c10d9d..ab5eb4fe7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -82,6 +82,16 @@ import { type MetadataFilterNegation, type MetadataCondition, } from "./metadata-filter.js"; +import { + listMetadata as storeListMetadata, + type ListMetadataOptions, + type ListMetadataResult, + type MetadataKeySummary, + type MetadataKeyTypeSummary, + type MetadataValueCount, + type MetadataKeyOverview, + type MetadataValueType, +} from "./metadata-store.js"; import { setConfigSource, loadConfig, @@ -136,6 +146,17 @@ export type { }; export { parseMetadataFilter, MetadataFilterError }; +// Re-export metadata discovery types (listMetadata() and status metadata keys) +export type { + ListMetadataOptions, + ListMetadataResult, + MetadataKeySummary, + MetadataKeyTypeSummary, + MetadataValueCount, + MetadataKeyOverview, + MetadataValueType, +}; + // Re-export the internal Store type for advanced consumers export type { InternalStore }; @@ -303,6 +324,14 @@ export interface QMDStore { /** List all collections with document stats */ listCollections(): Promise<{ name: string; pwd: string; glob_pattern: string; doc_count: number; active_count: number; last_modified: string | null; includeByDefault: boolean }[]>; + /** + * Discover metadata keys, types, and value counts across the documents in + * scope. `key` and `value` are picomatch patterns selecting where to look; + * `filter` selects which documents are counted. Every value reported is + * one an `eq` filter can match under the same scope. + */ + listMetadata(options?: ListMetadataOptions): Promise; + /** Get names of collections included by default in queries */ getDefaultCollectionNames(): Promise; @@ -510,6 +539,10 @@ export async function createStore(options: StoreOptions): Promise { return result; }, listCollections: async () => storeListCollections(db), + listMetadata: async (opts) => storeListMetadata(db, { + ...opts, + filter: opts?.filter === undefined ? undefined : parseMetadataFilter(opts.filter), + }), getDefaultCollectionNames: async () => { const collections = storeListCollections(db); return collections.filter(c => c.includeByDefault).map(c => c.name); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 73f3ee460..ee81cca81 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -28,8 +28,10 @@ import { type IndexStatus, type DocumentMetadata, type MetadataFilter, + type MetadataKeyOverview, } from "../index.js"; import { getConfigPath } from "../collections.js"; +import { formatMetadataKeySummaries } from "../metadata-format.js"; import { enableProductionMode } from "../store.js"; import { checkRequestOrigin, resolveOriginGuard } from "./origin-guard.js"; @@ -71,6 +73,7 @@ type StatusResult = { pattern: string | null; documents: number; lastUpdated: string; + metadataKeys: MetadataKeyOverview[]; }[]; }; @@ -608,6 +611,14 @@ Intent-aware lex (C++ performance, not sports): for (const col of status.collections) { summary.push(` - ${col.name}: ${col.path} (${col.documents} docs)`); + if (col.metadataKeys.length > 0) { + const keyLabels = col.metadataKeys.map(overview => `${overview.key} (${overview.types.join(" | ")})`); + summary.push(` metadata keys: ${keyLabels.join(", ")}`); + } + } + + if (status.collections.some(col => col.metadataKeys.length > 0)) { + summary.push(` Metadata: call the 'metadata' tool to see values and counts before writing a 'filter'`); } return { @@ -617,6 +628,83 @@ Intent-aware lex (C++ performance, not sports): }) ); + // --------------------------------------------------------------------------- + // Tool: qmd_metadata (Metadata discovery) + // --------------------------------------------------------------------------- + + server.registerTool( + "metadata", + { + title: "Metadata Discovery", + description: `Discover which metadata keys exist, what types they hold, and how many documents share each value, so you can write a precise \`filter\` for the query tool. + +Documents carry metadata as \`qmd.metadata\` frontmatter (strings, numbers, booleans, or arrays of one of those). This tool reports what is indexed, never guesses. + +## Mental model + +\`key\` and \`value\` are glob patterns that select WHERE to look in the key/value space. \`filter\` selects WHICH documents are counted. They compose: + +| key | value | Question answered | +|---|---|---| +| | | Which keys exist, with a window of values each | +| \`topics\` | | Everything about one key | +| \`mem-*\` | | Which keys look like this | +| | \`docs-team\` | Which keys hold this value (reverse lookup) | +| | \`2025-*\` | Which keys hold values shaped like this | +| \`topics\` | \`type*\` | Values of one key matching a pattern | + +Add \`filter\` to any of these to see what remains after narrowing, e.g. the topics among published documents. The header then reports how many documents pass the filter. + +## Reading the result + +One entry per key, ordered by coverage. Each key splits by type. Metadata is validated per document, never across documents, so a key can hold numbers in some files and strings in others within a single collection as easily as across collections. A key with more than one type reports each type separately with its own document count and contributing collections, so you can see how many documents a typed filter would reach. Per type: \`documents\` holding it, \`distinctValues\`, the windowed \`values\` with document counts, and \`remaining\` values not shown. \`remaining\` is exact: raise \`limit\` to see them. Numbers also report \`range\` (min, median, max) for writing gt/lt thresholds. Counts are documents, not values: a document with \`topics: [a, b]\` counts once for each. + +Every value reported here can be matched with \`{key, operator: 'eq', value}\` under the same collections.`, + annotations: { readOnlyHint: true, openWorldHint: false }, + inputSchema: z.object({ + collections: z.array(z.string()).optional().describe("Restrict to these collections (default: the same collections query searches)"), + key: z.string().optional().describe("Glob over key names (picomatch). 'topics' matches exactly that key, 'mem-*' a family. Default: every key"), + value: z.string().optional().describe("Glob over values in text form (numbers as digits, booleans as true/false). Default: every value"), + filter: z.record(z.string(), z.unknown()).optional().describe( + "Count only documents matching this metadata filter. Same recursive AST as the query tool's 'filter'. " + + "Example: {\"key\":\"status\",\"operator\":\"eq\",\"value\":\"published\"}" + ), + limit: z.number().int().positive().optional().default(10).describe("Values shown per key (default: 10). 'remaining' reports how many were left out"), + sort: z.enum(["count", "value"]).optional().default("count").describe("Order values by document count descending (default) or by value ascending"), + minCount: z.number().int().positive().optional().default(1).describe("Hide values held by fewer documents than this (default: 1)"), + }), + }, + track(async ({ collections, key, value, filter, limit, sort, minCount }) => { + const filterValidation = validateFilterArgument(filter); + if (filterValidation.error) { + return { + content: [{ type: "text" as const, text: `Error: ${filterValidation.error}` }], + isError: true, + }; + } + + const effectiveCollections = collections ?? defaultCollectionNames; + const result = await store.listMetadata({ + collection: effectiveCollections.length > 0 ? effectiveCollections : undefined, + key, + value, + filter: filterValidation.filter, + limit, + sort, + minCount, + }); + + const text = result.keys.length === 0 + ? "No metadata matches. Call without key/value/filter to see every key, or check the status tool for collections with metadata." + : formatMetadataKeySummaries(result, { showCollections: effectiveCollections.length !== 1, limitHint: "a higher 'limit'" }); + + return { + content: [{ type: "text", text }], + structuredContent: result, + }; + }) + ); + return server; } @@ -1103,6 +1191,66 @@ export async function startMcpHttpServer( return; } + // REST endpoint: POST /metadata — metadata discovery, same body as the metadata tool + if (pathname === "/metadata" && nodeReq.method === "POST") { + const rawBody = await collectBody(nodeReq); + let parsedParams: unknown; + try { + parsedParams = rawBody.trim() === "" ? {} : JSON.parse(rawBody); + } catch { + nodeRes.writeHead(400, { "Content-Type": "application/json" }); + nodeRes.end(JSON.stringify({ error: "Invalid JSON body" })); + return; + } + if (typeof parsedParams !== "object" || parsedParams === null || Array.isArray(parsedParams)) { + nodeRes.writeHead(400, { "Content-Type": "application/json" }); + nodeRes.end(JSON.stringify({ error: "JSON body must be an object" })); + return; + } + const params = parsedParams as Record; + + // Optional metadata filter — must be an object and a valid filter AST + let restFilter: MetadataFilter | undefined; + if (params.filter !== undefined) { + if (typeof params.filter !== "object" || params.filter === null || Array.isArray(params.filter)) { + nodeRes.writeHead(400, { "Content-Type": "application/json" }); + nodeRes.end(JSON.stringify({ error: "Invalid field: filter (must be an object)" })); + return; + } + const filterValidation = validateFilterArgument(params.filter); + if (filterValidation.error) { + nodeRes.writeHead(400, { "Content-Type": "application/json" }); + nodeRes.end(JSON.stringify({ error: filterValidation.error })); + return; + } + restFilter = filterValidation.filter; + } + + if (params.sort !== undefined && params.sort !== "count" && params.sort !== "value") { + nodeRes.writeHead(400, { "Content-Type": "application/json" }); + nodeRes.end(JSON.stringify({ error: "Invalid field: sort (must be 'count' or 'value')" })); + return; + } + + // Use default collections if none specified + const effectiveCollections = Array.isArray(params.collections) ? params.collections.map(String) : defaultCollectionNames; + + const result = await store.listMetadata({ + collection: effectiveCollections.length > 0 ? effectiveCollections : undefined, + key: typeof params.key === "string" ? params.key : undefined, + value: typeof params.value === "string" ? params.value : undefined, + filter: restFilter, + limit: typeof params.limit === "number" ? params.limit : undefined, + sort: params.sort, + minCount: typeof params.minCount === "number" ? params.minCount : undefined, + }); + + nodeRes.writeHead(200, { "Content-Type": "application/json" }); + nodeRes.end(JSON.stringify(result)); + log(`${ts()} POST /metadata ${result.keys.length} keys (${Date.now() - reqStart}ms)`); + return; + } + if (pathname === "/mcp") { const rawBody = nodeReq.method !== "GET" && nodeReq.method !== "HEAD" ? await collectBody(nodeReq) diff --git a/src/metadata-format.ts b/src/metadata-format.ts new file mode 100644 index 000000000..a7ecf5a3c --- /dev/null +++ b/src/metadata-format.ts @@ -0,0 +1,227 @@ +/** + * QMD Metadata Format - Plain-text rendering of metadata discovery results. + * + * Shared by the CLI and the MCP `metadata` tool so both print one shape: a + * header per key, a body per type, and a footer naming the remainder and the + * option that removes the cap whenever a value list is truncated. + */ + +import type { + ListMetadataResult, + MetadataKeySummary, + MetadataKeyTypeSummary, + MetadataValueCount, +} from "./metadata-store.js"; + +export interface MetadataFormatColors { + reset: string; + dim: string; + bold: string; + cyan: string; +} + +export interface FormatMetadataOptions { + /** Show which collections contribute each type on type-split keys. */ + showCollections?: boolean; + /** How the caller raises the value window, e.g. `-n or --all`. */ + limitHint: string; + /** ANSI sequences; omit for plain text. */ + colors?: MetadataFormatColors; +} + +const NO_COLORS: MetadataFormatColors = { reset: "", dim: "", bold: "", cyan: "" }; + +/** Render every key summary as a block, separated by blank lines. */ +export function formatMetadataKeySummaries(result: ListMetadataResult, options: FormatMetadataOptions): string { + return result.keys.map(summary => formatMetadataKeySummary(summary, result, options)).join("\n\n"); +} + +export function formatMetadataKeySummary(summary: MetadataKeySummary, result: ListMetadataResult, options: FormatMetadataOptions): string { + const colors = options.colors ?? NO_COLORS; + const typeLabel = summary.types.map(typeLabelOf).join(" | "); + const coverage = `${formatCount(summary.documents)} of ${formatCount(result.documents)} documents${result.filteredDocuments === undefined ? "" : " match filter"}`; + const header = [`${colors.cyan}${colors.bold}${summary.key}${colors.reset}`, `${colors.dim}${typeLabel}${colors.reset}`, coverage]; + const lines: string[] = []; + + if (summary.types.length === 1) { + const typeSummary = summary.types[0]!; + if (typeSummary.type !== "boolean") header.push(`${formatCount(typeSummary.distinctValues)} distinct`); + lines.push(header.join(" "), ...formatTypeBody(typeSummary)); + } else { + lines.push(header.join(" "), ...formatTypeSplit(summary.types, options)); + } + + const remaining = summary.types.reduce((sum, typeSummary) => sum + typeSummary.remaining, 0); + if (remaining > 0) { + lines.push(`${colors.dim}${formatCount(remaining)} more values, use ${options.limitHint}${colors.reset}`); + } + + return lines.join("\n"); +} + +export interface FormatMetadataOverviewOptions { + /** Active, extracted documents declaring at least one key. */ + documentsWithMetadata: number; + /** Active documents awaiting extraction, mentioned so the coverage reads honestly. */ + pendingMetadata: number; + /** Keys detailed before the "more keys" pointer. */ + keyLimit: number; + /** Command that shows the rest, e.g. `qmd collection metadata notes`. */ + drillDownHint: string; + colors?: MetadataFormatColors; +} + +/** + * The `Metadata:` section of `collection show`: a coverage line, then the + * top keys by coverage as aligned rows with a short value preview, then a + * pointer at the drill-down when keys were left out. Indented to sit under + * the other `show` fields. + */ +export function formatMetadataOverview(result: ListMetadataResult, options: FormatMetadataOverviewOptions): string { + const colors = options.colors ?? NO_COLORS; + const pendingNote = options.pendingMetadata > 0 ? ` (${formatCount(options.pendingMetadata)} pending extraction)` : ""; + + if (result.keys.length === 0) return ` Metadata: none${pendingNote}`; + + const keyLabel = result.keys.length === 1 ? "key" : "keys"; + const lines = [` Metadata: ${formatCount(result.keys.length)} ${keyLabel}, ${formatCount(options.documentsWithMetadata)} of ${formatCount(result.documents)} documents${pendingNote}`]; + + const shownKeys = result.keys.slice(0, options.keyLimit); + const keyWidth = Math.max(...shownKeys.map(summary => summary.key.length)); + const typeWidth = Math.max(...shownKeys.map(summary => summary.types.map(typeLabelOf).join(" | ").length)); + const documentsWidth = Math.max(...shownKeys.map(summary => formatCount(summary.documents).length)); + const distinctWidth = Math.max(...shownKeys.map(summary => formatCount(distinctValuesOf(summary)).length)); + + for (const summary of shownKeys) { + const typeLabel = summary.types.map(typeLabelOf).join(" | "); + const columns = [ + `${colors.cyan}${summary.key.padEnd(keyWidth)}${colors.reset}`, + `${colors.dim}${typeLabel.padEnd(typeWidth)}${colors.reset}`, + `${formatCount(summary.documents).padStart(documentsWidth)} ${documentsLabelOf(summary.documents)}`, + `${formatCount(distinctValuesOf(summary)).padStart(distinctWidth)} distinct`, + ]; + const preview = formatValuePreview(summary, options.drillDownHint); + if (preview) columns.push(preview); + lines.push(` ${columns.join(" ")}`); + } + + const hiddenKeys = result.keys.length - shownKeys.length; + if (hiddenKeys > 0) { + lines.push(` ${colors.dim}${formatCount(hiddenKeys)} more ${hiddenKeys === 1 ? "key" : "keys"}, see '${options.drillDownHint}'${colors.reset}`); + } + + return lines.join("\n"); +} + +/** + * One-line value preview for the overview row. Strings list the window with + * a trailing ellipsis when truncated, and nothing at all when every value is + * unique (a value list would be noise). Numbers give the range, booleans the + * two counts, and a type conflict points at the drill-down. + */ +function formatValuePreview(summary: MetadataKeySummary, drillDownHint: string): string { + if (summary.types.length > 1) return `types disagree, see '${drillDownHint} --key ${summary.key}'`; + + const typeSummary = summary.types[0]!; + if (typeSummary.type === "boolean") return formatBooleanCounts(typeSummary.values).replace(" ", ", "); + if (typeSummary.type === "number") { + const range = typeSummary.range!; + return `${formatValue(range.min)} to ${formatValue(range.max)}, median ${formatValue(range.median)}`; + } + if (typeSummary.distinctValues === typeSummary.documents) return ""; + + const preview = typeSummary.values.map(count => `${formatValue(count.value)} (${formatCount(count.documents)})`); + if (typeSummary.remaining > 0) preview.push("..."); + return preview.join(", "); +} + +function distinctValuesOf(summary: MetadataKeySummary): number { + return summary.types.reduce((sum, typeSummary) => sum + typeSummary.distinctValues, 0); +} + +/** Body for a key with one type: vertical values for strings, a range for numbers, one line for booleans. */ +function formatTypeBody(typeSummary: MetadataKeyTypeSummary): string[] { + if (typeSummary.type === "boolean") return [` ${formatBooleanCounts(typeSummary.values)}`]; + + if (typeSummary.type === "number") { + const lines = [` ${formatRange(typeSummary)}`]; + if (typeSummary.values.length > 0) lines.push(` ${formatInlineValues(typeSummary).join(" ")}`); + return lines; + } + + const valueWidth = Math.max(...typeSummary.values.map(count => formatValue(count.value).length)); + const countWidth = Math.max(...typeSummary.values.map(count => formatCount(count.documents).length)); + return typeSummary.values.map(count => ` ${formatValue(count.value).padEnd(valueWidth)} ${formatCount(count.documents).padStart(countWidth)}`); +} + +/** + * Body for a key whose documents disagree on type: one line per type with + * its own document count and a compact value summary, plus the contributing + * collections when the view spans more than one. + */ +function formatTypeSplit(typeSummaries: MetadataKeyTypeSummary[], options: FormatMetadataOptions): string[] { + const typeWidth = Math.max(...typeSummaries.map(typeSummary => typeSummary.type.length)); + const documentsWidth = Math.max(...typeSummaries.map(typeSummary => formatCount(typeSummary.documents).length)); + + const rows = typeSummaries.map(typeSummary => { + let valuesSummary: string; + if (typeSummary.type === "boolean") valuesSummary = formatBooleanCounts(typeSummary.values); + else if (typeSummary.type === "number") valuesSummary = formatRange(typeSummary); + else { + const inlineValues = typeSummary.values.map(count => `${formatValue(count.value)} (${formatCount(count.documents)})`); + if (typeSummary.remaining > 0) inlineValues.push(`${formatCount(typeSummary.remaining)} more`); + valuesSummary = inlineValues.join(", "); + } + const row = ` ${typeSummary.type.padEnd(typeWidth)} ${formatCount(typeSummary.documents).padStart(documentsWidth)} ${documentsLabelOf(typeSummary.documents)} ${valuesSummary}`; + return { row, collections: typeSummary.collections.join(", ") }; + }); + + if (!options.showCollections) return rows.map(({ row }) => row); + + const rowWidth = Math.max(...rows.map(({ row }) => row.length)); + return rows.map(({ row, collections }) => `${row.padEnd(rowWidth)} ${collections}`); +} + +function typeLabelOf(typeSummary: MetadataKeyTypeSummary): string { + return typeSummary.multiValued ? `${typeSummary.type}[]` : typeSummary.type; +} + +function formatRange(typeSummary: MetadataKeyTypeSummary): string { + const range = typeSummary.range!; + return `min ${formatValue(range.min)} median ${formatValue(range.median)} max ${formatValue(range.max)}`; +} + +/** + * Numbers enumerate inline as `value (documents)`. When the whole + * distribution fits it reads in value order; a truncated window keeps the + * requested order so the most common values stay visible. + */ +function formatInlineValues(typeSummary: MetadataKeyTypeSummary): string[] { + const values = typeSummary.remaining === 0 + ? [...typeSummary.values].sort((a, b) => Number(a.value) - Number(b.value)) + : typeSummary.values; + return values.map(count => `${formatValue(count.value)} (${formatCount(count.documents)})`); +} + +function formatBooleanCounts(values: MetadataValueCount[]): string { + const trueCount = values.find(count => count.value === true); + const falseCount = values.find(count => count.value === false); + const parts: string[] = []; + if (trueCount) parts.push(`true ${formatCount(trueCount.documents)}`); + if (falseCount) parts.push(`false ${formatCount(falseCount.documents)}`); + return parts.join(" "); +} + +/** Padded so `doc` and `docs` rows stay column-aligned. */ +function documentsLabelOf(documents: number): string { + return documents === 1 ? "doc " : "docs"; +} + +function formatValue(value: string | number | boolean): string { + if (value === "") return '""'; + return String(value); +} + +function formatCount(count: number): string { + return count.toLocaleString("en-US"); +} diff --git a/src/metadata-store.ts b/src/metadata-store.ts index 704b2268b..b318ae98d 100644 --- a/src/metadata-store.ts +++ b/src/metadata-store.ts @@ -14,13 +14,17 @@ * for filtering. */ -import type { Database } from "./db.js"; +import picomatch from "picomatch"; + +import type { Database, SQLiteValue } from "./db.js"; import { extractDocumentMetadata, METADATA_EXTRACTION_VERSION, type DocumentMetadata, type MetadataExtractionResult, + type MetadataScalar, } from "./metadata.js"; +import { compileMetadataFilter, type MetadataFilter } from "./metadata-filter.js"; // ============================================================================= // Schema @@ -163,9 +167,11 @@ function isDocumentMetadataCurrent(db: Database, documentId: number): boolean { /** * Count active documents without a current, error-free metadata extraction. * These documents are excluded from filtered search until `qmd update` runs. + * Scoped to `collectionNames` when given, otherwise the whole index. */ -export function countDocumentsPendingMetadata(db: Database): number { - const row = db.prepare(` +export function countDocumentsPendingMetadata(db: Database, collectionNames?: string[]): number { + const params: SQLiteValue[] = [METADATA_EXTRACTION_VERSION]; + let sql = ` SELECT COUNT(*) as c FROM documents d WHERE d.active = 1 AND NOT EXISTS ( @@ -173,8 +179,12 @@ export function countDocumentsPendingMetadata(db: Database): number { WHERE dm.document_id = d.id AND dm.extraction_version = ? AND dm.extraction_error IS NULL - ) - `).get(METADATA_EXTRACTION_VERSION) as { c: number }; + )`; + if (collectionNames) { + sql += ` AND d.collection IN (SELECT value FROM json_each(?))`; + params.push(JSON.stringify(collectionNames)); + } + const row = db.prepare(sql).get(...params) as { c: number }; return row.c; } @@ -210,3 +220,497 @@ export function parseMetadataJson(metadataJson: string | null | undefined): Docu return {}; } } + +// ============================================================================= +// Discovery +// ============================================================================= + +export interface ListMetadataOptions { + /** Restrict to these collections. Undefined means every collection in the index. */ + collection?: string | string[]; + /** picomatch pattern over key names. Undefined matches every key. */ + key?: string; + /** + * picomatch pattern over values in their text form (`String(number)`, + * `"true"`/`"false"`). Undefined matches every value. + */ + value?: string; + /** Count only documents matching this filter. Same AST as search. */ + filter?: MetadataFilter; + /** Per-key value window (default 10). `Infinity` removes the window. */ + limit?: number; + /** Order of values within a key (default "count"). */ + sort?: "count" | "value"; + /** Drop values held by fewer documents than this (default 1). */ + minCount?: number; +} + +export interface ListMetadataResult { + /** Active documents in scope — the denominator for every coverage count. */ + documents: number; + /** Documents in scope that pass `filter`. Present only when a filter was given. */ + filteredDocuments?: number; + /** One entry per matching key, by documents descending then key ascending. */ + keys: MetadataKeySummary[]; +} + +export interface MetadataKeySummary { + key: string; + /** Distinct documents declaring the key with any type. */ + documents: number; + /** One entry per value_type present. Length > 1 is a type conflict. */ + types: MetadataKeyTypeSummary[]; +} + +export interface MetadataKeyTypeSummary { + type: MetadataValueType; + /** + * True when any document holds more than one value for this key. A + * one-element array is indistinguishable from a scalar in the index. + */ + multiValued: boolean; + /** Distinct documents holding a matching value of this type. */ + documents: number; + /** Distinct matching values that meet `minCount`. */ + distinctValues: number; + /** Windowed by `limit`, filtered by `value` and `minCount`, ordered by `sort`. */ + values: MetadataValueCount[]; + /** Distinct values not shown: `distinctValues - values.length`. */ + remaining: number; + /** Numbers only. Computed over every matching value row, so array elements each count. */ + range?: { min: number; median: number; max: number }; + /** Collections contributing a matching value of this type, ascending. */ + collections: string[]; +} + +export interface MetadataValueCount { + value: MetadataScalar; + documents: number; +} + +/** The light form of a key summary for status views: name, coverage, and types, no values. */ +export interface MetadataKeyOverview { + key: string; + /** Distinct documents declaring the key with any type. */ + documents: number; + /** By documents descending then name. Length > 1 is a type conflict. */ + types: MetadataValueType[]; +} + +export type MetadataValueType = "string" | "number" | "boolean"; + +const DEFAULT_METADATA_VALUE_LIMIT = 10; + +/** Same glob dialect as `multi-get`; `dot` because values are not file paths. */ +const PATTERN_OPTIONS = { dot: true }; + +/** + * The value rows discovery aggregates over. `withSql` defines the `eligible` + * CTE (and `selected_values` when a value pattern is set); `fromSql` joins + * `document_metadata_values mv` to them. Each query supplies its own SELECT + * list, WHERE, and GROUP BY around these two parts. + */ +interface Region { + withSql: string; + withParams: SQLiteValue[]; + fromSql: string; + fromParams: SQLiteValue[]; +} + +type SelectedValue = { key: string; type: MetadataValueType; value: string | number }; + +type ValueRow = { + key: string; + value_type: MetadataValueType; + text_value: string | null; + number_value: number | null; + boolean_value: number | null; +}; + +/** + * Summarize metadata keys, types, and value counts for the documents in + * scope. Discovery sees exactly what filtering sees: the same extraction gate, + * active-document rule, and collection scope, so every value reported here is + * a value an `eq` filter can match. + * + * `key` and `value` are picomatch patterns selecting a region of the key/value + * space; `filter` selects which documents are counted. Counts are documents, + * not values — a document with `topics: [a, b]` contributes one to each. + */ +export function listMetadata(db: Database, options: ListMetadataOptions = {}): ListMetadataResult { + const collectionNames = options.collection === undefined ? undefined : [options.collection].flat(); + const eligible = buildEligibleCte(collectionNames, options.filter); + + const result: ListMetadataResult = { documents: countActiveDocuments(db, collectionNames), keys: [] }; + if (options.filter) result.filteredDocuments = countEligibleDocuments(db, eligible); + + const keyNames = options.key ? selectKeyNames(db, eligible, options.key) : undefined; + if (keyNames?.length === 0) return result; + + const keyRegion = buildRegion(eligible, keyNames); + const selectedValues = options.value ? selectValues(db, keyRegion, options.value) : undefined; + if (selectedValues?.length === 0) return result; + + const region = selectedValues ? buildRegion(eligible, keyNames, selectedValues) : keyRegion; + const medianByKey = new Map(queryNumberMedians(db, region).map(row => [row.key, row.median])); + + const typeSummaryByKeyType = new Map(); + const typeSummariesByKey = new Map(); + + for (const row of queryTypeStats(db, region)) { + const typeSummary: MetadataKeyTypeSummary = { + type: row.value_type, + multiValued: false, + documents: row.documents, + distinctValues: 0, + values: [], + remaining: 0, + collections: [], + }; + if (row.value_type === "number") { + typeSummary.range = { min: row.min_value!, median: medianByKey.get(row.key)!, max: row.max_value! }; + } + typeSummaryByKeyType.set(`${row.key}\0${row.value_type}`, typeSummary); + + const typeSummaries = typeSummariesByKey.get(row.key) ?? []; + typeSummaries.push(typeSummary); + typeSummariesByKey.set(row.key, typeSummaries); + } + + for (const row of queryTypeArrayness(db, keyRegion)) { + const typeSummary = typeSummaryByKeyType.get(`${row.key}\0${row.value_type}`); + if (typeSummary) typeSummary.multiValued = row.multi_valued === 1; + } + + for (const row of queryTypeCollections(db, region)) { + typeSummaryByKeyType.get(`${row.key}\0${row.value_type}`)?.collections.push(row.collection); + } + + const window: ValueWindow = { + limit: options.limit ?? DEFAULT_METADATA_VALUE_LIMIT, + minCount: options.minCount ?? 1, + sort: options.sort ?? "count", + }; + for (const row of queryValueCounts(db, region, window)) { + const typeSummary = typeSummaryByKeyType.get(`${row.key}\0${row.value_type}`); + if (!typeSummary) continue; + typeSummary.distinctValues = row.distinct_values; + typeSummary.values.push({ value: scalarOf(row), documents: row.documents }); + } + + for (const typeSummary of typeSummaryByKeyType.values()) { + typeSummary.remaining = typeSummary.distinctValues - typeSummary.values.length; + } + + // A document holds a key under exactly one type (arrays are homogeneous), + // so per-type document counts partition the key's documents. + for (const [key, typeSummaries] of typeSummariesByKey) { + typeSummaries.sort((a, b) => b.documents - a.documents || a.type.localeCompare(b.type)); + result.keys.push({ + key, + documents: typeSummaries.reduce((sum, typeSummary) => sum + typeSummary.documents, 0), + types: typeSummaries, + }); + } + result.keys.sort((a, b) => b.documents - a.documents || a.key.localeCompare(b.key)); + + return result; +} + +/** + * Key names, coverage, and types for the documents in scope, in coverage + * order. One GROUP BY, no values: what `collection list`, `status`, and the + * MCP status tool print so a first look reveals that metadata exists. + */ +export function listMetadataKeys(db: Database, collectionNames?: string[]): MetadataKeyOverview[] { + const region = buildRegion(buildEligibleCte(collectionNames, undefined)); + + const documentsByKeyType = new Map>(); + for (const row of queryTypeStats(db, region)) { + const documentsByType = documentsByKeyType.get(row.key) ?? new Map(); + documentsByType.set(row.value_type, row.documents); + documentsByKeyType.set(row.key, documentsByType); + } + + const overviews: MetadataKeyOverview[] = []; + for (const [key, documentsByType] of documentsByKeyType) { + const typeEntries = [...documentsByType.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); + overviews.push({ + key, + documents: typeEntries.reduce((sum, [, documents]) => sum + documents, 0), + types: typeEntries.map(([type]) => type), + }); + } + return overviews.sort((a, b) => b.documents - a.documents || a.key.localeCompare(b.key)); +} + +/** Active, extracted documents in scope that declare at least one metadata key. */ +export function countDocumentsWithMetadata(db: Database, collectionNames?: string[]): number { + const region = buildRegion(buildEligibleCte(collectionNames, undefined)); + const row = db.prepare(` + ${region.withSql} + SELECT COUNT(DISTINCT mv.document_id) AS c + ${region.fromSql} + `).get(...region.withParams, ...region.fromParams) as { c: number }; + return row.c; +} + +/** + * Documents that filtered search can see: active, with a current, error-free + * extraction, in scope, and passing the filter. Lists bind as one JSON + * parameter so the SQLite variable limit never applies. + */ +function buildEligibleCte(collectionNames: string[] | undefined, filter: MetadataFilter | undefined): Region { + const withParams: SQLiteValue[] = []; + let withSql = ` + WITH eligible AS ( + SELECT d.id AS document_id, d.collection + FROM documents d + JOIN document_metadata dm ON dm.document_id = d.id + WHERE d.active = 1 + AND dm.extraction_version = ${METADATA_EXTRACTION_VERSION} + AND dm.extraction_error IS NULL`; + + if (collectionNames) { + withSql += ` + AND d.collection IN (SELECT value FROM json_each(?))`; + withParams.push(JSON.stringify(collectionNames)); + } + + if (filter) { + const compiledFilter = compileMetadataFilter(filter, "d"); + withSql += ` + AND ${compiledFilter.sql}`; + withParams.push(...compiledFilter.params); + } + + withSql += ` + )`; + + return { withSql, withParams, fromSql: "", fromParams: [] }; +} + +/** + * Join eligible documents' values, narrowed to the selected keys and, when a + * value pattern is set, to the values picomatch confirmed. The restrictions + * live in the JOIN so queries can add their own WHERE. + */ +function buildRegion(eligible: Region, keyNames?: string[], selectedValues?: SelectedValue[]): Region { + const region: Region = { + withSql: eligible.withSql, + withParams: [...eligible.withParams], + fromSql: ` + FROM document_metadata_values mv + JOIN eligible e ON e.document_id = mv.document_id`, + fromParams: [], + }; + + if (keyNames) { + region.fromSql += ` AND mv.key IN (SELECT value FROM json_each(?))`; + region.fromParams.push(JSON.stringify(keyNames)); + } + + if (selectedValues) { + region.withSql += `, + selected_values AS MATERIALIZED ( + SELECT json_extract(value, '$.key') AS key, json_extract(value, '$.type') AS value_type, json_extract(value, '$.value') AS scalar + FROM json_each(?) + )`; + region.withParams.push(JSON.stringify(selectedValues)); + region.fromSql += ` + JOIN selected_values sv + ON sv.key = mv.key AND sv.value_type = mv.value_type + AND sv.scalar = COALESCE(mv.text_value, mv.number_value, mv.boolean_value)`; + } + + return region; +} + +function countActiveDocuments(db: Database, collectionNames: string[] | undefined): number { + let sql = `SELECT COUNT(*) AS c FROM documents d WHERE d.active = 1`; + const params: SQLiteValue[] = []; + if (collectionNames) { + sql += ` AND d.collection IN (SELECT value FROM json_each(?))`; + params.push(JSON.stringify(collectionNames)); + } + const row = db.prepare(sql).get(...params) as { c: number }; + return row.c; +} + +function countEligibleDocuments(db: Database, eligible: Region): number { + const row = db.prepare(`${eligible.withSql} SELECT COUNT(*) AS c FROM eligible`).get(...eligible.withParams) as { c: number }; + return row.c; +} + +/** The distinct key set is small, so the pattern is matched in JS. */ +function selectKeyNames(db: Database, eligible: Region, keyPattern: string): string[] { + const isMatch = picomatch(keyPattern, PATTERN_OPTIONS); + const rows = db.prepare(` + ${eligible.withSql} + SELECT DISTINCT mv.key + FROM document_metadata_values mv + JOIN eligible e ON e.document_id = mv.document_id + `).all(...eligible.withParams) as { key: string }[]; + // Not `.filter(isMatch)`: picomatch reads a second argument as `returnObject`. + return rows.map(row => row.key).filter(key => isMatch(key)); +} + +/** + * Resolve a value pattern to the distinct (key, type, value) triples it + * matches. Values can be high-cardinality, so string candidates are pruned in + * SQL with a GLOB superset of the pattern before picomatch makes the final + * call. Numbers and booleans skip the pruning: SQLite's text rendering of a + * REAL does not agree with `String(number)`, so they are matched in JS only. + */ +function selectValues(db: Database, keyRegion: Region, valuePattern: string): SelectedValue[] { + const isMatch = picomatch(valuePattern, PATTERN_OPTIONS); + const globPrefilter = buildGlobPrefilter(valuePattern); + const params = [...keyRegion.withParams, ...keyRegion.fromParams]; + let sql = ` + ${keyRegion.withSql} + SELECT mv.key, mv.value_type, mv.text_value, mv.number_value, mv.boolean_value + ${keyRegion.fromSql}`; + + if (globPrefilter !== null) { + sql += ` + WHERE mv.value_type <> 'string' OR mv.text_value GLOB ?`; + params.push(globPrefilter); + } + + const rows = db.prepare(`${sql} + GROUP BY mv.key, mv.value_type, mv.text_value, mv.number_value, mv.boolean_value + `).all(...params) as ValueRow[]; + + const selectedValues: SelectedValue[] = []; + for (const row of rows) { + const scalar = scalarOf(row); + if (!isMatch(String(scalar))) continue; + selectedValues.push({ key: row.key, type: row.value_type, value: bindScalar(scalar) }); + } + return selectedValues; +} + +/** + * Translate a picomatch pattern to a SQLite GLOB pattern matching a superset + * of what picomatch matches, or null when the pattern uses features (escapes, + * groups, brackets, braces, extglobs, negation, globstars, `./` segments) + * where the superset cannot be guaranteed. Only literals, `*`, and `?` + * survive, and GLOB's wildcards match `/` where picomatch's do not — the + * superset direction. A pattern without wildcards degenerates to equality. + * Exported for the equivalence tests. + */ +export function buildGlobPrefilter(pattern: string): string | null { + if (/[\\()[\]{}|!+@]/.test(pattern) || pattern.includes("**") || pattern.includes("./")) return null; + return pattern; +} + +type TypeStatsRow = { key: string; value_type: MetadataValueType; documents: number; min_value: number | null; max_value: number | null }; + +function queryTypeStats(db: Database, region: Region): TypeStatsRow[] { + return db.prepare(` + ${region.withSql} + SELECT mv.key, mv.value_type, + COUNT(DISTINCT mv.document_id) AS documents, + MIN(mv.number_value) AS min_value, + MAX(mv.number_value) AS max_value + ${region.fromSql} + GROUP BY mv.key, mv.value_type + `).all(...region.withParams, ...region.fromParams) as TypeStatsRow[]; +} + +type TypeArraynessRow = { key: string; value_type: MetadataValueType; multi_valued: number }; + +/** Array-ness is a property of the key, so it is read before the value pattern narrows the rows. */ +function queryTypeArrayness(db: Database, keyRegion: Region): TypeArraynessRow[] { + return db.prepare(` + ${keyRegion.withSql} + SELECT mv.key, mv.value_type, MAX(mv.ordinal) > 0 AS multi_valued + ${keyRegion.fromSql} + GROUP BY mv.key, mv.value_type + `).all(...keyRegion.withParams, ...keyRegion.fromParams) as TypeArraynessRow[]; +} + +type NumberMedianRow = { key: string; median: number }; + +/** Median over value rows: the middle row for odd counts, the mean of the two middle rows for even. */ +function queryNumberMedians(db: Database, region: Region): NumberMedianRow[] { + return db.prepare(` + ${region.withSql} + SELECT key, AVG(number_value) AS median + FROM ( + SELECT mv.key, mv.number_value, + ROW_NUMBER() OVER (PARTITION BY mv.key ORDER BY mv.number_value) AS position, + COUNT(*) OVER (PARTITION BY mv.key) AS total + ${region.fromSql} + WHERE mv.value_type = 'number' + ) + WHERE position IN ((total + 1) / 2, (total + 2) / 2) + GROUP BY key + `).all(...region.withParams, ...region.fromParams) as NumberMedianRow[]; +} + +type TypeCollectionRow = { key: string; value_type: MetadataValueType; collection: string }; + +function queryTypeCollections(db: Database, region: Region): TypeCollectionRow[] { + return db.prepare(` + ${region.withSql} + SELECT mv.key, mv.value_type, e.collection + ${region.fromSql} + GROUP BY mv.key, mv.value_type, e.collection + ORDER BY e.collection + `).all(...region.withParams, ...region.fromParams) as TypeCollectionRow[]; +} + +type ValueCountRow = ValueRow & { documents: number; distinct_values: number }; + +interface ValueWindow { + limit: number; + minCount: number; + sort: "count" | "value"; +} + +/** + * Distinct values per key and type, ranked within each partition. Only one + * typed column is non-null per partition, so ordering by all three is stable. + * `distinct_values` counts the partition after `minCount`, so the caller's + * remainder is exact. + */ +function queryValueCounts(db: Database, region: Region, window: ValueWindow): ValueCountRow[] { + const valueOrder = "mv.text_value, mv.number_value, mv.boolean_value"; + const rankOrder = window.sort === "count" ? `COUNT(DISTINCT mv.document_id) DESC, ${valueOrder}` : valueOrder; + const params = [...region.withParams, ...region.fromParams, window.minCount]; + let sql = ` + ${region.withSql} + SELECT key, value_type, text_value, number_value, boolean_value, documents, distinct_values + FROM ( + SELECT mv.key, mv.value_type, mv.text_value, mv.number_value, mv.boolean_value, + COUNT(DISTINCT mv.document_id) AS documents, + ROW_NUMBER() OVER (PARTITION BY mv.key, mv.value_type ORDER BY ${rankOrder}) AS rank, + COUNT(*) OVER (PARTITION BY mv.key, mv.value_type) AS distinct_values + ${region.fromSql} + GROUP BY mv.key, mv.value_type, mv.text_value, mv.number_value, mv.boolean_value + HAVING COUNT(DISTINCT mv.document_id) >= ? + )`; + + if (Number.isFinite(window.limit)) { + sql += ` + WHERE rank <= ?`; + params.push(Math.max(1, Math.floor(window.limit))); + } + + return db.prepare(`${sql} + ORDER BY key, value_type, rank + `).all(...params) as ValueCountRow[]; +} + +function scalarOf(row: ValueRow): MetadataScalar { + if (row.value_type === "string") return row.text_value!; + if (row.value_type === "number") return row.number_value!; + return row.boolean_value === 1; +} + +/** Booleans are stored as 0/1, and JSON round-trips them the same way. */ +function bindScalar(scalar: MetadataScalar): string | number { + if (typeof scalar === "boolean") return scalar ? 1 : 0; + return scalar; +} diff --git a/src/store.ts b/src/store.ts index 3a45bbcb9..1f1bc5aef 100644 --- a/src/store.ts +++ b/src/store.ts @@ -44,7 +44,9 @@ import { syncDocumentMetadata, countDocumentsPendingMetadata, getMetadataByFilepath, + listMetadataKeys, parseMetadataJson, + type MetadataKeyOverview, } from "./metadata-store.js"; // ============================================================================= @@ -2528,6 +2530,8 @@ export type CollectionInfo = { pattern: string | null; documents: number; lastUpdated: string; + /** Metadata keys declared in this collection with coverage and types, by coverage. */ + metadataKeys: MetadataKeyOverview[]; }; export type IndexStatus = { @@ -5260,6 +5264,7 @@ export function getStatus(db: Database, model: string = DEFAULT_EMBED_MODEL): In pattern: config?.pattern ?? null, documents: row.active_count, lastUpdated: row.last_doc_update || new Date().toISOString(), + metadataKeys: listMetadataKeys(db, [row.name]), }; }); diff --git a/test/mcp.test.ts b/test/mcp.test.ts index dc0e4c523..ecb5f9664 100644 --- a/test/mcp.test.ts +++ b/test/mcp.test.ts @@ -1131,7 +1131,7 @@ describe.skipIf(!!process.env.CI)("MCP HTTP Transport", () => { expect(headers.get("mcp-session-id")).toBeNull(); const toolNames = json.result.tools.map((t: any) => t.name); - expect(toolNames).toEqual(["query", "get", "multi_get", "status"]); + expect(toolNames).toEqual(["query", "get", "multi_get", "status", "metadata"]); }); test("POST /mcp tools/call query returns results", async () => { @@ -1309,7 +1309,7 @@ describe("MCP HTTP Transport — 2026-07-28 protocol", () => { expect(json.result.ttlMs).toBe(60_000); expect(json.result.cacheScope).toBe("private"); const toolNames = json.result.tools.map((t: { name: string }) => t.name); - expect(toolNames).toEqual(["query", "get", "multi_get", "status"]); + expect(toolNames).toEqual(["query", "get", "multi_get", "status", "metadata"]); const serverInfo = json.result._meta?.["io.modelcontextprotocol/serverInfo"]; expect(serverInfo?.name).toBe("qmd"); }); diff --git a/test/metadata-cli.test.ts b/test/metadata-cli.test.ts index 213f0eede..a8f574c09 100644 --- a/test/metadata-cli.test.ts +++ b/test/metadata-cli.test.ts @@ -159,3 +159,205 @@ describe("qmd search --filter", () => { expect(stderr).toMatch(/unknown operator 'equal'/); }, 30000); }); + +describe("qmd collection metadata", () => { + beforeAll(async () => { + await writeFile(join(fixturesDir, "discovery.md"), [ + "---", + "qmd:", + " metadata:", + " status: published", + " topics: [typescript, sqlite, search, architecture, embeddings, mcp, cli, testing, agents, indexing, chunking, reranking]", + " priority: 5", + " owner: docs-team", + "---", + "", + "# Discovery doc", + "", + "cli filter keyword body", + "", + ].join("\n")); + await writeFile(join(fixturesDir, "priority.md"), [ + "---", + "qmd:", + " metadata:", + " status: draft", + " priority: 1", + " reviewed: false", + " reviewers: [docs-team, security-team]", + "---", + "", + "# Priority doc", + "", + "cli filter keyword body", + "", + ].join("\n")); + // Nothing enforces a type across documents, so one file in the same + // collection may spell priority as a label where the others use numbers. + await writeFile(join(fixturesDir, "conflict.md"), [ + "---", + "qmd:", + " metadata:", + " status: archived", + " priority: high", + "---", + "", + "# Conflict doc", + "", + "cli filter keyword body", + "", + ].join("\n")); + const updateResult = await runQmd(["update"]); + expect(updateResult.exitCode).toBe(0); + }, 60000); + + test("lists every key with coverage, type, and a value window", async () => { + const { stdout, exitCode } = await runQmd(["collection", "metadata", "notes"]); + expect(exitCode).toBe(0); + + // Keys arrive in coverage order; the denominator is every active document. + expect(stdout).toMatch(/^status {2}string {2}5 of 6 documents {2}3 distinct\n {2}draft {6}2\n {2}published {2}2\n {2}archived {3}1\n/); + expect(stdout).toContain("topics string[] 2 of 6 documents 13 distinct"); + expect(stdout).toContain("reviewed boolean 1 of 6 documents\n false 1"); + expect(stdout).toContain("reviewers string[] 1 of 6 documents 2 distinct\n docs-team 1\n security-team 1"); + }, 30000); + + test("splits a key whose documents disagree on type, even within one collection", async () => { + const { stdout, exitCode } = await runQmd(["collection", "metadata", "notes", "--key", "priority"]); + expect(exitCode).toBe(0); + + // One row per type with its own document count. The collection column + // is omitted since the view covers a single collection. + expect(stdout).toBe([ + "priority number | string 3 of 6 documents", + " number 2 docs min 1 median 3 max 5", + " string 1 doc high (1)", + "", + ].join("\n")); + }, 30000); + + test("truncates value lists with an explicit remainder and escape hatch", async () => { + const { stdout, exitCode } = await runQmd(["collection", "metadata", "notes", "--key", "topics"]); + expect(exitCode).toBe(0); + + const valueLines = stdout.split("\n").filter(line => /^ {2}\S/.test(line)); + expect(valueLines).toHaveLength(10); + expect(valueLines[0]).toBe(" typescript 2"); + expect(stdout).toContain("3 more values, use -n or --all"); + expect(stdout).not.toContain("status"); + }, 30000); + + test("-n and --all raise or remove the window", async () => { + const limited = await runQmd(["collection", "metadata", "notes", "--key", "topics", "-n", "2"]); + expect(limited.stdout).toContain("11 more values, use -n or --all"); + + const all = await runQmd(["collection", "metadata", "notes", "--key", "topics", "--all"]); + expect(all.stdout).not.toContain("more values"); + expect(all.stdout.split("\n").filter(line => /^ {2}\S/.test(line))).toHaveLength(13); + }, 30000); + + test("--value is a reverse lookup across keys", async () => { + const { stdout, exitCode } = await runQmd(["collection", "metadata", "notes", "--value", "docs-team"]); + expect(exitCode).toBe(0); + + expect(stdout).toBe([ + "owner string 1 of 6 documents 1 distinct", + " docs-team 1", + "", + "reviewers string[] 1 of 6 documents 1 distinct", + " docs-team 1", + "", + ].join("\n")); + }, 30000); + + test("--filter counts only matching documents and says so in the header", async () => { + const { stdout, exitCode } = await runQmd([ + "collection", "metadata", "notes", "--key", "priority", + "--filter", '{"key":"status","operator":"eq","value":"published"}', + ]); + expect(exitCode).toBe(0); + // Only the published document remains, so the type split above + // collapses to a flat number view. + expect(stdout).toContain("priority number 1 of 6 documents match filter 1 distinct\n min 5 median 5 max 5\n 5 (1)"); + }, 30000); + + test("--sort value and --min-count reshape the window", async () => { + const sorted = await runQmd(["collection", "metadata", "notes", "--key", "topics", "--sort", "value", "-n", "2"]); + expect(sorted.stdout).toContain(" agents 1\n architecture 1\n"); + + const common = await runQmd(["collection", "metadata", "notes", "--key", "topics", "--min-count", "2"]); + expect(common.stdout).toContain("topics string[] 2 of 6 documents 1 distinct\n typescript 2\n"); + expect(common.stdout).not.toContain("more values"); + }, 30000); + + test("omitting the collection covers the default collections", async () => { + const { stdout, exitCode } = await runQmd(["collection", "metadata"]); + expect(exitCode).toBe(0); + expect(stdout).toContain("status string 5 of 6 documents 3 distinct"); + }, 30000); + + test("reports when nothing matches the patterns", async () => { + const { stdout, exitCode } = await runQmd(["collection", "metadata", "notes", "--key", "missing-*"]); + expect(exitCode).toBe(0); + expect(stdout).toContain("No metadata matches"); + }, 30000); + + test("exits on an unknown collection", async () => { + const { stderr, exitCode } = await runQmd(["collection", "metadata", "missing"]); + expect(exitCode).toBe(1); + expect(stderr).toContain("Collection not found: missing"); + }, 30000); + + test("exits on an invalid filter", async () => { + const { stderr, exitCode } = await runQmd([ + "collection", "metadata", "notes", "--filter", '{"key":"status","operator":"equal","value":"x"}', + ]); + expect(exitCode).toBe(1); + expect(stderr).toMatch(/Invalid metadata filter at \$/); + }, 30000); + + test("exits on invalid -n, --min-count, and --sort values", async () => { + const badLimit = await runQmd(["collection", "metadata", "notes", "-n", "0"]); + expect(badLimit.exitCode).toBe(1); + expect(badLimit.stderr).toContain("Invalid -n value: 0"); + + const badMinCount = await runQmd(["collection", "metadata", "notes", "--min-count", "x"]); + expect(badMinCount.exitCode).toBe(1); + expect(badMinCount.stderr).toContain("Invalid --min-count value: x"); + + const badSort = await runQmd(["collection", "metadata", "notes", "--sort", "size"]); + expect(badSort.exitCode).toBe(1); + expect(badSort.stderr).toContain("Invalid --sort value: size"); + }, 30000); +}); + +describe("metadata in collection list, show, and status", () => { + test("collection list names the top keys and counts the rest", async () => { + const { stdout, exitCode } = await runQmd(["collection", "list"]); + expect(exitCode).toBe(0); + expect(stdout).toContain(" Metadata: status, priority, topics, owner, reviewed, +1 more\n"); + }, 30000); + + test("collection show details the top keys and points at the drill-down", async () => { + const { stdout, exitCode } = await runQmd(["collection", "show", "notes"]); + expect(exitCode).toBe(0); + + const metadataSection = stdout.slice(stdout.indexOf(" Metadata:")); + expect(metadataSection).toBe([ + " Metadata: 6 keys, 5 of 6 documents", + " status string 5 docs 3 distinct draft (2), published (2), archived (1)", + " priority number | string 3 docs 3 distinct types disagree, see 'qmd collection metadata notes --key priority'", + " topics string[] 2 docs 13 distinct typescript (2), agents (1), architecture (1), ...", + " owner string 1 doc 1 distinct", + " reviewed boolean 1 doc 1 distinct false 1", + " 1 more key, see 'qmd collection metadata notes'", + "", + ].join("\n")); + }, 30000); + + test("status summarizes metadata and points at the drill-down", async () => { + const { stdout, exitCode } = await runQmd(["status"]); + expect(exitCode).toBe(0); + expect(stdout).toContain(" Metadata: 6 keys across 5 files (explore with 'qmd collection metadata')\n"); + }, 30000); +}); diff --git a/test/metadata-discovery.test.ts b/test/metadata-discovery.test.ts new file mode 100644 index 000000000..a7401a26d --- /dev/null +++ b/test/metadata-discovery.test.ts @@ -0,0 +1,480 @@ +/** + * metadata-discovery.test.ts - Store-level metadata discovery: key summaries, + * per-type value windows, scope and gate agreement with filtered search, and + * the picomatch/GLOB pruning contract. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import picomatch from "picomatch"; +import { + createStore, + insertContent, + insertDocument, + hashContent, + searchFTS, + type Store, +} from "../src/store.js"; +import { + buildGlobPrefilter, + countDocumentsPendingMetadata, + countDocumentsWithMetadata, + listMetadata, + listMetadataKeys, + replaceDocumentMetadata, + type ListMetadataOptions, + type MetadataKeySummary, +} from "../src/metadata-store.js"; +import { METADATA_EXTRACTION_VERSION, type DocumentMetadata } from "../src/metadata.js"; + +let testDir: string; +let store: Store; + +beforeAll(async () => { + testDir = await mkdtemp(join(tmpdir(), "qmd-metadata-discovery-")); +}); + +afterAll(async () => { + await rm(testDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + const dbPath = join(testDir, `test-${Date.now()}-${Math.random().toString(36).slice(2)}.sqlite`); + store = createStore(dbPath); +}); + +afterEach(() => { + store.close(); +}); + +let documentCounter = 0; + +/** Insert an active document with extracted metadata. Every body contains "doc" so FTS can reach it. */ +async function insertMetadataDoc(collection: string, metadata: DocumentMetadata): Promise { + documentCounter += 1; + const path = `doc-${documentCounter}.md`; + const content = `# doc ${documentCounter}\n\nbody of doc ${documentCounter}\n`; + const now = new Date().toISOString(); + const hash = await hashContent(content); + insertContent(store.db, hash, content, now); + const documentId = insertDocument(store.db, collection, path, path, hash, now, now); + replaceDocumentMetadata(store.db, documentId, { metadata, extractionVersion: METADATA_EXTRACTION_VERSION }); + return documentId; +} + +function summaryOf(keys: MetadataKeySummary[], key: string): MetadataKeySummary { + const summary = keys.find(candidate => candidate.key === key); + if (!summary) throw new Error(`key ${key} missing from ${keys.map(candidate => candidate.key).join(", ")}`); + return summary; +} + +function valuesOf(keys: MetadataKeySummary[], key: string): [string | number | boolean, number][] { + return summaryOf(keys, key).types[0]!.values.map(count => [count.value, count.documents]); +} + +describe("listMetadata counting", () => { + test("counts documents, not values, and reports coverage per key", async () => { + await insertMetadataDoc("notes", { topics: ["a", "b"], status: "draft" }); + await insertMetadataDoc("notes", { topics: ["a"], status: "published" }); + await insertMetadataDoc("notes", { status: "published" }); + + const result = listMetadata(store.db); + + expect(result.documents).toBe(3); + expect(result.filteredDocuments).toBeUndefined(); + expect(result.keys.map(summary => summary.key)).toEqual(["status", "topics"]); + + const topics = summaryOf(result.keys, "topics"); + expect(topics.documents).toBe(2); + expect(topics.types).toHaveLength(1); + expect(topics.types[0]!.multiValued).toBe(true); + expect(topics.types[0]!.distinctValues).toBe(2); + expect(valuesOf(result.keys, "topics")).toEqual([["a", 2], ["b", 1]]); + + const status = summaryOf(result.keys, "status"); + expect(status.documents).toBe(3); + expect(status.types[0]!.multiValued).toBe(false); + expect(valuesOf(result.keys, "status")).toEqual([["published", 2], ["draft", 1]]); + }); + + test("returns an empty key list when nothing has metadata", async () => { + await insertMetadataDoc("notes", {}); + + const result = listMetadata(store.db); + + expect(result).toEqual({ documents: 1, keys: [] }); + }); + + test("applies the extraction gate filtered search applies", async () => { + await insertMetadataDoc("notes", { status: "visible" }); + const pendingId = await insertMetadataDoc("notes", { status: "pending" }); + const erroredId = await insertMetadataDoc("notes", { status: "errored" }); + const staleId = await insertMetadataDoc("notes", { status: "stale" }); + const inactiveId = await insertMetadataDoc("notes", { status: "inactive" }); + + store.db.prepare(`DELETE FROM document_metadata WHERE document_id = ?`).run(pendingId); + store.db.prepare(`UPDATE document_metadata SET extraction_error = 'boom' WHERE document_id = ?`).run(erroredId); + store.db.prepare(`UPDATE document_metadata SET extraction_version = ? WHERE document_id = ?`).run(METADATA_EXTRACTION_VERSION - 1, staleId); + store.db.prepare(`UPDATE documents SET active = 0 WHERE id = ?`).run(inactiveId); + + const result = listMetadata(store.db); + + // The denominator counts active documents whether or not they are extracted. + expect(result.documents).toBe(4); + expect(valuesOf(result.keys, "status")).toEqual([["visible", 1]]); + }); +}); + +describe("listMetadata scope and filter", () => { + beforeEach(async () => { + await insertMetadataDoc("notes", { status: "published", priority: 3 }); + await insertMetadataDoc("notes", { status: "draft", priority: 1 }); + await insertMetadataDoc("work", { status: "published", priority: 5 }); + await insertMetadataDoc("work", { status: "archived" }); + }); + + test("undefined collection means every collection", () => { + const result = listMetadata(store.db); + + expect(result.documents).toBe(4); + expect(summaryOf(result.keys, "status").documents).toBe(4); + expect(summaryOf(result.keys, "status").types[0]!.collections).toEqual(["notes", "work"]); + }); + + test("a single collection scopes counts and the denominator", () => { + const result = listMetadata(store.db, { collection: "notes" }); + + expect(result.documents).toBe(2); + expect(valuesOf(result.keys, "status")).toEqual([["draft", 1], ["published", 1]]); + expect(summaryOf(result.keys, "status").types[0]!.collections).toEqual(["notes"]); + }); + + test("a collection list scopes to exactly those collections", async () => { + await insertMetadataDoc("other", { status: "elsewhere" }); + + const result = listMetadata(store.db, { collection: ["notes", "work"] }); + + expect(result.documents).toBe(4); + expect(valuesOf(result.keys, "status").map(([value]) => value)).not.toContain("elsewhere"); + }); + + test("an unknown collection yields an empty scope", () => { + const result = listMetadata(store.db, { collection: "missing" }); + + expect(result).toEqual({ documents: 0, keys: [] }); + }); + + test("filter narrows which documents are counted and reports how many pass", () => { + const result = listMetadata(store.db, { + filter: { key: "status", operator: "eq", value: "published" }, + }); + + expect(result.documents).toBe(4); + expect(result.filteredDocuments).toBe(2); + expect(summaryOf(result.keys, "priority").documents).toBe(2); + expect(valuesOf(result.keys, "priority")).toEqual([[3, 1], [5, 1]]); + expect(valuesOf(result.keys, "status")).toEqual([["published", 2]]); + }); + + test("filter composes with the same AST search accepts", () => { + const result = listMetadata(store.db, { + filter: { + operator: "and", + operands: [ + { key: "status", operator: "eq", value: "published" }, + { key: "priority", operator: "gte", value: 4 }, + ], + }, + }); + + expect(result.filteredDocuments).toBe(1); + expect(summaryOf(result.keys, "status").types[0]!.collections).toEqual(["work"]); + }); +}); + +describe("listMetadata key and value patterns", () => { + beforeEach(async () => { + await insertMetadataDoc("notes", { topics: ["typescript", "sqlite"], owner: "docs-team", "mem-kind": "fact", priority: 3, reviewed: true }); + await insertMetadataDoc("notes", { topics: ["typescript"], owner: "search-team", reviewers: ["docs-team", "security-team"], "mem-scope": "user", priority: 10, reviewed: false }); + }); + + test("an exact key pattern matches only that key", () => { + const result = listMetadata(store.db, { key: "topics" }); + + expect(result.keys.map(summary => summary.key)).toEqual(["topics"]); + expect(valuesOf(result.keys, "topics")).toEqual([["typescript", 2], ["sqlite", 1]]); + }); + + test("a key glob selects a family of keys", () => { + const result = listMetadata(store.db, { key: "mem-*" }); + + expect(result.keys.map(summary => summary.key)).toEqual(["mem-kind", "mem-scope"]); + }); + + test("a key pattern matching nothing yields an empty key list", () => { + const result = listMetadata(store.db, { key: "missing-*" }); + + expect(result.keys).toEqual([]); + expect(result.documents).toBe(2); + }); + + test("an exact value pattern is a reverse lookup across keys", () => { + const result = listMetadata(store.db, { value: "docs-team" }); + + expect(result.keys.map(summary => summary.key)).toEqual(["owner", "reviewers"]); + expect(summaryOf(result.keys, "owner").documents).toBe(1); + expect(summaryOf(result.keys, "owner").types[0]!.distinctValues).toBe(1); + expect(valuesOf(result.keys, "reviewers")).toEqual([["docs-team", 1]]); + }); + + test("a value glob keeps only matching values and their document counts", () => { + const result = listMetadata(store.db, { key: "topics", value: "type*" }); + + expect(valuesOf(result.keys, "topics")).toEqual([["typescript", 2]]); + expect(summaryOf(result.keys, "topics").types[0]!.remaining).toBe(0); + // Array-ness describes the key, not the matched rows. + expect(summaryOf(result.keys, "topics").types[0]!.multiValued).toBe(true); + }); + + test("numbers and booleans match by their text form", () => { + expect(listMetadata(store.db, { value: "10" }).keys.map(summary => summary.key)).toEqual(["priority"]); + expect(valuesOf(listMetadata(store.db, { value: "1*" }).keys, "priority")).toEqual([[10, 1]]); + expect(listMetadata(store.db, { value: "true" }).keys.map(summary => summary.key)).toEqual(["reviewed"]); + expect(valuesOf(listMetadata(store.db, { value: "true" }).keys, "reviewed")).toEqual([[true, 1]]); + }); + + test("a numeric range reflects only the matched values", () => { + const result = listMetadata(store.db, { key: "priority", value: "1*" }); + + expect(summaryOf(result.keys, "priority").types[0]!.range).toEqual({ min: 10, median: 10, max: 10 }); + }); + + test("a value pattern matching nothing yields an empty key list", () => { + const result = listMetadata(store.db, { value: "missing" }); + + expect(result.keys).toEqual([]); + }); + + test("patterns needing the picomatch fallback still match", () => { + expect(listMetadata(store.db, { value: "{docs-team,search-team}" }).keys.map(summary => summary.key)).toEqual(["owner", "reviewers"]); + expect(valuesOf(listMetadata(store.db, { key: "topics", value: "!(typescript)" }).keys, "topics")).toEqual([["sqlite", 1]]); + }); +}); + +describe("listMetadata value window", () => { + beforeEach(async () => { + const tags = ["a", "a", "a", "b", "b", "c", "d", "d", "d", "d"]; + for (const tag of tags) await insertMetadataDoc("notes", { tag }); + }); + + test("defaults to count order with a stable tiebreak", () => { + expect(valuesOf(listMetadata(store.db).keys, "tag")).toEqual([["d", 4], ["a", 3], ["b", 2], ["c", 1]]); + }); + + test("sort by value orders ascending", () => { + expect(valuesOf(listMetadata(store.db, { sort: "value" }).keys, "tag")).toEqual([["a", 3], ["b", 2], ["c", 1], ["d", 4]]); + }); + + test("limit windows values and reports the exact remainder", () => { + const tag = summaryOf(listMetadata(store.db, { limit: 2 }).keys, "tag").types[0]!; + + expect(tag.values.map(count => count.value)).toEqual(["d", "a"]); + expect(tag.distinctValues).toBe(4); + expect(tag.remaining).toBe(2); + }); + + test("an infinite limit removes the window", () => { + const tag = summaryOf(listMetadata(store.db, { limit: Infinity }).keys, "tag").types[0]!; + + expect(tag.values).toHaveLength(4); + expect(tag.remaining).toBe(0); + }); + + test("minCount drops the tail from the values, distinct count, and remainder", () => { + const tag = summaryOf(listMetadata(store.db, { minCount: 3, limit: 1 }).keys, "tag").types[0]!; + + expect(tag.values).toEqual([{ value: "d", documents: 4 }]); + expect(tag.distinctValues).toBe(2); + expect(tag.remaining).toBe(1); + // Coverage still counts every document holding the key. + expect(tag.documents).toBe(10); + }); + + test("a minCount nothing meets leaves the key with an empty window", () => { + const tag = summaryOf(listMetadata(store.db, { minCount: 99 }).keys, "tag").types[0]!; + + expect(tag.values).toEqual([]); + expect(tag.distinctValues).toBe(0); + expect(tag.remaining).toBe(0); + }); +}); + +describe("listMetadata numbers", () => { + test("reports min, median, and max for an odd count", async () => { + for (const priority of [5, 1, 3]) await insertMetadataDoc("notes", { priority }); + + const priority = summaryOf(listMetadata(store.db).keys, "priority").types[0]!; + + expect(priority.range).toEqual({ min: 1, median: 3, max: 5 }); + expect(priority.values.map(count => count.value)).toEqual([1, 3, 5]); + }); + + test("averages the middle values for an even count", async () => { + for (const priority of [1, 2, 3, 10]) await insertMetadataDoc("notes", { priority }); + + expect(summaryOf(listMetadata(store.db).keys, "priority").types[0]!.range).toEqual({ min: 1, median: 2.5, max: 10 }); + }); + + test("median counts every value row, including array elements", async () => { + await insertMetadataDoc("notes", { scores: [1, 1, 1] }); + await insertMetadataDoc("notes", { scores: [9] }); + + const scores = summaryOf(listMetadata(store.db).keys, "scores").types[0]!; + + expect(scores.range).toEqual({ min: 1, median: 1, max: 9 }); + expect(scores.values).toEqual([{ value: 1, documents: 1 }, { value: 9, documents: 1 }]); + }); + + test("strings and booleans carry no range", async () => { + await insertMetadataDoc("notes", { status: "x", reviewed: true }); + + const result = listMetadata(store.db); + + expect(summaryOf(result.keys, "status").types[0]!.range).toBeUndefined(); + expect(summaryOf(result.keys, "reviewed").types[0]!.range).toBeUndefined(); + }); +}); + +describe("listMetadata type conflicts and attribution", () => { + test("splits a key by type and partitions its documents", async () => { + await insertMetadataDoc("notes", { priority: 3 }); + await insertMetadataDoc("notes", { priority: 1 }); + await insertMetadataDoc("work", { priority: "high" }); + + const priority = summaryOf(listMetadata(store.db).keys, "priority"); + + expect(priority.documents).toBe(3); + expect(priority.types.map(typeSummary => [typeSummary.type, typeSummary.documents])).toEqual([["number", 2], ["string", 1]]); + expect(priority.types[0]!.range).toEqual({ min: 1, median: 2, max: 3 }); + expect(priority.types[0]!.collections).toEqual(["notes"]); + expect(priority.types[1]!.collections).toEqual(["work"]); + expect(priority.types[1]!.values).toEqual([{ value: "high", documents: 1 }]); + }); + + test("orders equally covered types by name", async () => { + await insertMetadataDoc("notes", { flag: true }); + await insertMetadataDoc("notes", { flag: "yes" }); + + expect(summaryOf(listMetadata(store.db).keys, "flag").types.map(typeSummary => typeSummary.type)).toEqual(["boolean", "string"]); + }); + + test("orders keys by coverage descending, then name", async () => { + await insertMetadataDoc("notes", { zeta: 1, alpha: 1, mid: 1 }); + await insertMetadataDoc("notes", { zeta: 1, alpha: 1 }); + await insertMetadataDoc("notes", { zeta: 1 }); + + expect(listMetadata(store.db).keys.map(summary => summary.key)).toEqual(["zeta", "alpha", "mid"]); + }); +}); + +describe("listMetadata agrees with filtered search", () => { + test("every reported value is reachable through an eq filter under the same scope", async () => { + await insertMetadataDoc("notes", { topics: ["a", "b"], priority: 3, reviewed: true, owner: "docs-team" }); + await insertMetadataDoc("notes", { topics: ["b"], priority: 1.5, reviewed: false }); + await insertMetadataDoc("work", { topics: ["c"], priority: 3 }); + const pendingId = await insertMetadataDoc("notes", { topics: ["ghost"] }); + store.db.prepare(`DELETE FROM document_metadata WHERE document_id = ?`).run(pendingId); + + const scopes: ListMetadataOptions[] = [{}, { collection: "notes" }, { collection: ["notes", "work"] }]; + for (const scope of scopes) { + const result = listMetadata(store.db, { ...scope, limit: Infinity }); + expect(result.keys.length).toBeGreaterThan(0); + + for (const summary of result.keys) { + for (const typeSummary of summary.types) { + for (const count of typeSummary.values) { + const hits = searchFTS(store.db, "doc", 100, scope.collection, { key: summary.key, operator: "eq", value: count.value }); + expect(hits, `${summary.key} = ${String(count.value)}`).toHaveLength(count.documents); + } + } + } + } + }); +}); + +describe("status view helpers", () => { + beforeEach(async () => { + await insertMetadataDoc("notes", { status: "published", priority: 3 }); + await insertMetadataDoc("notes", { status: "draft" }); + await insertMetadataDoc("notes", {}); + await insertMetadataDoc("work", { priority: "high", source: "jira" }); + const pendingId = await insertMetadataDoc("work", { status: "ghost" }); + store.db.prepare(`DELETE FROM document_metadata WHERE document_id = ?`).run(pendingId); + }); + + test("listMetadataKeys reports names, coverage, and types in coverage order", () => { + expect(listMetadataKeys(store.db)).toEqual([ + { key: "priority", documents: 2, types: ["number", "string"] }, + { key: "status", documents: 2, types: ["string"] }, + { key: "source", documents: 1, types: ["string"] }, + ]); + expect(listMetadataKeys(store.db, ["work"])).toEqual([ + { key: "priority", documents: 1, types: ["string"] }, + { key: "source", documents: 1, types: ["string"] }, + ]); + expect(listMetadataKeys(store.db, ["missing"])).toEqual([]); + }); + + test("countDocumentsWithMetadata counts extracted documents declaring a key", () => { + expect(countDocumentsWithMetadata(store.db)).toBe(3); + expect(countDocumentsWithMetadata(store.db, ["notes"])).toBe(2); + }); + + test("countDocumentsPendingMetadata accepts a collection scope", () => { + expect(countDocumentsPendingMetadata(store.db)).toBe(1); + expect(countDocumentsPendingMetadata(store.db, ["notes"])).toBe(0); + expect(countDocumentsPendingMetadata(store.db, ["work"])).toBe(1); + }); +}); + +describe("buildGlobPrefilter", () => { + const values = [ + "typescript", "type", "sqlite", "a/b", "a/b/c", ".env", "x.env", "2025-01-02", "https://a.b/c", + "a b", "a*b", "a?b", "a[1]", "a{b}", "a(b)", "a|b", "a\\b", "TypeScript", "", "./a", "a", + ]; + + const prunable = ["*", "type*", "*script", "t?pe", "2025-*", "https://*", "a*b", "a", "a b", ".env", "*.env", "?", "a/*"]; + const fallback = ["{a,b}", "a[1]", "!(a)", "!a", "+(a)", "@(a)", "(a|b)", "a|b", "a\\*", "./a", "a/./b", "\\a", "**", "a/**"]; + + function globMatches(value: string, pattern: string): boolean { + const row = store.db.prepare(`SELECT ? GLOB ? AS matched`).get(value, pattern) as { matched: number }; + return row.matched === 1; + } + + test("prunable patterns pass through unchanged", () => { + for (const pattern of prunable) expect(buildGlobPrefilter(pattern), pattern).toBe(pattern); + }); + + test("patterns with features GLOB cannot mirror fall back to picomatch only", () => { + for (const pattern of fallback) expect(buildGlobPrefilter(pattern), pattern).toBeNull(); + }); + + test("GLOB never excludes a value picomatch would keep", () => { + for (const pattern of prunable) { + const isMatch = picomatch(pattern, { dot: true }); + for (const value of values) { + if (!isMatch(value)) continue; + expect(globMatches(value, pattern), `${JSON.stringify(pattern)} vs ${JSON.stringify(value)}`).toBe(true); + } + } + }); + + test("picomatch remains the final word where GLOB is looser", () => { + // GLOB's `*` crosses `/`, picomatch's does not. + expect(globMatches("a/b", "*")).toBe(true); + expect(picomatch("*", { dot: true })("a/b")).toBe(false); + expect(picomatch("**", { dot: true })("a/b")).toBe(true); + }); +}); diff --git a/test/metadata-surfaces.test.ts b/test/metadata-surfaces.test.ts index 3b5eec3cc..f23895f1f 100644 --- a/test/metadata-surfaces.test.ts +++ b/test/metadata-surfaces.test.ts @@ -94,6 +94,47 @@ describe("SDK metadata filter", () => { filter: { operator: "and", operands: [] }, })).rejects.toThrow(/non-empty 'operands'/); }); + + test("listMetadata summarizes keys, types, and value counts", async () => { + const result = await store.listMetadata(); + expect(result.documents).toBe(2); + expect(result.filteredDocuments).toBeUndefined(); + expect(result.keys.map(summary => summary.key)).toEqual(["status", "topics"]); + + const topics = result.keys[1]!.types[0]!; + expect(topics).toMatchObject({ type: "string", documents: 2, distinctValues: 1, remaining: 0, collections: ["docs"] }); + expect(topics.values).toEqual([{ value: "typescript", documents: 2 }]); + }); + + test("listMetadata scopes, narrows by filter, and matches patterns", async () => { + const filtered = await store.listMetadata({ + collection: "docs", + key: "status", + filter: { key: "status", operator: "eq", value: "published" }, + }); + expect(filtered.filteredDocuments).toBe(1); + expect(filtered.keys[0]!.types[0]!.values).toEqual([{ value: "published", documents: 1 }]); + + const reverse = await store.listMetadata({ value: "dra*" }); + expect(reverse.keys.map(summary => summary.key)).toEqual(["status"]); + + const scoped = await store.listMetadata({ collection: "missing" }); + expect(scoped).toEqual({ documents: 0, keys: [] }); + }); + + test("listMetadata validates filters at the SDK runtime boundary", async () => { + // Plain-JS callers bypass the declarations, so the SDK must reject a bad AST at runtime. + const untrustedOptions: import("../src/index.js").ListMetadataOptions = JSON.parse('{"filter":{"key":"status","operator":"equal","value":"x"}}'); + await expect(store.listMetadata(untrustedOptions)).rejects.toThrow(/unknown operator 'equal'/); + }); + + test("getStatus lists metadata keys per collection", async () => { + const status = await store.getStatus(); + expect(status.collections[0]!.metadataKeys).toEqual([ + { key: "status", documents: 2, types: ["string"] }, + { key: "topics", documents: 2, types: ["string"] }, + ]); + }); }); // ============================================================================= @@ -121,6 +162,7 @@ describe("MCP and HTTP metadata filter", () => { const internal = createInternalStore(dbPath); await seedDoc(internal.db, "published.md", "# Pub\n\nhttp keyword body", { status: "published" }); await seedDoc(internal.db, "draft.md", "# Draft\n\nhttp keyword body", { status: "draft" }); + await seedDoc(internal.db, "tagged.md", "# Tagged\n\nhttp keyword body", { status: "archived", topics: ["sqlite", "search"], priority: 3 }); const testConfig: CollectionConfig = { collections: { docs: { path: "/test/docs", pattern: "**/*.md" } }, @@ -160,6 +202,10 @@ describe("MCP and HTTP metadata filter", () => { } async function callQueryTool(args: Record): Promise<{ status: number; json: any }> { + return callTool("query", args); + } + + async function callTool(name: string, args: Record): Promise<{ status: number; json: any }> { const res = await fetch(`${baseUrl}/mcp`, { method: "POST", headers: { @@ -167,14 +213,14 @@ describe("MCP and HTTP metadata filter", () => { "Accept": "application/json, text/event-stream", "MCP-Protocol-Version": "2026-07-28", "Mcp-Method": "tools/call", - "Mcp-Name": "query", + "Mcp-Name": name, }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { - name: "query", + name, arguments: args, _meta: { "io.modelcontextprotocol/protocolVersion": "2026-07-28", @@ -256,4 +302,81 @@ describe("MCP and HTTP metadata filter", () => { expect(json.result.isError).toBe(true); expect(json.result.content[0].text).toMatch(/non-empty 'operands'/); }); + + test("MCP metadata tool returns key summaries as structured content and CLI-shaped text", async () => { + const { status, json } = await callTool("metadata", { key: "status" }); + expect(status).toBe(200); + expect(json.result.isError).toBeFalsy(); + + const result = json.result.structuredContent; + expect(result.documents).toBe(3); + expect(result.keys).toHaveLength(1); + expect(result.keys[0].types[0]).toMatchObject({ type: "string", documents: 3, distinctValues: 3, remaining: 0 }); + expect(result.keys[0].types[0].values).toEqual([ + { value: "archived", documents: 1 }, + { value: "draft", documents: 1 }, + { value: "published", documents: 1 }, + ]); + expect(json.result.content[0].text).toBe("status string 3 of 3 documents 3 distinct\n archived 1\n draft 1\n published 1"); + }); + + test("MCP metadata tool narrows by filter, windows values, and reports the remainder", async () => { + const { json } = await callTool("metadata", { + key: "topics", + limit: 1, + filter: { key: "status", operator: "eq", value: "archived" }, + }); + const result = json.result.structuredContent; + expect(result.filteredDocuments).toBe(1); + const topics = result.keys[0].types[0]; + expect(topics.multiValued).toBe(true); + expect(topics.values).toHaveLength(1); + expect(topics.remaining).toBe(1); + expect(json.result.content[0].text).toContain("1 more values, use a higher 'limit'"); + }); + + test("MCP metadata tool rejects invalid filters", async () => { + const { json } = await callTool("metadata", { filter: { key: "status", operator: "equal", value: "x" } }); + expect(json.result.isError).toBe(true); + expect(json.result.content[0].text).toMatch(/unknown operator 'equal'/); + }); + + test("MCP status tool lists metadata keys per collection", async () => { + const { json } = await callTool("status", {}); + expect(json.result.isError).toBeFalsy(); + expect(json.result.structuredContent.collections[0].metadataKeys).toEqual([ + { key: "status", documents: 3, types: ["string"] }, + { key: "priority", documents: 1, types: ["number"] }, + { key: "topics", documents: 1, types: ["string"] }, + ]); + expect(json.result.content[0].text).toContain("metadata keys: status (string), priority (number), topics (string)"); + expect(json.result.content[0].text).toContain("call the 'metadata' tool"); + }); + + test("POST /metadata returns the same result as the tool", async () => { + const { status, json } = await postJson("/metadata", { key: "priority" }); + expect(status).toBe(200); + expect(json.documents).toBe(3); + expect(json.keys[0].types[0]).toMatchObject({ type: "number", range: { min: 3, median: 3, max: 3 } }); + + const reverse = await postJson("/metadata", { value: "search", limit: 5 }); + expect(reverse.json.keys.map((summary: { key: string }) => summary.key)).toEqual(["topics"]); + }); + + test("POST /metadata rejects invalid bodies, filters, and sort with 400", async () => { + const stringFilter = await postJson("/metadata", { filter: "status = published" }); + expect(stringFilter.status).toBe(400); + expect(stringFilter.json.error).toMatch(/must be an object/); + + const invalidAst = await postJson("/metadata", { filter: { key: "status", operator: "equal", value: "x" } }); + expect(invalidAst.status).toBe(400); + expect(invalidAst.json.error).toMatch(/unknown operator 'equal'/); + + const badSort = await postJson("/metadata", { sort: "size" }); + expect(badSort.status).toBe(400); + expect(badSort.json.error).toMatch(/sort/); + + const arrayBody = await fetch(`${baseUrl}/metadata`, { method: "POST", headers: { "Content-Type": "application/json" }, body: "[]" }); + expect(arrayBody.status).toBe(400); + }); });