From b4b0c1dfd26a53d69d166cc99113df6ae69ed470 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 23 Oct 2026 10:46:58 +0300 Subject: [PATCH 1/6] fix(shared): anchor keyword-placement diagnostics to the offending keyword MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keyword-placement and unknown-keyword guard errors (§9.11, §9.7.1) from validateSchemaAgainstParent embed the offending keyword's JSON path inline ("… found at: a/b/c"), but that path was discarded and the diagnostic fell back to the schema's `$id` at the document root. This also let a self-referencing x-gts-ref elsewhere appear to own an unrelated field's error. Parse the first reported path (stripping the depth-guard's trailing "(nesting exceeds …)" suffix) and map it to an instancePath via fieldPathToInstancePath, so the squiggle lands on the actual keyword. Signed-off-by: Artifizer --- packages/shared/src/registry.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/shared/src/registry.ts b/packages/shared/src/registry.ts index b6c0c91..54d2cea 100644 --- a/packages/shared/src/registry.ts +++ b/packages/shared/src/registry.ts @@ -887,6 +887,15 @@ export class JsonRegistry { const regexPath = !propPath && !refMatch && !isTraitError && /regular expression|Unsupported pattern/.test(msg) ? findRegexPatternPath(entity.content, msg) : null + // The keyword-placement/unknown-keyword guards (§9.11, §9.7.1) report + // the offending keyword's JSON path(s) inline as "… found at: a/b/c, + // d/e/f". Anchor the diagnostic to the first such path so the squiggle + // lands on the actual keyword (e.g. properties/subject_type/x-gts-type) + // instead of the schema's `$id` — and so a self-referencing x-gts-ref + // elsewhere no longer appears to "own" an error from another field. + const foundAtMatch = !propPath && !refMatch && !isTraitError && !regexPath + ? msg.match(/ found at: (.+)$/) + : null let instancePath: string | null = null let params: Record = {} if (propPath) { @@ -902,6 +911,13 @@ export class JsonRegistry { params = traitName ? { trait: traitName } : {} } else if (regexPath) { instancePath = regexPath + } else if (foundAtMatch) { + // The reported path uses `/` separators and `[n]` array indices + // (e.g. "allOf[0]/properties/x/x-gts-type"); the depth-guard also + // appends " (nesting exceeds …)" which is not part of the path. + const firstPath = foundAtMatch[1].split(', ')[0].replace(/ \(.*\)$/, '').trim() + instancePath = fieldPathToInstancePath(firstPath) + params = { keywordPath: firstPath } } validation.errors.push({ instancePath: instancePath || '/$id', From 50ae1fcb5311a4d737870a5a088a77ce04f1d765 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Tue, 27 Oct 2026 14:01:22 +0200 Subject: [PATCH 2/6] fix(vscode): track validation state and handle 100K+ file workspaces Files without diagnostics were shown as valid before their first validation pass finished, so on large workspaces not-yet-processed invalid files turned green. Track which files have published results, keep the rest uncolored with a "validation pending" tooltip, and publish workspace results progressively. Replace push(...spread) over file and entity lists with loops/concat: spreading 100K+ items as call arguments overflowed the stack. Signed-off-by: Artifizer --- apps/vscode-extension/src/extension.ts | 8 +- apps/vscode-extension/src/gtsExplorer.ts | 26 +++- apps/vscode-extension/src/validation.ts | 153 +++++++++++++++---- apps/web/src/components/EntitiesListMenu.tsx | 11 +- apps/web/src/components/FileList.tsx | 12 +- 5 files changed, 165 insertions(+), 45 deletions(-) diff --git a/apps/vscode-extension/src/extension.ts b/apps/vscode-extension/src/extension.ts index 9f193f4..84b6e85 100644 --- a/apps/vscode-extension/src/extension.ts +++ b/apps/vscode-extension/src/extension.ts @@ -7,7 +7,7 @@ import { setLastScanFiles } from './scanStore' import { rebuildRegistry, indexFile as indexFileInRegistry, removeFile as removeFileFromRegistry, getRegistry } from './registryStore' import { getWorkspaceIgnore, resetWorkspaceIgnore, isGitIgnored, type FolderIgnore } from './gitignore' import { WorkspaceLayoutStorage } from './storage' -import { initValidation, resetValidationDiagnostics, validateOpenDocument, validateWorkspaceInBackground, revalidateDependents, onValidationCompleted } from './validation' +import { initValidation, resetValidationDiagnostics, validateOpenDocument, validateWorkspaceInBackground, revalidateDependents, onValidationCompleted, forgetValidationState } from './validation' import { isGtsCandidateFile, isIndexableGtsDocument } from './helpers' import { GtsLinkProvider } from './linkProvider' import { registerGtsExplorer, type GtsExplorer } from './gtsExplorer' @@ -97,7 +97,10 @@ async function findGtsFiles(ignores: Map, baseExclude: str if (capPerFolder !== undefined && found.length >= capPerFolder) { console.warn(`[GTS] Phase 1 hit its ${capPerFolder}-file cap in "${folder.name}"; the rest is indexed by phase 2`) } - uris.push(...found) + // Not `uris.push(...found)`: spreading passes every element as a call + // argument, which overflows the stack ("Maximum call stack size exceeded") + // once a folder holds more than ~120K files. + for (const uri of found) uris.push(uri) } return uris } @@ -197,6 +200,7 @@ function indexFileText(fsPath: string, text: string): void { function dropFileFromRegistry(fsPath: string): void { removeFileFromRegistry(fsPath) forgetIndexedPath(fsPath) + forgetValidationState(fsPath) } /** diff --git a/apps/vscode-extension/src/gtsExplorer.ts b/apps/vscode-extension/src/gtsExplorer.ts index b8e7f15..b9ca796 100644 --- a/apps/vscode-extension/src/gtsExplorer.ts +++ b/apps/vscode-extension/src/gtsExplorer.ts @@ -1,7 +1,7 @@ import * as vscode from 'vscode' import * as path from 'path' import { getRegistry, getMalformedIds, getPathsWithMalformedIds } from './registryStore' -import { countPublishedProblems } from './validation' +import { countPublishedProblems, isValidationKnown, onValidationStateChanged } from './validation' /** * Left-sidebar file browser for GTS: shows every discovered file that holds at @@ -175,9 +175,8 @@ export class GtsFileTreeProvider title: 'Open GTS File', arguments: [element.fsPath] } - item.tooltip = hasGtsErrors(item.resourceUri) - ? 'Has GTS validation errors' - : 'No GTS validation errors' + // The tooltip is filled in on hover (resolveTreeItem), so it reflects the + // file's validation state at that moment rather than when the row was built. return item } @@ -187,6 +186,16 @@ export class GtsFileTreeProvider return item } + resolveTreeItem(item: vscode.TreeItem, element: GtsTreeElement): vscode.TreeItem { + if (element.kind === 'file') { + const uri = vscode.Uri.file(element.fsPath) + item.tooltip = !isValidationKnown(element.fsPath) + ? 'GTS validation pending' + : hasGtsErrors(uri) ? 'Has GTS validation errors' : 'No GTS validation errors' + } + return item + } + getChildren(element?: GtsTreeElement): GtsTreeElement[] { const folder = element ? (element.kind === 'folder' ? element : undefined) : this.root if (!folder) return [] @@ -215,7 +224,8 @@ export class GtsFileTreeProvider * Colors every valid GTS file green (in the sidebar tree, the OS-style Explorer * and editor tabs). Files with GTS problems get no decoration of their own: * VS Code's Problems decoration colors them with its standard error color and - * count, so every GTS problem looks the same everywhere. + * count, so every GTS problem looks the same everywhere. Files that haven't + * been validated yet stay uncolored: no diagnostics doesn't mean valid yet. */ export class GtsFileDecorationProvider implements vscode.FileDecorationProvider { private readonly _onDidChangeFileDecorations = new vscode.EventEmitter() @@ -227,6 +237,7 @@ export class GtsFileDecorationProvider implements vscode.FileDecorationProvider provideFileDecoration(uri: vscode.Uri): vscode.FileDecoration | undefined { if (!isDiscoveredGtsFile(uri.fsPath)) return undefined + if (!isValidationKnown(uri.fsPath)) return undefined if (hasGtsErrors(uri)) return undefined return new vscode.FileDecoration(undefined, 'GTS: file is valid', new vscode.ThemeColor('charts.green')) @@ -302,6 +313,11 @@ export function registerGtsExplorer(context: vscode.ExtensionContext): GtsExplor decorationProvider.refresh(relevant) scheduleTreeUpdate() }), + // A file whose validation just became known turns green without any + // diagnostics change, so it has to be repainted explicitly. + onValidationStateChanged(uris => { + decorationProvider.refresh(uris) + }), new vscode.Disposable(() => { if (updateTimer) clearTimeout(updateTimer) }) ) diff --git a/apps/vscode-extension/src/validation.ts b/apps/vscode-extension/src/validation.ts index 8ab771e..23d38cc 100644 --- a/apps/vscode-extension/src/validation.ts +++ b/apps/vscode-extension/src/validation.ts @@ -5,7 +5,7 @@ import * as jsonc from 'jsonc-parser' import { ValidationError, DEFAULT_GTS_CONFIG, parseGtsFileContent, isYamlFileName } from '@gts/shared' import type { JsonRegistry, JsonSchema, JsonObj } from '@gts/shared' import { getLastScanFiles } from './scanStore' -import { getRegistry, rebuildRegistry, indexFile, removeFile, getMalformedIds, getPathsWithMalformedIds } from './registryStore' +import { getRegistry, getRegistryRevision, rebuildRegistry, indexFile, removeFile, getMalformedIds, getPathsWithMalformedIds } from './registryStore' import { malformedGtsIdMessage } from './gtsIdFormat' import { isGtsCandidateFile, isIndexableGtsDocument } from './helpers' @@ -17,6 +17,13 @@ const documentValidationGenerations = new Map() const validationCompletedListeners = new Set<(uri: vscode.Uri) => void>() let workspaceValidationGeneration = 0 +// Files whose GTS problems have been published at least once. Until then, a +// file without diagnostics is "not validated yet", not "valid": on a large +// workspace the first pass takes a while, and treating the absence of markers +// as valid colored every not-yet-processed invalid file green. +const validatedPaths = new Set() +const validationStateListeners = new Set<(uris: vscode.Uri[]) => void>() + export function getDocumentValidationErrors(uri: vscode.Uri): ValidationError[] { return documentValidationErrors.get(uri.toString()) || [] } @@ -26,6 +33,39 @@ export function onValidationCompleted(listener: (uri: vscode.Uri) => void): vsco return new vscode.Disposable(() => validationCompletedListeners.delete(listener)) } +/** True once the file's validation result has been published (its diagnostics are authoritative). */ +export function isValidationKnown(fsPath: string): boolean { + return validatedPaths.has(fsPath) +} + +/** Fires with the files whose validation result just became known. */ +export function onValidationStateChanged(listener: (uris: vscode.Uri[]) => void): vscode.Disposable { + validationStateListeners.add(listener) + return new vscode.Disposable(() => validationStateListeners.delete(listener)) +} + +/** The file left the index; if it comes back it is unvalidated again. */ +export function forgetValidationState(fsPath: string): void { + validatedPaths.delete(fsPath) +} + +function markValidated(fsPaths: Iterable): void { + const added: vscode.Uri[] = [] + for (const fsPath of fsPaths) { + if (validatedPaths.has(fsPath)) continue + validatedPaths.add(fsPath) + added.push(vscode.Uri.file(fsPath)) + } + if (added.length === 0) return + for (const listener of validationStateListeners) { + try { + listener(added) + } catch (err) { + console.error('[GTS Validation] Error in validation state listener:', err) + } + } +} + /** * A document's text plus its syntax tree, parsed lazily at most once and shared * by every error-position lookup of one validation (previously each error @@ -585,6 +625,7 @@ export async function validateOpenDocument(document: vscode.TextDocument) { documentValidationErrors.delete(document.uri.toString()) diagnosticCollection.delete(document.uri) } + markValidated([filePath]) for (const listener of validationCompletedListeners) { try { @@ -665,8 +706,8 @@ type RegistryEntity = JsonSchema | JsonObj */ function entitiesByFile(registry: JsonRegistry): RegistryEntity[] { const entities: RegistryEntity[] = [] - for (const list of registry.jsonFileSchemas.values()) entities.push(...list) - for (const list of registry.jsonFileObjs.values()) entities.push(...list) + for (const list of registry.jsonFileSchemas.values()) for (const entity of list) entities.push(entity) + for (const list of registry.jsonFileObjs.values()) for (const entity of list) entities.push(entity) return entities } @@ -791,54 +832,111 @@ function collectValidationErrors(registry: JsonRegistry, filePath: string, index ] } +// While a pass runs, the files it has finished are published at this interval, +// so a large workspace fills in progressively instead of all at the very end. +const WORKSPACE_PUBLISH_INTERVAL_MS = 1000 + +/** Paths of the GTS files currently open in an editor (they get precise diagnostics from validateOpenDocument). */ +function openGtsPaths(): Set { + const openPaths = new Set() + for (const doc of vscode.workspace.textDocuments) { + if (isGtsCandidateFile(doc)) openPaths.add(doc.uri.fsPath) + } + return openPaths +} + +/** Every file the workspace pass reports on. */ +function workspaceFilePaths(registry: JsonRegistry): Set { + const paths = new Set() + for (const p of registry.jsonFileSchemas.keys()) paths.add(p) + for (const p of registry.jsonFileObjs.keys()) paths.add(p) + for (const p of registry.invalidFiles.keys()) paths.add(p) + for (const p of getPathsWithMalformedIds()) paths.add(p) + return paths +} + /** One full validation pass. Returns false if it was superseded and must rerun. */ async function runWorkspaceValidationPass(): Promise { const registry = getRegistry() if (!registry || !workspaceDiagnosticCollection) return true + const collection = workspaceDiagnosticCollection const validationGeneration = workspaceValidationGeneration const isSuperseded = () => getRegistry() !== registry || workspaceValidationGeneration !== validationGeneration const startTime = Date.now() + // Files are validated one at a time and published in batches as the pass + // goes. Everything is indexed before validation starts, so the duplicate-id + // index is complete from the beginning; it is rebuilt when the registry + // changes mid-pass, and the final publish below recomputes every file anyway. + let index = buildDefinitionIndex(registry) + let indexRevision = getRegistryRevision() + let finished: string[] = [] + let lastPublish = Date.now() + const publishFinished = () => { + if (getRegistryRevision() !== indexRevision) { + index = buildDefinitionIndex(registry) + indexRevision = getRegistryRevision() + } + const openPaths = openGtsPaths() + const closed = finished.filter(filePath => !openPaths.has(filePath)) + const entries: Array<[vscode.Uri, vscode.Diagnostic[] | undefined]> = [] + for (const filePath of closed) { + const uri = vscode.Uri.file(filePath) + const problems = collectValidationErrors(registry, filePath, index) + if (problems.length > 0) entries.push([uri, problems.map(coarseDiagnostic)]) + else if (collection.has(uri)) entries.push([uri, undefined]) + } + if (entries.length > 0) collection.set(entries) + markValidated(closed) + finished = [] + lastPublish = Date.now() + } + // Re-indexing a file replaces its entity objects, so rather than aborting when // the registry changes mid-pass, keep sweeping until every *current* entity has // been validated. Edits that land during the pass are picked up by the next // sweep; replaced entities are skipped. const validated = new Set() let sliceStart = Date.now() + let pendingPaths: Iterable = workspaceFilePaths(registry) for (;;) { - const pending = entitiesByFile(registry).filter(entity => !validated.has(entity)) - if (pending.length === 0) break - for (const entity of pending) { - validated.add(entity) - if (!isIndexedInItsFile(registry, entity)) continue - await registry.validateEntity(entity) - if (Date.now() - sliceStart >= WORKSPACE_VALIDATION_SLICE_MS) { - await yieldToEventLoop() + let swept = 0 + for (const filePath of pendingPaths) { + swept++ + for (const entity of fileEntities(registry, filePath)) { + if (validated.has(entity)) continue + validated.add(entity) + if (!isIndexedInItsFile(registry, entity)) continue + await registry.validateEntity(entity) + if (Date.now() - sliceStart >= WORKSPACE_VALIDATION_SLICE_MS) { + await yieldToEventLoop() + if (isSuperseded()) return false + sliceStart = Date.now() + } + } + finished.push(filePath) + if (Date.now() - lastPublish >= WORKSPACE_PUBLISH_INTERVAL_MS) { if (isSuperseded()) return false - sliceStart = Date.now() + publishFinished() } } + if (swept === 0) break + pendingPaths = [...workspaceFilePaths(registry)].filter(filePath => + fileEntities(registry, filePath).some(entity => !validated.has(entity))) } if (isSuperseded()) return false // Collect from the registry's current state (open documents get precise // diagnostics from validateOpenDocument instead). - const openPaths = new Set() - for (const doc of vscode.workspace.textDocuments) { - if (isGtsCandidateFile(doc)) openPaths.add(doc.uri.fsPath) - } - - const index = buildDefinitionIndex(registry) - const filePaths = new Set([ - ...registry.jsonFileSchemas.keys(), - ...registry.jsonFileObjs.keys(), - ...registry.invalidFiles.keys(), - ...getPathsWithMalformedIds() - ]) + const openPaths = openGtsPaths() + index = buildDefinitionIndex(registry) + const filePaths = workspaceFilePaths(registry) + const closedPaths: string[] = [] const diagnosticsByPath = new Map() for (const filePath of filePaths) { if (openPaths.has(filePath)) continue + closedPaths.push(filePath) const problems = collectValidationErrors(registry, filePath, index) if (problems.length > 0) diagnosticsByPath.set(filePath, problems.map(coarseDiagnostic)) } @@ -847,13 +945,14 @@ async function runWorkspaceValidationPass(): Promise { // listed files, so files that became valid must be cleared explicitly or they // keep stale red markers. const entries: Array<[vscode.Uri, vscode.Diagnostic[] | undefined]> = [] - workspaceDiagnosticCollection.forEach(uri => { + collection.forEach(uri => { if (!diagnosticsByPath.has(uri.fsPath)) entries.push([uri, undefined]) }) for (const [filePath, diagnostics] of diagnosticsByPath.entries()) { entries.push([vscode.Uri.file(filePath), diagnostics]) } - workspaceDiagnosticCollection.set(entries) + collection.set(entries) + markValidated(closedPaths) console.log(`[GTS Validation] Workspace pass: ${validated.size} entities, ${diagnosticsByPath.size} file(s) with problems (${Date.now() - startTime}ms)`) return true } @@ -874,6 +973,7 @@ async function validateClosedFile(filePath: string, index?: DefinitionIndex): Pr const uri = vscode.Uri.file(filePath) workspaceDiagnosticCollection.set(uri, diagnostics.length > 0 ? diagnostics : undefined) + markValidated([filePath]) } /** @@ -954,6 +1054,7 @@ export function countPublishedProblems(): number { export function resetValidationDiagnostics(): void { documentValidationErrors.clear() documentValidationGenerations.clear() + validatedPaths.clear() workspaceValidationGeneration++ diagnosticCollection?.clear() workspaceDiagnosticCollection?.clear() diff --git a/apps/web/src/components/EntitiesListMenu.tsx b/apps/web/src/components/EntitiesListMenu.tsx index 1ffd688..4806d83 100644 --- a/apps/web/src/components/EntitiesListMenu.tsx +++ b/apps/web/src/components/EntitiesListMenu.tsx @@ -74,11 +74,12 @@ export function EntityList({ jsonObjs, schemas, invalidFiles, selectedEntity: se // Build the visible linear list in render order for keyboard navigation const visibleEntities: Array = useMemo(() => { - const list: Array = [] - if (openJsonList) list.push(...filteredJsonObjs) - if (openSchemaList) list.push(...filteredSchemas) - if (openInvalidList) list.push(...filteredInvalidFiles) - return list + // concat, not push(...): spreading 100K+ items as call arguments overflows the stack. + return ([] as Array).concat( + openJsonList ? filteredJsonObjs : [], + openSchemaList ? filteredSchemas : [], + openInvalidList ? filteredInvalidFiles : [] + ) }, [filteredJsonObjs, filteredSchemas, filteredInvalidFiles, openJsonList, openSchemaList, openInvalidList]) // Notify parent of filtered entities for keyboard navigation (only when content actually changes) diff --git a/apps/web/src/components/FileList.tsx b/apps/web/src/components/FileList.tsx index 6314d4b..6a0afca 100644 --- a/apps/web/src/components/FileList.tsx +++ b/apps/web/src/components/FileList.tsx @@ -38,13 +38,11 @@ export function EntityList({ jsonObjs, schemas, selectedEntity: selectedEntity, const filteredSchemas = schemas.filter(s => match(s.file?.name || s.id, s.id)) // Build the visible linear list in render order for keyboard navigation - const visibleEntities: Entity[] = [] - if (openJsonList) { - visibleEntities.push(...filteredJsonObjs) - } - if (openSchemaList) { - visibleEntities.push(...filteredSchemas) - } + // concat, not push(...): spreading 100K+ items as call arguments overflows the stack. + const visibleEntities: Entity[] = ([] as Entity[]).concat( + openJsonList ? filteredJsonObjs : [], + openSchemaList ? filteredSchemas : [] + ) // Notify parent of filtered entities for keyboard navigation useEffect(() => { From 30e2f15f7743fb4adb0c3f10cc5465f997d97bf1 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Tue, 27 Oct 2026 14:01:37 +0200 Subject: [PATCH 3/6] build(shared): rebase gts-kit onto gts-ts v0.11.0 gts-ts 0.11 implements GTS spec 0.17, which removes `x-gts-ref` (and the legacy `x-gts-type`) in favor of `x-gts-type-ref`, `x-gts-instance-ref` and `x-gts-pattern-ref`, and enforces the safe regex profile. - XGtsRefValidator is now registry-less: delegate instance reference checks to the registry's gts-ts store, and drive the any-valid transitive check from fields declaring an entity-reference keyword. - Strip/recognize the new keywords for Ajv, scalar-field detection, diagnostic anchoring, schema property descriptions and highlighting. - Match the new "Referenced type ... is invalid" / "... names ..., which is invalid" messages when anchoring schema diagnostics. - gts-ts now uses re2js, so stop copying re2.wasm into the vscode bundle. - Migrate examples to the new keywords and drop the lookahead from the ISO-8601 duration pattern. Signed-off-by: Artifizer --- apps/vscode-extension/README.md | 11 +- apps/vscode-extension/build.mjs | 25 -- apps/vscode-extension/src/linkProvider.ts | 4 +- apps/vscode-extension/src/validation.ts | 6 +- apps/web/src/lib/schemaParser.ts | 9 +- docs/GTS_TS_MIGRATION.md | 10 +- .../gts.x.core.events.topic.v1~.schema.json | 8 +- .../gts.x.core.events.type.v1~.schema.json | 6 +- ...mmerce.orders.order_placed.v1~.schema.json | 2 +- ...x.core.idp.contact_created.v1~.schema.json | 2 +- package-lock.json | 142 +++++--- packages/shared/package.json | 2 +- packages/shared/src/entities.ts | 13 +- packages/shared/src/gts-styling.ts | 2 +- packages/shared/src/registry.ts | 329 +++++++++++------- packages/shared/src/schemaParser.ts | 21 +- 16 files changed, 358 insertions(+), 234 deletions(-) diff --git a/apps/vscode-extension/README.md b/apps/vscode-extension/README.md index 7aa5c69..26179b8 100644 --- a/apps/vscode-extension/README.md +++ b/apps/vscode-extension/README.md @@ -18,8 +18,7 @@ GTS supports **type derivation** through chaining: a derived type extends a base **1. Base type** -- defines the common fields every user has: -```jsonc -// file: gts.x.core.idp.user.v1~.schema.json +```json { "$id": "gts://gts.x.core.idp.user.v1~", "$schema": "http://json-schema.org/draft-07/schema#", @@ -29,7 +28,7 @@ GTS supports **type derivation** through chaining: a derived type extends a base "required": ["id", "type", "email", "display_name", "status"], "properties": { "id": { "type": "string", "format": "uuid" }, - "type": { "type": "string", "x-gts-ref": "/$id" }, + "type": { "type": "string", "x-gts-type-ref": "/$id" }, "email": { "type": "string", "format": "email" }, "display_name": { "type": "string" }, "properties": { "type": "object" } @@ -39,8 +38,7 @@ GTS supports **type derivation** through chaining: a derived type extends a base **2. Derived type** -- extends the base with admin-specific fields. The chained `$id` shows that `platform_admin` inherits from `user`: -```jsonc -// file: gts.x.core.idp.user.v1~x.core.idp.platform_admin.v1~.schema.json +```json { "$id": "gts://gts.x.core.idp.user.v1~x.core.idp.platform_admin.v1~", "$schema": "http://json-schema.org/draft-07/schema#", @@ -67,8 +65,7 @@ GTS supports **type derivation** through chaining: a derived type extends a base **3. Instance** -- a concrete platform administrator that conforms to the derived type: -```jsonc -// file: gts.x.core.idp.user.v1~x.core.idp.platform_admin.v1~.examples.jsonc +```json { "id": "f0e1d2c3-b4a5-6789-0123-456789abcdef", "type": "gts.x.core.idp.user.v1~x.core.idp.platform_admin.v1~", diff --git a/apps/vscode-extension/build.mjs b/apps/vscode-extension/build.mjs index 56475d5..c7b4ae5 100644 --- a/apps/vscode-extension/build.mjs +++ b/apps/vscode-extension/build.mjs @@ -1,29 +1,5 @@ import * as esbuild from 'esbuild' -import { promises as fs } from 'fs' -import path from 'path' -import { createRequire } from 'module' -const require = createRequire(import.meta.url) - -/** - * gts-ts compiles schema patterns with re2-wasm, which loads its `re2.wasm` - * binary from the directory of the running script. Bundled, that is `dist/`, - * so the binary must sit next to `dist/extension.js` or the extension throws - * on load. Resolved through gts-ts so it's the exact copy gts-ts depends on. - * - * @type {esbuild.Plugin} - */ -const copyRe2WasmPlugin = { - name: 'copy-re2-wasm', - setup(build) { - build.onEnd(async result => { - if (result.errors.length > 0) return - const gtsTsDir = path.dirname(require.resolve('@globaltypesystem/gts-ts/package.json', { paths: ['../../packages/shared'] })) - const re2Dir = path.dirname(require.resolve('re2-wasm/package.json', { paths: [gtsTsDir] })) - await fs.copyFile(path.join(re2Dir, 'build', 'wasm', 're2.wasm'), path.join('dist', 're2.wasm')) - }) - }, -} const production = process.argv.includes('--production') const watch = process.argv.includes('--watch') @@ -64,7 +40,6 @@ async function main() { mainFields: ['module', 'main'], plugins: [ esbuildProblemMatcherPlugin, - copyRe2WasmPlugin, ], }) diff --git a/apps/vscode-extension/src/linkProvider.ts b/apps/vscode-extension/src/linkProvider.ts index fd09145..8292aaf 100644 --- a/apps/vscode-extension/src/linkProvider.ts +++ b/apps/vscode-extension/src/linkProvider.ts @@ -336,7 +336,7 @@ export class GtsLinkProvider implements vscode.DocumentLinkProvider=18" } }, - "node_modules/@globaltypesystem/gts-ts/node_modules/ajv-formats": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", - "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, "node_modules/@gts-viewer/electron": { "resolved": "apps/electron", "link": true @@ -1860,6 +1841,30 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/@hyperjump/json-schema-formats": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@hyperjump/json-schema-formats/-/json-schema-formats-1.1.0.tgz", + "integrity": "sha512-vLMIOwMThlgWgPJBV39Ds7kaTaygK0+UxjvQShTmKB8rRy8HXq472jnHP8/uaik+77HDx2M+W5Bvzhq6zFc2zQ==", + "license": "MIT", + "dependencies": { + "@hyperjump/uri": "^1.3.2", + "idn-hostname": "^15.1.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/hyperjump-io" + } + }, + "node_modules/@hyperjump/uri": { + "version": "1.3.9", + "resolved": "https://registry.npmjs.org/@hyperjump/uri/-/uri-1.3.9.tgz", + "integrity": "sha512-cxG5vyIdx3gUb/OQD/FDruQQcGNzF2aKTXWJ6LzlAC1kh97YZQSRsF3hL274xlTDJ1nWGJGRtknlW6ZpU3Ch+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/jdesrosiers" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -2955,6 +2960,12 @@ "win32" ] }, + "node_modules/@sagold/json-pointer": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/@sagold/json-pointer/-/json-pointer-7.2.1.tgz", + "integrity": "sha512-8EX4r5Royl5M3qNPTh5W5njdOtRqbWgQfVv26DbzjGj2/55b60EqvbiqUIglzw7fADfY/Io6jDJxGNJHmC+g8g==", + "license": "MIT" + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -7908,6 +7919,12 @@ "@types/yauzl": "^2.9.1" } }, + "node_modules/fast-copy": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.3.0.tgz", + "integrity": "sha512-oWBZLHWpUYt2A471n+ayC4+bJSGDWpl2h3COgBxYX5xHFnr7KqOSrYJZX82ZmrtAfVK+SqsOuNMgsoL+lJquuQ==", + "license": "MIT" + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -8873,6 +8890,18 @@ "node": "^8.11.2 || >=10" } }, + "node_modules/idn-hostname": { + "version": "15.1.19", + "resolved": "https://registry.npmjs.org/idn-hostname/-/idn-hostname-15.1.19.tgz", + "integrity": "sha512-pRGRbFpQWCptOBWw9Gn5DXtPdern5/XtdkcXdoEavjd+sh72jNm2eVI1eeysp64UisXPfaqoQrnLUKpSN4hJWw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=20.12.0 <21 || >=22.0.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -9216,6 +9245,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, "license": "MIT", "bin": { "jsesc": "bin/jsesc" @@ -9231,6 +9261,36 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-library": { + "version": "11.6.14", + "resolved": "https://registry.npmjs.org/json-schema-library/-/json-schema-library-11.6.14.tgz", + "integrity": "sha512-fN8X+ZBUOPDF7B9ZBjVq4ZKay0K/U9KQeJrQp4GqvjiPCMw/En8EJM0OHcFKBHynz6MDPJ0lIm/CZFbk0WhbZA==", + "license": "MIT", + "dependencies": { + "@hyperjump/json-schema-formats": "^1.0.8", + "@sagold/json-pointer": "^7.2.1", + "fast-copy": "^4.2.0", + "fast-deep-equal": "^3.1.3", + "fast-uri": "^4.2.2", + "valid-url": "^1.0.9" + } + }, + "node_modules/json-schema-library/node_modules/fast-uri": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.2.3.tgz", + "integrity": "sha512-I1PXwP5eAhlNlcdpjt9CgB59op2ZvZfxw+BkqGFVIUOgNpCVTcTqp3HhTX7brxj5vsip+nSY4nTvff+eIA76hg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/json-schema-ref-resolver": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz", @@ -10780,7 +10840,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -10905,13 +10964,13 @@ "node": ">=0.10.0" } }, - "node_modules/re2-wasm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/re2-wasm/-/re2-wasm-1.0.2.tgz", - "integrity": "sha512-VXUdgSiUrE/WZXn6gUIVVIsg0+Hp6VPZPOaHCay+OuFKy6u/8ktmeNEf+U5qSA8jzGGFsg8jrDNu1BeHpz2pJA==", - "license": "Apache-2.0", + "node_modules/re2js": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/re2js/-/re2js-2.9.0.tgz", + "integrity": "sha512-rZpqkd4ETP3GrQ5mDLzADr14OleeKdb/40lG84WE8absRfsVLJpWt2ifjtYYeoPqeouIkcxhTlurxIXfR5kR5Q==", + "license": "MIT", "engines": { - "node": ">=10" + "node": ">=20.0.0" } }, "node_modules/react": { @@ -11067,18 +11126,6 @@ "node": ">= 12.13.0" } }, - "node_modules/regjsparser": { - "version": "0.13.3", - "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.13.3.tgz", - "integrity": "sha512-ycwFAS14Jw4mppvmK4GR/J6u3WpWpjkEApehuHtLc/8VpPNpDMbQ4WjqwplXifGeyKOzHSFLmSPqzksDQE2Sfg==", - "license": "BSD-2-Clause", - "dependencies": { - "jsesc": "~3.1.0" - }, - "bin": { - "regjsparser": "bin/parser" - } - }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -12855,6 +12902,11 @@ "dev": true, "license": "MIT" }, + "node_modules/valid-url": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", + "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA==" + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -13816,7 +13868,7 @@ "name": "@gts/shared", "version": "0.2.7", "dependencies": { - "@globaltypesystem/gts-ts": "^0.8.1", + "@globaltypesystem/gts-ts": "^0.11.0", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "js-yaml": "^4.1.0", diff --git a/packages/shared/package.json b/packages/shared/package.json index 7d57a7b..f641cbf 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -16,7 +16,7 @@ "dev": "tsc --watch" }, "dependencies": { - "@globaltypesystem/gts-ts": "^0.8.1", + "@globaltypesystem/gts-ts": "^0.11.0", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "js-yaml": "^4.1.0", diff --git a/packages/shared/src/entities.ts b/packages/shared/src/entities.ts index 2c12daa..2677eff 100644 --- a/packages/shared/src/entities.ts +++ b/packages/shared/src/entities.ts @@ -3,7 +3,8 @@ import { validateGtsID as gtsValidateID, matchIDPattern as gtsMatchIDPattern, extractID as gtsExtractID, - Gts + Gts, + X_GTS_REF_SELF } from '@globaltypesystem/gts-ts' import type { ValidationResult as GtsValidationResult, @@ -116,17 +117,17 @@ export function isGtsPattern(value: string): boolean { } /** - * Check if a GTS x-gts-ref value is valid. Accepts: - * - Full GTS type IDs (e.g. "gts.vendor.pkg.ns.type.v1~") + * Check if a GTS reference keyword operand (`x-gts-type-ref`, + * `x-gts-instance-ref`, `x-gts-pattern-ref`) is valid. Accepts: + * - Full GTS IDs (e.g. "gts.vendor.pkg.ns.type.v1~") * - Wildcard patterns (e.g. "gts.*", "gts.vendor.pkg.*") - * - JSON Pointer references starting with "/" (e.g. "/$id") + * - The reserved selected-type operand "/$id" * * Uses the official gts-ts library for GTS ID/pattern validation. */ export function isValidXGtsRef(value: string): boolean { if (!value || typeof value !== 'string') return false - // JSON Pointer references are valid x-gts-ref values - if (value.startsWith('/')) return true + if (value === X_GTS_REF_SELF) return true // Otherwise validate as GTS ID or pattern return isGtsIdOrPattern(value) } diff --git a/packages/shared/src/gts-styling.ts b/packages/shared/src/gts-styling.ts index f6d2a05..5b669d6 100644 --- a/packages/shared/src/gts-styling.ts +++ b/packages/shared/src/gts-styling.ts @@ -66,7 +66,7 @@ export interface GtsStyleAnalysis { * authoritative gts-ts validation results surfaced through `entityLookup`: * a segment whose cumulative entity failed gts-ts validation (`isValid: false`) * is rendered as an error. GTS *rule* violations (abstract instantiation, - * derivation incompatibility, x-gts-ref, ...) are therefore not re-derived here; + * derivation incompatibility, reference keywords, ...) are therefore not re-derived here; * they are read back from `entityLookup`/the caller's validation errors, keeping * gts-ts the single source of truth. * diff --git a/packages/shared/src/registry.ts b/packages/shared/src/registry.ts index 54d2cea..284ccf1 100644 --- a/packages/shared/src/registry.ts +++ b/packages/shared/src/registry.ts @@ -4,7 +4,7 @@ import { isYamlFileName } from './parse.js' import { findSchemaPropertyPath, findXGtsRefPath, findTraitRequiredPath, findRegexPatternPath, getInstanceFieldSubschema } from './schemaParser.js' import Ajv, { type ValidateFunction, type ErrorObject } from 'ajv' import addFormats from 'ajv-formats' -import { GtsModifiers, GtsStore, GtsRefValidationMode, createJsonEntity } from '@globaltypesystem/gts-ts' +import { GtsModifiers, GtsStore, GtsRefValidationMode, createJsonEntity, GTS_REF_KEYWORDS, X_GTS_TYPE_REF, X_GTS_INSTANCE_REF } from '@globaltypesystem/gts-ts' // XGtsRefValidator is not re-exported from the package index, so import it from // its published subpath module. import { XGtsRefValidator } from '@globaltypesystem/gts-ts/dist/x-gts-ref.js' @@ -25,34 +25,46 @@ const JSON_SCHEMA_ANNOTATION_KEYWORDS = new Set([ 'contentSchema' ]) +/** The reference keywords whose values name concrete entities (§9.6.1). */ +const ENTITY_REF_KEYWORDS = [X_GTS_TYPE_REF, X_GTS_INSTANCE_REF] + +/** True for the GTS reference keywords (`x-gts-type-ref`, `x-gts-instance-ref`, `x-gts-pattern-ref`). */ +function isGtsRefKeyword(key: string): boolean { + return (GTS_REF_KEYWORDS as readonly string[]).includes(key) +} + +/** True when `schema` declares any GTS reference keyword. */ +function hasGtsRefKeyword(schema: any): boolean { + return GTS_REF_KEYWORDS.some(key => schema[key] !== undefined) +} + /** - * Prepare a schema for Ajv instance validation by removing the `x-gts-ref` - * keyword, mirroring gts-ts's own `GtsStore.normalizeSchema` (the reference - * implementation strips `x-gts-ref` "so Ajv never sees the unknown keyword" - * and then prunes combinator branches that were `x-gts-ref`-only). + * Prepare a schema for Ajv instance validation by removing the GTS reference + * keywords (`x-gts-type-ref`, `x-gts-instance-ref`, `x-gts-pattern-ref`) and + * pruning combinator branches that were reference-keyword-only. * - * This matters for combinators: a branch like `{ "x-gts-ref": "…" }` becomes an + * This matters for combinators: a branch like `{ "x-gts-type-ref": "…" }` becomes an * empty schema once the keyword is dropped, i.e. always-true. Left in place, an * `oneOf` of two such branches matches *both* and Ajv spuriously reports * "must match exactly one schema in oneOf" for a value that is perfectly valid. - * The actual `x-gts-ref` assertions — including correct oneOf/anyOf branch - * counting — are enforced separately by `XGtsRefValidator` (§9.6). + * The actual reference assertions — including correct oneOf/anyOf branch + * counting — are enforced separately by the gts-ts store (§9.6). * * Unlike gts-ts's full `normalizeSchema`, this intentionally does NOT rewrite * `$id`/`$ref` (it leaves any `gts://` prefixes untouched) because the * registry's Ajv instance resolves those via its own `gts://`-aware loader. */ -function stripXGtsRefForAjv(schema: any): any { +function stripGtsRefKeywordsForAjv(schema: any): any { if (schema === null || typeof schema !== 'object') return schema - if (Array.isArray(schema)) return schema.map(stripXGtsRefForAjv) + if (Array.isArray(schema)) return schema.map(stripGtsRefKeywordsForAjv) const normalized: Record = {} for (const [key, value] of Object.entries(schema)) { - if (key === 'x-gts-ref') continue - normalized[key] = value && typeof value === 'object' ? stripXGtsRefForAjv(value) : value + if (isGtsRefKeyword(key)) continue + normalized[key] = value && typeof value === 'object' ? stripGtsRefKeywordsForAjv(value) : value } - // Drop combinator subschemas whose only assertion was `x-gts-ref`, so Ajv + // Drop combinator subschemas whose only assertion was a reference keyword, so Ajv // doesn't treat their remaining annotations as always-true branches. for (const combinator of ['oneOf', 'anyOf', 'allOf'] as const) { if (!Array.isArray(normalized[combinator])) continue @@ -62,8 +74,8 @@ function stripXGtsRefForAjv(schema: any): any { original && typeof original === 'object' && !Array.isArray(original) && - original['x-gts-ref'] !== undefined && - Object.keys(original).every(key => key === 'x-gts-ref' || JSON_SCHEMA_ANNOTATION_KEYWORDS.has(key)) + hasGtsRefKeyword(original) && + Object.keys(original).every(key => isGtsRefKeyword(key) || JSON_SCHEMA_ANNOTATION_KEYWORDS.has(key)) return !hasOnlyXGtsRefAssertion }) if (normalized[combinator].length === 0) delete normalized[combinator] @@ -88,6 +100,110 @@ function matchesFormat(format: Format, value: string): boolean { return false } +/** Ajv configured for GTS: the options, formats and keywords every instance shares. */ +function createGtsAjv(options: { validateSchema?: boolean; loadSchema?: (uri: string) => Promise } = {}): Ajv { + const ajv = new Ajv({ + strict: false, + allErrors: true, + verbose: true, + // Enable full JSON Schema support + discriminator: true, + allowUnionTypes: true, + // Provide data alongside errors + $data: true, + // Disable code generation for CSP compliance (VS Code webviews) + code: { source: false }, + ...options + }) + + // Add format validation (email, uri, date-time, etc.). Default "full" mode + // validates real value ranges (e.g. rejects month 13, offset +25:00). + addFormats(ajv) + + // Tighten the temporal formats to strict RFC 3339. ajv-formats' full-mode + // date/time splits on `/t|\s/i`, so it accepts a space instead of `T` + // (permitted by RFC 3339 §5.6's NOTE, but not by the ABNF grammar GTS + // requires). Compose the two *standard* ajv-formats validators so a value + // must satisfy BOTH: the "fast" grammar (strict `T` separator + mandatory + // time-offset) AND the "full" validator (real calendar/clock ranges). + for (const name of ['date', 'time', 'date-time'] as const) { + const fast = addFormats.get(name, 'fast') + const full = addFormats.get(name, 'full') + ajv.addFormat(name, { + type: 'string', + validate: (value: string) => matchesFormat(fast, value) && matchesFormat(full, value), + }) + } + + // GTS id formats emitted by schema generators (e.g. gts-rust) as annotations + // next to an `x-gts-type-ref`/`x-gts-instance-ref` that does the actual checking. Registered as + // accept-all — exactly how Ajv treated them before (ignored as unknown) — + // so each compile no longer logs an "unknown format" warning. + // Same for the sized-integer formats Rust's schemars emits (int32/int64 are + // already provided by ajv-formats). + for (const name of ['gts-instance-id', 'gts-type-id', 'int8', 'int16', 'uint', 'uint8', 'uint16', 'uint32', 'uint64']) { + ajv.addFormat(name, true) + } + + // Add custom schema loader that resolves GTS IDs from the registry + ajv.addKeyword({ + keyword: 'gtsRef', + schemaType: 'string', + }) + + return ajv +} + +// Meta-schema validation depends only on the Ajv options, never on the +// registry, so one instance — whose meta-schemas are compiled once — serves +// every schema. Each schema still compiles in its own fresh instance, and that +// instance used to compile the draft-07 meta-schema from scratch first: ~80% of +// all validation time, minutes on a workspace with 100K+ schemas. +let sharedMetaValidator: Ajv | null = null + +/** True for the json-schema.org URIs the registry's schema loader resolves to `true` (accept-all). */ +function isStandardJsonSchemaUri(uri: string): boolean { + const id = decodeGtsId(uri) + return id.startsWith('https://json-schema.org') || id.startsWith('http://json-schema.org') +} + +/** + * Meta-validate `schema` exactly as Ajv does before adding it to an instance, + * but against the shared meta-schemas; throws Ajv's own "schema is invalid" + * error. Returns false, without validating, when the schema names a `$schema` + * that only the per-instance loader can resolve. + */ +function metaValidateShared(schema: any): boolean { + if (!schema || typeof schema !== 'object') return true + const ajv = (sharedMetaValidator ??= createGtsAjv()) + const $schema = schema.$schema + if ($schema !== undefined) { + if (typeof $schema !== 'string') return false + let known: boolean + try { + known = Boolean(ajv.getSchema($schema)) + } catch { + return false + } + // Any other json-schema.org meta-schema (e.g. draft 2020-12, which plain + // Ajv doesn't bundle) is loaded as `true`, so the schema is valid against it. + if (!known) return isStandardJsonSchemaUri($schema) + } + if (!ajv.validateSchema(schema)) throw new Error(`schema is invalid: ${ajv.errorsText(ajv.errors)}`) + return true +} + +/** + * A meta-validated schema without its `$schema`, so the instance compiling it + * doesn't load and compile that meta-schema again. `$schema` has no other + * effect on compilation. + */ +function withoutMetaSchemaRef(schema: any): any { + if (!schema || typeof schema !== 'object' || !('$schema' in schema)) return schema + const { $schema: _metaSchema, ...rest } = schema + return rest +} + /** * Convert an XGtsRefValidator field path (dot/bracket notation, e.g. * `value.tags[0]`) into a JSON-Pointer-style instancePath (`/value/tags/0`) @@ -108,13 +224,13 @@ function normalizeToArray(content: any): any[] { /** * True when a resolved subschema describes a *scalar data value* rather than a * GTS reference: it constrains the value with a scalar `type`, a `format`, an - * `enum`, or a `const`, and does NOT declare it as a reference via `x-gts-ref` + * `enum`, or a `const`, and does NOT declare it as a reference via a GTS reference keyword * or `$ref`. Used to suppress the syntactic "GTS reference not found" check for * fields whose GTS-id-shaped value is just data (e.g. a `format: uuid` string). */ function isScalarDataField(sub: any): boolean { if (!sub || typeof sub !== 'object') return false - if (sub['x-gts-ref'] !== undefined || sub['$ref'] !== undefined) return false + if (hasGtsRefKeyword(sub) || sub['$ref'] !== undefined) return false const scalarTypes = new Set(['string', 'number', 'integer', 'boolean', 'null']) const t = sub.type const hasScalarType = typeof t === 'string' @@ -131,7 +247,7 @@ function isScalarDataField(sub: any): boolean { * to `id` changes their meaning, so this relation is followed **transitively**. * * `references.get(id)` — entity ids that merely *point at* `id` by GTS id (any - * GTS id field, `x-gts-ref`). They must be re-checked when `id` changes/renames, + * GTS id field, `x-gts-type-ref`/`x-gts-instance-ref`). They must be re-checked when `id` changes/renames, * but their own shape is unaffected, so this relation is applied **depth-1**. */ interface DependencyGraph { @@ -343,8 +459,8 @@ export class JsonRegistry { * * references (depth-1 — Type #2 id reference): * - any GTS id used anywhere in the entity's content (JsonEntity.gtsRefs), - * which already includes `x-gts-ref` targets (their concrete values are - * valid GTS ids). Wildcard `x-gts-ref` *patterns* are authoring + * which already includes entity-reference targets (their concrete values + * are valid GTS ids). Wildcard reference *patterns* are authoring * constraints; the concrete instance value that matches carries the real * id edge via gtsRefs, so patterns need no separate reverse edge. */ @@ -668,7 +784,7 @@ export class JsonRegistry { /** * Validation result of an entity that an instance references through an - * `x-gts-ref` field, evaluated within the chain of instances being validated + * entity-reference field, evaluated within the chain of instances being validated * (`chain`). Returns null for a reference cycle, which is treated as valid * (like gts-ts's visiting guard). * @@ -705,12 +821,12 @@ export class JsonRegistry { const cache = this.instanceValidators.byContent let validator = cache.get(schemaContent) if (!validator) { - // Strip `x-gts-ref` first (mirroring gts-ts's normalizeSchema) so Ajv never - // sees the unknown keyword and, crucially, so `x-gts-ref`-only combinator - // branches don't collapse into always-true schemas and make e.g. - // `oneOf: [{x-gts-ref}, {x-gts-ref}]` fail. The `x-gts-ref` assertions - // themselves are enforced by XGtsRefValidator. - validator = this.createAjvInstance().compileAsync(stripXGtsRefForAjv(schemaContent)) + // Strip the reference keywords first so Ajv never sees them and, + // crucially, so reference-only combinator branches don't collapse into + // always-true schemas and make e.g. + // `oneOf: [{x-gts-type-ref}, {x-gts-instance-ref}]` fail. The reference + // assertions themselves are enforced by the gts-ts store. + validator = this.compileSchema(stripGtsRefKeywordsForAjv(schemaContent)) cache.set(schemaContent, validator) validator.catch(() => { if (cache.get(schemaContent) === validator) cache.delete(schemaContent) }) } @@ -776,7 +892,7 @@ export class JsonRegistry { // Schema-aware skip: when the instance's schema types this field as a // scalar data value (a `format`/scalar `type`/`enum`/`const`, and NOT a - // reference via `x-gts-ref`/`$ref`), its GTS-id-shaped value is data, + // reference via a reference keyword/`$ref`), its GTS-id-shaped value is data, // not a reference, so its existence must not be checked. Configured // id/schema fields are never exempted (they are references by design). const lastSegment = ref.sourcePath.replace(/\[\d+\]/g, '').split('.').pop() || '' @@ -816,9 +932,8 @@ export class JsonRegistry { if (entity instanceof JsonSchema) { // Validate the schema itself (meta-validation) try { - const ajv = this.createAjvInstance() // Try to compile the schema to check if it's valid (async to support $ref resolution) - await ajv.compileAsync(entity.content) + await this.compileSchema(entity.content) } catch (error: any) { // If Ajv provides a detailed errors array, use it for precise paths if (Array.isArray(error?.errors) && error.errors.length > 0) { @@ -869,12 +984,14 @@ export class JsonRegistry { for (const msg of rawMessages) { const propMatch = msg.match(/^Property '([^']+)'/) const propPath = propMatch ? propMatch[1] : null - // A "Referenced x-gts-ref entity '' is invalid: ..." message is - // caused by a specific `x-gts-ref` node, not by the schema's own - // `$id`; anchor it to that node so the squiggle lands on the ref - // (e.g. line with `"x-gts-ref": ""`) rather than the document - // root. Everything else keeps the previous `$id` fallback. - const refMatch = propPath ? null : msg.match(/^Referenced x-gts-ref entity '([^']+)'/) + // A "Referenced type '' is invalid: ..." / " names '', + // which is invalid: ..." message is caused by a specific reference + // node, not by the schema's own `$id`; anchor it to that node so the + // squiggle lands on the ref (e.g. line with `"x-gts-type-ref": ""`) + // rather than the document root. Everything else keeps the previous + // `$id` fallback. + const refMatch = propPath ? null : msg.match(/^Referenced (?:type|entity) '([^']+)' is invalid| names '([^']+)', which is invalid/) + const refId = refMatch ? refMatch[1] ?? refMatch[2] : null // A "trait validation: ..." message comes from the OP#13 trait // completeness check; anchor it to the unmet `x-gts-traits-schema` // requirement (the specific `required` entry when we can name the @@ -890,8 +1007,8 @@ export class JsonRegistry { // The keyword-placement/unknown-keyword guards (§9.11, §9.7.1) report // the offending keyword's JSON path(s) inline as "… found at: a/b/c, // d/e/f". Anchor the diagnostic to the first such path so the squiggle - // lands on the actual keyword (e.g. properties/subject_type/x-gts-type) - // instead of the schema's `$id` — and so a self-referencing x-gts-ref + // lands on the actual keyword (e.g. properties/subject_type/x-gts-type-ref) + // instead of the schema's `$id` — and so a self-referencing reference keyword // elsewhere no longer appears to "own" an error from another field. const foundAtMatch = !propPath && !refMatch && !isTraitError && !regexPath ? msg.match(/ found at: (.+)$/) @@ -902,8 +1019,8 @@ export class JsonRegistry { instancePath = findSchemaPropertyPath(entity.content, propPath) params = { property: propPath } } else if (refMatch) { - instancePath = findXGtsRefPath(entity.content, refMatch[1]) - params = { refValue: refMatch[1] } + instancePath = findXGtsRefPath(entity.content, refId!) + params = { refValue: refId } } else if (isTraitError) { const traitMatch = msg.match(/required (?:property|trait) '([^']+)'/) const traitName = traitMatch ? traitMatch[1] : undefined @@ -913,7 +1030,7 @@ export class JsonRegistry { instancePath = regexPath } else if (foundAtMatch) { // The reported path uses `/` separators and `[n]` array indices - // (e.g. "allOf[0]/properties/x/x-gts-type"); the depth-guard also + // (e.g. "allOf[0]/properties/x/x-gts-type-ref"); the depth-guard also // appends " (nesting exceeds …)" which is not part of the path. const firstPath = foundAtMatch[1].split(', ')[0].replace(/ \(.*\)$/, '').trim() instancePath = fieldPathToInstancePath(firstPath) @@ -930,15 +1047,14 @@ export class JsonRegistry { } } - // §9.6: validate the `x-gts-ref` *pattern declarations* inside the schema - // (e.g. a literal that is neither a valid GTS ID/pattern nor a resolvable - // JSON Pointer). This checks the schema authoring, not an instance value. + // §9.6.3: validate the reference keyword *operands* inside the schema + // (e.g. a literal that is neither a valid GTS ID/pattern nor `/$id`). This checks the schema authoring, not an instance value. const refDeclErrors = new XGtsRefValidator().validateSchema(entity.content) for (const err of refDeclErrors) { validation.errors.push({ instancePath: fieldPathToInstancePath(err.fieldPath), schemaPath: '#', - keyword: 'x-gts-ref', + keyword: err.keyword ?? 'x-gts-schema', message: err.reason, params: { value: err.value, refPattern: err.refPattern } }) @@ -1040,47 +1156,53 @@ export class JsonRegistry { validation.errors.push(...formatted) } - // §9.6: `x-gts-ref` is an assertion keyword on instance string values + // §9.6: the reference keywords are assertions on instance string values // (the value must be a GTS ID matching the declared prefix/pattern). - // Ajv treats it as an unknown keyword and silently ignores it, so it is + // Ajv treats them as unknown keywords and silently ignores them, so they are // enforced explicitly here (mirrors gts-ts store.validateInstance). - // The validator gets the registry's gts-ts store so it can follow - // `gts://` `$ref`s (e.g. `allOf: [{ $ref: }]`) into parent - // schemas: without one, every such `$ref` is unresolvable and gts-ts - // reports "Cannot resolve $ref ... for x-gts-ref traversal" (it fails - // closed rather than skip the parent's constraints). It runs in - // any-present mode to collect the referenced ids for the any-valid - // check below; its own "not found" errors repeat the gtsRefs registry - // check above and are dropped. - const refValidator = new XGtsRefValidator(this.getGtsStore(), GtsRefValidationMode.AnyPresent) - const xGtsRefErrors = refValidator.validateInstance(entity.content, schema.content, '', undefined, entity.id) - for (const err of xGtsRefErrors) { - if (/^Referenced entity '.+' not found in registry$/.test(err.reason)) continue + // Since gts-ts 0.11 XGtsRefValidator is registry-less, so the check is + // delegated to the registry's gts-ts store, which follows `gts://` + // `$ref`s into parent schemas. It runs in any-present mode (the + // any-valid check is done below); only reference-keyword issues are + // kept (Ajv already reported the rest), and "not found" errors repeat + // the gtsRefs registry check above and are dropped. + const store = this.getGtsStore() + const storeResult = store.get(entity.id) + ? store.validateInstance(entity.id, GtsRefValidationMode.AnyPresent) + : null + for (const issue of storeResult?.errors ?? []) { + if (!issue.keyword.startsWith('x-gts-')) continue + if (/not found/.test(issue.message)) continue validation.errors.push({ - instancePath: fieldPathToInstancePath(err.fieldPath), - schemaPath: '#', - keyword: 'x-gts-ref', - message: err.reason, - params: { value: err.value, refPattern: err.refPattern } + instancePath: issue.instancePath, + schemaPath: issue.schemaPath || '#', + keyword: issue.keyword, + message: issue.message, + params: issue.params }) } // §9.6 `gts-ref-validation=any-valid` (the spec's default policy): an - // entity referenced through an `x-gts-ref` field must itself be valid, + // entity referenced through an entity-reference field must itself be valid, // transitively, or the referencing instance is invalid too. - for (const referencedId of refValidator.getReferencedIds()) { + const checkedIds = new Set() + for (const { id: referencedId, sourcePath } of entity.gtsRefs ?? []) { + if (checkedIds.has(referencedId)) continue + const fieldSchema = getInstanceFieldSubschema(schema.content, sourcePath) + const keyword = ENTITY_REF_KEYWORDS.find(key => fieldSchema?.[key] !== undefined) + if (!keyword) continue + checkedIds.add(referencedId) const dependency = this.jsonObjs.get(referencedId) ?? this.jsonSchemas.get(referencedId) if (!dependency || dependency === entity) continue const dependencyValidation = await this.referencedEntityValidation(dependency, chain) const firstError = dependencyValidation?.errors[0] if (!firstError) continue - const sourcePath = entity.gtsRefs?.find(ref => ref.id === referencedId)?.sourcePath validation.errors.push({ instancePath: sourcePath && sourcePath !== 'root' ? '/' + sourcePath.replace(/\./g, '/').replace(/\[(\d+)\]/g, '/$1') : '/', schemaPath: '#', - keyword: 'x-gts-ref', + keyword, message: `Referenced entity '${referencedId}' is invalid: ${firstError.message}`, params: { value: referencedId } }) @@ -1169,30 +1291,37 @@ export class JsonRegistry { }) } + /** + * Compile `schema` with an Ajv instance that resolves GTS `$ref`s from the + * registry. Rejects exactly like `compileAsync` did when each fresh instance + * meta-validated schemas itself, but the meta-validation runs against the + * shared, compiled-once meta-schemas (see metaValidateShared). Schemas naming + * a `$schema` the shared instance doesn't know keep the full per-instance path. + */ + private async compileSchema(schema: any): Promise { + if (!metaValidateShared(schema)) return this.createAjvInstance(false).compileAsync(schema) + return this.createAjvInstance(true).compileAsync(withoutMetaSchemaRef(schema)) + } + /** * Create an Ajv instance with custom schema resolver. * This resolver handles GTS ID references and supports all JSON Schema features. + * + * With `sharedMeta`, the instance doesn't meta-validate schemas itself: the + * caller validated the root (compileSchema) and the loader validates every + * schema it loads, both against the shared meta-schemas. */ - private createAjvInstance(): Ajv { + private createAjvInstance(sharedMeta: boolean): Ajv { const registry = this - const ajv = new Ajv({ - strict: false, - allErrors: true, - verbose: true, - // Enable full JSON Schema support - discriminator: true, - allowUnionTypes: true, - // Provide data alongside errors - $data: true, - // Disable code generation for CSP compliance (VS Code webviews) - code: { source: false }, + return createGtsAjv({ + validateSchema: !sharedMeta, // Custom schema loader for GTS ID resolution loadSchema: async (uri: string): Promise => { const schemaId = decodeGtsId(uri) // Allow standard JSON Schema references - if (schemaId.startsWith('https://json-schema.org') || schemaId.startsWith('http://json-schema.org')) { + if (isStandardJsonSchemaUri(uri)) { return true } @@ -1202,46 +1331,10 @@ export class JsonRegistry { // Show human-readable error message with decoded URI throw new Error(`Schema not found for $ref: ${schemaId}`) } + if (sharedMeta && metaValidateShared(schema.content)) return withoutMetaSchemaRef(schema.content) return schema.content } }) - - // Add format validation (email, uri, date-time, etc.). Default "full" mode - // validates real value ranges (e.g. rejects month 13, offset +25:00). - addFormats(ajv) - - // Tighten the temporal formats to strict RFC 3339. ajv-formats' full-mode - // date/time splits on `/t|\s/i`, so it accepts a space instead of `T` - // (permitted by RFC 3339 §5.6's NOTE, but not by the ABNF grammar GTS - // requires). Compose the two *standard* ajv-formats validators so a value - // must satisfy BOTH: the "fast" grammar (strict `T` separator + mandatory - // time-offset) AND the "full" validator (real calendar/clock ranges). - for (const name of ['date', 'time', 'date-time'] as const) { - const fast = addFormats.get(name, 'fast') - const full = addFormats.get(name, 'full') - ajv.addFormat(name, { - type: 'string', - validate: (value: string) => matchesFormat(fast, value) && matchesFormat(full, value), - }) - } - - // GTS id formats emitted by schema generators (e.g. gts-rust) as annotations - // next to an `x-gts-ref` that does the actual checking. Registered as - // accept-all — exactly how Ajv treated them before (ignored as unknown) — - // so each compile no longer logs an "unknown format" warning. - // Same for the sized-integer formats Rust's schemars emits (int32/int64 are - // already provided by ajv-formats). - for (const name of ['gts-instance-id', 'gts-type-id', 'int8', 'int16', 'uint', 'uint8', 'uint16', 'uint32', 'uint64']) { - ajv.addFormat(name, true) - } - - // Add custom schema loader that resolves GTS IDs from the registry - ajv.addKeyword({ - keyword: 'gtsRef', - schemaType: 'string', - }) - - return ajv } /** diff --git a/packages/shared/src/schemaParser.ts b/packages/shared/src/schemaParser.ts index 884067e..408b60a 100644 --- a/packages/shared/src/schemaParser.ts +++ b/packages/shared/src/schemaParser.ts @@ -1,4 +1,5 @@ import { isGtsId, isGtsType as checkGtsType, isGtsObj as checkGtsObj, normalizeGtsId } from '@gts/shared' +import { GTS_REF_KEYWORDS, type GtsRefKeyword } from '@globaltypesystem/gts-ts' export interface PropertyInfo { name: string @@ -93,7 +94,7 @@ export function parseJsonToProperties(data: any, name = 'root'): PropertyInfo[] } } - // For plain nested objects, use recursive parsing without any x-gts-ref special handling + // For plain nested objects, use recursive parsing without any reference-keyword special handling return { name: key, type: 'object', @@ -158,9 +159,10 @@ export function parseSchemaToProperties(schema: any): PropertyInfo[] { description = description ? `${description}\n${patDescription}` : patDescription } - if (prop['x-gts-ref']) { - const gtsTypeDescription = `GTS Type: ${prop['x-gts-ref']}` - description = description ? `${description}\n${gtsTypeDescription}` : gtsTypeDescription + for (const [keyword, label] of [['x-gts-type-ref', 'GTS Type'], ['x-gts-instance-ref', 'GTS Instance'], ['x-gts-pattern-ref', 'GTS Pattern']] as const) { + if (!prop[keyword]) continue + const refDescription = `${label}: ${prop[keyword]}` + description = description ? `${description}\n${refDescription}` : refDescription isGtsType = true } @@ -402,11 +404,12 @@ export function findSchemaPropertyPath(content: any, propPath: string): string | } /** - * Locate the JSON Pointer instancePath of the `x-gts-ref` keyword whose value + * Locate the JSON Pointer instancePath of the reference keyword + * (`x-gts-type-ref` / `x-gts-instance-ref` / `x-gts-pattern-ref`) whose value * equals `refValue` (any `gts://` prefix ignored), searching every position in - * the schema document. Used to anchor a "Referenced x-gts-ref entity '' is - * invalid" diagnostic to the offending `x-gts-ref` node rather than the - * document root. Returns null when no matching `x-gts-ref` is present. + * the schema document. Used to anchor a "Referenced type '' is invalid" + * diagnostic to the offending reference node rather than the document root. + * Returns null when no matching reference keyword is present. */ export function findXGtsRefPath(content: any, refValue: string): string | null { const target = normalizeGtsId(refValue) @@ -422,7 +425,7 @@ export function findXGtsRefPath(content: any, refValue: string): string | null { } for (const [key, value] of Object.entries(node)) { const childPath = `${currentPath}/${key}` - if (key === 'x-gts-ref' && typeof value === 'string' && normalizeGtsId(value) === target) { + if (GTS_REF_KEYWORDS.includes(key as GtsRefKeyword) && typeof value === 'string' && normalizeGtsId(value) === target) { return childPath } const res = walk(value, childPath) From 4f221931b98f7383b8105fc7b532637f8051d04c Mon Sep 17 00:00:00 2001 From: Artifizer Date: Tue, 27 Oct 2026 22:20:44 +0200 Subject: [PATCH 4/6] perf(shared): optimize entity indexing and keep gts-ts store in sync incrementally - Fast-path prefix check in validateNormalizedGtsId to avoid throwing and catching errors in gtsValidateID for non-GTS strings during bulk file scans. - Avoid redundant string typecheck during JsonEntity declared reference traversal. - Mirror gts-ts GtsStore incrementally via GtsStoreMirror across entity add/remove instead of tearing down and re-registering every entity on each change. - Memoize matchingIds pattern queries until store contents change. - Skip re-indexing and invalidating memoized results when file contents are unchanged. - Skip non-existent entity checks for x-gts-pattern-ref operands and x-gts-traits values in schemas. Signed-off-by: Artifizer --- packages/shared/src/entities.ts | 38 ++--- packages/shared/src/registry.ts | 236 +++++++++++++++++++++++++------- 2 files changed, 203 insertions(+), 71 deletions(-) diff --git a/packages/shared/src/entities.ts b/packages/shared/src/entities.ts index 2677eff..48a3341 100644 --- a/packages/shared/src/entities.ts +++ b/packages/shared/src/entities.ts @@ -50,10 +50,21 @@ export function normalizeGtsId(id: string): string { * Does NOT accept wildcard patterns — use {@link isGtsIdOrPattern} for that. */ export function isGtsId(value: string): boolean { - if (!value || typeof value !== 'string') return false + const result = validateNormalizedGtsId(value) + return !!result && result.ok && !result.is_wildcard +} + +/** + * gts-ts validation of the normalized value, or null when it can't be a GTS id + * at all. gts-ts reports a missing "gts." prefix by throwing (and catching) an + * error, which made checking every string of every scanned file - nearly all of + * them plain words or URLs - the dominant cost of indexing a large workspace. + */ +function validateNormalizedGtsId(value: string): GtsValidationResult | null { + if (!value || typeof value !== 'string') return null const normalized = normalizeGtsId(value) - const result = gtsValidateID(normalized) - return result.ok && !result.is_wildcard + if (!normalized.startsWith('gts.')) return null + return gtsValidateID(normalized) } /** @@ -91,10 +102,7 @@ export function isGtsObj(value: string): boolean { * isGtsIdOrPattern('not-a-gts-id') // false */ export function isGtsIdOrPattern(value: string): boolean { - if (!value || typeof value !== 'string') return false - const normalized = normalizeGtsId(value) - const result = gtsValidateID(normalized) - return result.ok + return validateNormalizedGtsId(value)?.ok === true } /** @@ -110,10 +118,8 @@ export function isGtsIdOrPattern(value: string): boolean { * isGtsPattern('gts.x.core.events.type.v1~') // false (full ID, not a pattern) */ export function isGtsPattern(value: string): boolean { - if (!value || typeof value !== 'string') return false - const normalized = normalizeGtsId(value) - const result = gtsValidateID(normalized) - return result.ok && (result.is_wildcard === true) + const result = validateNormalizedGtsId(value) + return !!result && result.ok && (result.is_wildcard === true) } /** @@ -514,15 +520,9 @@ export class JsonEntity { return } if (typeof node === 'object') { + // A string value is checked (once) by the recursive call itself. Object.entries(node).forEach(([k, v]) => { - const nextPath = currentPath ? `${currentPath}.${k}` : k - if (typeof v === 'string') { - const normalized = normalizeGtsId(v) - if (isGtsId(normalized)) { - found.push({ id: normalized, sourcePath: nextPath }) - } - } - walk(v, nextPath) + walk(v, currentPath ? `${currentPath}.${k}` : k) }) } } diff --git a/packages/shared/src/registry.ts b/packages/shared/src/registry.ts index 284ccf1..0cc71fe 100644 --- a/packages/shared/src/registry.ts +++ b/packages/shared/src/registry.ts @@ -4,7 +4,7 @@ import { isYamlFileName } from './parse.js' import { findSchemaPropertyPath, findXGtsRefPath, findTraitRequiredPath, findRegexPatternPath, getInstanceFieldSubschema } from './schemaParser.js' import Ajv, { type ValidateFunction, type ErrorObject } from 'ajv' import addFormats from 'ajv-formats' -import { GtsModifiers, GtsStore, GtsRefValidationMode, createJsonEntity, GTS_REF_KEYWORDS, X_GTS_TYPE_REF, X_GTS_INSTANCE_REF } from '@globaltypesystem/gts-ts' +import { GtsModifiers, GtsStore, GtsRefValidationMode, createJsonEntity, GTS_REF_KEYWORDS, X_GTS_TYPE_REF, X_GTS_INSTANCE_REF, X_GTS_PATTERN_REF } from '@globaltypesystem/gts-ts' // XGtsRefValidator is not re-exported from the package index, so import it from // its published subpath module. import { XGtsRefValidator } from '@globaltypesystem/gts-ts/dist/x-gts-ref.js' @@ -25,6 +25,82 @@ const JSON_SCHEMA_ANNOTATION_KEYWORDS = new Set([ 'contentSchema' ]) +/** + * A gts-ts GtsStore holding the registry's winning entity for each id, plus + * what registering them reported. + */ +class GtsStoreMirror { + readonly store = new GtsStore() + // Modifier-declaration errors captured while registering schemas into the + // gts-ts store (register() throws these per §9.11.1), keyed by schema id. + readonly declErrors = new Map() + // Registry id -> id the entity is registered under in the store, so a + // removed/replaced entity can be unregistered without re-extracting its id. + private readonly storeIds = new Map() + // Ids matching a wildcard pattern, valid until the store's entity set changes. + private readonly matchCache = new Map() + + constructor() { + // gts-ts resolves a wildcard reference operand by testing every registered + // id under the pattern's literal prefix, on every validation that reaches + // it; with `gts.x.*`-style operands and 100K+ entities that dominated + // workspace validation. The result only depends on the registered ids, and + // this mirror performs every mutation of the store, so it is memoized here + // until the next add/remove. (Private gts-ts method: if it is renamed, the + // store simply runs unmemoized.) + const store = this.store as unknown as { matchingIds?: (pattern: string) => string[] } + const matchingIds = store.matchingIds + if (typeof matchingIds !== 'function') return + store.matchingIds = (pattern: string) => { + let ids = this.matchCache.get(pattern) + if (!ids) { + ids = matchingIds.call(this.store, pattern) + this.matchCache.set(pattern, ids) + } + return ids + } + } + + /** Register `entity`, replacing whatever was registered for its id. */ + add(entity: JsonEntity): void { + this.matchCache.clear() + this.remove(entity.id) + const storeEntity = createJsonEntity(entity.content) + try { + this.store.register(storeEntity) + this.storeIds.set(entity.id, storeEntity.id) + } catch (err) { + // A schema register() rejects (§9.11.1 declaration error) is reported by + // validateEntity; an instance that can't be registered (e.g. malformed or + // UUID-less id) is reported through the normal instance-validation path. + if (entity instanceof JsonSchema) { + this.declErrors.set(entity.id, err instanceof Error ? err.message : String(err)) + } + } + } + + /** Unregister whatever is registered for registry id `id`. */ + remove(id: string): void { + this.declErrors.delete(id) + const storeId = this.storeIds.get(id) + if (storeId === undefined) return + this.storeIds.delete(id) + this.matchCache.clear() + this.store.unregister(storeId) + } +} + +/** True when two file contents (parsed values, or raw text for unparsable files) are identical. */ +function sameFileContent(a: any, b: any): boolean { + if (a === b) return true + if (typeof a === 'string' || typeof b === 'string') return false + try { + return JSON.stringify(a) === JSON.stringify(b) + } catch { + return false + } +} + /** The reference keywords whose values name concrete entities (§9.6.1). */ const ENTITY_REF_KEYWORDS = [X_GTS_TYPE_REF, X_GTS_INSTANCE_REF] @@ -287,14 +363,11 @@ export class JsonRegistry { // Default file to open/select when displaying layout private defaultFilePath: string | null - // Lazily-built gts-ts store mirroring the registry's schemas, used to - // delegate GTS-specific schema validation (derivation §9.12, traits OP#13, - // modifier declaration/placement §9.11) that plain Ajv cannot express. It is - // invalidated whenever schemas change and rebuilt on next demand. - private gtsStore: GtsStore | null = null - // Modifier-declaration errors captured while registering schemas into the - // gts-ts store (register() throws these per §9.11.1), keyed by schema id. - private gtsStoreDeclErrors: Map = new Map() + // Lazily-built gts-ts store mirroring the registry's schemas and instances, + // used to delegate GTS-specific schema validation (derivation §9.12, traits + // OP#13, modifier declaration/placement §9.11) that plain Ajv cannot + // express. Once built it is kept in sync as entities are indexed/removed. + private gtsMirror: GtsStoreMirror | null = null // Cached reverse-dependency graph used by getDependentFilePaths(). Rebuilt // lazily and invalidated whenever the entity set changes (any indexFile / @@ -347,8 +420,7 @@ export class JsonRegistry { /** Drop the cached gts-ts store so it is rebuilt from current schemas on next use. */ private invalidateGtsStore(): void { - this.gtsStore = null - this.gtsStoreDeclErrors.clear() + this.gtsMirror = null } /** @@ -361,58 +433,93 @@ export class JsonRegistry { * rather than aborting the whole build; instances that fail to register are * skipped (they surface through the normal instance-validation path). */ - private getGtsStore(): GtsStore { - if (this.gtsStore) return this.gtsStore - const store = new GtsStore() - this.gtsStoreDeclErrors.clear() - for (const schema of this.jsonSchemas.values()) { - try { - store.register(createJsonEntity(schema.content)) - } catch (err) { - this.gtsStoreDeclErrors.set(schema.id, err instanceof Error ? err.message : String(err)) - } + private getGtsMirror(): GtsStoreMirror { + if (!this.gtsMirror) { + const mirror = new GtsStoreMirror() + for (const schema of this.jsonSchemas.values()) mirror.add(schema) + for (const obj of this.jsonObjs.values()) mirror.add(obj) + this.gtsMirror = mirror } - for (const obj of this.jsonObjs.values()) { - try { - store.register(createJsonEntity(obj.content)) - } catch { - // Instance couldn't be registered (e.g. malformed/UUID-less id); it is - // reported through validateEntity's normal path, not via the store. + return this.gtsMirror + } + + private getGtsStore(): GtsStore { + return this.getGtsMirror().store + } + + /** + * Build the gts-ts store ahead of validation, calling `tick` between + * entities so the caller can yield: built on demand, it blocks for seconds + * on a workspace with 100K+ entities. The store is only adopted when the + * registry didn't change while it was being built. + */ + async prepareGtsStore(tick: () => Promise): Promise { + for (let attempt = 0; attempt < 2 && !this.gtsMirror; attempt++) { + const generation = this.contentGeneration + const mirror = new GtsStoreMirror() + for (const entities of [Array.from(this.jsonSchemas.values()), Array.from(this.jsonObjs.values())]) { + for (const entity of entities) { + mirror.add(entity) + await tick() + } } + if (!this.gtsMirror && this.contentGeneration === generation) this.gtsMirror = mirror } - this.gtsStore = store - return store } /** - * Invalidate a file and remove its JsonFile and associated records from the registry. + * Mirror a newly winning entity (the one `jsonSchemas`/`jsonObjs` now hold + * for its id) into the gts-ts store, if it has been built. Updating it in + * place instead of dropping it matters on large workspaces: rebuilding it + * re-registers every entity, which took seconds and ran on every edit. */ - invalidateFile(path: string): void { - // Any schema set change invalidates the derived gts-ts store and the - // reverse-dependency graph (both are rebuilt lazily on next use). - this.invalidateGtsStore() + private addToGtsStore(entity: JsonEntity): void { + this.gtsMirror?.add(entity) + } + + /** Drop the entity registered for registry id `id` from the gts-ts store, if any. */ + private removeFromGtsStore(id: string): void { + this.gtsMirror?.remove(id) + } + + /** Record that the indexed content changed: derived caches and memoized results are stale. */ + private markContentChanged(): void { this.depGraph = null this.contentGeneration++ - if (this.jsonFiles.has(path)) { - this.jsonFiles.delete(path) - } - if (this.invalidFiles.has(path)) { - this.invalidFiles.delete(path) - } - if (this.jsonFileObjs.has(path)) { - for (const obj of this.jsonFileObjs.get(path)!) { + } + + /** + * Invalidate a file and remove its JsonFile and associated records from the registry. + * Returns false (and leaves every cache intact) when nothing was indexed for it. + */ + invalidateFile(path: string): boolean { + const objs = this.jsonFileObjs.get(path) + const schemas = this.jsonFileSchemas.get(path) + const hadRecords = this.jsonFiles.has(path) || this.invalidFiles.has(path) || !!objs?.length || !!schemas?.length + if (!hadRecords) return false + // Any entity set change invalidates the reverse-dependency graph (rebuilt + // lazily) and every memoized validation result; the gts-ts store is + // updated in place. + this.markContentChanged() + this.jsonFiles.delete(path) + this.invalidFiles.delete(path) + if (objs) { + for (const obj of objs) { this.jsonObjs.delete(obj.id) + this.removeFromGtsStore(obj.id) } this.jsonFileObjs.delete(path) this.jsonFileObjs.set(path, []) } - if (this.jsonFileSchemas.has(path)) { - for (const schema of this.jsonFileSchemas.get(path)!) { + if (schemas) { + for (const schema of schemas) { this.jsonSchemas.delete(schema.id) + this.removeFromGtsStore(schema.id) } this.jsonFileSchemas.delete(path) this.jsonFileSchemas.set(path, []) } + return true } /** @@ -673,9 +780,15 @@ export class JsonRegistry { * Process a file and store its entities if they are GTS entities. * This is a helper used by both scanFile and ingestFiles. */ - private processFileContent(path: string, name: string, content: any, cfg: GtsConfig): void { + private processFileContent(path: string, name: string, content: any, cfg: GtsConfig): boolean { + // Re-indexing identical content (opening a file, saving without changes, + // revalidating open documents) must not count as a change: a change throws + // away every memoized validation result and compiled validator. + const previous = this.jsonFiles.get(path) ?? this.invalidFiles.get(path) + if (previous && previous.name === name && sameFileContent(previous.content, content)) return false + // Cleanup any existing records for this file - this.invalidateFile(path) + let changed = this.invalidateFile(path) // Create JsonFile const jsonFile = new JsonFile(path, name, content) @@ -686,7 +799,8 @@ export class JsonRegistry { if (jsonFile.validation && jsonFile.validation.errors.length > 0) { // Store JsonFile if it's invalid to show it in the UI this.invalidFiles.set(path, jsonFile) - return + this.markContentChanged() + return true } // Use the file's parsed content: JsonFile parses raw string input (JSON/JSONC @@ -700,6 +814,10 @@ export class JsonRegistry { const hasSelectedId = entity?.selectedEntityIdField !== undefined || entity?.selectedSchemaIdField !== undefined if (entity && (!requireSelectedId || hasSelectedId) && entity.isGtsEntity()) { hasGtsEntities = true + if (!changed) { + this.markContentChanged() + changed = true + } if (entity instanceof JsonSchema) { this.jsonSchemas.set(entity.id, entity) this.jsonFileSchemas.set(path, [...this.jsonFileSchemas.get(path) || [], entity]) @@ -707,6 +825,7 @@ export class JsonRegistry { this.jsonObjs.set(entity.id, entity as JsonObj) this.jsonFileObjs.set(path, [...this.jsonFileObjs.get(path) || [], entity as JsonObj]) } + this.addToGtsStore(entity) } } @@ -733,6 +852,7 @@ export class JsonRegistry { if (hasGtsEntities && !this.jsonFiles.has(path)) { this.jsonFiles.set(path, jsonFile) } + return changed } /** @@ -741,16 +861,18 @@ export class JsonRegistry { * This is the incremental counterpart to ingestFiles: it upserts one file's * entities without re-processing the whole workspace and without running Ajv. * Use it to keep a long-lived registry in sync with editor/file changes. + * Returns whether the indexed state changed (false for identical content). */ - indexFile(path: string, name: string, content: any, cfg: GtsConfig): void { + indexFile(path: string, name: string, content: any, cfg: GtsConfig): boolean { // Skip files in the .gts-viewer directory (cross-platform, browser-safe) if (/(^|[\\\/])\.gts-viewer[\\\/]/.test(path)) { - return + return false } try { - this.processFileContent(path, name, content, getGtsConfig(cfg)) + return this.processFileContent(path, name, content, getGtsConfig(cfg)) } catch (error) { console.error(`Failed to index file ${path}:`, error) + return true } } @@ -890,6 +1012,16 @@ export class JsonRegistry { continue } + // In a schema, an `x-gts-pattern-ref` operand only constrains what + // values look like (§9.6: it names no entity, and gts-ts never looks it + // up), and `x-gts-traits` values are trait data, checked against the + // trait schema by gts-ts. Neither has to exist. (`x-gts-type-ref` / + // `x-gts-instance-ref` operands are checked by gts-ts as well.) + const segments = ref.sourcePath.replace(/\[\d+\]/g, '').split('.') + if (entity instanceof JsonSchema && (segments[segments.length - 1] === X_GTS_PATTERN_REF || segments.includes('x-gts-traits'))) { + continue + } + // Schema-aware skip: when the instance's schema types this field as a // scalar data value (a `format`/scalar `type`/`enum`/`const`, and NOT a // reference via a reference keyword/`$ref`), its GTS-id-shaped value is data, @@ -965,8 +1097,8 @@ export class JsonRegistry { // - §9.11 misplaced document-level keywords + final-base-in-chain guard // - §9.12 (OP#12) derivation constraint compatibility with the parent // - §9.7 (OP#13) trait schema/value validation across the chain - const store = this.getGtsStore() - const declError = this.gtsStoreDeclErrors.get(entity.id) + const { store, declErrors } = this.getGtsMirror() + const declError = declErrors.get(entity.id) if (declError) { // register() rejected this schema outright (§9.11.1); surface it and // skip chain validation (the schema isn't in the store). From 4da4cdb76fca18cb3d361c388f69b8843be951db Mon Sep 17 00:00:00 2001 From: Artifizer Date: Tue, 27 Oct 2026 22:21:25 +0200 Subject: [PATCH 5/6] feat(vscode): add folder rescan and time-slice background scans - Implement cooperative time slicing via TimeSlicer and adaptive PublishSchedule to keep the extension host responsive during large scans and validations. - Add "Rescan Folder" command (gts-kit.rescanFolder) to the GTS file explorer context menu with row-level progress indication. - Add status row and progress reporting at the top of the GTS file explorer during scanning and validation passes. - Implement incremental revalidation (revalidateFiles) for rescanned folders and their dependents. - Optimize canonical path resolution with reverse real-path index and cache tree element sorting. Signed-off-by: Artifizer --- apps/vscode-extension/package.json | 22 ++ apps/vscode-extension/src/extension.ts | 337 ++++++++++++++++----- apps/vscode-extension/src/gtsExplorer.ts | 299 +++++++++++++----- apps/vscode-extension/src/registryStore.ts | 57 +++- apps/vscode-extension/src/timeSlice.ts | 94 ++++++ apps/vscode-extension/src/validation.ts | 227 +++++++++++--- 6 files changed, 835 insertions(+), 201 deletions(-) create mode 100644 apps/vscode-extension/src/timeSlice.ts diff --git a/apps/vscode-extension/package.json b/apps/vscode-extension/package.json index aa193d4..39481dd 100644 --- a/apps/vscode-extension/package.json +++ b/apps/vscode-extension/package.json @@ -59,6 +59,12 @@ "category": "GTS", "icon": "$(refresh)" }, + { + "command": "gts-kit.rescanFolder", + "title": "Rescan Folder", + "category": "GTS", + "icon": "$(refresh)" + }, { "command": "gts-kit.openToSide", "title": "Open to the Side" @@ -137,7 +143,23 @@ "when": "resourceExtname =~ /\\.(json|jsonc|gts)$/" } ], + "commandPalette": [ + { + "command": "gts-kit.rescanFolder", + "when": "false" + } + ], "view/item/context": [ + { + "command": "gts-kit.rescanFolder", + "group": "inline", + "when": "view == gts-kit.fileExplorer && viewItem == gtsFolder" + }, + { + "command": "gts-kit.rescanFolder", + "group": "navigation@10", + "when": "view == gts-kit.fileExplorer && viewItem == gtsFolder" + }, { "command": "gts-kit.openToSide", "group": "navigation@10", diff --git a/apps/vscode-extension/src/extension.ts b/apps/vscode-extension/src/extension.ts index 84b6e85..063fbab 100644 --- a/apps/vscode-extension/src/extension.ts +++ b/apps/vscode-extension/src/extension.ts @@ -4,13 +4,14 @@ import * as fs from 'fs' import { parseGtsFileContent, DEFAULT_GTS_CONFIG } from '@gts/shared' import type { EntityValidationDto, ObjValidationDto, InvalidFileValidationDto, ValidationRelayPayload } from '@gts/shared' import { setLastScanFiles } from './scanStore' -import { rebuildRegistry, indexFile as indexFileInRegistry, removeFile as removeFileFromRegistry, getRegistry } from './registryStore' +import { rebuildRegistry, indexFile as indexFileInRegistry, indexFiles, removeFile as removeFileFromRegistry, getRegistry, getPathsWithMalformedIds, type RegistryFileInput } from './registryStore' import { getWorkspaceIgnore, resetWorkspaceIgnore, isGitIgnored, type FolderIgnore } from './gitignore' import { WorkspaceLayoutStorage } from './storage' -import { initValidation, resetValidationDiagnostics, validateOpenDocument, validateWorkspaceInBackground, revalidateDependents, onValidationCompleted, forgetValidationState } from './validation' +import { initValidation, resetValidationDiagnostics, validateOpenDocument, validateWorkspaceInBackground, revalidateDependents, revalidateFiles, onValidationCompleted, forgetValidationState, whenWorkspaceValidationIdle } from './validation' import { isGtsCandidateFile, isIndexableGtsDocument } from './helpers' import { GtsLinkProvider } from './linkProvider' -import { registerGtsExplorer, type GtsExplorer } from './gtsExplorer' +import { registerGtsExplorer, type GtsExplorer, type GtsTreeElement, type FolderProgress } from './gtsExplorer' +import { TimeSlicer, CancelledError, filterSliced, yieldToEventLoop } from './timeSlice' import type { LayoutSaveRequest, LayoutTarget, LayoutSnapshot } from '@gts/layout-storage' // Glob used for all GTS workspace scans and the on-disk file watcher. @@ -113,33 +114,38 @@ async function findGtsFiles(ignores: Map, baseExclude: str // once and let the most-recently-scanned/edited path win (so an open file, which // is scanned first, stays canonical and gets its in-editor diagnostics). const realPathIndex = new Map() +// Reverse of realPathIndex (indexed path -> real path), so forgetting a path is +// a lookup instead of a scan of every indexed file. +const realPathOfIndexedPath = new Map() + +function setCanonicalPath(real: string, fsPath: string): void { + const previous = realPathIndex.get(real) + if (previous !== undefined) realPathOfIndexedPath.delete(previous) + realPathIndex.set(real, fsPath) + realPathOfIndexedPath.set(fsPath, real) +} + +function clearCanonicalPaths(): void { + realPathIndex.clear() + realPathOfIndexedPath.clear() +} /** Resolve a path to its canonical real path; fall back to the input on error. */ function resolveRealPath(fsPath: string): string { try { return fs.realpathSync.native(fsPath) } catch { return fsPath } } -/** Drop any canonical-path entries that point at `fsPath` (on delete/rename). */ -function forgetIndexedPath(fsPath: string): void { - for (const [real, p] of realPathIndex) { - if (p === fsPath) realPathIndex.delete(real) - } +/** Async resolveRealPath, for bulk scans (the sync one blocked the extension host for seconds). */ +async function resolveRealPathAsync(fsPath: string): Promise { + try { return await fs.promises.realpath(fsPath) } catch { return fsPath } } -/** - * Keep only one URI per physical file, recording the canonical path chosen. - * First-seen wins, so callers should pass higher-priority paths (open files) - * first. Duplicates reached through other symlinks are dropped. - */ -function dedupeUrisByRealPath(uris: vscode.Uri[]): vscode.Uri[] { - const out: vscode.Uri[] = [] - for (const uri of uris) { - const real = resolveRealPath(uri.fsPath) - if (realPathIndex.has(real)) continue - realPathIndex.set(real, uri.fsPath) - out.push(uri) - } - return out +/** Drop the canonical-path entry that points at `fsPath` (on delete/rename). */ +function forgetIndexedPath(fsPath: string): void { + const real = realPathOfIndexedPath.get(fsPath) + if (real === undefined) return + realPathOfIndexedPath.delete(fsPath) + if (realPathIndex.get(real) === fsPath) realPathIndex.delete(real) } /** @@ -155,7 +161,7 @@ function claimCanonicalPath(fsPath: string): void { removeFileFromRegistry(existing) forgetIndexedPath(existing) } - realPathIndex.set(real, fsPath) + setCanonicalPath(real, fsPath) } let viewerPanel: vscode.WebviewPanel | null = null @@ -175,6 +181,57 @@ function enqueueScan(operation: () => Promise): Promise { return run } +// Bumped by every full scan request. Queued and running scans (full or folder) +// check it at each time slice and stop once a newer full scan was requested, +// so a Rescan click takes over right away instead of waiting behind them. +let scanGeneration = 0 + +/** + * Run a scan step in the scan queue. With a `status`, the GTS files view + * shows its progress bar and the status from the moment this is called, even + * while the step still waits in the queue (folder rescans report progress on + * their own row instead). The step is abandoned (at its next time slice) when + * a newer full scan supersedes `generation`. + */ +async function runScan(status: string | undefined, generation: number, step: (slicer: TimeSlicer) => Promise): Promise { + const isCurrent = () => generation === scanGeneration + const run = () => enqueueScan(async () => { + if (!isCurrent()) return + if (status) gtsExplorer?.setScanStatus(status) + // Let the progress bar and status reach the UI before the heavy lifting. + await yieldToEventLoop() + pathsTouchedDuringScan = new Set() + try { + await step(new TimeSlicer(() => !isCurrent())) + } catch (error) { + if (!(error instanceof CancelledError)) throw error + console.log('[GTS] Scan superseded by a newer one') + } finally { + pathsTouchedDuringScan = null + if (status && isCurrent()) gtsExplorer?.setScanStatus(undefined) + } + }) + if (!status) return run() + gtsExplorer?.setScanStatus(status) + await vscode.window.withProgress({ location: { viewId: 'gts-kit.fileExplorer' } }, run) +} + +/** Calls `report(done)` at most every 200ms (and always for the last file). */ +function throttledProgress(total: number, report: (done: number) => void): (done: number) => void { + let last = 0 + return done => { + const now = Date.now() + if (done < total && now - last < 200) return + last = now + report(done) + } +} + +/** Reports "