From 14048c0d8d7fea250c9de32f9ee8d48782eda6b8 Mon Sep 17 00:00:00 2001 From: Abe Wheeler Date: Wed, 14 Oct 2026 21:21:35 -0500 Subject: [PATCH] Reuse retention calculations within each assessment and release 0.18.7 --- docs/architecture.md | 4 +- package-lock.json | 8 +-- package.json | 2 +- packages/create-filegrc/package.json | 2 +- packages/create-filegrc/src/index.js | 4 +- packages/create-filegrc/template/package.json | 2 +- packages/filegrc/package.json | 2 +- packages/filegrc/src/calculation-cache.js | 8 ++- packages/filegrc/src/program-readiness.js | 5 ++ packages/filegrc/src/workflow.js | 3 +- .../filegrc/test/calculation-cache.test.js | 67 +++++++++++++++++++ 11 files changed, 93 insertions(+), 14 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 37360ee..693002b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,6 +122,8 @@ Calculated review revisions follow the facts being reviewed. A source binding in When an older binding is stale only because its calculation included review bookkeeping, FileGRC checks committed Git history for the original revision and compares the reviewed facts with the current files. This can clear a spurious stale result without changing a record or recording another approval. Run `filegrc validate` and inspect `review-bindings` or `review-collection --scaffold` with the current engine. Keep the full Git history available; a missing historical source or a changed commitment, applicability decision, retention decision, or governed Markdown leaves the review stale and calls for the normal review workflow. +Public workflow and Program readiness assessments share a bounded calculation cache for the lifetime of one invocation. Nested assessments reuse it, while parallel and subsequent calls receive separate caches. This avoids rebuilding the same retention review graph and parsing the same Markdown for every schedule row. Ordinary calls discard the cache after success or failure, so changes to records and Markdown remain visible on the next assessment. Verified state sessions keep their existing snapshot cache and live-source verification. + Historical review calculations share a workspace-scoped history context. Parsed records are reused by their exact committed source bytes, models are loaded once per historical version, and reconstructed snapshots retain an ID index. Applicability recovery reuses both successful and unsuccessful matches for the same record, Program, model version, current scope revision, and stored review revision. The current scope is recalculated before reuse, and lifecycle and decision checks still run first. New workspace loads receive a fresh context, so these caches remain disposable and cannot replace source validation. The page-loading deadline stays at ten seconds. Bulk applicability calculations build a resource and Program-selection index once, then read only each record’s linked Requirements, Frameworks, and decisions. Current calculations rebuild this index at their boundary; immutable historical snapshots reuse it. Indexing preserves input order and duplicates so stored revision hashes remain unchanged. Historical JSON classification uses the complete owning Evidence state at each Git revision. Owned attachments remain source bytes, including JSON exports, rather than domain records. Ownership changes recheck unchanged files, and record renames and deletions retain their historical identities. An unreadable historical record reports its commit and path; the same immutable failure is reused while HEAD stays unchanged, and legacy review recovery does not reread every historical workspace after that failure. @@ -134,7 +136,7 @@ Program Overview renders its scope, navigation, and program path from bootstrap State sessions reuse section calculations within a token. New bootstrap requests independently fingerprint all source records, governed Markdown, and attachments and check repository state before sharing the unchanged loaded snapshot and a bounded Git read cache. Retaining the exact snapshot also reuses its historical review comparisons; current lifecycle and decision checks still run. Async reads share in-flight work and retain successful bounded output; writes and failed reads are never cached. Section responses still verify the live files and Git after calculation. A rejected revision verification retires the shared cache and every token using it, including when source files are subsequently restored. Browser mutations retire sessions and the server cache before and after writing. Time-sensitive readiness, validation, and Obligation calculations run again for each bootstrap rather than carrying an earlier clock forward. Program state calculates Collection Reviews once and passes the same assessments into readiness. Bounded timezone formatter and exact local-time conversion caches reuse stable timezone rules. They do not cache the current date or bypass checks for ambiguous and nonexistent local times; implicit host timezones and failed conversions remain uncached. -Browser read calculations run in one server-owned worker, which stops with the server. Validation, readiness, obligations, audit checks, and workflow use the same domain functions as direct callers. Repository synchronization and mutations stay on the HTTP server. Each snapshot crosses the worker boundary once; subsequent jobs send its ID. Both sides retain at most two registered snapshots and clear registrations when the worker stops. Evicted snapshots are sent again when needed. Within a snapshot, bounded calculation results reuse collection assessments, source indexes and graphs, parsed Markdown, applicability projections, and historical-equivalence results. Shared resource indexes also serve type, ID, and relationship lookups across applicability, reviewer selection, Obligations, and workflow. Review basis keys retain section selection, projection modes, source revisions, and Program identity. Exact content approvals and substantive comparisons still apply. Ordinary mutable domain calls do not use this cache. The [scaling benchmark](performance.md) measures calculation growth, memory, and cloning at increasing record counts. +Browser read calculations run in one server-owned worker, which stops with the server. Validation, readiness, obligations, audit checks, and workflow use the same domain functions as direct callers. Repository synchronization and mutations stay on the HTTP server. Each snapshot crosses the worker boundary once; subsequent jobs send its ID. Both sides retain at most two registered snapshots and clear registrations when the worker stops. Evicted snapshots are sent again when needed. Within a snapshot, bounded calculation results reuse collection assessments, source indexes and graphs, parsed Markdown, applicability projections, and historical-equivalence results. Shared resource indexes also serve type, ID, and relationship lookups across applicability, reviewer selection, Obligations, and workflow. Review basis keys retain section selection, projection modes, source revisions, and Program identity. Exact content approvals and substantive comparisons still apply. Ordinary workflow and Program readiness calls share a fresh cache only within their invocation and discard it afterward. The [scaling benchmark](performance.md) measures calculation growth, memory, and cloning at increasing record counts. Repository signatures include effective configuration, system and global attributes, worktree and common-directory attributes, and every relevant in-tree `.gitattributes` source, including ignored files. Attribute checks reuse successful results only inside verified read calculations. Git versions that cannot report the system attribute file keep those checks uncached and verify effective attributes in repository signatures. Every section still verifies the live snapshot before returning, so edits to an already-dirty attribute file also retire cached calculations. diff --git a/package-lock.json b/package-lock.json index ec5e388..12707da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "filegrc-monorepo", - "version": "0.18.6", + "version": "0.18.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "filegrc-monorepo", - "version": "0.18.6", + "version": "0.18.7", "workspaces": [ "packages/filegrc", "packages/create-filegrc" @@ -24,7 +24,7 @@ "link": true }, "packages/create-filegrc": { - "version": "0.18.6", + "version": "0.18.7", "license": "MIT", "bin": { "create-filegrc": "bin/create-filegrc.js" @@ -34,7 +34,7 @@ } }, "packages/filegrc": { - "version": "0.18.6", + "version": "0.18.7", "license": "MIT", "bin": { "filegrc": "bin/filegrc.js" diff --git a/package.json b/package.json index f5c516a..ed6132d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "filegrc-monorepo", - "version": "0.18.6", + "version": "0.18.7", "private": true, "workspaces": [ "packages/filegrc", diff --git a/packages/create-filegrc/package.json b/packages/create-filegrc/package.json index 4639551..5feeed4 100644 --- a/packages/create-filegrc/package.json +++ b/packages/create-filegrc/package.json @@ -1,6 +1,6 @@ { "name": "create-filegrc", - "version": "0.18.6", + "version": "0.18.7", "description": "Create a filegrc workspace for a SOC 2 program", "license": "MIT", "repository": { diff --git a/packages/create-filegrc/src/index.js b/packages/create-filegrc/src/index.js index 1af6484..7cd2da7 100644 --- a/packages/create-filegrc/src/index.js +++ b/packages/create-filegrc/src/index.js @@ -524,13 +524,13 @@ async function runCombinedSetup(target, input) { async function writeMinimalLockfile(target, name, versionRange) { const lock = { name, - version: "0.18.6", + version: "0.18.7", lockfileVersion: 3, requires: true, packages: { "": { name, - version: "0.18.6", + version: "0.18.7", dependencies: { filegrc: versionRange } } } diff --git a/packages/create-filegrc/template/package.json b/packages/create-filegrc/template/package.json index 6f7f7d3..b166dfa 100644 --- a/packages/create-filegrc/template/package.json +++ b/packages/create-filegrc/template/package.json @@ -1,6 +1,6 @@ { "name": "{{project_name}}", - "version": "0.18.6", + "version": "0.18.7", "private": true, "description": "filegrc workspace for a SOC 2 program", "type": "module", diff --git a/packages/filegrc/package.json b/packages/filegrc/package.json index 5402c46..33cd1ce 100644 --- a/packages/filegrc/package.json +++ b/packages/filegrc/package.json @@ -1,6 +1,6 @@ { "name": "filegrc", - "version": "0.18.6", + "version": "0.18.7", "description": "Zero-dependency Git-native GRC engine", "license": "MIT", "repository": { diff --git a/packages/filegrc/src/calculation-cache.js b/packages/filegrc/src/calculation-cache.js index 849b748..bba7fe2 100644 --- a/packages/filegrc/src/calculation-cache.js +++ b/packages/filegrc/src/calculation-cache.js @@ -5,8 +5,12 @@ const sizes = new WeakMap(); const MAX_RESULTS = 200_000; const limits = new WeakMap(); -// Only explicit, verified read snapshots may retain results. Ordinary domain -// calls continue to observe mutable inputs and files on every invocation. +// Only explicit, verified read snapshots may retain results between calls. +// Ordinary assessments share work within one invocation, then discard it. +export function withAssessmentCache(callback) { + return hasCalculationCache() ? callback() : withCalculationCache(new WeakMap(), callback); +} + export function withCalculationCache(cache, callback, { maxResults = MAX_RESULTS } = {}) { limits.set(cache, maxResults); return calculations.run(cache, callback); diff --git a/packages/filegrc/src/program-readiness.js b/packages/filegrc/src/program-readiness.js index d5df3cb..d3246e1 100644 --- a/packages/filegrc/src/program-readiness.js +++ b/packages/filegrc/src/program-readiness.js @@ -1,3 +1,4 @@ +import { withAssessmentCache } from "./calculation-cache.js"; import { assessOperatingPreparation, firstOperatingTasks } from "./operating-preparation.js"; import { reportingCutoverIssues } from "./activation-readiness.js"; import { recordBelongsToProgram } from "./program.js"; @@ -134,6 +135,10 @@ function calculateSetupActionProgress(stages, target, windowMilestoneComplete) { } export async function assessProgramReadiness(input, options = {}) { + return withAssessmentCache(() => assessProgramReadinessUncached(input, options)); +} + +async function assessProgramReadinessUncached(input, options = {}) { const loaded = input?.resources && input?.model && input?.entries ? input : await loadWorkspace(input); diff --git a/packages/filegrc/src/workflow.js b/packages/filegrc/src/workflow.js index 5d10a99..fdf6761 100644 --- a/packages/filegrc/src/workflow.js +++ b/packages/filegrc/src/workflow.js @@ -1,3 +1,4 @@ +import { withAssessmentCache } from "./calculation-cache.js"; import { resourceIndex } from "./resource-index.js"; import { assertWorkspaceEngineVersion } from "./runtime-version.js"; import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; @@ -90,7 +91,7 @@ const OWNER_FIELDS = [ * the complete contract for cross-interface workflow guidance. */ export async function assessWorkflow(input, options = {}) { - return measureTiming("workflow-assessment", () => assessWorkflowUnmeasured(input, options)); + return withAssessmentCache(() => measureTiming("workflow-assessment", () => assessWorkflowUnmeasured(input, options))); } async function assessWorkflowUnmeasured(input, options = {}) { diff --git a/packages/filegrc/test/calculation-cache.test.js b/packages/filegrc/test/calculation-cache.test.js index aa80057..7181c88 100644 --- a/packages/filegrc/test/calculation-cache.test.js +++ b/packages/filegrc/test/calculation-cache.test.js @@ -198,3 +198,70 @@ test("large projection workloads retain completed collection assessments at the assert.strictEqual(assessCollectionReview(loaded, "person"), assessment); }, { maxResults: 128 }); }); + +test("assessment caches share nested work and discard results after success or failure", async () => { + const { withAssessmentCache, hasCalculationCache } = await import("../src/calculation-cache.js"); + const owner = {}; let calculated = 0; + const read = () => calculationMemo(owner, "assessment", [], () => ++calculated); + await withAssessmentCache(async () => { + assert.equal(read(), 1); + await Promise.resolve(); + assert.equal(withAssessmentCache(read), 1); + }); + assert.equal(hasCalculationCache(), false); + assert.equal(withAssessmentCache(read), 2); + await assert.rejects(withAssessmentCache(async () => { assert.equal(read(), 3); throw new Error("failed"); }), /failed/); + assert.equal(hasCalculationCache(), false); + assert.equal(withAssessmentCache(read), 4); + withCalculationCache(new WeakMap(), () => { + assert.equal(read(), 5); + assert.equal(withAssessmentCache(read), 5); + assert.equal(read(), 5); + }); +}); + +test("parallel assessments cannot reuse another invocation's mutable inputs", async () => { + const { withAssessmentCache } = await import("../src/calculation-cache.js"); + const owner = {}; let release; + const pending = new Promise(resolve => { release = resolve; }); + const first = withAssessmentCache(async () => { + assert.equal(calculationMemo(owner, "value", [], () => "first"), "first"); + await pending; + return calculationMemo(owner, "value", [], () => "unexpected"); + }); + assert.equal(await withAssessmentCache(async () => calculationMemo(owner, "value", [], () => "second")), "second"); + release(); assert.equal(await first, "first"); +}); + +test("public workflow bounds review Markdown reads and observes edits on its next invocation", async context => { + const { createRequire, syncBuiltinESMExports } = await import("node:module"); + const { assessWorkflow, validateWorkspace } = await import("../src/index.js"); + const { resolve } = await import("node:path"); + const root = await workspace(context), loaded = await loadWorkspace(root); + const policy = loaded.entries.find(({record}) => record.type === "policy"); + const rule = loaded.resources.find(record => record.type === "retention-schedule-item"); + rule.status = "active"; + rule.sourceResourceIds = [policy.record.id]; + rule.reviewedSourceRevisions = { [policy.record.id]: "stale" }; + for (let index=0;index<20;index++) { + const record={...rule,id:`retention-row-${index}`}; + loaded.resources.push(record); + loaded.entries.push({record,relativePath:`retention-schedule-items/${record.id}.json`,source:JSON.stringify(record)}); + } + const markdownPath = resolve(loaded.root, "data", policy.relativePath.replace(/\.json$/, ".md")); + const fs = createRequire(import.meta.url)("node:fs"), originalRead = fs.readFileSync; + const validation = await validateWorkspace(loaded); + let reads = 0; + fs.readFileSync = function(path, ...args) { if(String(path) === markdownPath)reads++; return originalRead.call(this,path,...args); }; + syncBuiltinESMExports(); + try { + const options = { asOf:"2026-10-14", now:"2026-10-14T20:00:00.000Z", validation }; + await assessWorkflow(loaded, options); + const firstReads=reads; + assert.ok(firstReads>0 && firstReads<=2, "source reads must stay bounded as schedule rows share a review graph"); + const original = originalRead(markdownPath,"utf8"); + await writeFile(markdownPath, `${original}\nChanged operating procedure.\n`); + await assessWorkflow(loaded, options); + assert.equal(reads, firstReads*2, "the next assessment must re-read mutable source content"); + } finally { fs.readFileSync=originalRead;syncBuiltinESMExports(); } +});