Allow independent verification of finalized Evidence and release 0.20.3

package.json

@@ -1,6 +1,6 @@
 {
   "name": "filegrc-monorepo",
-  "version": "0.20.2",
+  "version": "0.20.3",
   "private": true,
   "workspaces": [
     "packages/filegrc",

packages/create-filegrc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-filegrc",
-  "version": "0.20.2",
+  "version": "0.20.3",
   "description": "Create a filegrc workspace for a SOC 2 program",
   "license": "MIT",
   "repository": {

packages/create-filegrc/src/index.js

@@ -524,13 +524,13 @@ async function runCombinedSetup(target, input) {
 async function writeMinimalLockfile(target, name, versionRange) {
   const lock = {
     name,
-    version: "0.20.2",
+    version: "0.20.3",
     lockfileVersion: 3,
     requires: true,
     packages: {
       "": {
         name,
-        version: "0.20.2",
+        version: "0.20.3",
         dependencies: { filegrc: versionRange }
       }
     }

packages/create-filegrc/template/package.json

@@ -1,6 +1,6 @@
 {
   "name": "{{project_name}}",
-  "version": "0.20.2",
+  "version": "0.20.3",
   "private": true,
   "description": "filegrc workspace for a SOC 2 program",
   "type": "module",

packages/filegrc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "filegrc",
-  "version": "0.20.2",
+  "version": "0.20.3",
   "description": "Zero-dependency Git-native GRC engine",
   "license": "MIT",
   "repository": {

packages/filegrc/src/evidence-observation.js

@@ -0,0 +1,68 @@
+import { getDataRecordHistoryIndex, isDataHistoryAncestor } from "./git.js";
+import { reviewHistoryContext, reviewHistoricalWorkspace } from "./historical-workspace.js";
+import { currentCalendarDate, isRfc3339Timestamp } from "./time.js";
+import { personWasActiveOn } from "./soc2.js";
+
+const VERIFICATION_FIELDS = new Set(["status", "verifiedOn", "verifiedAt", "verifierIds"]);
+const same = (left, right) => JSON.stringify(canonical(left)) === JSON.stringify(canonical(right));
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (!value || typeof value !== "object") return value;
+  return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]));
+}
+
+// Finalized decisions bind the observation. A later first independent
+// verification may add only its own metadata, which then becomes immutable.
+export function evidenceObservationMatches(original, current, loaded = null, reliedOnCommit = null) {
+  if (!original || !current) return false;
+  const index = loaded ? getDataRecordHistoryIndex(loaded.root) : null;
+  if (loaded && original.status === "collected" && !index.available) return false;
+  let verificationCommit = null, verificationTimestamp = null;
+  if (original.status === "collected" && loaded && reliedOnCommit) {
+    for (const version of [...(index.historiesById.get(original.id) || [])].reverse()) {
+      if (!isDataHistoryAncestor(loaded, reliedOnCommit, version.commit)) continue;
+      const record = index.recordsByCommit.get(version.commit)?.get(original.id)?.record;
+      if (record?.status === "verified") {
+        if (!same(record, current)) return false;
+        verificationCommit = version.commit;
+        verificationTimestamp = version.timestamp;
+        break;
+      }
+    }
+  }
+  if (same(original, current)) return true;
+  if (original.type !== "evidence" || current.type !== "evidence"
+    || original.status !== "collected" || current.status !== "verified") return false;
+  if (![...new Set([...Object.keys(original), ...Object.keys(current)])]
+    .every((key) => VERIFICATION_FIELDS.has(key) || same(original[key], current[key]))) return false;
+  // Existing metadata is never replaced, even on a collected observation.
+  for (const key of ["verifiedOn", "verifiedAt", "verifierIds"]) {
+    const prior = original[key];
+    const empty = prior === undefined || prior === null || (key === "verifierIds" && Array.isArray(prior) && !prior.length);
+    if (!empty && !same(prior, current[key])) return false;
+  }
+  if (!loaded) return true;
+  const context = verificationCommit
+    ? reviewHistoricalWorkspace(reviewHistoryContext(loaded.root, loaded), verificationCommit) : loaded;
+  if (!context) return false;
+  const timezone = context.workspace.timezone;
+  // Git records whole seconds; allow the subsecond part of that same instant.
+  const now = verificationTimestamp ? new Date(new Date(verificationTimestamp).getTime() + 999) : new Date();
+  const today = currentCalendarDate(timezone, now);
+  const verifiers = current.verifierIds;
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(current.verifiedOn || "")
+    || current.verifiedOn < original.collectedOn || current.verifiedOn > today
+    || !Array.isArray(verifiers) || !verifiers.length
+    || verifiers.some((id) => (original.collectorIds || []).includes(id)
+      || !personWasActiveOn(context.resources.find((record) => record.id === id), current.verifiedOn))) return false;
+  if (current.verifiedAt && (!isRfc3339Timestamp(current.verifiedAt)
+    || new Date(current.verifiedAt) > now
+    || currentCalendarDate(timezone, new Date(current.verifiedAt)) !== current.verifiedOn
+    || (original.collectedAt && new Date(current.verifiedAt) < new Date(original.collectedAt)))) return false;
+  if (reliedOnCommit) {
+    const reliedOn = index.changesByCommit.get(reliedOnCommit)?.[0]?.summary.timestamp;
+    if (!reliedOn || current.verifiedOn < currentCalendarDate(timezone, new Date(reliedOn))
+      || (current.verifiedAt && new Date(current.verifiedAt) < new Date(reliedOn))) return false;
+  }
+  return true;
+}

packages/filegrc/src/files.js

@@ -1,3 +1,4 @@
+import { evidenceObservationMatches } from "./evidence-observation.js";
 import { workflowProofIds } from "./workflow-proof.js";
 import { randomUUID } from "node:crypto";
 import { constants, link, lstat, mkdir, open, readFile, rename, rm, stat } from "node:fs/promises";
@@ -386,7 +387,7 @@ async function applyResourceBatchUnlocked(input, changes = {}, lifecycleOperatio
       requireExpectedRevisions: hasContentUpdate
     });
     if (hasContentUpdate) preparedContentIds.add(record.id);
-    if (recordContentWrites.length) assertWorkflowContentMutable(loaded, existing.record);
+    if (recordContentWrites.some(({ source, previous }) => source !== previous)) assertWorkflowContentMutable(loaded, existing.record);
     contentWrites.push(...recordContentWrites);
     const nextRecord = hasContentUpdate
       ? await prepareApprovalBinding(loaded, record, recordContentWrites, existing.record)
@@ -654,7 +655,7 @@ async function updateResourceUnlocked(input, type, id, record, options) {
   });
   const existing = loaded.entries.find(({ record: candidate }) => candidate.id === id)?.record;
   assertImmutableWorkflowRecord(existing, record, options, loaded);
-  if (contentWrites.length) assertWorkflowContentMutable(loaded, existing);
+  if (contentWrites.some(({ source, previous }) => source !== previous)) assertWorkflowContentMutable(loaded, existing);
   const nextRecord = await prepareApprovalBinding(loaded, record, contentWrites, existing);
   assertGovernedContentLifecycleMutation(existing, nextRecord, loaded.model, null);
   try {
@@ -761,9 +762,10 @@ function assertImmutableWorkflowRecord(existing, next, options = {}, loaded = nu
     throw new Error(`Obligation occurrence "${existing.id}" is workflow-managed. Scaffold and save its reconciliation instead of editing it directly.`);
   }
   const finalizedOccurrence = finalizedOccurrenceUsingProof(loaded, existing.id);
-  if (finalizedOccurrence && !preservesExisting()) {
-    throw new Error(
-      `Completion record "${existing.id}" is immutable because finalized occurrence "${finalizedOccurrence.id}" relies on it. `
+  if (finalizedOccurrence && !preservesExisting()
+    && !evidenceObservationMatches(existing, next, loaded)) {
+    throw finalizedProofError(
+      `${existing.type === "evidence" ? "Proof" : "Completion"} record "${existing.id}" is immutable because finalized occurrence "${finalizedOccurrence.id}" relies on it. `
       + "Create a corrected completion and superseding occurrence instead."
     );
   }
@@ -840,10 +842,14 @@ function assertImmutableWorkflowRecord(existing, next, options = {}, loaded = nu
   }
 }
 
+function finalizedProofError(message) {
+  return Object.assign(new Error(message), { code: "FILEGRC_FINALIZED_PROOF" });
+}
+
 function assertFinalizedOccurrenceProofMutable(loaded, record) {
   const occurrence = finalizedOccurrenceUsingProof(loaded, record.id);
   if (occurrence) {
-    throw new Error(
+    throw finalizedProofError(
       `Proof record "${record.id}" is immutable because finalized occurrence "${occurrence.id}" relies on it. `
       + "Create corrected proof and a superseding occurrence instead."
     );
@@ -873,6 +879,8 @@ function finalizedOccurrenceUsingProof(loaded, targetId) {
     (record.type === "obligation-occurrence" && ["reconciled", "superseded"].includes(record.status))
     || (record.type === "audit-population" && ["reconciled", "not-applicable", "superseded"].includes(record.status))
     || (record.type === "attestation" && record.status === "completed")
+    || record.type === "nonmaterial-amendment"
+    || (record.type === "exception" && ["approved", "revoked", "closed"].includes(record.status))
   ))) {
     if (workflowProofIds(owner, (id) => byId.get(id)).has(targetId)) return owner;
   }

packages/filegrc/src/nonmaterial-amendments.js

@@ -1,3 +1,4 @@
+import { evidenceObservationMatches as amendmentEvidenceObservationMatches } from "./evidence-observation.js";
 import { readFileSync } from "node:fs";
 import { validateWorkspace } from "./validate.js";
 import { join } from "node:path";
@@ -137,20 +138,7 @@ export function nonmaterialAmendmentIssues(loaded, amendment) {
   return issues;
 }
 
-// Independent verification can be added later without changing the observation
-// the owner relied on. Captures, collectors, dates and conclusions stay fixed.
-export function amendmentEvidenceObservationMatches(original, current, loaded = null, reliedOnCommit = null) {
-  if (!original || !current) return false;
-  if (original.status === "collected" && loaded && reliedOnCommit) {
-    const history = [...(getDataRecordHistoryIndex(loaded.root).historiesById.get(original.id) || [])].reverse();
-    const firstVerified = history.filter(({ commit }) => isDataHistoryAncestor(loaded, reliedOnCommit, commit))
-      .map(({ commit }) => historicalRecord(loaded, original.id, commit)).find((record) => record?.status === "verified");
-    if (firstVerified && (!amendmentEvidenceObservationMatches(original, firstVerified) || JSON.stringify(firstVerified) !== JSON.stringify(current))) return false;
-  }
-  const verification = original.status === "collected" && current.status === "verified"
-    ? new Set(["status", "verifiedOn", "verifiedAt", "verifierIds"]) : new Set();
-  return [...new Set([...Object.keys(original), ...Object.keys(current)])].every((key) => verification.has(key) || JSON.stringify(original[key]) === JSON.stringify(current[key]));
-}
+export { evidenceObservationMatches as amendmentEvidenceObservationMatches } from "./evidence-observation.js";
 
 function fixedSupport(loaded, evidence, amendment, date, commit) {
   return evidence?.type === "evidence" && ["collected", "verified"].includes(evidence.status)

packages/filegrc/src/server.js

@@ -1421,6 +1421,7 @@ function urlHost(host) {
 }
 
 function statusFor(error) {
+  if (error?.code === "FILEGRC_FINALIZED_PROOF") return 400;
   if (error?.code === "FILEGRC_RETENTION_APPLICATION") return 400;
   if (error?.code === "FILEGRC_ACTIVATION_NOT_READY") return 400;
   if (error?.code === "FILEGRC_STATE_EXPIRED") return 409;

packages/filegrc/src/web.js

@@ -6019,10 +6019,7 @@ function readGuidedRecord(dialog, base, fields) {
   const record = structuredClone(base);
   for (const group of dialog.querySelectorAll("[data-field-group]")) {
     const name = group.dataset.fieldGroup;
-    if (group.hidden) {
-      delete record[name];
-      continue;
-    }
+    if (group.hidden) continue;
     const kind = group.dataset.kind;
     let value;
     if (kind === "relation-array" || kind === "choice-array") value = [...group.querySelectorAll('input[type="checkbox"]:checked')].map((input) => input.value);
@@ -6050,6 +6047,11 @@ function readGuidedRecord(dialog, base, fields) {
     if (empty && group.dataset.required !== "true") delete record[name];
     else record[name] = value;
   }
+  // Visibility controls the form, while allowedWhen controls stored facts.
+  // Keep valid existing optional values when their editor is hidden.
+  for (const [name, field] of Object.entries(fields)) {
+    if (field.allowedWhen && !conditionMatchesValues(field.allowedWhen, (key) => record[key])) delete record[name];
+  }
   return record;
 }
 

packages/filegrc/src/workflow-history-integrity.js

@@ -1,5 +1,5 @@
 import { workflowProofIds } from "./workflow-proof.js";
-import { amendmentEvidenceObservationMatches } from "./nonmaterial-amendments.js";
+import { evidenceObservationMatches } from "./evidence-observation.js";
 import { readFile } from "node:fs/promises";
 import {
   getDataRecordHistoryIndex,
@@ -283,8 +283,8 @@ export async function validateWorkflowHistoryIntegrity(loaded, diagnostics) {
         diagnostics.push(integrityError(ownerEntry, `Proof record "${id}" must exist in the commit where ${owner.type} "${owner.id}" was first finalized.`));
         continue;
       }
-      if (!(owner.type === "nonmaterial-amendment" && historical.record.type === "evidence"
-        ? amendmentEvidenceObservationMatches(historical.record, currentEntry.record, loaded, finalized.commit)
+      if (!(historical.record.type === "evidence"
+        ? evidenceObservationMatches(historical.record, currentEntry.record, loaded, finalized.commit)
         : sameJson(currentEntry.record, historical.record))) {
         diagnostics.push(integrityError(currentEntry, `Proof record "${id}" differs from the version used when ${owner.type} "${owner.id}" was first finalized. Create corrected proof and a superseding record.`));
         continue;