From 495e09e7923ed008f6e5d938ea897d2af806de83 Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Fri, 25 Sep 2026 20:37:45 +0000 Subject: [PATCH] Add runnable examples, a working quickstart, metadata and the 0.1.0 changeset The README quickstart read two variables it never defined, so the published block failed with "ReferenceError: name is not defined" from a clean install. It now declares synthetic inputs inline, prints what the pass returned, and shows that output beside it. Verified from a packed tarball in an empty directory with node quickstart.mts: exit 0. Four examples cover the jobs the README presents, each importing @cosyte/deid by name (resolved through the package's own exports to dist/) and checking its own output: the core engine on a locus model, the HL7 v2 adapter on the repository's synthetic ORU^R01, the fail-closed cases (free text blocked, a throwing or empty redactor still blocked, no key refused, a mislabelled policy refused), and the Expert Determination support report, whose determination is always null. The examples README says plainly what the library does not do: no certification, no determination. Inputs are the ZZ-placeholder shapes of test/fixtures. pnpm examples (scripts/run-examples.ts) runs them all and refuses when there is no build or no example. A new examples job in ci.yml installs, builds and runs it on Node 22 and 24, actions pinned by SHA, not yet a required context. examples/tsconfig.json maps the package name to src/ for tsc and typescript-eslint only, so pnpm typecheck, pnpm lint and format cover examples/ before any build; at run time nothing maps it. package.json: the description names the formats and the fail-closed posture, keywords widen, homepage moves to docs.cosyte.com/deid. The status lines in the README and two docs pages say 0.1 rather than pre-alpha 0.0.x, and .changeset/README.md states the 0.1.x ladder. The new changeset tells a user what 0.1 covers, what the version promises, what it is not, and what is not covered yet. Claude-Session: https://claude.ai/code/session_01U24bnHKNxQWMxmfN7ny7w9 --- .changeset/README.md | 13 ++- .changeset/what-0-1-means.md | 13 +++ .github/workflows/ci.yml | 26 ++++++ README.md | 38 ++++++--- docs-content/installation.md | 2 +- docs-content/intro.md | 7 +- eslint.config.js | 14 ++- examples/README.md | 36 ++++++++ examples/deidentify-a-model.ts | 87 +++++++++++++++++++ examples/deidentify-an-hl7-message.ts | 81 ++++++++++++++++++ examples/expert-determination-support.ts | 66 ++++++++++++++ examples/fail-closed.ts | 104 +++++++++++++++++++++++ examples/tsconfig.json | 15 ++++ package.json | 22 +++-- scripts/run-examples.ts | 83 ++++++++++++++++++ 15 files changed, 581 insertions(+), 26 deletions(-) create mode 100644 .changeset/what-0-1-means.md create mode 100644 examples/README.md create mode 100644 examples/deidentify-a-model.ts create mode 100644 examples/deidentify-an-hl7-message.ts create mode 100644 examples/expert-determination-support.ts create mode 100644 examples/fail-closed.ts create mode 100644 examples/tsconfig.json create mode 100644 scripts/run-examples.ts diff --git a/.changeset/README.md b/.changeset/README.md index 6e26675..6cc8aba 100644 --- a/.changeset/README.md +++ b/.changeset/README.md @@ -24,5 +24,14 @@ Add a changeset for every meaningful change: pnpm changeset ``` -During pre-alpha, pick **patch**: that keeps the package on the `0.0.x` ladder until its first -alpha. See the cosyte version ladder in the meta-repo's `documentation/conventions.md`. +Pick the bump type from what the change does to the published package. On the `0.1.x` ladder: + +- **minor** for anything a consumer gains: a newly exported symbol, a new option, a new stable + code, or a new published artifact they can use. It takes `0.1.x` to `0.2.0`. +- **patch** for a fix, for documentation of a surface that already ships, and for contributor-only + tooling. It takes `0.1.0` to `0.1.1`. +- **major** only when you mean to declare `1.0.0`. Before that, a breaking change is a **minor**, + and the break has to be spelled out in the summary, because the number alone tells a consumer on + a `0.x` release nothing. + +See the cosyte version ladder in the meta-repo's `documentation/conventions.md`. diff --git a/.changeset/what-0-1-means.md b/.changeset/what-0-1-means.md new file mode 100644 index 0000000..b49f289 --- /dev/null +++ b/.changeset/what-0-1-means.md @@ -0,0 +1,13 @@ +--- +"@cosyte/deid": minor +--- + +`0.1.0` is the first release of `@cosyte/deid` whose public API we ask you to build on. + +**What is covered, and what you can depend on.** Applying a de-identification policy to a structurally located model of a healthcare document and getting back the transformed model plus a value-free manifest. That means the built-in HIPAA Safe Harbor policy across all 18 identifier categories, `defineDeidPolicy`, the two named profiles and `defineDeidProfile` (which can only tighten a profile), the five transforms (redact, generalize, keyed date-shift, keyed pseudonymize, keyed hash), the adapters for HL7 v2, C-CDA, FHIR R4, X12, NCPDP Telecom and DICOM metadata at their own subpaths, the corpus registry for consistent keyed surrogates and date shifts across documents, the free-text redactor interface, and the Expert Determination support report. Failing closed is part of the contract: an uncertain value is blocked, free text is blocked unless your redactor handles it, and a keyed transform with no key is a fatal `DEID_NO_KEY`. The stable `DEID_*` codes are public API, so renaming or removing one is a breaking change. Node.js 22 and 24, ESM and CommonJS, with type declarations for both, and no third-party runtime dependency. + +**What the version promises.** Until 1.0, a breaking change raises the minor version (0.1 to 0.2), and the changelog entry says what broke and what to change. A patch release (0.1.x) does not break you, so a `^0.1.0` range takes the patches and stops before 0.2.0. + +**What it is not, and what is not covered yet.** A result is "Safe-Harbor-transformed per the configured policy". It is not a certification that data is de-identified and not a compliance claim: the actual-knowledge condition of Safe Harbor stays with you, and an Expert Determination is supported, never rendered. Not covered yet: NCPDP SCRIPT, which is refused with `DEID_FORMAT_UNSUPPORTED`; DICOM pixel data, where burned-in text is flagged and never cleaned; a built-in free-text scrubber; and value-bearing positions no rule names, which are counted as unexamined residuals rather than cleaned. The documentation at https://docs.cosyte.com/deid lists the known limitations in full. + +The repository now carries runnable examples under `examples/`, including the fail-closed cases, run against the built package on every change. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fefd977..11da9dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,3 +64,29 @@ jobs: uses: cosyte/.github/.github/workflows/ci.yml@main with: run-phi-scan: true + + # The programs under examples/, run against the BUILT package the way a consumer imports it: + # `pnpm examples` (scripts/run-examples.ts) fails when any example exits non-zero, and each + # example checks its own output, the fail-closed cases included. Added on top of the caller above, + # never instead of it. Its contexts, `examples (22)` and `examples (24)`, are new and deliberately + # NOT in the `ci-required-checks` ruleset yet: let them run on `main` first, then read the real + # names off a live check run before requiring them. + examples: + strategy: + fail-fast: false + matrix: + node: ["22", "24"] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ matrix.node }} + cache: pnpm + - run: pnpm install --frozen-lockfile + - name: Build (the examples import the built package) + run: pnpm build + - name: Run every example + run: pnpm examples diff --git a/README.md b/README.md index 70d43cc..711ee17 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,17 @@ but **inverts the reflex**: where a parser is liberal on input, a de-identifier > (§164.514(b)(1)) is supported, never rendered or certified.** The certification is > always the consumer's. -> **Status:** pre-alpha (`0.0.x`), published on npm. This release ships the **format-agnostic -> core** plus six format bindings: the **HL7 v2 adapter** (`@cosyte/deid/hl7`), the **C-CDA adapter** +> **Status:** `0.1`, published on npm. Below 1.0, a breaking change raises the minor version and the +> changelog says what broke. This release ships the **format-agnostic core** plus six format +> bindings: the **HL7 v2 adapter** (`@cosyte/deid/hl7`), the **C-CDA adapter** > (`@cosyte/deid/ccda`), the **FHIR R4 adapter** (`@cosyte/deid/fhir`), the **X12 EDI adapter** -> (`@cosyte/deid/x12`), the **NCPDP Telecom adapter** (`@cosyte/deid/ncpdp`), and the **DICOM adapter** -> (`@cosyte/deid/dicom`), plus the **longitudinal layer**, the corpus registry (`createDeidRegistry`) -> for cross-document consistency and the formalized key contract, and the **Expert-Determination support -> report** (`buildExpertDeterminationSupportReport`) that structures the manifest for a statistician -> **without ever rendering a determination**. NCPDP SCRIPT is **not** supported, and an entry point -> handed one refuses it outright rather than returning a partial pass. +> (`@cosyte/deid/x12`), the **NCPDP Telecom adapter** (`@cosyte/deid/ncpdp`), and the **DICOM +> adapter** (`@cosyte/deid/dicom`), plus the **longitudinal layer**, the corpus registry +> (`createDeidRegistry`) for cross-document consistency and the formalized key contract, and the +> **Expert-Determination support report** (`buildExpertDeterminationSupportReport`) that structures +> the manifest for a statistician **without ever rendering a determination**. NCPDP SCRIPT is +> **not** supported, and an entry point handed one refuses it outright rather than returning a +> partial pass. ## Install @@ -44,6 +46,10 @@ npm install @cosyte/deid ```ts import { deidentify, SAFE_HARBOR_CATEGORIES } from "@cosyte/deid"; +// Synthetic inputs: placeholders in the shape of an HL7 v2 name and date of birth, not a person. +const name = "ZZFAMILY^ZZGIVEN"; +const dob = "19900215"; + const { document, manifest } = deidentify( { loci: [ @@ -55,12 +61,20 @@ const { document, manifest } = deidentify( {}, ); -// document.loci[0].value === null (name removed) -// document.loci[1].value === "" (date generalized) -// document.loci[2].value === "5.4 mmol/L" (clinical value retained, the over-scrub guard) -// manifest records each category + locus + disposition, never a value. +console.log(document.loci.map((locus) => locus.value)); +console.log(manifest.map((entry) => `${entry.locus} ${entry.category} ${entry.disposition}`)); +``` + +```text +[ null, '1990', '5.4 mmol/L' ] +[ 'PID-5 NAMES removed', 'PID-7 DATES transformed' ] ``` +The name is removed, the date keeps only its year, and the clinical value is retained untouched (the +over-scrub guard). The manifest records the category, locus and disposition of each decision, never +the value. [`examples/`](https://github.com/cosyte/deid/tree/main/examples) has runnable programs for +the HL7 v2 adapter, the fail-closed cases and the Expert Determination support report. + ## Keyed transforms Pseudonymization and keyed hashing use a **keyed HMAC-SHA-256**; the key is the consumer's and never diff --git a/docs-content/installation.md b/docs-content/installation.md index 0740d13..1231348 100644 --- a/docs-content/installation.md +++ b/docs-content/installation.md @@ -11,7 +11,7 @@ sidebar_position: 1 builds with per-condition type declarations, so it works from either module system without configuration. -> **Status:** pre-alpha (`0.0.x`), published to npm. The command below installs the current release. +> **Status:** `0.1`, published to npm. The command below installs the current release. > `npm view @cosyte/deid version` is the authority on which one that is. ## Prerequisites diff --git a/docs-content/intro.md b/docs-content/intro.md index 7ea4e67..4aa698d 100644 --- a/docs-content/intro.md +++ b/docs-content/intro.md @@ -22,9 +22,10 @@ through as safe. > Determination (§164.514(b)(1)) is *supported*, never *rendered* or certified. The > certification is always the consumer's. -> **Status:** pre-alpha (`0.0.x`), published to npm. This release ships the **format-agnostic -> core** (policy engine, five transforms, 18-category Safe Harbor model, fail-closed rule, value-free -> manifest) **plus six format bindings**: HL7 v2 (`@cosyte/deid/hl7`), C-CDA (`@cosyte/deid/ccda`), +> **Status:** `0.1`, published to npm. Below 1.0, a breaking change raises the minor version and +> the changelog says what broke. This release ships the **format-agnostic core** (policy engine, +> five transforms, 18-category Safe Harbor model, fail-closed rule, value-free manifest) **plus six +> format bindings**: HL7 v2 (`@cosyte/deid/hl7`), C-CDA (`@cosyte/deid/ccda`), > FHIR R4 (`@cosyte/deid/fhir`), X12 EDI (`@cosyte/deid/x12`), NCPDP Telecom (`@cosyte/deid/ncpdp`) and > DICOM (`@cosyte/deid/dicom`). NCPDP SCRIPT is **not** supported, and an entry point handed one > refuses it outright rather than returning a partial pass. diff --git a/eslint.config.js b/eslint.config.js index aa85719..cc8a160 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,3 +1,15 @@ import cosyte from "@cosyte/eslint-config"; -export default cosyte(import.meta.dirname); +export default [ + ...cosyte(import.meta.dirname, { + files: ["src/**/*.ts", "test/**/*.ts", "scripts/**/*.ts", "examples/**/*.ts", "*.config.ts"], + }), + + // The examples are programs a reader runs: printing what they show is their job. + { + files: ["examples/**/*.ts"], + rules: { + "no-console": "off", + }, + }, +]; diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..28d2268 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,36 @@ +# Examples + +Four small programs that show what `@cosyte/deid` does, each runnable as it stands. Every example +imports `@cosyte/deid` by its published name, which resolves through this package's own `exports` +to the built `dist/`, so build first. Each one checks its own output and exits non-zero on a +mismatch. + +```bash +pnpm install +pnpm build +pnpm examples # runs all four +pnpm tsx examples/deidentify-a-model.ts # or one at a time +``` + +| Example | What it shows | +| ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`deidentify-a-model.ts`](./deidentify-a-model.ts) | The core engine on a generic locus model: a name removed, a date kept to its year, a ZIP kept to three digits, a clinical value retained, free text and an unknown value blocked, and a manifest that carries no value. | +| [`deidentify-an-hl7-message.ts`](./deidentify-an-hl7-message.ts) | The `@cosyte/deid/hl7` adapter on a parsed ORU^R01: identifiers located by structure, structured results kept, string and narrative results and the NTE comment blocked, and no input identifier left in the output. | +| [`fail-closed.ts`](./fail-closed.ts) | What happens when the library cannot be sure: free text blocked with no redactor, and again when your redactor throws or returns nothing; a keyed transform with no key refused; a mislabelled policy refused. | +| [`expert-determination-support.ts`](./expert-determination-support.ts) | The Expert Determination support report built from a manifest. `determination` is always `null`: the library structures the facts for a qualified expert and never renders a determination or a risk score. | + +**What these examples do not show, because the library does not do it.** A result is +"Safe-Harbor-transformed per the configured policy". It is not a certification that data is +de-identified, and nothing here makes the actual-knowledge judgment Safe Harbor also requires, or an +Expert Determination. Those stay with you. See the +[known limitations](../docs-content/limitations.md) before relying on this for data that leaves your +control. + +Every input is synthetic, in the shape of the repository's own fixtures under `test/fixtures/`: `ZZ` +placeholder names and identifiers, reserved `555` phone numbers, and `900`-range numbers that are +never issued. `@cosyte/hl7` is an optional peer dependency of the package and a development +dependency here, which is what the HL7 examples parse with. + +`tsconfig.json` in this folder maps `@cosyte/deid` to the source, so `pnpm typecheck` and +`pnpm lint` check the examples before anything is built. At run time nothing maps the name: Node +resolves it through `exports`, as it does in your project. diff --git a/examples/deidentify-a-model.ts b/examples/deidentify-a-model.ts new file mode 100644 index 0000000..ad9c458 --- /dev/null +++ b/examples/deidentify-a-model.ts @@ -0,0 +1,87 @@ +/** + * Apply the built-in Safe Harbor policy to a generic locus model and read the value-free manifest. + * + * The core works on a flat list of structurally located values; the per-format adapters build that + * list for you. Identifiers are removed, a date keeps its year, a ZIP keeps a safe three-digit + * prefix, a clinical value is kept untouched, and anything uncertain (free text, a value with no + * known category) is blocked. The result is "Safe-Harbor-transformed per the configured policy": + * a record of what the code did, not a certification that the output is de-identified. + * + * Run it after `pnpm build`: + * + * pnpm tsx examples/deidentify-a-model.ts + */ + +import assert from "node:assert/strict"; + +import { deidentify, OUTPUT_LABEL, SAFE_HARBOR_CATEGORIES } from "@cosyte/deid"; + +// Synthetic values in the shape of the repository's own fixtures: none of them is a person. +const syntheticName = "ZZLABFAMILY^ZZLABGIVEN"; +const syntheticDateOfBirth = "19850302"; +const syntheticZip = "90210"; +const narrative = "Reported by ZZLABFAMILY on behalf of the ordering provider."; + +const { document, manifest } = deidentify( + { + loci: [ + { + path: "PID-5", + kind: "identifier", + category: SAFE_HARBOR_CATEGORIES.NAMES, + value: syntheticName, + }, + { + path: "PID-7", + kind: "date", + category: SAFE_HARBOR_CATEGORIES.DATES, + value: syntheticDateOfBirth, + }, + { + path: "PID-11", + kind: "zip", + category: SAFE_HARBOR_CATEGORIES.GEOGRAPHIC, + value: syntheticZip, + }, + { path: "OBX-5", kind: "clinical", value: "140 mmol/L" }, + { path: "NTE-3", kind: "freetext", value: narrative }, + { path: "ZPI-2", kind: "unknown", value: "ZZSECRETNOTE" }, + ], + }, + {}, +); + +for (const locus of document.loci) { + console.log( + `${locus.path.padEnd(7)} ${locus.disposition.padEnd(12)} ${JSON.stringify(locus.value)}`, + ); +} + +const byPath = new Map(document.loci.map((locus) => [locus.path, locus])); +assert.equal(byPath.get("PID-5")?.value, null, "the name is removed"); +assert.equal(byPath.get("PID-7")?.value, "1985", "the date of birth keeps its year only"); +assert.equal(byPath.get("PID-11")?.value, "902", "the ZIP keeps a safe three-digit prefix"); +assert.equal(byPath.get("OBX-5")?.value, "140 mmol/L", "the clinical value is retained untouched"); +assert.equal(byPath.get("NTE-3")?.disposition, "blocked", "free text fails closed"); +assert.equal(byPath.get("ZPI-2")?.disposition, "blocked", "a value of unknown kind fails closed"); + +// The manifest records category, transform, locus, count, disposition and code: never a value. +console.log("manifest:"); +for (const entry of manifest) { + console.log( + ` ${entry.locus.padEnd(7)} ${entry.category.padEnd(16)} ${entry.transform.padEnd(11)} ${entry.code}`, + ); +} +const recorded = JSON.stringify(manifest); +for (const input of [ + syntheticName, + syntheticDateOfBirth, + syntheticZip, + narrative, + "ZZSECRETNOTE", +]) { + assert.ok(!recorded.includes(input), "the manifest carries no input value"); +} + +console.log(`label: ${OUTPUT_LABEL}`); +assert.equal(OUTPUT_LABEL, "Safe-Harbor-transformed per the configured policy"); diff --git a/examples/deidentify-an-hl7-message.ts b/examples/deidentify-an-hl7-message.ts new file mode 100644 index 0000000..08cd369 --- /dev/null +++ b/examples/deidentify-an-hl7-message.ts @@ -0,0 +1,81 @@ +/** + * De-identify a parsed HL7 v2 ORU^R01 with the `@cosyte/deid/hl7` adapter. + * + * The adapter locates identifiers by the message's structure (PID-5 is the patient's name because + * HL7 v2 says so), never by a regular expression over the bytes, and returns a transformed + * `Hl7Message` plus the value-free manifest. `@cosyte/hl7` is an optional peer dependency: install + * it beside `@cosyte/deid` to use this subpath. The built-in Safe Harbor policy uses no keyed + * transform, so no key is needed here. + * + * Run it after `pnpm build`: + * + * pnpm tsx examples/deidentify-an-hl7-message.ts + */ + +import assert from "node:assert/strict"; + +import { parseHL7 } from "@cosyte/hl7"; + +import { deidentifyHl7 } from "@cosyte/deid/hl7"; + +// The repository's synthetic fixture test/fixtures/hl7/oru-r01.hl7: every identifier is a ZZ +// placeholder, a reserved 555 phone number or a 900-range number that is never issued. +const message = [ + "MSH|^~\\&|LABAPP|LABFAC|EHRAPP|EHRFAC|20200102080000||ORU^R01|SYNTHMSG002|P|2.5", + "PID|1||ZZMRN002^^^HOSP^MR||ZZLABFAMILY^ZZLABGIVEN||19850302|F|||ZZLABSTREET^^ZZLABCITY^MA^90210|049|5550000010|||||ZZACCT200|900000010", + "OBR|1|PLACER001|FILLER001|2951-2^Sodium^LN|||20200102080000", + "OBX|1|NM|2951-2^Sodium^LN|1|140|mmol/L|135-145|N|||F", + "OBX|2|NM|2823-3^Potassium^LN|1|4.2|mmol/L|3.5-5.1|N|||F", + "OBX|3|CWE|883-9^ABO group^LN|1|O^O positive^L||||||F", + "OBX|4|ST|11557-6^Interpretation^LN|1|Within normal limits|||N|||F", + "OBX|5|TX|1234-5^Pathology note^LN|1|Specimen from ZZLABFAMILY reviewed; contact 5550000010 for questions|||||F", + "NTE|1|L|Reported by ZZLABFAMILY on behalf of the ordering provider.", +].join("\r"); + +const { document, manifest, unexaminedResiduals } = deidentifyHl7(parseHL7(message)); +const output = document.toString(); +console.log(output.split("\r").join("\n")); + +const segments = output.split("\r").map((segment) => segment.split("|")); +const pid = segments.find((fields) => fields[0] === "PID") ?? []; +const obx = segments.filter((fields) => fields[0] === "OBX"); + +assert.equal(pid[5], "", "PID-5, the patient's name, is removed"); +assert.equal(pid[7], "1985", "PID-7, the date of birth, keeps its year only"); +assert.equal(pid[11], "^^^^902", "PID-11 keeps only a safe three-digit ZIP"); +assert.deepEqual( + obx.map((fields) => fields[5]), + ["140", "4.2", "O^O positive^L", "", ""], + "structured clinical results survive; string and narrative results are blocked", +); + +// Not one identifier from the input survives anywhere in the output, or in the manifest. +const identifiers = [ + "ZZMRN002", + "ZZLABFAMILY", + "ZZLABGIVEN", + "19850302", + "ZZLABSTREET", + "ZZLABCITY", + "5550000010", + "ZZACCT200", + "900000010", +]; +const recorded = JSON.stringify(manifest); +for (const identifier of identifiers) { + assert.ok(!output.includes(identifier), "no input identifier survives in the output"); + assert.ok(!recorded.includes(identifier), "the manifest carries no input value"); +} + +const removed = manifest + .filter((entry) => entry.disposition === "removed") + .map((entry) => entry.locus); +const blocked = manifest + .filter((entry) => entry.disposition === "blocked") + .map((entry) => entry.locus); +console.log(`removed: ${removed.join(", ")}`); +console.log(`blocked (fail closed): ${blocked.join(", ")}`); +console.log( + `positions no rule examined, counted and never cleaned: ${String(unexaminedResiduals.length)}`, +); +assert.ok(blocked.includes("NTE-3"), "the NTE comment is blocked"); diff --git a/examples/expert-determination-support.ts b/examples/expert-determination-support.ts new file mode 100644 index 0000000..f16d6f6 --- /dev/null +++ b/examples/expert-determination-support.ts @@ -0,0 +1,66 @@ +/** + * Build the Expert Determination support report from a manifest, and see what it will not say. + * + * HIPAA has two routes to de-identification. Safe Harbor is mechanical, and this library applies + * it. Expert Determination (45 CFR 164.514(b)(1)) is a qualified statistician's judgment about a + * specific dataset and its recipient. `@cosyte/deid` supports that judgment and never renders it: + * the report structures what a pass did and what it left in place, `determination` is always + * `null`, and no re-identification risk score is computed. The determination, and any + * certification, stays with you and your expert. + * + * Run it after `pnpm build`: + * + * pnpm tsx examples/expert-determination-support.ts + */ + +import assert from "node:assert/strict"; + +import { parseHL7 } from "@cosyte/hl7"; + +import { + buildExpertDeterminationSupportReport, + EXPERT_DETERMINATION_DISCLAIMER, + formatExpertDeterminationSupportReport, +} from "@cosyte/deid"; +import { deidentifyHl7 } from "@cosyte/deid/hl7"; + +// A synthetic ADT^A01 in the shape of the repository's fixtures: ZZ placeholders, a reserved 555 +// phone number, a 900-range number that is never issued. +const message = [ + "MSH|^~\\&|SENDAPP|SENDFAC|RECVAPP|RECVFAC|20200101120000||ADT^A01|SYNTHMSG001|P|2.5", + "PID|1||ZZMRN001^^^HOSP^MR||ZZFAMILY^ZZGIVEN||19900215|M|||ZZSTREET^^ZZCITY^MA^90210||5550000001|||||ZZACCT018|900000005", + "PV1|1|I|WARD^ROOM^BED", +].join("\r"); + +const { manifest, unexaminedResiduals } = deidentifyHl7(parseHL7(message)); +const report = buildExpertDeterminationSupportReport(manifest, { + policy: "safe-harbor", + unexaminedResiduals, +}); + +console.log(`determination: ${String(report.determination)}`); +console.log(`output label: ${report.outputLabel}`); +console.log( + `loci acted on: ${String(report.totals.loci)}, Safe Harbor categories acted on: ${String(report.totals.categoriesActedOn)} of 18`, +); +console.log( + `retained quasi-identifiers: ${report.retainedQuasiIdentifiers.map((q) => q.locus).join(", ")}`, +); +console.log( + `positions no rule examined: ${String(report.unexaminedResiduals.length)} (measured: ${String(report.unexaminedResidualsMeasured)})`, +); + +assert.equal(report.determination, null, "the library never renders a determination"); +assert.equal( + report.unexaminedResidualsMeasured, + true, + "the residual inventory was handed over and measured", +); +assert.equal(report.disclaimer, EXPERT_DETERMINATION_DISCLAIMER); +assert.ok(report.disclaimer.includes("NOT a determination")); + +// The Markdown rendering for a statistician leads with the same disclaimer. +const markdown = formatExpertDeterminationSupportReport(report); +console.log(markdown.split("\n").slice(0, 3).join("\n")); +assert.ok(markdown.includes("NOT A DETERMINATION")); +assert.ok(!markdown.includes("ZZFAMILY"), "the report is value-free"); diff --git a/examples/fail-closed.ts b/examples/fail-closed.ts new file mode 100644 index 0000000..94510d4 --- /dev/null +++ b/examples/fail-closed.ts @@ -0,0 +1,104 @@ +/** + * Fail closed: what `@cosyte/deid` does when it cannot be sure, shown case by case. + * + * A parser is liberal about what it accepts. A de-identifier is the opposite: anything it cannot + * confidently handle is blocked or refused, never passed through as safe. + * + * - Free text is blocked by default. The library ships no NLP model and no regex scrub. + * - A redactor you bring is used, and its output is recorded as consumer-asserted: the engine does + * not re-scan it. When it throws or returns nothing, the prose is blocked, never leaked. + * - A keyed transform with no key is a fatal `DEID_NO_KEY`, never a silent unkeyed fallback. + * - A policy that wears the `safe-harbor` label but derives its output from the value itself is + * refused with `DEID_POLICY_INVALID`. + * + * Run it after `pnpm build`: + * + * pnpm tsx examples/fail-closed.ts + */ + +import assert from "node:assert/strict"; + +import { + DeidError, + defineDeidPolicy, + deidentify, + type FreeTextRedactor, + type LocusModel, + SAFE_HARBOR_CATEGORIES, +} from "@cosyte/deid"; + +// Synthetic prose in the shape of the repository's own fixtures. +const note: LocusModel = { + loci: [{ path: "NTE-3", kind: "freetext", value: "Reported by ZZLABFAMILY, call 5550000010." }], +}; + +/** One line: what happened to the single locus, the value left in place, and the manifest's code. */ +function report(label: string, result: ReturnType): void { + const locus = result.document.loci[0]; + const entry = result.manifest[0]; + console.log( + `${label.padEnd(24)} ${String(locus?.disposition)} ${JSON.stringify(locus?.value)} ${String(entry?.code)}`, + ); +} + +// 1. No redactor: the prose is blocked. +const noRedactor = deidentify(note, {}); +report("no redactor:", noRedactor); +assert.equal(noRedactor.document.loci[0]?.value, null); +assert.equal(noRedactor.manifest[0]?.code, "DEID_FREETEXT_BLOCKED"); + +// 2. A redactor that throws: still blocked. +const failing: FreeTextRedactor = () => { + throw new Error("the redaction service is unavailable"); +}; +const afterThrow = deidentify(note, { redactor: failing }); +report("redactor threw:", afterThrow); +assert.equal(afterThrow.document.loci[0]?.value, null); +assert.equal(afterThrow.manifest[0]?.code, "DEID_FREETEXT_BLOCKED"); + +// 3. A redactor that returns nothing: still blocked. +const afterNothing = deidentify(note, { redactor: () => undefined }); +report("redactor returned none:", afterNothing); +assert.equal(afterNothing.document.loci[0]?.value, null); + +// 4. A redactor that returns text: written back, and recorded as YOUR assertion, not verified here. +const yours: FreeTextRedactor = () => ({ text: "Reported by [NAME], call [PHONE]." }); +const redacted = deidentify(note, { redactor: yours }); +report("redactor returned text:", redacted); +assert.equal(redacted.document.loci[0]?.value, "Reported by [NAME], call [PHONE]."); +assert.equal(redacted.manifest[0]?.code, "DEID_FREETEXT_CONSUMER_REDACTED"); + +// 5. A keyed transform with no key is fatal. +const research = defineDeidPolicy({ + name: "research", + transforms: { [SAFE_HARBOR_CATEGORIES.MRN]: "pseudonymize" }, +}); +assert.throws( + () => + deidentify( + { + loci: [ + { + path: "PID-3", + kind: "identifier", + category: SAFE_HARBOR_CATEGORIES.MRN, + value: "ZZMRN002", + }, + ], + }, + { policy: research }, + ), + (error: unknown) => error instanceof DeidError && error.code === "DEID_NO_KEY", +); +console.log(`${"keyed transform, no key:".padEnd(24)} refused, DEID_NO_KEY`); + +// 6. A safe-harbor-labelled policy that shifts dates is refused when it is defined. +assert.throws( + () => + defineDeidPolicy({ + name: "safe-harbor", + transforms: { [SAFE_HARBOR_CATEGORIES.DATES]: "date-shift" }, + }), + (error: unknown) => error instanceof DeidError && error.code === "DEID_POLICY_INVALID", +); +console.log(`${"safe-harbor, date-shift:".padEnd(24)} refused, DEID_POLICY_INVALID`); diff --git a/examples/tsconfig.json b/examples/tsconfig.json new file mode 100644 index 0000000..8c09276 --- /dev/null +++ b/examples/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "paths": { + "@cosyte/deid": ["../src/index.ts"], + "@cosyte/deid/hl7": ["../src/hl7/index.ts"], + "@cosyte/deid/ccda": ["../src/ccda/index.ts"], + "@cosyte/deid/fhir": ["../src/fhir/index.ts"], + "@cosyte/deid/x12": ["../src/x12/index.ts"], + "@cosyte/deid/ncpdp": ["../src/ncpdp/index.ts"], + "@cosyte/deid/dicom": ["../src/dicom/index.ts"] + } + }, + "include": ["*.ts"] +} diff --git a/package.json b/package.json index 9cfcbe4..64be67b 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,25 @@ { "name": "@cosyte/deid", "version": "0.0.9", - "description": "Healthcare de-identification for Node.js and TypeScript. A HIPAA Safe Harbor policy engine that fails closed and emits a value-free manifest.", + "description": "Healthcare de-identification for Node.js and TypeScript: applies a HIPAA Safe Harbor policy to HL7 v2, C-CDA, FHIR R4, X12, NCPDP Telecom and DICOM, fails closed, and emits a value-free manifest.", "keywords": [ "deid", "de-identification", "hipaa", "safe-harbor", "phi", + "pseudonymization", "healthcare", "interoperability", + "hl7", + "fhir", + "ccda", + "x12", + "ncpdp", + "dicom", "typescript" ], - "homepage": "https://github.com/cosyte/deid#readme", + "homepage": "https://docs.cosyte.com/deid", "bugs": { "url": "https://github.com/cosyte/deid/issues" }, @@ -152,20 +159,21 @@ "scripts": { "build": "tsup", "changeset": "changeset", - "typecheck": "tsc --noEmit", + "typecheck": "tsc --noEmit && tsc --noEmit -p examples", "phi-scan": "tsx scripts/phi-scan.ts", "check": "pnpm run check:test-selection && pnpm run check:agent-notes", "check:test-selection": "tsx scripts/check-test-selection.ts", "check:agent-notes": "tsx scripts/check-agent-notes.ts", "check:no-emdash": "bash scripts/check-no-emdash.sh", "check:no-internal-refs": "bash scripts/check-no-internal-refs.sh", - "lint": "eslint \"src/**/*.ts\" \"scripts/**/*.ts\" \"test/**/*.ts\" --max-warnings=0 --no-error-on-unmatched-pattern", - "lint:fix": "eslint \"src/**/*.ts\" \"scripts/**/*.ts\" \"test/**/*.ts\" --fix --no-error-on-unmatched-pattern", - "format": "prettier --write \"src/**/*.{ts,md}\" \"test/**/*.ts\" \"scripts/**/*.{ts,mjs}\" \"*.{json,md,yml}\"", - "format:check": "prettier --check \"src/**/*.{ts,md}\" \"test/**/*.ts\" \"scripts/**/*.{ts,mjs}\" \"*.{json,md,yml}\"", + "lint": "eslint \"src/**/*.ts\" \"scripts/**/*.ts\" \"test/**/*.ts\" \"examples/**/*.ts\" --max-warnings=0 --no-error-on-unmatched-pattern", + "lint:fix": "eslint \"src/**/*.ts\" \"scripts/**/*.ts\" \"test/**/*.ts\" \"examples/**/*.ts\" --fix --no-error-on-unmatched-pattern", + "format": "prettier --write \"src/**/*.{ts,md}\" \"test/**/*.ts\" \"scripts/**/*.{ts,mjs}\" \"examples/**/*.{ts,json,md}\" \"*.{json,md,yml}\"", + "format:check": "prettier --check \"src/**/*.{ts,md}\" \"test/**/*.ts\" \"scripts/**/*.{ts,mjs}\" \"examples/**/*.{ts,json,md}\" \"*.{json,md,yml}\"", "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", + "examples": "tsx scripts/run-examples.ts", "clean": "rm -rf dist coverage", "pack:docs": "bash scripts/build-docs-artifacts.sh", "smoke": "node scripts/smoke.mjs", diff --git a/scripts/run-examples.ts b/scripts/run-examples.ts new file mode 100644 index 0000000..6142f76 --- /dev/null +++ b/scripts/run-examples.ts @@ -0,0 +1,83 @@ +/** + * Runs every example in `examples/` against the BUILT package, and fails if any of them fails. + * + * Each top-level `examples/*.ts` (a name starting with `_` is skipped) imports this package by its + * published name, which Node resolves through the package's own `exports` map to `dist/`: the files + * a consumer installs, not the source tree. Every example checks its own key output and exits + * non-zero on a mismatch, so the exit status is the verdict here. The output of every example is + * printed, a failing one's stderr included. + * + * It refuses (exit 2) rather than reporting green when there is nothing real to run: no build to + * import, or no example at all. + * + * pnpm build && pnpm examples + * + * SECURITY: every path reaches spawnSync as an argv element, never through a shell. + */ + +import { spawnSync } from "node:child_process"; +import { existsSync, readdirSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; + +const ROOT = join(import.meta.dirname, ".."); +const EXAMPLES_DIR = join(ROOT, "examples"); +const BUILT_ENTRY = join(ROOT, "dist", "index.mjs"); +const TIMEOUT_MS = 60_000; + +/** tsx runs the TypeScript. It resolves `tsconfig.json` from the working directory (the repo root), which maps no paths, so the package name resolves through `exports`. */ +const TSX_CLI = createRequire(import.meta.url).resolve("tsx/cli"); + +function refuse(reason: string): never { + console.error(`[examples] REFUSED: ${reason}`); + process.exit(2); +} + +function indent(text: string): string { + return text + .trimEnd() + .split("\n") + .map((line) => ` ${line}`) + .join("\n"); +} + +if (!existsSync(BUILT_ENTRY)) { + refuse( + `${BUILT_ENTRY} does not exist. The examples import the built package: run \`pnpm build\` first.`, + ); +} + +const examples = readdirSync(EXAMPLES_DIR, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".ts") && !entry.name.startsWith("_")) + .map((entry) => entry.name) + .sort(); + +if (examples.length === 0) refuse(`no example found under ${EXAMPLES_DIR}`); + +let failed = 0; +for (const file of examples) { + const result = spawnSync(process.execPath, [TSX_CLI, join("examples", file)], { + cwd: ROOT, + encoding: "utf8", + shell: false, + timeout: TIMEOUT_MS, + }); + + if (result.status === 0) { + console.log(`OK ${file}`); + if (result.stdout.trim().length > 0) console.log(indent(result.stdout)); + continue; + } + + failed += 1; + const how = + result.error !== undefined + ? result.error.message + : `exit ${String(result.status)}${result.signal === null ? "" : `, signal ${result.signal}`}`; + console.error(`FAIL ${file} (${how})`); + if (result.stdout.trim().length > 0) console.error(indent(result.stdout)); + if (result.stderr.trim().length > 0) console.error(indent(result.stderr)); +} + +console.log(`[examples] ${String(examples.length - failed)} of ${String(examples.length)} passed`); +process.exit(failed === 0 ? 0 : 1);