From 988d7aea142b001dc4eb382228b5dcf6a082c65f Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Mon, 7 Sep 2026 12:25:08 +1000 Subject: [PATCH 1/2] =?UTF-8?q?feat(media,rig,agents,abilities)!:=20docume?= =?UTF-8?q?nts=20=E2=80=94=20PDF=20ingress,=20a=20documents=20ability,=20a?= =?UTF-8?q?ssets=20on=20the=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field Note can attach a PDF. The content plane gains a document ingress (PDFium): one text/markdown representation, the PDF retained as source, a facts-only sidecar (sections, page map, per-page counts, page renders as their own image roots). A new first-party ability, `documents`, searches, reads and views those documents; evidence is cited by content address (`attachment://<12 hex>/page/`). media - `createContentIngress` dispatches on bytes: image → normalizer, %PDF- → document ingress. - `pdf.ts` / `pdf-layout.ts`: PDFium codec per document, one shared gate, deterministic coverage under declared bounds; layout from glyph geometry (advance boxes, text-matrix size, baseline grouping, small-caps headings, corroborated titles). Hardened on two real PDFs (Springer chapter, arXiv paper); fixtures + `matrix.pdf` pin the rules. - Sidecar type + `pagesOf`; renders keep the page's aspect ratio. rig - `fitChunks`, `BM25Index`, ranges lifted to rig; `loadDocuments`; content routes gain `GET /v1/media/`, `/config`, `/source`; the registry seeds the store into abilities. - `createReranker` KV default q4_0 → q8_0. Measured on real document windows: at q4_0 ten identical passages spread 4–6 logits across the leaves and the verdict changed sign; at q8_0 the spread is 0.05–0.12 at the same pass time. `reranker-resolution.test.ts` (weights-gated) holds the default to that floor. agents - Assets available to a run: `AgentPoolOptions.attachments`, `ToolContext.attachments` (staged ∪ admitted by any agent), `Source.promptData(attachments)`; tool media may be a descriptor (`TOOL_ATTACHMENTS_KEY`); rail follows bitmaps; `book()` carries roots. - `agent:prefilled` rides the bus beside the `branch:prefill` trace: a host books and shows admissions from the stream it already consumes (announced from `settle()`, since `emit.trace` refuses bus projections). - Exploit mode takes ONE extra pass: `min(tool score, scoreEntailmentBatch)`. BREAKING: `EntailmentScorer.scoreRelevanceBatch` is removed (its only caller). abilities - `documents` (new, 0.1.0): `search_documents` (top-K within a token budget, explore mode always — the attached document is the on-topic universe), `read_document`, `view_page` (projection rule over sidecar facts). Skill: figures by caption, generic names, no silent substitution, failed tools named. - corpus adopts `fitChunks`; web's `fetch_page` says to attach a PDF instead. Gates: npm test (156 files) green, `tsc -p tsconfig.test.json` clean, verify:packed and verify:oci ok, lockfile reconciled; proved end to end on real weights in the scaffold's web target (evidence in the PR). --- package-lock.json | 41 +- packages/abilities/corpus/src/index.ts | 15 +- .../abilities/corpus/src/tools/read-file.ts | 58 +- packages/abilities/corpus/src/tools/search.ts | 2 +- .../abilities/corpus/test/ability.test.ts | 58 +- packages/abilities/documents/LICENSE | 107 +++ packages/abilities/documents/LICENSE-FAQ.md | 257 +++++++ packages/abilities/documents/README.md | 13 + packages/abilities/documents/ability.json | 10 + packages/abilities/documents/package.json | 33 + packages/abilities/documents/skill.eta | 30 + .../documents/src/documents-index.ts | 130 ++++ packages/abilities/documents/src/index.ts | 70 ++ packages/abilities/documents/src/source.ts | 70 ++ .../documents/src/tools/read-document.ts | 83 ++ .../documents/src/tools/search-documents.ts | 126 +++ .../documents/src/tools/view-page.ts | 93 +++ .../abilities/documents/test/ability.test.ts | 65 ++ .../documents/test/helpers/fixture.ts | 120 +++ .../abilities/documents/test/index.test.ts | 58 ++ .../documents/test/read-document.test.ts | 58 ++ .../documents/test/search-documents.test.ts | 83 ++ .../abilities/documents/test/source.test.ts | 51 ++ .../documents/test/view-page.test.ts | 78 ++ packages/abilities/documents/tsconfig.json | 14 + .../abilities/web/src/tools/fetch-page.ts | 4 +- packages/abilities/web/test/ability.test.ts | 19 +- packages/agents/src/AgentPolicy.ts | 2 +- packages/agents/src/Tool.ts | 43 +- packages/agents/src/admission.ts | 16 +- packages/agents/src/agent-pool.ts | 7 +- packages/agents/src/create-agent-pool.ts | 4 + packages/agents/src/emit.ts | 16 +- packages/agents/src/execute.ts | 52 +- packages/agents/src/index.ts | 2 +- packages/agents/src/prepare-content.ts | 9 +- packages/agents/src/replay.ts | 11 +- packages/agents/src/source.ts | 20 +- packages/agents/src/state.ts | 5 +- packages/agents/src/trace-types.ts | 2 +- packages/agents/src/types.ts | 23 +- packages/agents/test/admission.test.ts | 26 +- packages/agents/test/agent-pool.test.ts | 156 +++- packages/agents/test/attachments.test.ts | 26 +- packages/agents/test/helpers/media.ts | 40 +- ...al-replay-admitted-by-fit.scenario.test.ts | 4 +- .../no-projector-says-so.scenario.test.ts | 2 +- packages/agents/test/prepare-content.test.ts | 2 +- packages/agents/test/source.test.ts | 76 -- packages/agents/test/spawn-agents.test.ts | 8 +- packages/agents/test/spine-multimodal.test.ts | 2 +- packages/agents/test/tool-media.test.ts | 29 +- packages/media/README.md | 102 ++- packages/media/package.json | 15 +- packages/media/src/content-ingress.ts | 45 ++ packages/media/src/document.ts | 148 ++++ packages/media/src/gate.ts | 151 ++++ packages/media/src/image.ts | 139 +--- packages/media/src/index.ts | 5 + packages/media/src/ingress.ts | 13 +- packages/media/src/media-type.ts | 17 +- packages/media/src/node.ts | 8 + packages/media/src/pdf-layout.ts | Bin 0 -> 23984 bytes packages/media/src/pdf.ts | 719 ++++++++++++++++++ packages/media/test/content-ingress.test.ts | 43 ++ packages/media/test/document.test.ts | 62 ++ .../media/test/fixtures/pdf/encrypted.pdf | Bin 0 -> 4095 bytes packages/media/test/fixtures/pdf/make.sh | 144 ++++ packages/media/test/fixtures/pdf/matrix.pdf | Bin 0 -> 752 bytes packages/media/test/fixtures/pdf/scanned.pdf | Bin 0 -> 10515 bytes packages/media/test/fixtures/pdf/tagged.pdf | Bin 0 -> 54184 bytes packages/media/test/fixtures/pdf/untagged.pdf | Bin 0 -> 3847 bytes packages/media/test/gate.test.ts | 81 ++ packages/media/test/ingress.test.ts | 32 + packages/media/test/normalize.test.ts | 21 +- packages/media/test/pdf-ingress.test.ts | 126 +++ packages/media/test/pdf-layout.test.ts | 340 +++++++++ packages/media/test/pdf-render.test.ts | 112 +++ packages/media/test/pdfium-load.test.ts | 63 ++ packages/media/tsconfig.json | 6 +- .../{abilities/corpus => rig}/src/bm25.ts | 0 packages/rig/src/content-routes.ts | 61 +- packages/rig/src/index.ts | 9 + packages/rig/src/ranges.ts | 46 ++ packages/rig/src/registry.ts | 7 +- packages/rig/src/reranker.ts | 16 +- packages/rig/src/resources/documents.ts | 80 ++ packages/rig/src/resources/files.ts | 35 +- packages/rig/src/resources/fit.ts | 129 ++++ packages/rig/src/sources/chunking.ts | 58 +- .../corpus => rig}/test/bm25.test.ts | 0 packages/rig/test/chunking.test.ts | 45 ++ packages/rig/test/content-routes.test.ts | 98 +++ packages/rig/test/documents.test.ts | 79 ++ packages/rig/test/fit-chunks.test.ts | 104 +++ packages/rig/test/helpers/rerank-model.ts | 17 + packages/rig/test/ranges.test.ts | 34 + packages/rig/test/registry.test.ts | 21 + packages/rig/test/reranker-capacity.test.ts | 90 +++ packages/rig/test/reranker-resolution.test.ts | 43 ++ scripts/cut-alpha.test.ts | 8 +- scripts/verify-packed-install.sh | 15 +- tsconfig.test.json | 3 +- 103 files changed, 5242 insertions(+), 477 deletions(-) create mode 100644 packages/abilities/documents/LICENSE create mode 100644 packages/abilities/documents/LICENSE-FAQ.md create mode 100644 packages/abilities/documents/README.md create mode 100644 packages/abilities/documents/ability.json create mode 100644 packages/abilities/documents/package.json create mode 100644 packages/abilities/documents/skill.eta create mode 100644 packages/abilities/documents/src/documents-index.ts create mode 100644 packages/abilities/documents/src/index.ts create mode 100644 packages/abilities/documents/src/source.ts create mode 100644 packages/abilities/documents/src/tools/read-document.ts create mode 100644 packages/abilities/documents/src/tools/search-documents.ts create mode 100644 packages/abilities/documents/src/tools/view-page.ts create mode 100644 packages/abilities/documents/test/ability.test.ts create mode 100644 packages/abilities/documents/test/helpers/fixture.ts create mode 100644 packages/abilities/documents/test/index.test.ts create mode 100644 packages/abilities/documents/test/read-document.test.ts create mode 100644 packages/abilities/documents/test/search-documents.test.ts create mode 100644 packages/abilities/documents/test/source.test.ts create mode 100644 packages/abilities/documents/test/view-page.test.ts create mode 100644 packages/abilities/documents/tsconfig.json create mode 100644 packages/media/src/content-ingress.ts create mode 100644 packages/media/src/document.ts create mode 100644 packages/media/src/gate.ts create mode 100644 packages/media/src/pdf-layout.ts create mode 100644 packages/media/src/pdf.ts create mode 100644 packages/media/test/content-ingress.test.ts create mode 100644 packages/media/test/document.test.ts create mode 100644 packages/media/test/fixtures/pdf/encrypted.pdf create mode 100755 packages/media/test/fixtures/pdf/make.sh create mode 100644 packages/media/test/fixtures/pdf/matrix.pdf create mode 100644 packages/media/test/fixtures/pdf/scanned.pdf create mode 100644 packages/media/test/fixtures/pdf/tagged.pdf create mode 100644 packages/media/test/fixtures/pdf/untagged.pdf create mode 100644 packages/media/test/gate.test.ts create mode 100644 packages/media/test/pdf-ingress.test.ts create mode 100644 packages/media/test/pdf-layout.test.ts create mode 100644 packages/media/test/pdf-render.test.ts create mode 100644 packages/media/test/pdfium-load.test.ts rename packages/{abilities/corpus => rig}/src/bm25.ts (100%) create mode 100644 packages/rig/src/ranges.ts create mode 100644 packages/rig/src/resources/documents.ts create mode 100644 packages/rig/src/resources/fit.ts rename packages/{abilities/corpus => rig}/test/bm25.test.ts (100%) create mode 100644 packages/rig/test/chunking.test.ts create mode 100644 packages/rig/test/documents.test.ts create mode 100644 packages/rig/test/fit-chunks.test.ts create mode 100644 packages/rig/test/helpers/rerank-model.ts create mode 100644 packages/rig/test/ranges.test.ts create mode 100644 packages/rig/test/reranker-capacity.test.ts create mode 100644 packages/rig/test/reranker-resolution.test.ts diff --git a/package-lock.json b/package-lock.json index 34e0483d..d4dcfac4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,6 +46,13 @@ "node": ">=18" } }, + "node_modules/@embedpdf/pdfium": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/@embedpdf/pdfium/-/pdfium-2.15.0.tgz", + "integrity": "sha512-KgpRND2MYcdbhzb2EMb4WzWcJYrR0A6JXvhMv4WthEHKt6qmNo2v/MC68bpYvpveYT9GNnUnY/+TG5MpXY3pRw==", + "dev": true, + "license": "MIT" + }, "node_modules/@emnapi/runtime": { "version": "1.11.3", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", @@ -1063,6 +1070,10 @@ "resolved": "packages/dev-tools", "link": true }, + "node_modules/@lloyal-labs/documents-ability": { + "resolved": "packages/abilities/documents", + "link": true + }, "node_modules/@lloyal-labs/host": { "resolved": "packages/host", "link": true @@ -1246,12 +1257,6 @@ "win32" ] }, - "node_modules/@lloyal-labs/lloyal.node/node_modules/@lloyal-labs/lloyal.node-darwin-x64": { - "optional": true - }, - "node_modules/@lloyal-labs/lloyal.node/node_modules/@lloyal-labs/lloyal.node-linux-x64": { - "optional": true - }, "node_modules/@lloyal-labs/media": { "resolved": "packages/media", "link": true @@ -1637,6 +1642,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.6", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.6.tgz", + "integrity": "sha512-uN+9i8bFT5CUcZfyIEYDrSueACEyKGbUs5kC/72DGlZZoinh84sJfVV0i8UOJD1asdzkvLPBRrKs41kZ8MdEXg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -3842,6 +3854,17 @@ "effection": "^4.0.2" } }, + "packages/abilities/documents": { + "name": "@lloyal-labs/documents-ability", + "version": "0.1.0", + "license": "SEE LICENSE IN LICENSE", + "peerDependencies": { + "@lloyal-labs/lloyal-agents": "^5.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/media": ">=0.2.0-0 <0.3.0", + "@lloyal-labs/rig": "^5.5.0 || >=5.6.0-0 <6.0.0", + "effection": "^4.0.2" + } + }, "packages/abilities/web": { "name": "@lloyal-labs/web-ability", "version": "2.0.2", @@ -3926,12 +3949,18 @@ "version": "0.2.0-alpha.3", "license": "SEE LICENSE IN LICENSE", "devDependencies": { + "@embedpdf/pdfium": "^2.15.0", + "@types/emscripten": "^1.41.6", "sharp": "^0.35.4" }, "peerDependencies": { + "@embedpdf/pdfium": "^2.15.0", "sharp": "^0.35.4" }, "peerDependenciesMeta": { + "@embedpdf/pdfium": { + "optional": true + }, "sharp": { "optional": true } diff --git a/packages/abilities/corpus/src/index.ts b/packages/abilities/corpus/src/index.ts index 8c4ba4db..99f84641 100644 --- a/packages/abilities/corpus/src/index.ts +++ b/packages/abilities/corpus/src/index.ts @@ -1,8 +1,8 @@ /** * `@lloyal-labs/corpus-ability` — HDK reference ability: local-corpus research. * - * Requires a reranker (its `search` tool scores chunks); loads + tokenizes the - * corpus at construction, and returns a validated {@link Ability} whose + * Requires a reranker (its `search` tool scores chunks); loads the corpus and + * fits it into reranker-sized windows at construction, and returns a validated {@link Ability} whose * {@link CorpusSource} is already-bound. * * @packageDocumentation @@ -13,15 +13,13 @@ import { join } from "node:path"; import { call } from "effection"; import { AbilityConfigStoreCtx, RerankerCtx } from "@lloyal-labs/lloyal-agents"; import type { AbilityManifest, Tool } from "@lloyal-labs/lloyal-agents"; -import { defineAbility } from "@lloyal-labs/rig"; +import { defineAbility, fitChunks, DEFAULT_CHUNK_TOKENS } from "@lloyal-labs/rig"; import type { Reranker } from "@lloyal-labs/rig"; import { loadResources, chunkResources } from "@lloyal-labs/rig/node"; import { CorpusSource } from "./source"; export { CorpusSource } from "./source"; export type { CorpusSourceOpts, CorpusPromptData } from "./source"; -export { BM25Index } from "./bm25"; -export type { Bm25Opts, Bm25Hit } from "./bm25"; // The declarative manifest + skill template, read once at module load. The // manifest is handed to defineAbility, which advertises it on the factory — so the @@ -63,8 +61,11 @@ export const createCorpusAbility = defineAbility(manifest, function* () { } const resources = loadResources(corpusPath); - const chunks = chunkResources(resources); - yield* call(() => reranker.tokenizeChunks(chunks)); + // Sections become windows the reranker scores whole; the size is a retrieval + // choice (see DEFAULT_CHUNK_TOKENS), the tokens are the reranker's own. + const chunks = yield* call(() => + fitChunks(chunkResources(resources), { maxTokens: DEFAULT_CHUNK_TOKENS, tokenize: (t) => reranker.tokenize(t) }), + ); const source = new CorpusSource(resources, chunks, reranker); const tools: Record = {}; diff --git a/packages/abilities/corpus/src/tools/read-file.ts b/packages/abilities/corpus/src/tools/read-file.ts index 4f453f2c..c2cba480 100644 --- a/packages/abilities/corpus/src/tools/read-file.ts +++ b/packages/abilities/corpus/src/tools/read-file.ts @@ -2,64 +2,8 @@ import type { Operation } from 'effection'; import { Tool } from '@lloyal-labs/lloyal-agents'; import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; import type { Resource, Chunk } from '@lloyal-labs/rig'; +import { mergeRanges, subtractRanges } from '@lloyal-labs/rig'; -/** - * Subtract previously-covered ranges from a target range - * - * Given a target half-open interval `[s, e)` and an array of - * already-covered intervals, returns the sub-ranges of `[s, e)` - * that have not yet been covered. Used by {@link ReadFileTool} - * to avoid re-reading lines the agent has already seen. - * - * @param range - Target range `[start, end)` (0-indexed) - * @param covered - Array of previously-covered `[start, end)` ranges - * @returns Uncovered sub-ranges of the target - * - * @category Rig - */ -export function subtractRanges( - [s, e]: [number, number], - covered: [number, number][], -): [number, number][] { - let ranges: [number, number][] = [[s, e]]; - for (const [cs, ce] of covered) { - ranges = ranges.flatMap(([a, b]): [number, number][] => { - if (ce <= a || cs >= b) return [[a, b]]; - const result: [number, number][] = []; - if (a < cs) result.push([a, cs]); - if (ce < b) result.push([ce, b]); - return result; - }); - } - return ranges; -} - -/** - * Merge overlapping or adjacent half-open ranges into a minimal set - * - * Sorts the input ranges by start position, then collapses any - * overlapping or touching intervals. Used by {@link ReadFileTool} - * to maintain a compact record of lines already read per agent. - * - * @param ranges - Array of `[start, end)` ranges to merge - * @returns Merged non-overlapping ranges sorted by start - * - * @category Rig - */ -export function mergeRanges(ranges: [number, number][]): [number, number][] { - if (ranges.length === 0) return []; - const sorted = [...ranges].sort((a, b) => a[0] - b[0]); - const merged: [number, number][] = [sorted[0]]; - for (let i = 1; i < sorted.length; i++) { - const last = merged[merged.length - 1]; - if (sorted[i][0] <= last[1]) { - last[1] = Math.max(last[1], sorted[i][1]); - } else { - merged.push(sorted[i]); - } - } - return merged; -} /** * Read content from corpus files by line range diff --git a/packages/abilities/corpus/src/tools/search.ts b/packages/abilities/corpus/src/tools/search.ts index 851e7832..9d451f8b 100644 --- a/packages/abilities/corpus/src/tools/search.ts +++ b/packages/abilities/corpus/src/tools/search.ts @@ -4,7 +4,7 @@ import { Tool, Trace, admitChunks } from '@lloyal-labs/lloyal-agents'; import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; import type { Chunk } from '@lloyal-labs/rig'; import type { Reranker } from '@lloyal-labs/rig'; -import { BM25Index } from '../bm25'; +import { BM25Index } from '@lloyal-labs/rig'; /** * Default score floor for search hits — a useful **discrimination signal**, diff --git a/packages/abilities/corpus/test/ability.test.ts b/packages/abilities/corpus/test/ability.test.ts index f4a8cd18..a491edc7 100644 --- a/packages/abilities/corpus/test/ability.test.ts +++ b/packages/abilities/corpus/test/ability.test.ts @@ -10,9 +10,12 @@ import { createInMemoryConfigStore } from '@lloyal-labs/rig'; import { createCorpusAbility } from '../src/index'; import { SearchTool } from '../src/tools/search'; -// The factory only calls reranker.tokenizeChunks at construction; search -// scoring (which needs a real cross-encoder) isn't exercised here. -const mockReranker = { tokenizeChunks() {} } as unknown as Reranker; +// The factory fits the corpus into windows at construction, which needs only +// the reranker's tokenizer; search scoring (a real cross-encoder) isn't +// exercised here. Words stand in for tokens. +const mockReranker = { + tokenize: async (text: string) => text.split(/\s+/).filter(Boolean).map((_, i) => i + 1), +} as unknown as Reranker; let dir: string; beforeAll(() => { @@ -47,6 +50,55 @@ describe('createCorpusAbility', () => { ).rejects.toThrow(/requires a reranker/); }); + it('fits a long section into more than one window with real line ranges', async () => { + // One heading over 360 words on 60 lines: longer than DEFAULT_CHUNK_TOKENS + // under the word tokenizer, so the factory must cut it into windows the + // reranker can score whole. The source keeps its chunks private; the + // search envelope's `totalScored` is the chunk count, and each hit carries + // the window's real lines. + const longDir = mkdtempSync(join(tmpdir(), 'corpus-long-')); + const body = Array.from({ length: 60 }, (_, i) => `line ${i + 1} alpha beta gamma delta`).join('\n'); + writeFileSync(join(longDir, 'long.md'), `# Long\n\n${body}\n`); + try { + const wordy: Reranker = { + ...mkScoringReranker(new Map()), + tokenize: async (text: string) => text.split(/\s+/).filter(Boolean).map((_, i) => i + 1), + score(_query: string, chunks: Chunk[]) { + return (async function* () { + yield { + filled: chunks.length, total: chunks.length, + results: chunks.map((c) => ({ + file: c.resource, heading: c.heading, section: c.section, snippet: c.text, + score: 1, startLine: c.startLine, endLine: c.endLine, + })), + }; + })(); + }, + }; + const result = (await run(function* () { + yield* Trace.set(new NullTraceWriter()); + const store = createInMemoryConfigStore(); + yield* store.set('corpus', { corpusPath: longDir }); + yield* AbilityConfigStoreCtx.set(store); + yield* RerankerCtx.set(wordy); + const ability = yield* createCorpusAbility(); + const search = ability.tools.find((t) => t.name === 'search')!; + return yield* search.execute({ query: 'alpha' }); + })) as { hits: ScoredChunk[]; totalScored: number }; + + expect(result.totalScored).toBeGreaterThan(1); + const starts = result.hits.map((h) => h.startLine); + expect(new Set(starts).size).toBe(starts.length); + for (const h of result.hits) { + expect(h.endLine).toBeGreaterThanOrEqual(h.startLine); + expect(h.startLine).toBeGreaterThanOrEqual(1); + expect(h.endLine).toBeLessThanOrEqual(62); + } + } finally { + rmSync(longDir, { recursive: true, force: true }); + } + }); + it('throws when corpusPath config is missing', async () => { await expect( run(function* () { diff --git a/packages/abilities/documents/LICENSE b/packages/abilities/documents/LICENSE new file mode 100644 index 00000000..d19bdb84 --- /dev/null +++ b/packages/abilities/documents/LICENSE @@ -0,0 +1,107 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2026 Lloyal Labs + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publish, and distribute the +Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A "Permitted Purpose" is any purpose other than a Competing Use. A +"Competing Use" means making the Software available to others in a +commercial product or service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a Licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives +of the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, +INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO +THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL +DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software +under the Apache License, Version 2.0 that is effective on the second +anniversary of the date we make the Software available. On or after that +date, you may use the Software under the Apache License, Version 2.0, in +which case the following will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/abilities/documents/LICENSE-FAQ.md b/packages/abilities/documents/LICENSE-FAQ.md new file mode 100644 index 00000000..cd89f587 --- /dev/null +++ b/packages/abilities/documents/LICENSE-FAQ.md @@ -0,0 +1,257 @@ +# Licensing FAQ + +> Canonical version at https://docs.lloyal.ai/licensing/faq. +> This file is a synced copy. Edit the canonical source and re-run +> `scripts/sync-license-faq.sh` in lloyal-sdk to update all copies. + + +**You can build and sell commercial products using HDK.** + +> HDK is free to build products with; it is not free to become the +> replacement HDK platform. + +That single sentence is the entire restriction reduced to one line. The rest +of this page is illustration. + +## The short version + +HDK 3.0 runtime packages — `liblloyal`, `lloyal-node`, and the lloyal-sdk +packages (`agents`, `sdk`, `rig`, `abilities/corpus`, `abilities/web`) — are +**[Fair Source](https://fair.io)** under **FSL-1.1-Apache-2.0** (the Functional +Source License, Apache 2.0 future grant). + +Each version converts to Apache 2.0 two years after its release. The +restriction during those two years is narrow: **you cannot offer a competing +HDK runtime, a managed HDK service, or an alternative Ability distribution +channel.** Everything else — commercial use, redistribution, modification, +sale, embedding in shipped products — is freely permitted. + +The reason the channel restriction exists is consumer-protective: every Ability +listed on `apps.lloyal.ai` is reviewed by Lloyal Labs for tool-safety, +manifest conformance, and signature provenance before publication. Pinning +the Ability ecosystem to one verified channel keeps the AI-safety review meaningful +(consumers can rely on a single trust boundary) and prevents protocol +fragmentation (an Ability that works on one harness works on every harness). + +The `lloyal-ai` CLI is licensed under **MIT**; the `hdk-create-app` +scaffolder under **Apache 2.0**. Both are unrestricted, and neither is +part of the runtime stack. + +## Can I ship a commercial product built on HDK? + +**Yes.** This is the question everyone has and the answer is straightforward. +Concretely: + +- **Shipping a paid intelligent inbox app to consumers** — permitted ✅ +- **Selling an Excel-with-AI desktop app to enterprises** — permitted ✅ +- **Embedding HDK in a medical device sold commercially** — permitted ✅ +- **A consulting firm building a custom intelligent harness for a Fortune + 500 client, charging $500K for the engagement, deploying on client + infrastructure** — permitted ✅ +- **An indie dev shipping a paid productivity app on the Mac App Store** — + permitted ✅ +- **An OEM shipping HDK inside an infotainment system or industrial device** + — permitted ✅ +- **A startup building an end-user product on top of HDK** — including + vertical research apps, workflow tools, and agent applications — permitted + ✅, *as long as the product is not offering HDK itself as a substitute + runtime, managed HDK service, or competing Ability distribution channel*. +- **A research lab using HDK in published academic work** — permitted ✅ +- **Forking HDK on GitHub to learn, modify, demo, or contribute** — permitted ✅ +- **Running HDK internally inside your company for any business use** — + permitted ✅ + +If you are building something *with* HDK, you are almost certainly fine. + +## What is actually restricted? + +The restriction is narrow and specific: **don't become the replacement +platform vendor**. Concretely: + +- **AWS / Google / Microsoft launching "Bedrock Managed HDK" or "Vertex HDK" + as a hosted runtime service** — restricted ❌ +- **A competitor publishing "OpenHDK" as a forked harness runtime under a + different name** — restricted ❌ +- **A clean-room reimplementation of the HDK runtime in Python / Rust / Go + intended as a drop-in replacement** — restricted ❌ (Competing Use doesn't + require forking source code — a reimplementation that competes is the + same problem) +- **Launching `apps.competitor.com` as an alternative Ability distribution + channel** — restricted ❌ +- **Offering "managed HDK hosting" or "HDK-as-a-Service" as a competing + SaaS** — restricted ❌ +- **A hosted orchestration service exposing HDK-compatible APIs as a + substitute for the HDK runtime** — restricted ❌ + +Notice the pattern: every restricted scenario is "become the platform +vendor," not "build products with HDK." If your project doesn't compete +directly with the HDK runtime or its distribution channel, FSL doesn't +affect you. + +## Why does the LICENSE list four narrow Permitted Purposes then? + +You may read the FSL LICENSE and see this section: + +> Permitted Purposes specifically include using the Software: +> 1. for your internal use and access; +> 2. for non-commercial education; +> 3. for non-commercial research; and +> 4. in connection with professional services that you provide to a +> Licensee using the Software in accordance with these Terms and +> Conditions. + +A careful first reading can mistake this list for the *exclusive* set of +permitted uses — leading to the (incorrect) conclusion that commercial +product distribution is prohibited. + +It isn't. **The operative definition is broader.** Earlier in the same +section, the license states: + +> A "Permitted Purpose" is any purpose other than a Competing Use. + +The four enumerated items are **illustrative examples** added because those +specific cases are ones a careful reader might otherwise hesitate about +("is research permitted? is consulting permitted?"). The four items are +additive clarifications, not a closing of the open-ended definition. + +Sentry, who authored FSL and uses it on their own software, [confirms this +explicitly in their FAQ](https://fsl.software): + +> "You can do anything with FSL software except undermine its producer. You +> can run it for almost all purposes, study it, modify it, and distribute +> your changes…" + +If the license felt restrictive on first read, that's a documented +[FSL adoption hazard](https://fair.io) — many developers hit the same wall. +The answer is to read the "any purpose other than a Competing Use" line as +the operative definition and treat the four enumerated items as examples, +not as a closed list. + +## Will it become Apache 2.0? + +**Yes — automatically, on a per-version schedule.** Each released version of +the runtime stack converts to Apache 2.0 exactly two years after its release +date. The conversion is irrevocable and written into the license text — it's +not a promise from Lloyal Labs, it's a contractual clause. + +For example: if `lloyal-sdk @lloyal-labs/lloyal-agents` v3.0.0 is released +on 2026-06-01, that exact version becomes available under Apache 2.0 on +2028-06-01. Any consumer can elect to use that version under Apache 2.0 +from that date forward — Lloyal Labs takes no action; the grant is +automatic. + +New versions released after v3.0.0 start their own two-year clock from +their own release dates. There is no single global Change Date. + +## Is this OSI-approved open source? + +**No, and we want to be honest about that.** FSL is not OSI-approved +because the OSI definition of open source (clause 6, "No Discrimination +Against Fields of Endeavor") does not permit restrictions on specific +use cases. FSL restricts Competing Use. That restriction takes it out of +strict OSI compliance. + +FSL falls under the [Fair Source](https://fair.io) classification — +source-available licenses that are explicitly developer-friendly: +commercial use permitted, free redistribution, eventual open-source +conversion. Fair Source is a more developer-friendly framing than the +generic "source-available" label, which has been tainted by the +SSPL / Elastic / MongoDB relicensing trauma cycles. + +What this means practically: + +- **You can read the source.** ✅ +- **You can modify it.** ✅ +- **You can sell products built with it.** ✅ +- **You can redistribute it (with the same FSL terms).** ✅ +- **It will be Apache 2.0 in two years.** ✅ +- Some enterprise procurement policies that strictly require OSI-approved + licenses will require an exception for this. We're working on making + that exception easy to grant. + +## Why FSL specifically — why not stay Apache? + +HDK 3.0 introduces installable Abilities. Every Ability declares against a +specific Ability protocol — the bytes-locked intro, catalog format, +tool-selection rule, and boundary marker that the runtime renders into +the spine. Your Ability's reliability depends on every HDK runtime your users +install agreeing on the same protocol. + +Under a permissive license alone, the protocol is forkable. A +well-resourced redistributor could fork the runtime, modify the protocol +surface, and distribute a variant under a different name with captive +distribution. Ability developers then face a fragmented ecosystem: target one +protocol, target both, or pick the bigger distribution and abandon the +others. The cost of that split is paid by Ability developers in testing +burden, divergent behavior, and reliability degradation across runtimes. + +FSL's two-year Competing Use restriction is shaped to block that +fragmentation specifically. After the conversion, anyone can build +whatever they want — by which time the protocol has had enough time to +stabilize through ecosystem use and the protection is no longer the load- +bearing thing keeping it coherent. + +For the longer treatment of this argument, see +[Why FSL](./why-fsl). + +We could have used Apache 2.0 and tried to protect only the channel via +terms-of-service. We could have written a custom license. We chose +standard FSL because it's: + +- **Off-the-shelf** — no bespoke license review at every adopter +- **Recognizable** — Sentry, PowerSync, and others use it +- **Documented** — the FAQ, definitions, and edge cases have been + litigated publicly +- **Time-bounded** — the protocolual Apache 2.0 conversion is the answer + to the "is this just source-available forever?" critique +- **Pre-launch** — relicensing at HDK 3.0 launch is structurally + different from MongoDB / Elastic / HashiCorp relicensing under an + existing installed base, which is what causes the backlash cycle + +## What about the lloyal stack — what's under FSL and what's not? + +| Component | License | Why | +|---|---|---| +| `liblloyal` (C++ engine) | FSL-1.1-Apache-2.0 | Native primitives the runtime is built on | +| `lloyal-node` (N-API binding) | FSL-1.1-Apache-2.0 | The binding that lets Effection drive llama.cpp | +| `@lloyal-labs/lloyal-agents` | FSL-1.1-Apache-2.0 | Runtime framework | +| `@lloyal-labs/lloyal-sdk` | FSL-1.1-Apache-2.0 | Runtime framework | +| `@lloyal-labs/rig` | FSL-1.1-Apache-2.0 | Runtime framework — holds the Ability protocol | +| `@lloyal-labs/corpus`, `@lloyal-labs/web` | FSL-1.1-Apache-2.0 | Reference Abilities shipped in-tree | +| **`lloyal-ai`** (the CLI) | **MIT** | Scaffolder — unrestricted for scaffolding new harnesses and Abilities | +| **`hdk-create-app`** (when shipped) | **Apache 2.0** | Scaffolder — unrestricted, not part of the runtime stack | +| `llama.cpp` (vendored dependency) | MIT (unchanged) | External upstream library; we don't relicense their code | + +## Can I contribute to HDK? + +**Yes.** Contributions are welcome under the same FSL license terms. If you +submit a PR, you're granting Lloyal Labs the right to distribute your +contribution under FSL-1.1-Apache-2.0 (and automatically under Apache 2.0 +two years after each release that includes it). The CONTRIBUTING file in +each repo has the details. + +## I have a use case that's borderline — who do I ask? + +Email [legal@lloyal.ai](mailto:legal@lloyal.ai) (or open a discussion in the repo). The runtime +team will help you confirm whether your use case falls under Permitted +Purpose or Competing Use. We'd rather give you a quick yes than have you +worry about it. + +## Further reading + +- [FSL official site](https://fsl.software) — Sentry's canonical FSL + resources and FAQ +- [Fair Source](https://fair.io) — the category FSL belongs to +- [The FSL template, instantiated for each repo](./fsl-template) +- [Why we chose FSL over BSL, Apache, or a custom license](./why-fsl) + +## Is there a safe harbor for building products with the HDK? + +Yes. The [Lloyal Harness Builder Grant](https://github.com/lloyal-ai/hdk/blob/main/GRANT.md) +irrevocably guarantees that building, selling, and hosting Harnesses and Abilities +is a Permitted Purpose and never a Competing Use — even products that compete +head-on with Lloyal's own (including reasoning.run). Only three uses remain +restricted: offering the HDK itself as a developer framework, hosting the HDK +as-a-service for third-party developers, and operating a general-purpose Ability +distribution channel. Private/internal Ability distribution and your Harness's +own plugin system are explicitly permitted. diff --git a/packages/abilities/documents/README.md b/packages/abilities/documents/README.md new file mode 100644 index 00000000..e9a926c5 --- /dev/null +++ b/packages/abilities/documents/README.md @@ -0,0 +1,13 @@ +# @lloyal-labs/documents-ability + +**Give your agents the documents the user attached.** + +HDK Ability `lloyal/documents` — a PDF attached to the conversation becomes searchable passages with page numbers, readable sections, and page images an agent can look at when a table or chart matters. Retrieval is the corpus stack: BM25 first stage, cross-encoder rerank, honest scores. Evidence is cited by content address, so a page an agent quotes opens in the UI at the exact bytes the run used. + +| Tool | What agents get | +| --- | --- | +| `search_documents` | Passages ranked by the reranker, each with its document, heading, line range, pages and a `cite` URL | +| `read_document` | The exact text of a page or a line range — the verification step after search | +| `view_page` | The page (or one figure on it) as an image, for tables and charts; text-only pages say so | + +Protocol: `document_research` · Documents reach the ability as the assets available to a run (`ToolContext.attachments`) and are indexed once per digest. The table of contents is published via `Source.promptData(attachments)` for spine placement. Distributed through the signed channel. diff --git a/packages/abilities/documents/ability.json b/packages/abilities/documents/ability.json new file mode 100644 index 00000000..50e9d40d --- /dev/null +++ b/packages/abilities/documents/ability.json @@ -0,0 +1,10 @@ +{ + "name": "documents", + "abilityProtocolVersion": "3.0", + "protocol": { + "name": "document_research", + "useWhen": "reading the documents attached to this conversation — PDFs, papers, reports: finding passages, quoting sections with page numbers, and checking a table or figure on a specific page.", + "tools": ["search_documents", "read_document", "view_page"] + }, + "services": ["reranker"] +} diff --git a/packages/abilities/documents/package.json b/packages/abilities/documents/package.json new file mode 100644 index 00000000..d6550f02 --- /dev/null +++ b/packages/abilities/documents/package.json @@ -0,0 +1,33 @@ +{ + "name": "@lloyal-labs/documents-ability", + "version": "0.1.0", + "private": true, + "description": "HDK reference ability — research over the documents attached to a conversation (PDFs through the content plane). Distributed via the signed-bundle channel, never public npm.", + "license": "SEE LICENSE IN LICENSE", + "type": "commonjs", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist/", + "ability.json", + "skill.eta", + "attention-surface.json", + "LICENSE", + "LICENSE-FAQ.md" + ], + "scripts": { + "build": "tsc -b" + }, + "peerDependencies": { + "@lloyal-labs/lloyal-agents": "^5.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/media": ">=0.2.0-0 <0.3.0", + "@lloyal-labs/rig": "^5.5.0 || >=5.6.0-0 <6.0.0", + "effection": "^4.0.2" + } +} diff --git a/packages/abilities/documents/skill.eta b/packages/abilities/documents/skill.eta new file mode 100644 index 00000000..d7c62bb0 --- /dev/null +++ b/packages/abilities/documents/skill.eta @@ -0,0 +1,30 @@ +You are a thorough research assistant answering from the documents attached to this conversation. +<% if (it.agentCount > 1) { -%> + +You are one of <%= it.agentCount %> parallel agents. The other agents are covering: +<%= it.siblingTasks.map(function(q) { return '- ' + q }).join('\n') %> + +Stay focused on your own task. The other agents handle the rest. +<% } -%> + +You have <%= it.maxTurns %> tool calls. Plan your investigation within this budget. + +The attached documents are listed under `document_research` in your briefing, each with its page count and topics. + +RULES FOR TOOL OPTIMAL USE: +- search_documents returns the best-matching passages with their scores, each with its document, line range and pages — always some, ranked, never a verdict. Pass `document` (the id or title from a previous result) to search one document only. +- Read before you quote: read_document returns the exact text of a page or a line range. Search snippets are truncated. +- view_page is expensive — it puts the page image in front of you. Use it for a table, chart or figure after reading the text around it, and only on pages a result pointed you to. +- A figure or table named in the question (Fig. 2.2, Table 1): search for its caption text, then view_page the page the hit names. +- A brand or product name that returns nothing: search its generic or chemical name, then search the document again. +- Never present a different figure, table or section as the one asked for. If you did not find the one named, say so and describe what you did find. +- If a tool returns an error, name the failure in your report. +- If a query returns `Resource unavailable`, another agent already issued that exact call. Try different terms. +- Don't repeat the same query, read the same lines twice, or view the same page twice. +- When a tool returns an error about time limit, KV limit or word limit, stop and call report() with all your findings so far. + +PROCESS: +1. search_documents to find the passages that bear on your task. +2. read_document to verify every passage in context, with its page. +3. For a figure, chart or table you are asked about, view_page the page a result pointed you to, after reading around it. +4. Call report() with comprehensive findings: direct quotes with page numbers and data points. Put every page you cite in the report's sources with its `cite` URL. State what you found AND what you checked but could not find. Do not summarize — preserve the detail. diff --git a/packages/abilities/documents/src/documents-index.ts b/packages/abilities/documents/src/documents-index.ts new file mode 100644 index 00000000..a7827794 --- /dev/null +++ b/packages/abilities/documents/src/documents-index.ts @@ -0,0 +1,130 @@ +/** + * @file From the assets available to a run to a searchable value. + * + * A function from attachments to an index: the same attachments give the same + * index. Each document is fitted once per digest and remembered, so a second + * call in the same session — the next tool call, the next run of the thread — + * re-tokenizes nothing. No sync step, no state machine: what a tool sees is + * whatever is available to the run at the call, indexed. + */ +import { BM25Index, fitChunks, DEFAULT_CHUNK_TOKENS, loadDocuments } from '@lloyal-labs/rig'; +import type { Document } from '@lloyal-labs/rig'; +import type { Chunk } from '@lloyal-labs/lloyal-agents'; +import type { Attachment, AttachmentStore } from '@lloyal-labs/media'; + +/** How many hex digits of the root digest make a document's handle. */ +export const ID_LENGTH = 12; + +/** + * A document's handle: the first twelve hex digits of its root digest. It is + * the asset's own identity, copied from tool output like any id — never an + * ordinal three parties must order the same way. The UI resolves it against + * the digests it already holds and requires exactly one match. + */ +export function documentId(root: Attachment): string { + const hex = root.digest.slice(root.digest.indexOf(':') + 1); + return hex.slice(0, ID_LENGTH); +} + +/** The citation for a page: what the model copies into its report. */ +export function pageCite(root: Attachment, page: number): string { + return `attachment://${documentId(root)}/page/${page}`; +} + +/** A loaded document with its handle. Its chunks and resource are named by the handle. */ +export interface IndexedDocument extends Document { + id: string; +} + +export interface DocumentsIndex { + documents: IndexedDocument[]; + /** Every document's fitted chunks, `resource` = the document id. */ + chunks: Chunk[]; + /** Lexical first stage over `chunks`; null when there is nothing to index. */ + bm25: BM25Index | null; + /** By id, or by title (exact, case-insensitive). */ + find(document: string): IndexedDocument | undefined; +} + +export type Tokenize = (text: string) => Promise; + +/** + * Build the indexer for one session. Returns the function from attachments to + * an index. Roots that are not documents (an image) are skipped. A root whose + * content is missing or drifted throws, naming the digest — an empty index + * behind a digest that promises content would be a silent wrong answer. + */ +export function documentIndexer( + store: AttachmentStore, + tokenize: Tokenize, +): (attachments: readonly Attachment[]) => Promise { + type Fitted = { document: IndexedDocument; chunks: Chunk[] }; + const fitted = new Map>(); + const indices = new Map>(); + + const fit = (root: Attachment): Promise => { + let p = fitted.get(root.digest); + if (!p) { + p = (async () => { + const [loaded] = loadDocuments(store, [root]); + if (!loaded) return null; + const id = documentId(root); + const chunks = await fitChunks( + loaded.chunks.map((c) => ({ ...c, resource: id })), + { maxTokens: DEFAULT_CHUNK_TOKENS, tokenize }, + ); + const document: IndexedDocument = { ...loaded, id, resource: { name: id, content: loaded.resource.content } }; + return { document, chunks }; + })(); + fitted.set(root.digest, p); + // A failure is not remembered: the next call asks the store again. + p.catch(() => { fitted.delete(root.digest); }); + } + return p; + }; + + return (attachments) => { + // The same root attached twice is one document. + const roots = [...new Map(attachments.map((a) => [a.digest, a] as const)).values()]; + const key = roots.map((r) => r.digest).join('\n'); + let index = indices.get(key); + if (!index) { + index = (async () => { + const entries = (await Promise.all(roots.map(fit))).filter((e): e is Fitted => e !== null); + const documents = entries.map((e) => e.document); + const byId = new Map(); + for (const d of documents) { + const other = byId.get(d.id); + if (other) { + throw new Error( + `documents: two attached documents share the handle ${d.id} ` + + `(${other.attachment.digest} and ${d.attachment.digest}); a citation could not name one of them.`, + ); + } + byId.set(d.id, d); + } + const chunks = entries.flatMap((e) => e.chunks); + const bm25 = chunks.length > 0 ? new BM25Index(chunks.map((c) => c.tokens)) : null; + return { + documents, chunks, bm25, + find(document: string) { + const needle = document.trim().toLowerCase(); + return byId.get(needle) ?? documents.find((d) => d.meta.title.toLowerCase() === needle); + }, + }; + })(); + indices.set(key, index); + index.catch(() => { indices.delete(key); }); + } + return index; + }; +} + +/** The error a tool returns for a document it cannot find: it lists what is attached. */ +export function unknownDocument(asked: string, index: DocumentsIndex): string { + const attached = index.documents.map((d) => `${d.meta.title} (${d.id})`).join(', '); + return `Unknown document: ${asked}. Attached: ${attached || 'none'}.`; +} + +/** The message when nothing is attached — the same from every tool. */ +export const NO_DOCUMENTS = 'No documents are attached to this conversation.'; diff --git a/packages/abilities/documents/src/index.ts b/packages/abilities/documents/src/index.ts new file mode 100644 index 00000000..c4d42dd6 --- /dev/null +++ b/packages/abilities/documents/src/index.ts @@ -0,0 +1,70 @@ +/** + * `@lloyal-labs/documents-ability` — HDK reference ability: research over the + * documents attached to a conversation. + * + * Documents reach the ability as the assets available to a run: every tool + * reads `ToolContext.attachments` at the call and indexes what is there, once + * per digest. The table of contents the harness places on the spine comes + * from `Source.promptData(attachments)`. Nothing here touches the native + * addon or a node-only entry of a sibling package. + * + * @packageDocumentation + */ + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { RerankerCtx, Attachments } from '@lloyal-labs/lloyal-agents'; +import type { AbilityManifest, Tool } from '@lloyal-labs/lloyal-agents'; +import { defineAbility } from '@lloyal-labs/rig'; +import type { Reranker } from '@lloyal-labs/rig'; +import { DocumentsSource } from './source'; +import { documentIndexer } from './documents-index'; +import { SearchDocumentsTool } from './tools/search-documents'; +import { ReadDocumentTool } from './tools/read-document'; +import { ViewPageTool } from './tools/view-page'; + +export { DocumentsSource, buildToc } from './source'; +export type { DocumentsPromptData } from './source'; +export { documentIndexer, documentId, pageCite, NO_DOCUMENTS } from './documents-index'; +export type { DocumentsIndex, IndexedDocument } from './documents-index'; +export { SearchDocumentsTool, locate } from './tools/search-documents'; +export type { DocumentHit, IndexFor } from './tools/search-documents'; +export { ReadDocumentTool } from './tools/read-document'; +export { ViewPageTool, projectable } from './tools/view-page'; + +// The declarative manifest + skill template, read once at module load and +// handed to defineAbility, which advertises the manifest on the factory. +const dir = join(__dirname, '..'); +const manifest = JSON.parse(readFileSync(join(dir, 'ability.json'), 'utf8')) as AbilityManifest; +const skill = readFileSync(join(dir, 'skill.eta'), 'utf8'); + +/** + * Construct the documents ability. Reads the reranker from `RerankerCtx` and + * the content store from `Attachments`, builds the per-session indexer, and + * wires the three tools. Under `lloyal describe` the default store answers + * null and the ability is simply empty. + */ +export const createDocumentsAbility = defineAbility(manifest, function* () { + let reranker: Reranker; + try { + reranker = yield* RerankerCtx.expect(); + } catch { + throw new Error( + 'createDocumentsAbility: the documents ability requires a reranker (its `search_documents` tool ' + + 'scores passages), but RerankerCtx is unset. The harness boot normally provisions it from ' + + "the ability's `services: ['reranker']` — call provisionAbilityModels({ abilities, projectRoot }) " + + '(or otherwise set RerankerCtx) before enabling this ability.', + ); + } + const store = yield* Attachments.expect(); + const indexFor = documentIndexer(store, (text) => reranker.tokenize(text)); + const tools: Tool[] = [ + new SearchDocumentsTool(indexFor, reranker), + new ReadDocumentTool(indexFor), + new ViewPageTool(indexFor), + ]; + const source = new DocumentsSource(store, tools, reranker); + const byName: Record = {}; + for (const t of tools) byName[t.name] = t; + return { source, tools: byName, skill }; +}); diff --git a/packages/abilities/documents/src/source.ts b/packages/abilities/documents/src/source.ts new file mode 100644 index 00000000..b08e944f --- /dev/null +++ b/packages/abilities/documents/src/source.ts @@ -0,0 +1,70 @@ +import { Source } from '@lloyal-labs/lloyal-agents'; +import type { Tool, Chunk } from '@lloyal-labs/lloyal-agents'; +import type { Reranker, Document } from '@lloyal-labs/rig'; +import { loadDocuments } from '@lloyal-labs/rig'; +import type { Attachment, AttachmentStore } from '@lloyal-labs/media'; + +/** Data for the harness to place — once, in shared KV. */ +export type DocumentsPromptData = { + /** One line per attached document: title, page count, top-level topics. */ + toc: string; +}; + +/** Titles and headings are model-facing prose from a user's file: control + * characters (Unicode category Cc) go, whitespace collapses, length is capped. */ +const MAX_LABEL = 120; +const MAX_TOPICS = 8; +function label(text: string): string { + const clean = text.replace(/\p{Cc}/gu, ' ').replace(/\s+/g, ' ').trim(); + return clean.length > MAX_LABEL ? `${clean.slice(0, MAX_LABEL - 1)}…` : clean; +} + +/** The table of contents for a set of documents: ` — <n> pages (topics: …)`. */ +export function buildToc(documents: readonly Document[]): string { + return documents.map((d) => { + const title = label(d.meta.title); + const topics: string[] = []; + for (const s of d.meta.sections) { + if (s.path.includes(' > ')) continue; + const heading = label(s.heading); + if (heading === title || topics.includes(heading)) continue; + topics.push(heading); + if (topics.length >= MAX_TOPICS) break; + } + const pages = `${d.meta.pageCount} page${d.meta.pageCount === 1 ? '' : 's'}`; + return topics.length > 0 ? `${title} — ${pages} (topics: ${topics.join(', ')})` : `${title} — ${pages}`; + }).join('\n'); +} + +/** + * The documents source: `search_documents`, `read_document` and `view_page` + * over the documents available to a run. The tools read the run's assets per + * call; this class holds only what is per session — the store and the tools. + */ +export class DocumentsSource extends Source<Chunk> { + /** @inheritDoc */ + readonly name = 'documents'; + private readonly _store: AttachmentStore; + private readonly _tools: Tool[]; + + constructor(store: AttachmentStore, tools: Tool[], reranker: Reranker) { + super(); + this._store = store; + this._tools = tools; + this._reranker = reranker; + } + + /** @inheritDoc */ + get tools(): Tool[] { + return this._tools; + } + + /** + * The table of contents of the documents among `attachments` — the assets + * the run is being staged with — for the harness to place on the spine. + * Built from the sidecars alone: no tokenizing, no index. + */ + override promptData(attachments: readonly Attachment[] = []): DocumentsPromptData { + return { toc: buildToc(loadDocuments(this._store, attachments)) }; + } +} diff --git a/packages/abilities/documents/src/tools/read-document.ts b/packages/abilities/documents/src/tools/read-document.ts new file mode 100644 index 00000000..bf9218c9 --- /dev/null +++ b/packages/abilities/documents/src/tools/read-document.ts @@ -0,0 +1,83 @@ +import { call } from 'effection'; +import type { Operation } from 'effection'; +import { Tool } from '@lloyal-labs/lloyal-agents'; +import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; +import { mergeRanges, subtractRanges } from '@lloyal-labs/rig'; +import { pagesOf } from '@lloyal-labs/media'; +import { pageCite, unknownDocument, NO_DOCUMENTS } from '../documents-index'; +import type { DocumentsIndex } from '../documents-index'; +import type { IndexFor } from './search-documents'; + +/** + * The exact text of a document: a page, or a line range from a search hit. + * A page reads as the whole sections that touch it, so a table split by a + * page break stays whole. Per-agent read tracking returns only the unread + * part, as the corpus reader does. + */ +export class ReadDocumentTool extends Tool<{ document: string; page?: number; startLine?: number; endLine?: number }> { + readonly name = 'read_document'; + readonly protected = false; + readonly description = 'Read the exact text of an attached document: a whole page, or a line range from search results. Name the document by id or title.'; + readonly parameters: JsonSchema = { + type: 'object', + properties: { + document: { type: 'string', description: 'Document id or title, from search results or the briefing' }, + page: { type: 'number', description: 'Page number — returns every section that touches the page' }, + startLine: { type: 'number', description: 'Start line (1-indexed, from search results)' }, + endLine: { type: 'number', description: 'End line (1-indexed, from search results)' }, + }, + required: ['document'], + }; + + private readonly _indexFor: IndexFor; + private readonly _defaultMaxLines: number; + private readonly _read = new Map<string, [number, number][]>(); + + constructor(indexFor: IndexFor, opts?: { defaultMaxLines?: number }) { + super(); + this._indexFor = indexFor; + this._defaultMaxLines = opts?.defaultMaxLines ?? 100; + } + + *execute( + args: { document: string; page?: number; startLine?: number; endLine?: number }, + context?: ToolContext, + ): Operation<unknown> { + const index: DocumentsIndex = yield* call(() => this._indexFor(context?.attachments ?? [])); + if (index.documents.length === 0) return { error: NO_DOCUMENTS }; + const doc = index.find(args.document ?? ''); + if (!doc) return { error: unknownDocument(args.document ?? '', index) }; + const lines = doc.resource.content.split('\n'); + + // [s, e): 0-based start, exclusive end — the range helpers' convention. + let s: number; + let e: number; + if (args.page !== undefined) { + const page = doc.meta.pages.find((p) => p.page === args.page); + if (!page) return { error: `Page ${args.page} is out of range: ${doc.meta.title} has ${doc.meta.pageCount} pages.` }; + const covering = doc.meta.sections.filter((sec) => sec.pageStart <= page.page && page.page <= sec.pageEnd); + s = (covering.length > 0 ? Math.min(...covering.map((c) => c.startLine)) : page.startLine) - 1; + e = covering.length > 0 ? Math.max(...covering.map((c) => c.endLine)) : page.endLine; + } else { + s = Math.max(0, (args.startLine ?? 1) - 1); + e = Math.min(lines.length, args.endLine ?? s + this._defaultMaxLines); + } + if (e <= s) return { error: `Nothing to read: lines ${s + 1}-${e} of ${doc.meta.title}.` }; + + const key = `${context?.agentId ?? ''}:${doc.id}`; + const prev = this._read.get(key) ?? []; + const unread = subtractRanges([s, e], prev); + if (unread.length === 0) return { document: doc.meta.title, id: doc.id, note: `Lines ${s + 1}-${e} already read` }; + this._read.set(key, mergeRanges([...prev, [s, e]])); + + const content = unread.map(([a, b]) => lines.slice(a, b).join('\n')).join('\n...\n'); + const { pageStart, pageEnd } = pagesOf(doc.meta, s + 1, e); + return { + document: doc.meta.title, id: doc.id, + lines: unread.map(([a, b]) => `${a + 1}-${b}`), + pageStart, pageEnd, + cite: pageCite(doc.attachment, pageStart), + content, + }; + } +} diff --git a/packages/abilities/documents/src/tools/search-documents.ts b/packages/abilities/documents/src/tools/search-documents.ts new file mode 100644 index 00000000..f64035aa --- /dev/null +++ b/packages/abilities/documents/src/tools/search-documents.ts @@ -0,0 +1,126 @@ +import { call } from 'effection'; +import type { Operation } from 'effection'; +import { Tool, Trace, admitChunks } from '@lloyal-labs/lloyal-agents'; +import type { JsonSchema, ToolContext, Chunk } from '@lloyal-labs/lloyal-agents'; +import type { Reranker, ScoredChunk } from '@lloyal-labs/rig'; +import { pagesOf } from '@lloyal-labs/media'; +import type { Attachment } from '@lloyal-labs/media'; +import { pageCite, unknownDocument, NO_DOCUMENTS } from '../documents-index'; +import type { DocumentsIndex } from '../documents-index'; + +/** The function from a run's attachments to their index (see `documentIndexer`). */ +export type IndexFor = (attachments: readonly Attachment[]) => Promise<DocumentsIndex>; + +/** How many passages a search returns: the top K within a token budget, the + * allowance `fetch_page` gives a page. No score floor: the reranker is a + * relative judge — its scale shifts from passage to passage and its verdict + * on an attached document's own text runs within a few logits of zero — so + * the best K are returned with their scores and the agent reads them. */ +const DEFAULT_TOP_K = 5; +const DEFAULT_TOKEN_BUDGET = 2048; +/** First-stage cap: BM25 narrows to the top-K lexical matches before the + * cross-encoder, whose cost is linear in candidates. */ +const DEFAULT_FIRST_STAGE_K = 100; + +/** A hit as this ability returns it: the corpus shape plus where it is. */ +export interface DocumentHit extends ScoredChunk { + document: string; + id: string; + pageStart: number; + pageEnd: number; + /** `attachment://<id>/page/<pageStart>` — copied into report sources as-is. */ + cite: string; +} + +/** Place a scored chunk in its document: title, id, pages, citation. */ +export function locate(index: DocumentsIndex, hit: ScoredChunk): DocumentHit { + const doc = index.documents.find((d) => d.id === hit.file); + if (!doc) throw new Error(`search_documents: hit names ${hit.file}, which is not a document in the index`); + const { pageStart, pageEnd } = pagesOf(doc.meta, hit.startLine, hit.endLine); + return { ...hit, document: doc.meta.title, id: doc.id, pageStart, pageEnd, cite: pageCite(doc.attachment, pageStart) }; +} + +/** + * Semantic search over the documents available to the run: BM25 first stage, + * then the platform's admission pipeline (cross-encoder, top-K within a token + * budget, trace events). What is document-shaped lives here: the per-call + * index, the optional scope to one document, the pages on every hit — and the + * stance: a document search always scores in explore mode. The run's exploit + * stance exists to keep off-topic web pages out; an attached document is the + * on-topic universe, and a min() with the original question would veto the + * passage that answers a sub-question of it. + */ +export class SearchDocumentsTool extends Tool<{ query: string; document?: string }> { + readonly name = 'search_documents'; + readonly protected = false; + // Reranker (its own llama_context) + in-memory BM25 — no op on the MAIN + // context, so it runs off the loop fiber under concurrent dispatch. + readonly fanout = true; + readonly description = 'Search the attached documents. Returns passages ranked by relevance, each with its document, line range and pages for read_document. Pass `document` to search one document only.'; + readonly parameters: JsonSchema = { + type: 'object', + properties: { + query: { type: 'string', description: 'Search query' }, + document: { type: 'string', description: 'Document id or title, to search one document only' }, + }, + required: ['query'], + }; + + private readonly _indexFor: IndexFor; + private readonly _reranker: Reranker; + private readonly _topK: number; + private readonly _tokenBudget: number; + private readonly _firstStageK: number; + + constructor(indexFor: IndexFor, reranker: Reranker, opts?: { topK?: number; tokenBudget?: number; firstStageK?: number }) { + super(); + this._indexFor = indexFor; + this._reranker = reranker; + this._topK = opts?.topK ?? DEFAULT_TOP_K; + this._tokenBudget = opts?.tokenBudget ?? DEFAULT_TOKEN_BUDGET; + this._firstStageK = opts?.firstStageK ?? DEFAULT_FIRST_STAGE_K; + } + + *execute(args: { query: string; document?: string }, context?: ToolContext): Operation<unknown> { + const query = args.query?.trim(); + if (!query) return { error: 'query must not be empty' }; + const index: DocumentsIndex = yield* call(() => this._indexFor(context?.attachments ?? [])); + if (index.documents.length === 0) return { error: NO_DOCUMENTS }; + const scope = args.document ? index.find(args.document) : undefined; + if (args.document && !scope) return { error: unknownDocument(args.document, index) }; + const inScope = (c: Chunk): boolean => !scope || c.resource === scope.id; + const pool = index.chunks.filter(inScope); + + const tw = yield* Trace.expect(); + let candidates = pool; + if (index.bm25 && this._firstStageK < pool.length) { + const bm25Start = performance.now(); + tw.write({ + traceId: tw.nextId(), parentTraceId: null, ts: bm25Start, + type: 'bm25:start', query, candidateCount: pool.length, firstStageK: this._firstStageK, + }); + const queryTokens: number[] = yield* call(() => this._reranker.tokenize(query)); + // The index spans every document; a scoped search keeps the hits in + // scope and takes its K from those. + const hits = index.bm25.score(queryTokens, scope ? index.chunks.length : this._firstStageK); + candidates = hits.map((h) => index.chunks[h.index]).filter(inScope).slice(0, this._firstStageK); + tw.write({ + traceId: tw.nextId(), parentTraceId: null, ts: performance.now(), + type: 'bm25:end', candidateCount: pool.length, keptCount: candidates.length, durationMs: performance.now() - bm25Start, + }); + } + + const admitted = yield* admitChunks(this._reranker, candidates, query, context ? { ...context, explore: true } : context, { + tool: 'search_documents', + select: { mode: 'budget', topK: this._topK, tokenBudget: this._tokenBudget }, + }); + // The budget decides how many; the hits keep their addresses. The + // selection is a prefix of the ranking (every window has text), so the + // first `passages.length` scored chunks are the passages, with lines. + const kept = admitted.passages?.length ?? 0; + return { + hits: admitted.scored.slice(0, kept).map((h) => locate(index, h)), + totalScored: admitted.totalScored, + }; + } +} diff --git a/packages/abilities/documents/src/tools/view-page.ts b/packages/abilities/documents/src/tools/view-page.ts new file mode 100644 index 00000000..6081233f --- /dev/null +++ b/packages/abilities/documents/src/tools/view-page.ts @@ -0,0 +1,93 @@ +import { call } from 'effection'; +import type { Operation } from 'effection'; +import { Tool, TOOL_ATTACHMENTS_KEY } from '@lloyal-labs/lloyal-agents'; +import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { DocumentMeta } from '@lloyal-labs/media'; +import { pageCite, unknownDocument, NO_DOCUMENTS } from '../documents-index'; +import type { DocumentsIndex } from '../documents-index'; +import type { IndexFor } from './search-documents'; + +/** The per-page facts the sidecar records. */ +export type PageFacts = DocumentMeta['pages'][number]; + +/** How many drawing operations make a page "has graphics". */ +const PATH_OBJECTS_FOR_GRAPHICS = 20; + +/** + * The projection rule: which pages are worth a model's attention as an image. + * The facts are the sidecar's; the rule lives here, with the tool that spends + * the cells. Page one always (the title page carries the figure of record + * more often than not); a page with no extractable text (scanned); a page + * with images, drawings, tagged tables or tagged figures. + */ +export function projectable(page: PageFacts): boolean { + return page.page === 1 + || page.chars === 0 + || page.imageObjects > 0 + || page.pathObjects >= PATH_OBJECTS_FOR_GRAPHICS + || page.taggedTables > 0 + || page.taggedFigures > 0; +} + +/** + * Put a page — or one figure on it — in front of the model. The result names + * the page root (or figure root) under the framework's attachment key as a + * DESCRIPTOR: no bytes, no ingress, no normalizer permit; the pool resolves it + * through the store and admits it on the media rail. A text-only page says + * so instead, and a page past the render bound says it is not archived. + */ +export class ViewPageTool extends Tool<{ document: string; page: number; figure?: number }> { + readonly name = 'view_page'; + readonly protected = false; + readonly description = 'Look at a page of an attached document as an image — for a table, chart or figure, after reading the text around it. `figure` picks one figure on the page by its number.'; + readonly parameters: JsonSchema = { + type: 'object', + properties: { + document: { type: 'string', description: 'Document id or title' }, + page: { type: 'number', description: 'Page number, from search or read_document results' }, + figure: { type: 'number', description: 'Which figure on the page (1 = first), to view just the figure' }, + }, + required: ['document', 'page'], + }; + + private readonly _indexFor: IndexFor; + private readonly _viewed = new Set<string>(); + + constructor(indexFor: IndexFor) { + super(); + this._indexFor = indexFor; + } + + *execute(args: { document: string; page: number; figure?: number }, context?: ToolContext): Operation<unknown> { + const index: DocumentsIndex = yield* call(() => this._indexFor(context?.attachments ?? [])); + if (index.documents.length === 0) return { error: NO_DOCUMENTS }; + const doc = index.find(args.document ?? ''); + if (!doc) return { error: unknownDocument(args.document ?? '', index) }; + const page = doc.meta.pages.find((p) => p.page === args.page); + if (!page) return { error: `Page ${args.page} is out of range: ${doc.meta.title} has ${doc.meta.pageCount} pages.` }; + + const where = { document: doc.meta.title, id: doc.id, page: page.page }; + const key = `${context?.agentId ?? ''}:${doc.id}:${page.page}:${args.figure ?? ''}`; + if (this._viewed.has(key)) { + return { ...where, note: args.figure !== undefined ? `Already viewed figure ${args.figure} on page ${page.page}` : `Already viewed page ${page.page}` }; + } + + if (args.figure !== undefined) { + const figures = doc.meta.figures.filter((f) => f.page === page.page); + const fig = figures[args.figure - 1]; + if (!fig) { + return { error: figures.length === 0 + ? `Page ${page.page} has no figures.` + : `Page ${page.page} has ${figures.length} figure(s); figure must be between 1 and ${figures.length}.` }; + } + this._viewed.add(key); + return { ...where, figure: args.figure, ...(fig.caption ? { caption: fig.caption } : {}), + cite: pageCite(doc.attachment, page.page), [TOOL_ATTACHMENTS_KEY]: [fig.root] }; + } + + if (!projectable(page)) return { ...where, note: `Page ${page.page} is text only — read_document gives you its text.` }; + if (!page.render) return { ...where, note: `Page ${page.page} is not archived as an image.` }; + this._viewed.add(key); + return { ...where, cite: pageCite(doc.attachment, page.page), [TOOL_ATTACHMENTS_KEY]: [page.render] }; + } +} diff --git a/packages/abilities/documents/test/ability.test.ts b/packages/abilities/documents/test/ability.test.ts new file mode 100644 index 00000000..fa4fde2d --- /dev/null +++ b/packages/abilities/documents/test/ability.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { run } from 'effection'; +import { RerankerCtx, Attachments } from '@lloyal-labs/lloyal-agents'; +import type { Reranker } from '@lloyal-labs/rig'; +import { createDocumentsAbility } from '../src/index'; +import { makeFixture, wordTokenize } from './helpers/fixture'; + +const mockReranker = { tokenize: wordTokenize } as unknown as Reranker; +const pkg = join(__dirname, '..'); + +describe('createDocumentsAbility', () => { + it('builds document_research with the three tools and the documents source', async () => { + const { store } = makeFixture(); + const ability = await run(function* () { + yield* RerankerCtx.set(mockReranker); + yield* Attachments.set(store); + return yield* createDocumentsAbility(); + }); + expect(ability.manifest.protocol.name).toBe('document_research'); + expect(ability.manifest.services).toEqual(['reranker']); + expect(ability.manifest.configSchema).toBeUndefined(); + expect(ability.source.name).toBe('documents'); + expect(ability.tools.map((t) => t.name).sort()).toEqual(['read_document', 'search_documents', 'view_page']); + }); + + it('ships a skill in the corpus register: no boundary marker, no toc variable', () => { + const skill = readFileSync(join(pkg, 'skill.eta'), 'utf8'); + expect(skill).not.toContain('Apply the **'); + expect(skill).not.toContain('it.toc'); + expect(skill).toContain('view_page is expensive'); + }); + + it('throws a clear error when no reranker is set', async () => { + await expect(run(function* () { return yield* createDocumentsAbility(); })).rejects.toThrow(/requires a reranker/); + }); + + it('constructs with the default store and no attachments, and the toc is empty', async () => { + const source = await run(function* () { + yield* RerankerCtx.set(mockReranker); + return (yield* createDocumentsAbility()).source; + }); + expect(source.promptData()).toEqual({ toc: '' }); + expect(source.promptData([])).toEqual({ toc: '' }); + }); + + it('never reaches the native addon: no lloyal.node peer and no node-only sibling entry in src', () => { + const manifest = JSON.parse(readFileSync(join(pkg, 'package.json'), 'utf8')) as { peerDependencies: Record<string, string> }; + expect(Object.keys(manifest.peerDependencies)).not.toContain('@lloyal-labs/lloyal.node'); + const files: string[] = []; + const walk = (d: string): void => { + for (const n of readdirSync(d)) { + const p = join(d, n); + if (statSync(p).isDirectory()) walk(p); else if (p.endsWith('.ts')) files.push(p); + } + }; + walk(join(pkg, 'src')); + expect(files.length).toBeGreaterThan(3); + for (const f of files) { + const text = readFileSync(f, 'utf8'); + expect(text, f).not.toMatch(/lloyal\.node|@lloyal-labs\/(rig|media)\/node/); + } + }); +}); diff --git a/packages/abilities/documents/test/helpers/fixture.ts b/packages/abilities/documents/test/helpers/fixture.ts new file mode 100644 index 00000000..b47625bb --- /dev/null +++ b/packages/abilities/documents/test/helpers/fixture.ts @@ -0,0 +1,120 @@ +/** + * One document in a real file store: four pages, a tagged table on page two + * with a figure, a text-only page three, and a page four past the render + * bound. Page roots are the PNG-header idiom — nothing in these tests decodes. + */ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { FileAttachmentStore } from '@lloyal-labs/media/node'; +import { DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, AttachmentStore, DocumentMeta } from '@lloyal-labs/media'; +import type { Reranker, Chunk } from '@lloyal-labs/rig'; + +export const PNG_BYTES = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + +export const MARKDOWN = [ + '# Fixture Paper', '', // 1-2 page 1 + '## Introduction', '', 'Intro text alpha beta.', '', // 3-6 page 1 + '## Results', '', '| Config | Cells |', '| --- | --- |', '| baseline | 803 |', '', // 7-12 page 2 + '## Discussion', '', 'Closing words gamma.', '', // 13-16 page 3 + '## Appendix', '', 'Chart described here.', // 17-19 page 4 (+ line 20 blank from the trailing newline) +].join('\n') + '\n'; + +const DERIVE: DocumentMeta['derive'] = { + profile: 'pdf.v1', pdfium: 'test', dpi: 150, maxSide: 2048, maxPixels: 4194304, format: 'image/png', + renderedPages: 3, maxFigures: 16, maxTextPages: 400, tagged: true, structCoverage: 1, truncated: true, +}; + +export interface Fixture { + store: FileAttachmentStore; + doc: Attachment; + image: Attachment; + renders: Record<1 | 2 | 3, Attachment>; + figure: Attachment; + meta: DocumentMeta; +} + +export function makeFixture(): Fixture { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-ability-'))); + const png = (tag: number): Attachment => + store.putAttachment({ representations: [store.putBlob(new Uint8Array([...PNG_BYTES, tag]), 'image/png')] }); + const image = png(100); + const renders = { 1: png(1), 2: png(2), 3: png(3) } as const; + const figure = png(9); + const meta: DocumentMeta = { + title: 'Fixture Paper', + pageCount: 4, + sections: [ + { heading: 'Fixture Paper', path: 'Fixture Paper', origin: 'struct', startLine: 1, endLine: 2, pageStart: 1, pageEnd: 1 }, + { heading: 'Introduction', path: 'Introduction', origin: 'struct', startLine: 3, endLine: 6, pageStart: 1, pageEnd: 1 }, + { heading: 'Results', path: 'Results', origin: 'struct', startLine: 7, endLine: 12, pageStart: 2, pageEnd: 2 }, + { heading: 'Discussion', path: 'Discussion', origin: 'struct', startLine: 13, endLine: 16, pageStart: 3, pageEnd: 3 }, + { heading: 'Appendix', path: 'Appendix', origin: 'struct', startLine: 17, endLine: 20, pageStart: 4, pageEnd: 4 }, + ], + pages: [ + { page: 1, startLine: 1, endLine: 6, chars: 40, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0, render: renders[1] }, + { page: 2, startLine: 7, endLine: 12, chars: 30, imageObjects: 0, pathObjects: 30, taggedTables: 1, taggedFigures: 0, render: renders[2] }, + { page: 3, startLine: 13, endLine: 16, chars: 20, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0, render: renders[3] }, + { page: 4, startLine: 17, endLine: 20, chars: 21, imageObjects: 1, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }, + ], + figures: [{ page: 2, index: 0, bbox: [72, 300, 372, 500], caption: 'Figure 1. Cells per configuration.', root: figure }], + tables: [{ page: 2, startLine: 9, endLine: 11 }], + derive: DERIVE, + }; + const doc = store.putAttachment({ + representations: [store.putBlob(new TextEncoder().encode(MARKDOWN), 'text/markdown')], + source: store.putBlob(new TextEncoder().encode('%PDF-1.4 fixture'), 'application/pdf'), + config: { bytes: new TextEncoder().encode(JSON.stringify(meta)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); + return { store, doc, image, renders, figure, meta }; +} + +/** A second, one-page document in the same store. */ +export function makeSecond(store: AttachmentStore, title = 'Second Report'): Attachment { + const md = `# ${title}\n\n## Findings\n\nDelta epsilon zeta.\n`; + const meta: DocumentMeta = { + title, pageCount: 1, + sections: [ + { heading: title, path: title, origin: 'heuristic', startLine: 1, endLine: 2, pageStart: 1, pageEnd: 1 }, + { heading: 'Findings', path: 'Findings', origin: 'heuristic', startLine: 3, endLine: 6, pageStart: 1, pageEnd: 1 }, + ], + pages: [{ page: 1, startLine: 1, endLine: 6, chars: 30, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }], + figures: [], tables: [], derive: { ...DERIVE, renderedPages: 0, tagged: false, structCoverage: 0, truncated: false }, + }; + return store.putAttachment({ + representations: [store.putBlob(new TextEncoder().encode(md), 'text/markdown')], + config: { bytes: new TextEncoder().encode(JSON.stringify(meta)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); +} + +/** Words stand in for tokens: fitting needs only a tokenizer. */ +export const wordTokenize = async (text: string): Promise<number[]> => + text.split(/\s+/).filter(Boolean).map((_, i) => i + 1); + +/** A reranker whose scorer admits everything with score 1 — the funnel's + * shape is under test here, not the cross-encoder. */ +export function scoringReranker(): Reranker { + return { + tokenize: wordTokenize, + tokenizeChunks: async (chunks: Chunk[]) => chunks, + // The judge's verdict, shaped like the real one: log-odds, positive when + // a query word occurs in the passage, negative when none does — so a + // query the document never mentions is judged "no" everywhere. + score(query: string, chunks: Chunk[]) { + const words = query.toLowerCase().split(/\W+/).filter((w) => w.length > 2); + return (async function* () { + yield { + filled: chunks.length, total: chunks.length, + results: chunks.map((c) => ({ + file: c.resource, heading: c.heading, section: c.section, snippet: c.text, + score: words.some((w) => c.text.toLowerCase().includes(w)) ? 1 : -1, + startLine: c.startLine, endLine: c.endLine, + })), + }; + })(); + }, + scoreBatch: async () => [], + dispose: () => {}, + } as unknown as Reranker; +} diff --git a/packages/abilities/documents/test/index.test.ts b/packages/abilities/documents/test/index.test.ts new file mode 100644 index 00000000..95dc8dd8 --- /dev/null +++ b/packages/abilities/documents/test/index.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from 'vitest'; +import { MANIFEST_TYPE } from '@lloyal-labs/media'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId, pageCite } from '../src/documents-index'; +import { makeFixture, makeSecond, wordTokenize } from './helpers/fixture'; + +describe('documentIndexer', () => { + it('the same attachments give the same index, and a digest is fitted once', async () => { + const { store, doc, image } = makeFixture(); + let calls = 0; + const indexFor = documentIndexer(store, async (t) => { calls++; return wordTokenize(t); }); + const a = await indexFor([doc]); + const b = await indexFor([doc]); + expect(b).toBe(a); + expect(a.documents).toHaveLength(1); + expect(a.documents[0].id).toBe(documentId(doc)); + expect(a.chunks.length).toBeGreaterThan(0); + expect(a.chunks.every((c) => c.resource === a.documents[0].id)).toBe(true); + const fitted = calls; + expect(fitted).toBeGreaterThan(0); + // A different set — the image adds no document — builds a new index but re-tokenizes nothing. + const c = await indexFor([image, doc]); + expect(c).not.toBe(a); + expect(c.documents.map((d) => d.id)).toEqual([documentId(doc)]); + expect(calls).toBe(fitted); + // The same root twice is one document. + expect((await indexFor([doc, doc])).documents).toHaveLength(1); + }); + + it('a second document is searchable once it is available to the run, by id or title', async () => { + const { store, doc } = makeFixture(); + const second = makeSecond(store); + const indexFor = documentIndexer(store, wordTokenize); + expect((await indexFor([doc])).documents).toHaveLength(1); + const both = await indexFor([doc, second]); + expect(both.documents.map((d) => d.meta.title)).toEqual(['Fixture Paper', 'Second Report']); + expect(both.find('second report')?.id).toBe(documentId(second)); + expect(both.find(documentId(second))?.meta.title).toBe('Second Report'); + expect(both.find('nope')).toBeUndefined(); + }); + + it('a root that is not in the store throws naming the digest, and the failure is not remembered', async () => { + const { store, doc } = makeFixture(); + const indexFor = documentIndexer(store, wordTokenize); + const gone = { digest: 'sha256:' + 'e'.repeat(64), mediaType: MANIFEST_TYPE, size: 9 } as Attachment; + await expect(indexFor([doc, gone])).rejects.toThrow(/sha256:e{12}… is not in the content store/); + // The good document still indexes on its own afterwards. + expect((await indexFor([doc])).documents).toHaveLength(1); + }); + + it('handles and citations derive from the digest', () => { + const { doc } = makeFixture(); + const id = documentId(doc); + expect(id).toMatch(/^[0-9a-f]{12}$/); + expect(doc.digest).toContain(id); + expect(pageCite(doc, 7)).toBe(`attachment://${id}/page/7`); + }); +}); diff --git a/packages/abilities/documents/test/read-document.test.ts b/packages/abilities/documents/test/read-document.test.ts new file mode 100644 index 00000000..99996fc1 --- /dev/null +++ b/packages/abilities/documents/test/read-document.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from 'vitest'; +import { run } from 'effection'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId, NO_DOCUMENTS } from '../src/documents-index'; +import { ReadDocumentTool } from '../src/tools/read-document'; +import { makeFixture, wordTokenize } from './helpers/fixture'; + +type Read = { document?: string; id?: string; lines?: string[]; pageStart?: number; pageEnd?: number; cite?: string; content?: string; note?: string; error?: string }; + +function setup() { + const fx = makeFixture(); + const tool = new ReadDocumentTool(documentIndexer(fx.store, wordTokenize)); + const read = (args: { document: string; page?: number; startLine?: number; endLine?: number }, attachments: readonly Attachment[] = [fx.doc], agentId = 1) => + run(function* () { return (yield* tool.execute(args, { agentId, attachments } as ToolContext)) as Read; }); + return { ...fx, read }; +} + +describe('read_document', () => { + it('reads a line range by id, and by title regardless of case', async () => { + const { doc, read } = setup(); + const byId = await read({ document: documentId(doc), startLine: 3, endLine: 5 }); + expect(byId).toMatchObject({ + document: 'Fixture Paper', id: documentId(doc), lines: ['3-5'], + pageStart: 1, pageEnd: 1, cite: `attachment://${documentId(doc)}/page/1`, + content: '## Introduction\n\nIntro text alpha beta.', + }); + const byTitle = await read({ document: 'fixture paper', startLine: 3, endLine: 5 }, [doc], 2); + expect(byTitle.content).toBe(byId.content); + }); + + it('reads a page as the whole sections that touch it', async () => { + const { doc, read } = setup(); + const r = await read({ document: documentId(doc), page: 2 }); + expect(r.lines).toEqual(['7-12']); + expect(r.content!.startsWith('## Results')).toBe(true); + expect(r.content).toContain('| baseline | 803 |'); + expect([r.pageStart, r.pageEnd]).toEqual([2, 2]); + expect(r.cite).toBe(`attachment://${documentId(doc)}/page/2`); + }); + + it('the same agent re-reading gets a note; another agent reads', async () => { + const { doc, read } = setup(); + await read({ document: documentId(doc), page: 2 }); + const again = await read({ document: documentId(doc), page: 2 }); + expect(again.note).toMatch(/already read/); + expect(again.content).toBeUndefined(); + const other = await read({ document: documentId(doc), page: 2 }, [doc], 7); + expect(other.content).toContain('| baseline | 803 |'); + }); + + it('names what is attached for an unknown document, and bounds the page', async () => { + const { doc, read } = setup(); + expect((await read({ document: 'Nope' })).error).toMatch(/Unknown document: Nope\. Attached: Fixture Paper \([0-9a-f]{12}\)\./); + expect((await read({ document: documentId(doc), page: 9 })).error).toMatch(/out of range.*4 pages/); + expect((await read({ document: documentId(doc) }, [])).error).toBe(NO_DOCUMENTS); + }); +}); diff --git a/packages/abilities/documents/test/search-documents.test.ts b/packages/abilities/documents/test/search-documents.test.ts new file mode 100644 index 00000000..1c5b63a5 --- /dev/null +++ b/packages/abilities/documents/test/search-documents.test.ts @@ -0,0 +1,83 @@ +import { describe, it, expect } from 'vitest'; +import { run } from 'effection'; +import { Trace, NullTraceWriter } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId, NO_DOCUMENTS } from '../src/documents-index'; +import { SearchDocumentsTool } from '../src/tools/search-documents'; +import type { DocumentHit } from '../src/tools/search-documents'; +import { makeFixture, makeSecond, scoringReranker } from './helpers/fixture'; + +type Envelope = { hits: DocumentHit[]; totalScored: number; error?: string }; + +function setup() { + const fx = makeFixture(); + const reranker = scoringReranker(); + const indexFor = documentIndexer(fx.store, (t) => reranker.tokenize(t)); + const tool = new SearchDocumentsTool(indexFor, reranker); + const search = (args: { query: string; document?: string }, attachments: readonly Attachment[], agentId = 1) => + run(function* () { + yield* Trace.set(new NullTraceWriter()); + return (yield* tool.execute(args, { agentId, attachments } as ToolContext)) as Envelope; + }); + return { ...fx, tool, search }; +} + +describe('search_documents', () => { + it('hits carry the document, its id, its pages and a cite URL', async () => { + const { doc, image, search } = setup(); + const r = await search({ query: 'baseline cells' }, [image, doc]); + expect(r.error).toBeUndefined(); + expect(r.totalScored).toBeGreaterThan(0); + const results = r.hits.find((h) => h.heading === 'Results'); + expect(results).toMatchObject({ + document: 'Fixture Paper', id: documentId(doc), file: documentId(doc), + pageStart: 2, pageEnd: 2, cite: `attachment://${documentId(doc)}/page/2`, + }); + expect(results!.startLine).toBe(7); + }); + + it('scopes to one document by title or id, and names the attached ones for an unknown scope', async () => { + const { store, doc, search } = setup(); + const second = makeSecond(store); + const byTitle = await search({ query: 'delta', document: 'Second Report' }, [doc, second]); + expect(byTitle.hits.length).toBeGreaterThan(0); + expect(byTitle.hits.every((h) => h.id === documentId(second))).toBe(true); + const byId = await search({ query: 'delta', document: documentId(doc) }, [doc, second]); + expect(byId.hits.every((h) => h.id === documentId(doc))).toBe(true); + const unknown = await search({ query: 'delta', document: 'Third' }, [doc, second]); + expect(unknown.error).toMatch(/Unknown document: Third\. Attached: Fixture Paper \([0-9a-f]{12}\), Second Report/); + }); + + it('answers honestly when nothing is attached, and when only an image is', async () => { + const { image, search } = setup(); + expect((await search({ query: 'anything' }, [])).error).toBe(NO_DOCUMENTS); + expect((await search({ query: 'anything' }, [image])).error).toBe(NO_DOCUMENTS); + expect((await search({ query: ' ' }, [])).error).toMatch(/must not be empty/); + }); + + it('returns the best passages even when the judge says no to all of them — the ranker is relative, there is no floor', async () => { + const { doc, search } = setup(); + const r = await search({ query: 'zxqv plorth wibble' }, [doc]); + expect(r.error).toBeUndefined(); + expect(r.hits.length).toBeGreaterThan(0); + expect(r.hits.length).toBeLessThanOrEqual(5); + expect(r.hits.every((h) => h.score < 0)).toBe(true); + // The hits are the ranking's prefix: addresses intact, in score order. + expect(r.hits.every((h) => h.startLine >= 1 && h.cite.startsWith('attachment://'))).toBe(true); + }); + + it("scores in explore mode whatever the run's stance — the attached document is the on-topic universe, so the original question never vetoes a passage", async () => { + const { doc, tool } = setup(); + const scorer = { + scoreEntailmentBatch: async () => { throw new Error('the entailment scorer must not be consulted for a document search'); }, + }; + const r = await run(function* () { + yield* Trace.set(new NullTraceWriter()); + return (yield* tool.execute({ query: 'baseline cells' }, { agentId: 1, attachments: [doc], explore: false, scorer } as unknown as ToolContext)) as Envelope; + }); + expect(r.error).toBeUndefined(); + expect(r.hits.length).toBeGreaterThan(0); + }); +}); + diff --git a/packages/abilities/documents/test/source.test.ts b/packages/abilities/documents/test/source.test.ts new file mode 100644 index 00000000..c6f868d3 --- /dev/null +++ b/packages/abilities/documents/test/source.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect } from 'vitest'; +import type { Reranker, Document } from '@lloyal-labs/rig'; +import { DocumentsSource, buildToc } from '../src/source'; +import { makeFixture, makeSecond, wordTokenize } from './helpers/fixture'; + +const mockReranker = { tokenize: wordTokenize } as unknown as Reranker; + +describe('DocumentsSource.promptData', () => { + it('lists each document with its page count and top-level topics, skipping an image root', () => { + const { store, doc, image } = makeFixture(); + const second = makeSecond(store); + const source = new DocumentsSource(store, [], mockReranker); + expect(source.promptData([image, doc, second]).toc).toBe( + 'Fixture Paper — 4 pages (topics: Introduction, Results, Discussion, Appendix)\n' + + 'Second Report — 1 page (topics: Findings)', + ); + }); + + it('is empty for no attachments and for images only', () => { + const { store, image } = makeFixture(); + const source = new DocumentsSource(store, [], mockReranker); + expect(source.promptData().toc).toBe(''); + expect(source.promptData([image]).toc).toBe(''); + }); +}); + +describe('buildToc', () => { + const docWith = (title: string, headings: string[]): Document => ({ + meta: { + title, pageCount: 2, + sections: headings.map((h, i) => ({ heading: h, path: h, origin: 'heuristic' as const, startLine: i + 1, endLine: i + 1, pageStart: 1, pageEnd: 1 })), + pages: [], figures: [], tables: [], + derive: { profile: 'pdf.v1', pdfium: 't', dpi: 150, maxSide: 2048, maxPixels: 1, format: 'image/png', renderedPages: 0, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false }, + }, + } as unknown as Document); + + it('strips control characters, skips nested paths and the title, and caps topics at eight', () => { + const headings = Array.from({ length: 12 }, (_, i) => `Topic ${i + 1}`); + const toc = buildToc([docWith('BadTitle\nHere', ['Bad Title Here', 'Outer > Inner', ...headings])]); + expect(toc).not.toMatch(/\p{Cc}/u); + expect(toc.startsWith('BadTitle Here — 2 pages (topics: Bad Title Here, Topic 1,')).toBe(true); + expect(toc).not.toContain('Inner'); + expect(toc.match(/Topic \d+/g)).toHaveLength(7); + }); + + it('caps a long title', () => { + const toc = buildToc([docWith('T'.repeat(300), [])]); + expect(toc.length).toBeLessThan(140); + expect(toc).toContain('…'); + }); +}); diff --git a/packages/abilities/documents/test/view-page.test.ts b/packages/abilities/documents/test/view-page.test.ts new file mode 100644 index 00000000..1e891edd --- /dev/null +++ b/packages/abilities/documents/test/view-page.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, vi } from 'vitest'; +import { run } from 'effection'; +import { TOOL_ATTACHMENTS_KEY } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId } from '../src/documents-index'; +import { ViewPageTool, projectable } from '../src/tools/view-page'; +import { makeFixture, wordTokenize } from './helpers/fixture'; + +type View = Record<string, unknown> & { note?: string; error?: string; cite?: string; caption?: string }; + +function setup() { + const fx = makeFixture(); + const tool = new ViewPageTool(documentIndexer(fx.store, wordTokenize)); + const view = (args: { document: string; page: number; figure?: number }, attachments: readonly Attachment[] = [fx.doc], agentId = 1) => + run(function* () { return (yield* tool.execute(args, { agentId, attachments } as ToolContext)) as View; }); + return { ...fx, view }; +} + +describe('view_page', () => { + it('returns the page root as a descriptor under the attachment key, reading no blob for it', async () => { + const { doc, renders, store, view } = setup(); + // The first call builds the index (which reads the sidecar and markdown). + const text = await view({ document: documentId(doc), page: 3 }); + expect(text.note).toMatch(/text only/); + const reads = vi.spyOn(store, 'get'); + const r = await view({ document: documentId(doc), page: 2 }); + expect(r[TOOL_ATTACHMENTS_KEY]).toEqual([renders[2]]); + expect(r.cite).toBe(`attachment://${documentId(doc)}/page/2`); + expect(r).toMatchObject({ document: 'Fixture Paper', id: documentId(doc), page: 2 }); + expect(reads).not.toHaveBeenCalled(); + }); + + it('a figure returns its own root and caption; a figure that is not there is named', async () => { + const { doc, figure, view } = setup(); + const r = await view({ document: documentId(doc), page: 2, figure: 1 }); + expect(r[TOOL_ATTACHMENTS_KEY]).toEqual([figure]); + expect(r.caption).toBe('Figure 1. Cells per configuration.'); + expect(r.cite).toBe(`attachment://${documentId(doc)}/page/2`); + expect((await view({ document: documentId(doc), page: 2, figure: 2 })).error).toMatch(/1 figure\(s\); figure must be between 1 and 1/); + expect((await view({ document: documentId(doc), page: 3, figure: 1 })).error).toMatch(/no figures/); + }); + + it('a text-only page says so and carries no media key', async () => { + const { doc, view } = setup(); + const r = await view({ document: documentId(doc), page: 3 }); + expect(r.note).toMatch(/text only/); + expect(TOOL_ATTACHMENTS_KEY in r).toBe(false); + }); + + it('a page past the render bound is "not archived" — no range is ever named', async () => { + const { doc, view } = setup(); + const r = await view({ document: documentId(doc), page: 4 }); + expect(r.note).toMatch(/not archived/); + expect(r.note).not.toMatch(/\d+\s*[–-]\s*\d+/); + expect(TOOL_ATTACHMENTS_KEY in r).toBe(false); + }); + + it('page one is always projectable; a repeat by the same agent is a note, another agent sees it', async () => { + const { doc, renders, view } = setup(); + expect((await view({ document: documentId(doc), page: 1 }))[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); + expect((await view({ document: documentId(doc), page: 1 })).note).toMatch(/Already viewed page 1/); + expect((await view({ document: documentId(doc), page: 1 }, [doc], 2))[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); + expect((await view({ document: documentId(doc), page: 5 })).error).toMatch(/out of range/); + }); + + it('the projection rule is a rule over the sidecar facts', () => { + const base = { page: 2, startLine: 1, endLine: 1, chars: 10, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }; + expect(projectable(base)).toBe(false); + expect(projectable({ ...base, page: 1 })).toBe(true); + expect(projectable({ ...base, chars: 0 })).toBe(true); + expect(projectable({ ...base, imageObjects: 1 })).toBe(true); + expect(projectable({ ...base, pathObjects: 20 })).toBe(true); + expect(projectable({ ...base, pathObjects: 19 })).toBe(false); + expect(projectable({ ...base, taggedTables: 1 })).toBe(true); + expect(projectable({ ...base, taggedFigures: 1 })).toBe(true); + }); +}); diff --git a/packages/abilities/documents/tsconfig.json b/packages/abilities/documents/tsconfig.json new file mode 100644 index 00000000..4f93912f --- /dev/null +++ b/packages/abilities/documents/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"], + "references": [ + { "path": "../../sdk" }, + { "path": "../../agents" }, + { "path": "../../media" }, + { "path": "../../rig" } + ] +} diff --git a/packages/abilities/web/src/tools/fetch-page.ts b/packages/abilities/web/src/tools/fetch-page.ts index bb98dd15..eda41c0d 100644 --- a/packages/abilities/web/src/tools/fetch-page.ts +++ b/packages/abilities/web/src/tools/fetch-page.ts @@ -96,7 +96,7 @@ export class FetchPageTool extends Tool<{ url: string; query?: string }> { ) { return { error: - "PDF documents cannot be extracted. Try searching for an HTML version of this content.", + "This is a PDF, which fetch_page cannot read. Ask the user to attach the file to the conversation.", url, }; } @@ -135,7 +135,7 @@ export class FetchPageTool extends Tool<{ url: string; query?: string }> { if (contentType.includes("application/pdf")) { return { error: - "PDF documents cannot be extracted. Try searching for an HTML version of this content.", + "This is a PDF, which fetch_page cannot read. Ask the user to attach the file to the conversation.", url, } as const; } diff --git a/packages/abilities/web/test/ability.test.ts b/packages/abilities/web/test/ability.test.ts index c7e798e9..d29161aa 100644 --- a/packages/abilities/web/test/ability.test.ts +++ b/packages/abilities/web/test/ability.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import { run } from 'effection'; -import { AbilityConfigStoreCtx } from '@lloyal-labs/lloyal-agents'; +import { AbilityConfigStoreCtx, Trace, NullTraceWriter } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; import { createInMemoryConfigStore } from '@lloyal-labs/rig'; import { createWebAbility } from '../src/index'; @@ -24,3 +25,19 @@ describe('createWebAbility', () => { expect(agentSrc).not.toContain('Apply the **'); }); }); + +describe('fetch_page on a PDF', () => { + it('names the way in — attach the file — instead of a dead end', async () => { + const ability = await run(function* () { + yield* AbilityConfigStoreCtx.set(createInMemoryConfigStore()); + yield* Trace.set(new NullTraceWriter()); + return yield* createWebAbility(); + }); + const fetchPage = ability.tools.find((t) => t.name === 'fetch_page')!; + const r = (await run(function* () { + yield* Trace.set(new NullTraceWriter()); + return yield* fetchPage.execute({ url: 'https://example.com/paper.pdf' }, { agentId: 1 } as ToolContext); + })) as { error?: string }; + expect(r.error).toBe('This is a PDF, which fetch_page cannot read. Ask the user to attach the file to the conversation.'); + }); +}); diff --git a/packages/agents/src/AgentPolicy.ts b/packages/agents/src/AgentPolicy.ts index 7c2f6b28..03c7c048 100644 --- a/packages/agents/src/AgentPolicy.ts +++ b/packages/agents/src/AgentPolicy.ts @@ -250,7 +250,7 @@ export interface AgentPolicy { * DISPATCH phase: should this tool call explore or exploit? * * When true (explore), content-boundary tools use agent-local scoring only. - * When false (exploit), tools apply dual scoring via scoreRelevanceBatch. + * When false (exploit), admission also scores against the original question and ranks by min(). * Non-monotonic — flips with live pressure. Separate from lifecycle. * Optional — defaults to true (explore) when absent. */ diff --git a/packages/agents/src/Tool.ts b/packages/agents/src/Tool.ts index 4e5100c2..ce823d50 100644 --- a/packages/agents/src/Tool.ts +++ b/packages/agents/src/Tool.ts @@ -1,5 +1,7 @@ import type { Operation } from 'effection'; import type { JsonSchema, ToolSchema, ToolContext } from './types'; +import { asAttachment } from '@lloyal-labs/media'; +import type { Attachment } from '@lloyal-labs/media'; /** * Abstract base class for tools usable by agents in the runtime @@ -188,7 +190,7 @@ export class ToolRetryError extends Error { * * | key | direction | does the model read it? | * |---|---|---| - * | `_images` | OUT of the result, before serializing | **no** — that is the point | + * | `_attachments` | OUT of the result, before serializing | **no** — that is the point | * | `_contextAvailablePercent` | INTO the result | yes | * | `_imageError` | INTO the result | yes — it exists to be read | * @@ -203,8 +205,15 @@ export class ToolRetryError extends Error { * Taken OUT before serializing, because these bytes must reach the cache down * the embedding rail and must never reach it as JSON. A 180 KB image * stringifies to ~700k characters of digits, which is not a degraded prefill - * but a destroyed one. */ -export const TOOL_MEDIA_KEY = '_images'; + * but a destroyed one. + * + * An entry is raw bytes — admitted through the ingress door — or a root + * descriptor already in the content store (a page render a tool looked up, a + * document a tool fetched): no bytes, no door, the ingest-time digest. The + * rail follows what the root materializes to: bitmaps ride the embedding + * rail; a root with none rides the token rail beside the tool's text and is + * booked as an asset available to the run. */ +export const TOOL_ATTACHMENTS_KEY = '_attachments'; /** Split a tool result into the images it carried and the result WITHOUT them. * @@ -215,30 +224,34 @@ export const TOOL_MEDIA_KEY = '_images'; * halves and hands each to exactly one consumer. * * `result` is returned unchanged when there is no media, so a text-only tool - * copies nothing. Entries that are not `Uint8Array` are ignored — one marker - * is emitted per SURVIVING entry, so the prompt and the bitmap list stay in - * step whatever a tool hands over. + * copies nothing. An entry is bytes or a root descriptor the store would + * recognise; anything else is ignored — markers are emitted per SURVIVING + * representation, so the prompt and the bitmap list stay in step whatever a + * tool hands over. * * @category Agents */ export function takeToolMedia( result: unknown, -): { media: Uint8Array[]; result: unknown } { +): { media: (Uint8Array | Attachment)[]; result: unknown } { if (!result || typeof result !== 'object' || Array.isArray(result)) { return { media: [], result }; } - if (!(TOOL_MEDIA_KEY in result)) return { media: [], result }; - const { [TOOL_MEDIA_KEY]: raw, ...rest } = result as Record<string, unknown>; + if (!(TOOL_ATTACHMENTS_KEY in result)) return { media: [], result }; + const { [TOOL_ATTACHMENTS_KEY]: raw, ...rest } = result as Record<string, unknown>; // The reserved key never survives into the serialized result, even when its // value is malformed — returning the original would JSON-encode byte // indices onto the token rail, the exact failure this helper exists to // prevent. An invalid value is simply zero media entries. - return { - media: Array.isArray(raw) - ? raw.filter((b): b is Uint8Array => b instanceof Uint8Array) - : [], - result: rest, - }; + const media: (Uint8Array | Attachment)[] = []; + if (Array.isArray(raw)) { + for (const entry of raw) { + if (entry instanceof Uint8Array) { media.push(entry); continue; } + const root = asAttachment(entry); + if (root) media.push(root); + } + } + return { media, result: rest }; } /** diff --git a/packages/agents/src/admission.ts b/packages/agents/src/admission.ts index 74a95f57..aa096f98 100644 --- a/packages/agents/src/admission.ts +++ b/packages/agents/src/admission.ts @@ -135,9 +135,11 @@ function selectTopChunks( * greps; discovery signals (`alsoOnPage`) compensate. * * **Exploit mode** (`context.explore === false`, set by - * `policy.shouldExplore`): every chunk is re-scored by the entailment scorer - * against the query and re-sorted — `min(toolQueryScore, originalQueryScore)` - * tightens focus when KV or time is short, at the cost of serendipity. + * `policy.shouldExplore`): every scored chunk is also scored by the + * entailment scorer against the ORIGINAL question — one extra pass; the + * tool-query scores are already in hand — and re-sorted by + * `min(toolQueryScore, originalQueryScore)`, which tightens focus when KV or + * time is short, at the cost of serendipity. * * Callers own everything around it: fetching/chunking (and * `tokenizeChunks` for budget mode), any first-stage narrowing (BM25), @@ -189,11 +191,13 @@ export function* admitChunks( ); return chunk?.text ?? ''; }); - const combinedScores: number[] = yield* call(() => - context.scorer!.scoreRelevanceBatch(chunkTexts, query), + // One more pass, against the original question; the tool-query scores + // are the ones already in hand. Both must be high to rank: min(). + const originalScores: number[] = yield* call(() => + context.scorer!.scoreEntailmentBatch(chunkTexts), ); const reordered: ScoredWithOriginal[] = scored - .map((sc, i) => ({ ...sc, score: combinedScores[i], _toolQueryScore: sc.score })) + .map((sc, i) => ({ ...sc, score: Math.min(sc.score, originalScores[i]), _toolQueryScore: sc.score })) .sort((a, b) => b.score - a.score); scored = reordered; diff --git a/packages/agents/src/agent-pool.ts b/packages/agents/src/agent-pool.ts index 86bdf665..8060b2a9 100644 --- a/packages/agents/src/agent-pool.ts +++ b/packages/agents/src/agent-pool.ts @@ -1,6 +1,7 @@ import { resource, ensure, createSignal, createChannel, createQueue, spawn, each, sleep, action, race } from 'effection'; import type { Operation, Subscription, Task, Signal } from 'effection'; import type { SessionContext, BranchStore } from '@lloyal-labs/sdk'; +import type { Attachment } from '@lloyal-labs/media'; import { buildTurnDelta } from '@lloyal-labs/sdk'; import { Ctx, Store, Trace, TraceParent, GrantStoreCtx, WindDown, CancelAgent, Pause, Attachments, Ingress } from './context'; import { useTraceScope } from './trace-scope'; @@ -75,6 +76,10 @@ export function useAgentPool(opts: AgentPoolOptions): Operation<Subscription<Age const tw = yield* Trace.expect(); const attachments = yield* Attachments.expect(); const ingress = yield* Ingress.expect(); + // Assets available to the run: staged by the host now, grown by every + // tool result that admits a root. One list, owned here — it outlives any + // agent, so a root agent A admitted stays available after A is pruned. + const available: Attachment[] = [...(opts.attachments ?? [])]; const { spine, orchestrate, toolsJson, tools, maxTurns = 100, terminalToolName, trace = false, pruneOnReturn = false, enableThinking = true, eagerGrammar } = opts; const toolIndexMap = new Map([...tools.keys()].map((name, i) => [name, i])); @@ -192,7 +197,7 @@ export function useAgentPool(opts: AgentPoolOptions): Operation<Subscription<Age permits: makePermits(opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS), completed, wake, progress, scorer: opts.scorer, toolIndexMap, toolkitSize: tools.size, terminalGrammar, eagerGrammar, enableThinking, spine, runNow, counters, totals, policy, - pressureOpts, ingress, attachments, ladder, trace, + pressureOpts, ingress, attachments, available, ladder, trace, }); // ── PoolContext — the orchestrator's API ───────────────────── diff --git a/packages/agents/src/create-agent-pool.ts b/packages/agents/src/create-agent-pool.ts index 15cfd899..323f5168 100644 --- a/packages/agents/src/create-agent-pool.ts +++ b/packages/agents/src/create-agent-pool.ts @@ -10,6 +10,7 @@ import { Events } from './context'; import { createToolkit } from './toolkit'; import { withSpine } from './spine'; import { useAgentPool } from './agent-pool'; +import type { Attachment } from '@lloyal-labs/media'; // ── CreateAgentPool opts ──────────────────────────────────── @@ -56,6 +57,8 @@ export interface CreateAgentPoolOpts { session?: Session; /** Entailment scorer for semantic coherence across recursive depths. */ scorer?: EntailmentScorer; + /** Assets available to the run — see {@link AgentPoolOptions.attachments}. */ + attachments?: readonly Attachment[]; /** Echo detection threshold. @default 0.8 */ echoThreshold?: number; /** Check ancestor tasks for echo. @default false */ @@ -160,6 +163,7 @@ export function* agentPool(opts: CreateAgentPoolOpts): Operation<AgentPoolResult trace: opts.trace, policy: opts.policy, scorer: opts.scorer, + attachments: opts.attachments, enableThinking: opts.enableThinking, }); diff --git a/packages/agents/src/emit.ts b/packages/agents/src/emit.ts index 5529b6fa..599c1494 100644 --- a/packages/agents/src/emit.ts +++ b/packages/agents/src/emit.ts @@ -172,11 +172,17 @@ export function project(t: Transition): Emission[] { ...(t.rc !== undefined ? { rc: t.rc } : {}), attempt: t.attempt, pressure: pressureRecord(t.pressure), } }]; case 'prefilled': - return [{ trace: { - type: 'branch:prefill', branchHandle: t.agent.id, cells: t.cells, role: t.role, - ...(t.attachments ? { attachments: t.attachments } : {}), - ...(t.probeText !== undefined ? { probeText: t.probeText } : {}), - } }]; + return [ + { bus: { + type: 'agent:prefilled', agentId: t.agent.id, cells: t.cells, role: t.role, + ...(t.attachments ? { attachments: t.attachments } : {}), + } }, + { trace: { + type: 'branch:prefill', branchHandle: t.agent.id, cells: t.cells, role: t.role, + ...(t.attachments ? { attachments: t.attachments } : {}), + ...(t.probeText !== undefined ? { probeText: t.probeText } : {}), + } }, + ]; case 'settleOrder': return [{ trace: { type: 'tool:settle_order', batch: t.batch } }]; case 'pruned': diff --git a/packages/agents/src/execute.ts b/packages/agents/src/execute.ts index b0385cd6..8055a9cb 100644 --- a/packages/agents/src/execute.ts +++ b/packages/agents/src/execute.ts @@ -5,13 +5,13 @@ import { CHAT_FORMAT_CONTENT_ONLY, CHAT_FORMAT_GENERIC, GrammarTriggerType, buildToolResultDelta, buildToolResultDeltaMultimodal, decodeErrorOf, deltaCells, } from '@lloyal-labs/sdk'; -import type { Attachment, AttachmentStore, ContentIngress } from '@lloyal-labs/media'; +import type { Attachment, AttachmentStore, ContentIngress, PreparedContent } from '@lloyal-labs/media'; import { waitUntilSettled } from './combinators'; import { Trace, TraceParent, CallingAgent, SpineFmt } from './context'; import { Agent, type FormatConfig } from './Agent'; import type { AgentPolicy, ToolRetryAction } from './AgentPolicy'; import { Tool, ToolRetryError, takeToolMedia, TOOL_CONTEXT_KEY, TOOL_IMAGE_ERROR_KEY } from './Tool'; -import type { Emitter } from './emit'; +import type { Emitter, Transition } from './emit'; import { ContextPressure } from './pressure'; import { prepareBatch } from './prepare-content'; import { runReplay } from './replay'; @@ -177,6 +177,10 @@ export interface ExecDeps { pressureOpts: PressureThresholds; ingress: ContentIngress; attachments: AttachmentStore; + /** Assets available to the run: the roots the host staged, then every root + * a tool result admitted, in admission order. Grows in `book()`; read by + * every tool call. Roots only — the pool never materializes an entry. */ + available: Attachment[]; ladder: Ladder; trace: boolean; } @@ -255,6 +259,10 @@ export class Executor { const landed: Agent[] = []; const order: { agentId: number; callId: string; cells: number }[] = []; const probes = new Map<number, string>(); + // Admissions to announce once the rails are done: `prefilled` rides the + // bus as well as the trace, and a bus send may suspend, so it leaves the + // synchronous bookkeeping below and is emitted from this generator. + const admissions: Transition[] = []; const tokenItems = items.filter((it): it is PrefillItem & { rail: 'token' } => it.rail === 'token'); const mediaItems = items.filter((it): it is PrefillItem & { rail: 'media' } => it.rail === 'media'); @@ -265,6 +273,11 @@ export class Executor { a.records.push({ kind: 'toolResult', resultStr: it.resultStr, callId: it.callId, ...(refs && refs.length > 0 ? { attachments: refs } : {}) }); } + // Admitted roots become assets available to the whole run — pool state, + // so they outlive the agent that admitted them. + for (const r of refs ?? []) { + if (!d.available.some((x) => x.digest === r.digest)) d.available.push(r); + } landed.push(a); order.push({ agentId: a.id, callId: it.callId, cells }); if (it.probe) probes.set(a.id, it.probe); @@ -272,7 +285,7 @@ export class Executor { const after = new ContextPressure(d.ctx, d.pressureOpts); a.recordToolResult({ name: it.toolName, args: it.args, resultCells: cells, contextAfterPercent: after.percentAvailable, timestamp: performance.now() }); - d.emit.trace({ kind: 'prefilled', agent: a, cells, + admissions.push({ kind: 'prefilled', agent: a, cells, role: it.kind === 'recovery' ? 'recovery' : 'toolResult', attachments: refs }); }; @@ -282,7 +295,7 @@ export class Executor { d.counters.warmPrefillCalls++; d.counters.warmPrefillBranches += tokenItems.length; d.ladder.consecutiveFatalRc = 0; - for (const t of tokenItems) book(t, t.tokens.length); + for (const t of tokenItems) book(t, t.tokens.length, t.attachments); out.tokenRail = { items: tokenItems, outcome: { ok: true } }; } catch (err) { const de = decodeErrorOf(err); @@ -312,6 +325,8 @@ export class Executor { } } + for (const t of admissions) yield* d.emit.emit(t); + if (landed.length > 0) { d.emit.trace({ kind: 'settleOrder', batch: order }); const probePairs: [Branch, number[]][] = []; @@ -326,7 +341,7 @@ export class Executor { if (probePairs.length > 0) { yield* prefill(d.store, probePairs); for (const m of probeMeta) { - d.emit.trace({ kind: 'prefilled', agent: m.agent, cells: m.cells, role: 'probe', probeText: m.text }); + yield* d.emit.emit({ kind: 'prefilled', agent: m.agent, cells: m.cells, role: 'probe', probeText: m.text }); m.agent.records.push({ kind: 'probe', text: m.text }); } } @@ -509,6 +524,7 @@ export class Executor { scorer: d.scorer, explore, pressurePercentAvailable: reading.percentAvailable, peerHistory, + attachments: [...d.available], }; if (tool?.fanout) { @@ -629,27 +645,39 @@ export class Executor { if (Array.isArray(obj.results)) agent.addNestedResults((obj.results as unknown[]).filter((f): f is string => typeof f === 'string')); if (Array.isArray(obj.nestedResults)) agent.addNestedResults((obj.nestedResults as unknown[]).filter((f): f is string => typeof f === 'string')); } - // Images come OUT before serializing; a model with no projector is TOLD. + // Media comes OUT before serializing. Bytes go through the ingress door; + // roots resolve through the store. The rail follows what the batch + // MATERIALIZES to, never the fact that media was there. const { media, result: told } = takeToolMedia(result); - if (media.length > 0 && !d.ctx.supportsVision()) { + const vision = d.ctx.supportsVision(); + const hasBytes = media.some((m) => m instanceof Uint8Array); + // THE BARRIER: normalized and committed before a marker exists, before + // admission, before any KV moves. A failure here is not a tool retry. + // A model with no projector never ingests bytes; it still resolves roots, + // because a root that expands to no bitmaps — a document — asks nothing + // of sight and is admitted on the token rail. + let prepared: PreparedContent | null = null; + if (media.length > 0 && (vision || !hasBytes)) { + prepared = yield* prepareBatch(d.ingress, d.attachments, media); + } + if (media.length > 0 && !vision && (hasBytes || (prepared?.bitmaps.length ?? 0) > 0)) { (told as Record<string, unknown>)[TOOL_IMAGE_ERROR_KEY] = `${tc.name} returned ${media.length} image(s), but this model cannot see images. ` + `Work from the text, or use a different source.`; + prepared = null; } const resultStr = JSON.stringify(told); yield* d.emit.emit({ kind: 'toolTold', agent, tool: tc.name, resultStr, contextAvailablePercent }); const common = { agent, toolName: tc.name, callId, args: tc.arguments, resultStr, probe: tool?.probe(told) ?? undefined }; let item: PrefillItem; - if (media.length > 0 && d.ctx.supportsVision()) { - // THE BARRIER: normalized and committed before a marker exists, before - // admission, before any KV moves. A failure here is not a tool retry. - const prepared = yield* prepareBatch(d.ingress, d.attachments, media); + if (prepared && prepared.bitmaps.length > 0) { const delta = buildToolResultDeltaMultimodal(d.ctx, resultStr, callId, prepared.bitmaps as Uint8Array[], { enableThinking: agent.fmt.enableThinking }); const cells = yield* measureCells(d.ctx, delta); item = { kind: 'toolResult', rail: 'media', ...common, media: { delta, cells, attachments: prepared.attachments } }; } else { const tokens = buildToolResultDelta(d.ctx, resultStr, callId, { enableThinking: agent.fmt.enableThinking }); - item = { kind: 'toolResult', rail: 'token', ...common, tokens }; + item = { kind: 'toolResult', rail: 'token', ...common, tokens, + ...(prepared && prepared.attachments.length > 0 ? { attachments: prepared.attachments } : {}) }; } d.emit.trace({ kind: 'toolResult', agent, tool: tc.name, result: told, cells: item.rail === 'media' ? item.media.cells : item.tokens.length, diff --git a/packages/agents/src/index.ts b/packages/agents/src/index.ts index d6da3009..69b95ace 100644 --- a/packages/agents/src/index.ts +++ b/packages/agents/src/index.ts @@ -40,7 +40,7 @@ export { prepareBatch } from './prepare-content'; // The one member of the framework-channel namespace a TOOL writes. The other // two are framework→model and no tool author ever sets them, so they stay // internal rather than growing the surface to describe a convention. -export { TOOL_MEDIA_KEY } from './Tool'; +export { TOOL_ATTACHMENTS_KEY } from './Tool'; export { useTraceScope } from './trace-scope'; export { admitChunks } from './admission'; export type { AdmitOpts, AdmitResult, AdmitSelect, AdmittedPassage } from './admission'; diff --git a/packages/agents/src/prepare-content.ts b/packages/agents/src/prepare-content.ts index 6d9f45d6..06bb322c 100644 --- a/packages/agents/src/prepare-content.ts +++ b/packages/agents/src/prepare-content.ts @@ -18,7 +18,9 @@ import { materialize } from '@lloyal-labs/media'; * * The barrier this enforces, in order: * - * 1. **Prepare** every item, committing each root manifest. + * 1. **Prepare** every item, committing each root manifest. An item that is + * already a root — a descriptor a tool read from the store — takes its + * place in order without touching the door; only bytes are ingested. * 2. **Materialize** every representation. * 3. **Flatten** preserving attachment order, then representation order within * each — markers correspond to REPRESENTATIONS, so a video contributes its @@ -54,7 +56,7 @@ import { materialize } from '@lloyal-labs/media'; export function* prepareBatch( ingress: ContentIngress, store: AttachmentStore, - items: readonly Uint8Array[], + items: readonly (Uint8Array | Attachment)[], ): Operation<PreparedContent> { // The scope's own signal, hoisted once. `call()` makes a halt OBSERVABLE at // this boundary but cannot stop the promise behind it — that is the leaked @@ -65,7 +67,8 @@ export function* prepareBatch( // Concurrent, in input order: normalization is the expensive step and the // normalizer already bounds itself process-wide, so a batch of N must not // cost the sum of N decodes while permits sit idle. `all` keeps the order. - const roots: Attachment[] = yield* all(items.map((bytes) => call(() => ingress.ingest(bytes, signal)))); + const roots: Attachment[] = yield* all(items.map((item) => + (item instanceof Uint8Array ? call(() => ingress.ingest(item, signal)) : call(() => item)))); // Resolve from the store rather than trusting what ingest returned, through // the SAME call replay uses — so a batch that materializes here is one that // can be rebuilt later, by construction rather than by assertion. diff --git a/packages/agents/src/replay.ts b/packages/agents/src/replay.ts index d808d928..de4d8c11 100644 --- a/packages/agents/src/replay.ts +++ b/packages/agents/src/replay.ts @@ -273,8 +273,9 @@ export type AgentTurnRecord = | { kind: 'assistant'; text: string } | { kind: 'toolResult'; resultStr: string; callId: string; - /** Roots for a media-bearing result — resolved through the run's - * attachment store at replay, exactly as the seed's are. */ + /** Roots admitted with this result — resolved through the run's + * attachment store at replay, exactly as the seed's are. The rail + * follows what they materialize to, as it did live. */ attachments?: readonly Attachment[]; } | { kind: 'probe'; text: string }; @@ -313,7 +314,13 @@ export function* prepareReplay( } else if (r.kind === 'probe') { tokenStep(ctx.tokenizeSync(r.text, false)); } else if (r.attachments && r.attachments.length > 0) { + // Rail by what the roots MATERIALIZE to: a document root expands to no + // bitmaps and replays as the tool text it was. const { bitmaps } = materialize(attachments, r.attachments); + if (bitmaps.length === 0) { + tokenStep(buildToolResultDelta(ctx, r.resultStr, r.callId, opts)); + continue; + } const delta = buildToolResultDeltaMultimodal(ctx, r.resultStr, r.callId, [...bitmaps], opts); const priced = yield* measureCells(ctx, delta); steps.push({ kind: 'media', delta, cells: priced }); diff --git a/packages/agents/src/source.ts b/packages/agents/src/source.ts index 83ca61d7..3b76190d 100644 --- a/packages/agents/src/source.ts +++ b/packages/agents/src/source.ts @@ -1,4 +1,5 @@ import type { Tool } from './Tool'; +import type { Attachment } from '@lloyal-labs/media'; /** * Entailment scorer — scores texts against an original query to @@ -11,12 +12,12 @@ import type { Tool } from './Tool'; * * | Concept | Scope | Field name in code | * |--------------------|--------------------------|-------------------------------------------| - * | **Tool query** | Per tool call | `localQuery` param of scoreRelevanceBatch | + * | **Tool query** | Per tool call | scored by the tool's own reranker pass | * | **Agent task** | Per agent lifetime | `reference` param of scoreSimilarityBatch | * | **Original query** | Per research invocation | Captured in closure by createScorer | * * - `scoreEntailmentBatch` scores against the **original query** (steering boundaries) - * - `scoreRelevanceBatch` combines **tool query** + **original query** via min() (exploit mode) + * - exploit mode (`admitChunks`) takes `min(tool-query score, scoreEntailmentBatch)` — one extra pass, never two * - `scoreSimilarityBatch` scores against an arbitrary **reference** (echo detection uses agent task) * * Conflating these produces wrong scores. When adding new scoring @@ -33,7 +34,6 @@ export interface EntailmentScorer { * @param texts - Content chunks to score * @param localQuery - The tool call's query argument (NOT the agent task) */ - scoreRelevanceBatch(texts: string[], localQuery: string): Promise<number[]>; /** Score texts against an arbitrary reference string. Returns 0–1 per text. */ scoreSimilarityBatch(reference: string, texts: string[]): Promise<number[]>; /** Threshold gate — returns true if the score is high enough to proceed. */ @@ -43,7 +43,6 @@ export interface EntailmentScorer { /** No-op scorer — all scores 1.0, all proceed. Used when no reranker is available. */ export const NULL_SCORER: EntailmentScorer = { scoreEntailmentBatch: async (texts) => texts.map(() => 1), - scoreRelevanceBatch: async (texts) => texts.map(() => 1), scoreSimilarityBatch: async (_ref, texts) => texts.map(() => 0), shouldProceed: () => true, }; @@ -109,18 +108,11 @@ export abstract class Source<TChunk = unknown> { if (!reranker || !originalQuery) return NULL_SCORER; const floor = this._entailmentFloor; - const combine = (local: number, orig: number) => Math.min(local, orig); return { async scoreEntailmentBatch(texts: string[]): Promise<number[]> { return reranker.scoreBatch(originalQuery, texts); }, - async scoreRelevanceBatch(texts: string[], localQuery: string): Promise<number[]> { - // SEQUENTIAL — single llama_context, no concurrent scoreBatch calls - const origScores = await reranker.scoreBatch(originalQuery, texts); - const localScores = await reranker.scoreBatch(localQuery, texts); - return texts.map((_, i) => combine(localScores[i], origScores[i])); - }, async scoreSimilarityBatch(reference: string, texts: string[]): Promise<number[]> { return reranker.scoreBatch(reference, texts); }, @@ -147,6 +139,10 @@ export abstract class Source<TChunk = unknown> { * (first-party corpora); it must not blanket-append data from untrusted * third-party abilities — `renderSpine` itself stays prose-free for exactly * this reason. + * + * `attachments` are the assets available to the run being staged — roots, + * as the host holds them. A source keyed on attachments builds its data + * from them; a source that is not ignores the argument. */ - promptData(): Record<string, unknown> { return {}; } + promptData(_attachments: readonly Attachment[] = []): Record<string, unknown> { return {}; } } diff --git a/packages/agents/src/state.ts b/packages/agents/src/state.ts index 00167906..9ecaf297 100644 --- a/packages/agents/src/state.ts +++ b/packages/agents/src/state.ts @@ -41,7 +41,10 @@ export type PrefillItem = { resultStr?: string; } & ( /** The token rail: the delta tokenized here and prefilled as tokens. */ - | { rail: 'token'; tokens: number[]; media?: never } + | { rail: 'token'; tokens: number[]; media?: never; + /** Roots admitted with a token-rail result — ones that materialize to + * no bitmaps — booked as assets available to the run. */ + attachments?: readonly Attachment[] } /** The embedding rail. `llama_batch` is token-XOR-embd, so this cannot * join a token batch — a separate call, not a separate strategy. */ | { rail: 'media'; tokens?: never; media: { delta: MultimodalDelta; cells: number; attachments: readonly Attachment[] } } diff --git a/packages/agents/src/trace-types.ts b/packages/agents/src/trace-types.ts index 388fba4e..1909d886 100644 --- a/packages/agents/src/trace-types.ts +++ b/packages/agents/src/trace-types.ts @@ -447,7 +447,7 @@ export type TraceEvent = | TraceEventBase & { /** Exploit-mode dual scoring at a content boundary (search/fetch_page). * Emitted when policy.shouldExplore() returns false and the tool - * applies scoreRelevanceBatch to tighten focus. */ + * ranks by min(tool score, original-question score) to tighten focus. */ type: 'entailment:content:exploit'; tool: string; /** Pressure snapshot that triggered exploit mode. Only the field the diff --git a/packages/agents/src/types.ts b/packages/agents/src/types.ts index 47173092..4dc0f53d 100644 --- a/packages/agents/src/types.ts +++ b/packages/agents/src/types.ts @@ -5,6 +5,7 @@ import type { AgentPolicy } from './AgentPolicy'; import type { EntailmentScorer } from './source'; import type { ToolHistoryEntry } from './Agent'; import type { TraceEvent } from './trace-types'; +import type { Attachment } from '@lloyal-labs/media'; // ── Tool base class types ────────────────────────────────────── @@ -78,7 +79,7 @@ export interface ToolContext { scorer?: EntailmentScorer; /** * When false, content-boundary tools apply dual scoring - * (scoreRelevanceBatch) for tighter focus. Computed per-DISPATCH + * (min with the original-question score) for tighter focus. Computed per-DISPATCH * by policy.shouldExplore(). @default true */ explore?: boolean; @@ -95,6 +96,14 @@ export interface ToolContext { * a "resource unavailable" error to force diversification. */ peerHistory?: ToolHistoryEntry[]; + /** + * Assets available to the run at this call: the roots the host staged the + * pool with, then every root any agent's tool result has admitted so far, + * in admission order. Roots only — reading one costs no KV; the content + * store resolves it. Availability is not projection: nothing here is in + * the model's context unless a branch admitted it. + */ + attachments?: readonly Attachment[]; } // ── Trace types ─────────────────────────────────────────────── @@ -301,6 +310,12 @@ export interface AgentPoolOptions { /** Entailment scorer for semantic coherence across recursive depths. * Passed to every tool via {@link ToolContext.scorer}. */ scorer?: EntailmentScorer; + /** + * Assets available to the run from the start — roots the host stages the + * pool with. Zero KV: the pool never materializes them. Every tool call + * reads them, plus whatever the run admits, as {@link ToolContext.attachments}. + */ + attachments?: readonly Attachment[]; /** * Eager GBNF grammar applied to every spawned agent's generating branch — * constrains generation from the first sampled token (no trigger). Used for @@ -405,6 +420,12 @@ export type AgentEvent = | { type: 'agent:tool_result'; agentId: number; tool: string; result: string; contextAvailablePercent?: number } | { type: 'agent:tool_progress'; agentId: number; tool: string; filled: number; total: number } | { type: 'agent:tool_retry'; agentId: number; tool: string; retryAfterMs: number; attempt: number } + /** A prefill LANDED on the agent's branch — a tool result, a recovery + * prompt or a probe — with the roots it admitted (`attachments`, present + * when the result carried any). Admission is a run-record fact the host + * books and shows, so it rides the bus like every other one; the trace's + * `branch:prefill` is the same moment written for replay. */ + | { type: 'agent:prefilled'; agentId: number; cells: number; role: 'toolResult' | 'recovery' | 'probe'; attachments?: readonly Attachment[] } | { type: 'agent:return'; agentId: number; result: string } | { type: 'agent:recovered'; agentId: number; result: string } | { type: 'agent:failed'; agentId: number; reason: string } diff --git a/packages/agents/test/admission.test.ts b/packages/agents/test/admission.test.ts index d260cb5f..ac7f51da 100644 --- a/packages/agents/test/admission.test.ts +++ b/packages/agents/test/admission.test.ts @@ -111,8 +111,8 @@ describe('admitChunks — exploit re-rank', () => { agentId: 1, explore: false, pressurePercentAvailable: 33, - // Reverses the tool-query order: chunk 2's text entails the query best. - scorer: { scoreRelevanceBatch: async (texts: string[]) => texts.map((_, i) => i) } as any, + // Original-question scores reverse the tool-query order: chunk 2's text entails the question best. + scorer: { scoreEntailmentBatch: async (texts: string[]) => texts.map((_, i) => i) } as any, }, ); expect(result.scored.map(s => s.heading)).toEqual(['H2', 'H1', 'H0']); @@ -123,11 +123,29 @@ describe('admitChunks — exploit re-rank', () => { expect(exploit.chunks[0]).toMatchObject({ heading: 'H2', toolQueryScore: 8, combinedScore: 2 }); }); + it('exploit asks the entailment scorer ONCE for the original-question scores and takes min() with the tool scores it already has', async () => { + const calls: string[][] = []; + const scorer = { + // Original-question scores only: chunk i entails the question at i. + scoreEntailmentBatch: async (texts: string[]) => { calls.push(texts); return texts.map((_, i) => i); }, + }; + const { result, tw } = await admit( + { tool: 'fetch_page', select: { mode: 'budget', topK: 3, tokenBudget: 10_000 } }, + mkChunks(3), + { agentId: 1, explore: false, scorer } as any, + ); + expect(calls).toHaveLength(1); + expect(calls[0]).toHaveLength(3); + // min(tool, original): H2 = min(8, 2) = 2, H1 = min(9, 1) = 1, H0 = min(10, 0) = 0. + expect(result.scored.map(s => [s.heading, s.score])).toEqual([['H2', 2], ['H1', 1], ['H0', 0]]); + expect(tw.ofType('entailment:content:exploit')[0].chunks[0]).toMatchObject({ heading: 'H2', toolQueryScore: 8, combinedScore: 2 }); + }); + it('an OMITTED explore flag defaults to explore — never exploit by accident', async () => { const { result, tw } = await admit( { tool: 'fetch_page', select: { mode: 'budget', topK: 3, tokenBudget: 10_000 } }, mkChunks(3), - { agentId: 1, scorer: { scoreRelevanceBatch: async () => [9, 9, 9] } as any }, + { agentId: 1, scorer: { scoreEntailmentBatch: async () => [9, 9, 9] } as any }, ); expect(result.scored.map(s => s.heading)).toEqual(['H0', 'H1', 'H2']); expect(tw.ofType('entailment:content:exploit')).toHaveLength(0); @@ -137,7 +155,7 @@ describe('admitChunks — exploit re-rank', () => { const { result, tw } = await admit( { tool: 'fetch_page', select: { mode: 'budget', topK: 3, tokenBudget: 10_000 } }, mkChunks(3), - { agentId: 1, explore: true, scorer: { scoreRelevanceBatch: async () => [9, 9, 9] } as any }, + { agentId: 1, explore: true, scorer: { scoreEntailmentBatch: async () => [9, 9, 9] } as any }, ); expect(result.scored.map(s => s.heading)).toEqual(['H0', 'H1', 'H2']); expect(tw.ofType('entailment:content:exploit')).toHaveLength(0); diff --git a/packages/agents/test/agent-pool.test.ts b/packages/agents/test/agent-pool.test.ts index 8881fded..4d118af8 100644 --- a/packages/agents/test/agent-pool.test.ts +++ b/packages/agents/test/agent-pool.test.ts @@ -9,7 +9,8 @@ * transitions, trace events, event emissions, ToolContext fields, recovery. */ import { describe, it, expect } from 'vitest'; -import { MediaTool, PNG_BYTES, MEDIA_TEST_NCTX, mediaFailures } from './helpers/media'; +import { MediaTool, RootTool, imageRoot, documentRoot, PNG_BYTES, MEDIA_TEST_NCTX, mediaFailures } from './helpers/media'; +import type { Attachment } from '@lloyal-labs/media'; import { run, createChannel, createSignal, spawn, each, scoped, call } from 'effection'; import type { Operation, Channel } from 'effection'; import { MockSessionContext, createMockSdk } from '../../sdk/src/testing.js'; @@ -58,11 +59,17 @@ async function runPool(opts: { mutateCtx?: (ctx: MockSessionContext) => void; /** Install an ingress that refuses everything, to exercise the barrier. */ refusingIngress?: boolean; + /** Assets available to the run from the start — what a host stages the pool with. */ + attachments?: readonly Attachment[]; + /** A content store the test committed roots into beforehand. */ + contentStore?: MemoryAttachmentStore; }): Promise<{ result: AgentPoolResult; events: AgentEvent[]; trace: CapturingTraceWriter; ctx: MockSessionContext; + /** How many times a tool result went through the ingress door. */ + ingressCalls: number; }> { const { ctx, store, root } = createMockSdk({ nCtx: opts.nCtx ?? 16384, @@ -103,6 +110,7 @@ async function runPool(opts: { const traceWriter = new CapturingTraceWriter(); const collectedEvents: AgentEvent[] = []; + let ingressCalls = 0; // Prefill root to simulate withSpine system prompt const rootTokens = ctx.tokenizeSync('system prompt'); @@ -117,15 +125,16 @@ async function runPool(opts: { // A real store: media paths now REFUSE to run without one, because // unaddressed media makes a run unreplayable. Tests that exercise them // must be configured the way a real harness is. - const contentStore = new MemoryAttachmentStore(); + const contentStore = opts.contentStore ?? new MemoryAttachmentStore(); yield* Attachments.set(contentStore); // Media now refuses to run without an ingress, because unnormalized, // unaddressed bytes make a run unreplayable. Tests use a raw one — they // exercise the rail, not the normalizer. + const raw = rawIngress(contentStore); yield* Ingress.set( opts.refusingIngress ? { ingest: () => Promise.reject(new Error('ingress refused')) } - : rawIngress(contentStore), + : { ingest: (bytes, signal) => { ingressCalls++; return raw.ingest(bytes, signal); } }, ); const windDownSignal = createSignal<void, void>(); @@ -152,6 +161,7 @@ async function runPool(opts: { terminalToolName: opts.terminalTool, trace: opts.trace ?? false, pruneOnReturn: opts.pruneOnReturn ?? false, + attachments: opts.attachments, }); // Drain Subscription — collect events, return close value let windDownFired = false; @@ -168,7 +178,7 @@ async function runPool(opts: { }); }); - return { result, events: collectedEvents, trace: traceWriter, ctx }; + return { result, events: collectedEvents, trace: traceWriter, ctx, ingressCalls }; } /** Minimal policy stub — every method overridable */ @@ -2359,7 +2369,7 @@ describe('tool results carrying images', () => { const shown = events.filter(e => e.type === 'agent:tool_result') .map(e => (e as { result: string }).result).join(''); expect(shown).not.toContain('137,80,78,71'); - expect(shown).not.toContain('_images'); + expect(shown).not.toContain('_attachments'); expect(shown).toContain('p1'); }); @@ -2506,3 +2516,139 @@ describe('tool results carrying images', () => { expect(shown).not.toContain('137,80,78,71'); }); }); + +// ── Assets available to the run ───────────────────────────────── +// Availability is not projection. A root the host stages, or one a tool +// admits through the key, is readable by every tool call in the run through +// ToolContext.attachments and costs no KV. Only admission onto a branch +// projects — and the rail follows what a root MATERIALIZES to, never the +// fact that a root was there. +describe('assets available to the run', () => { + const policy = () => stubPolicy({ + shouldExit: () => false, + onProduced: (_a, parsed) => { + if (parsed.toolCalls.length > 0) return { type: 'tool_call', tc: parsed.toolCalls[0] }; + if (parsed.content) return { type: 'free_text_return', content: parsed.content }; + return { type: 'idle', reason: 'free_text_stop' }; + }, + }); + /** A turn whose raw carries 't1' calls `first`; one carrying 't2' calls + * `second`; any other finishes. Keyed off raw, as the ladder tests are. */ + const calls = (first: string, second: string) => (raw: string) => + raw.includes('t2') + ? { content: '', reasoningContent: '', toolCalls: [{ name: second, arguments: '{}', id: 'c2' }] } + : raw.includes('t1') + ? { content: '', reasoningContent: '', toolCalls: [{ name: first, arguments: '{}', id: 'c1' }] } + : { content: 'done', reasoningContent: '', toolCalls: [] }; + const booked = (trace: CapturingTraceWriter) => + trace.events.filter(e => e.type === 'branch:prefill' && (e as { role: string }).role === 'toolResult') as { attachments?: readonly Attachment[] }[]; + const shownTo = (events: AgentEvent[]) => + events.filter(e => e.type === 'agent:tool_result').map(e => (e as { result: string }).result).join(''); + /** The roots each admission announced on the bus — the host's view, which + * the run record books and the UI shows without reading the trace. */ + const announced = (events: AgentEvent[]) => + events.filter(e => e.type === 'agent:prefilled' && ((e as { attachments?: readonly Attachment[] }).attachments?.length ?? 0) > 0) + .map(e => (e as { attachments?: readonly Attachment[] }).attachments); + + it('the first call reads the staged roots, and none of them is projected', async () => { + const store = new MemoryAttachmentStore(); + const img = imageRoot(store); + const doc = documentRoot(store); + const spy = new SpyTool('look'); + const { ctx, trace } = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, STOP]], + parseChatOutputFn: calls('look', 'look'), policy: policy(), + tools: new Map<string, Tool>([['look', spy]]), + contentStore: store, attachments: [img, doc], trace: true, + }); + expect(spy.capturedContexts).toHaveLength(1); + expect(spy.capturedContexts[0].attachments).toEqual([img, doc]); + // Zero projection from availability — even for the image. + expect(ctx.multimodalPrefills).toHaveLength(0); + expect(booked(trace).flatMap(p => p.attachments ?? [])).toEqual([]); + }); + + it('a result carrying a document root rides the token rail, is booked on branch:prefill, and the next call sees it', async () => { + const store = new MemoryAttachmentStore(); + const doc = documentRoot(store); + const spy = new SpyTool('look'); + const { ctx, trace, ingressCalls, events } = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, 2, STOP, STOP]], + parseChatOutputFn: calls('open', 'look'), policy: policy(), + tools: new Map<string, Tool>([['open', new RootTool([doc], 'open')], ['look', spy]]), + contentStore: store, trace: true, + }); + expect(ingressCalls).toBe(0); + expect(ctx.multimodalPrefills).toHaveLength(0); + expect(booked(trace)[0]?.attachments).toEqual([doc]); + // The same admission rides the bus: one event, the same roots, so a host + // books and shows it from the stream it already consumes. + expect(announced(events)).toEqual([[doc]]); + expect(spy.capturedContexts).toHaveLength(1); + expect(spy.capturedContexts[0].attachments).toEqual([doc]); + const cost = trace.events.find(e => e.type === 'tool:result') as { cells: number } | undefined; + expect(cost?.cells).toBeGreaterThan(0); + }); + + it('a result carrying a page root rides the media rail through the store — the ingress is never called', async () => { + const store = new MemoryAttachmentStore(); + const page = imageRoot(store); + const { ctx, trace, ingressCalls, events } = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, STOP]], + parseChatOutputFn: calls('view', 'view'), policy: policy(), + tools: new Map<string, Tool>([['view', new RootTool([page], 'view')]]), + contentStore: store, trace: true, + }); + expect(ingressCalls).toBe(0); + expect(ctx.multimodalPrefills).toHaveLength(1); + expect(ctx.multimodalPrefills[0].bitmapCounts).toEqual([1]); + expect(booked(trace)[0]?.attachments).toEqual([page]); + expect(announced(events)).toEqual([[page]]); + }); + + it('a root one agent admits is available to its siblings, and stays so after the admitting agent returns and is pruned', async () => { + const store = new MemoryAttachmentStore(); + const doc = documentRoot(store); + const spy = new SpyTool('look'); + const { result } = await runPool({ + nCtx: MEDIA_TEST_NCTX, taskCount: 2, + // A admits on its first turn and finishes. B generates for a while, + // then calls — well after A's result landed and A was pruned. + forkTokenQueues: [[1, STOP, STOP], [7, 7, 7, 7, 7, 7, 2, STOP, STOP]], + parseChatOutputFn: calls('open', 'look'), policy: policy(), + tools: new Map<string, Tool>([['open', new RootTool([doc], 'open')], ['look', spy]]), + contentStore: store, pruneOnReturn: true, trace: true, + }); + expect(result.agents[0].result).toBe('done'); + expect(spy.capturedContexts).toHaveLength(1); + expect(spy.capturedContexts[0].attachments).toEqual([doc]); + }); + + it('without a projector a document root is still admitted, while a page root is refused with the note', async () => { + const store = new MemoryAttachmentStore(); + const doc = documentRoot(store); + const page = imageRoot(store); + const spy = new SpyTool('look'); + const text = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, 2, STOP, STOP]], + parseChatOutputFn: calls('open', 'look'), policy: policy(), + tools: new Map<string, Tool>([['open', new RootTool([doc], 'open')], ['look', spy]]), + contentStore: store, trace: true, + mutateCtx: (c) => { c.mockSupportsVision = false; }, + }); + expect(spy.capturedContexts[0]?.attachments).toEqual([doc]); + expect(shownTo(text.events)).not.toContain('cannot see images'); + expect(booked(text.trace)[0]?.attachments).toEqual([doc]); + + const picture = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, STOP]], + parseChatOutputFn: calls('view', 'view'), policy: policy(), + tools: new Map<string, Tool>([['view', new RootTool([page], 'view')]]), + contentStore: store, trace: true, + mutateCtx: (c) => { c.mockSupportsVision = false; }, + }); + expect(shownTo(picture.events)).toContain('cannot see images'); + expect(picture.ctx.multimodalPrefills).toHaveLength(0); + expect(booked(picture.trace)[0]?.attachments).toBeUndefined(); + }); +}); diff --git a/packages/agents/test/attachments.test.ts b/packages/agents/test/attachments.test.ts index a028ae1d..b319a4c9 100644 --- a/packages/agents/test/attachments.test.ts +++ b/packages/agents/test/attachments.test.ts @@ -20,7 +20,7 @@ import { BranchStore } from '../../sdk/src/BranchStore'; import { NullAttachmentStore } from '@lloyal-labs/media'; import type { AttachmentStore } from '@lloyal-labs/media'; import { MemoryAttachmentStore } from './helpers/memory-store'; -import { representationsOf, sourceOf, ATTACHMENT_ARTIFACT_TYPE, EMPTY_DESCRIPTOR, MANIFEST_TYPE } from '@lloyal-labs/media'; +import { representationsOf, sourceOf, ATTACHMENT_ARTIFACT_TYPE, EMPTY_DESCRIPTOR, MANIFEST_TYPE, DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; import type { Attachment } from '@lloyal-labs/media'; /** A manifest descriptor, shaped the way a store would have returned one. @@ -187,6 +187,30 @@ describe('reconstructBranch', () => { }, store); }); + it('replayAgentTurns rails a record by what its roots MATERIALIZE to — a document root goes down the token rail', async () => { + // A root is not a picture. A document root (one text/markdown + // representation) expands to no bitmaps, so its record replays as the + // tool text it was — the same rule the live path applies at intake. + const store = new MemoryAttachmentStore(); + const doc = store.putAttachment({ + representations: [store.putBlob(new TextEncoder().encode('# Paper\n\nBody.\n'), 'text/markdown')], + config: { bytes: new TextEncoder().encode('{}'), mediaType: DOCUMENT_CONFIG_TYPE }, + }); + await withCtx(function*(ctx) { + const spine = yield* reconstructBranch(cp()); + const fork = spine.forkSync(); + let tokenPrefills = 0; + const orig = ctx._storePrefill.bind(ctx); + ctx._storePrefill = async (h, t) => { tokenPrefills++; return orig(h, t); }; + yield* replayAgentTurns(fork, [ + { kind: 'toolResult', resultStr: '{"page":"p1"}', callId: 'c1', attachments: [doc] }, + ], { enableThinking: false }); + expect(tokenPrefills).toBe(1); + expect(ctx.multimodalPrefills).toHaveLength(0); + return null; + }, store); + }); + it('refuses a marker with no attachment references', async () => { // The pre-attachments behaviour, preserved: a trace that recorded only // the marker still cannot be replayed. diff --git a/packages/agents/test/helpers/media.ts b/packages/agents/test/helpers/media.ts index a9402710..43c4c023 100644 --- a/packages/agents/test/helpers/media.ts +++ b/packages/agents/test/helpers/media.ts @@ -1,6 +1,8 @@ import type { Operation } from 'effection'; +import { DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, AttachmentStore } from '@lloyal-labs/media'; import { MockTool } from './mock-tool'; -import { TOOL_MEDIA_KEY } from '../../src/Tool'; +import { TOOL_ATTACHMENTS_KEY } from '../../src/Tool'; import type { AgentEvent } from '../../src/types'; /** @@ -18,7 +20,7 @@ export class MediaTool extends MockTool { *execute(): Operation<unknown> { // Through the constant, like a real tool author would: a fixture spelling // the literal is a fixture that keeps passing after the key changes. - return { page: 'p1', [TOOL_MEDIA_KEY]: this._bytes }; + return { page: 'p1', [TOOL_ATTACHMENTS_KEY]: this._bytes }; } } @@ -37,3 +39,37 @@ export const MEDIA_TEST_NCTX = 32768; export const mediaFailures = (events: AgentEvent[]): AgentEvent[] => events.filter(e => e.type === 'agent:failed' && (e as { reason?: string }).reason === 'media_prefill_failed'); + +/** A tool returning attachment ROOTS under the same key — what a tool that + * reads the content store hands back: descriptors, no bytes, no ingress. */ +export class RootTool extends MockTool { + constructor(private _roots: readonly Attachment[], name = 'view') { super(name); } + *execute(): Operation<unknown> { + return { page: 'p1', [TOOL_ATTACHMENTS_KEY]: this._roots }; + } +} + +/** An image root: one PNG representation — the shape a page render has. + * Materializes to one bitmap. */ +export function imageRoot(store: AttachmentStore, bytes: Uint8Array = PNG_BYTES): Attachment { + return store.putAttachment({ representations: [store.putBlob(bytes, 'image/png')] }); +} + +/** A document root: one `text/markdown` representation and the document + * sidecar as config. Materializes to NO bitmaps — that is the property the + * rail tests lean on; the sidecar's content is the media package's affair. */ +export function documentRoot(store: AttachmentStore, title = 'A Paper'): Attachment { + const markdown = new TextEncoder().encode(`# ${title}\n\nBody.\n`); + const sidecar = { + title, pageCount: 1, + sections: [{ heading: title, path: title, origin: 'heuristic', startLine: 1, endLine: 3, pageStart: 1, pageEnd: 1 }], + pages: [{ page: 1, startLine: 1, endLine: 3, chars: 5, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }], + figures: [], tables: [], + derive: { profile: 'pdf.v1', pdfium: 'test', dpi: 150, maxSide: 2048, maxPixels: 1, format: 'image/png', + renderedPages: 0, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false }, + }; + return store.putAttachment({ + representations: [store.putBlob(markdown, 'text/markdown')], + config: { bytes: new TextEncoder().encode(JSON.stringify(sidecar)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); +} diff --git a/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts b/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts index 4698275d..cd4beaf5 100644 --- a/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts +++ b/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts @@ -23,7 +23,7 @@ import { describe, it, expect } from 'vitest'; import type { Operation } from 'effection'; import type { AgentPolicy } from '../../../src/AgentPolicy'; -import { Tool, TOOL_MEDIA_KEY } from '../../../src/Tool'; +import { Tool, TOOL_ATTACHMENTS_KEY } from '../../../src/Tool'; import type { JsonSchema } from '../../../src/types'; import { runPool, STOP } from '../harness'; import { PNG_BYTES, mediaFailures } from '../../helpers/media'; @@ -38,7 +38,7 @@ class WideThenImage extends Tool<{ query: string }> { *execute(): Operation<unknown> { this.calls++; if (this.calls <= 4) return { results: ['x'.repeat(600)] }; - return { page: 'p1', [TOOL_MEDIA_KEY]: [PNG_BYTES] }; + return { page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES] }; } } diff --git a/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts b/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts index dcc37d56..5167238b 100644 --- a/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts +++ b/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts @@ -54,7 +54,7 @@ describe('scenario: a model with no projector is TOLD, not silently shorted', () const body = (run.channelEvents.find(e => e.type === 'agent:tool_result') as { result: string }).result; - expect(body).not.toContain('_images'); + expect(body).not.toContain('_attachments'); expect(body.length).toBeLessThan(2_000); }); diff --git a/packages/agents/test/prepare-content.test.ts b/packages/agents/test/prepare-content.test.ts index b2c357d2..05fd56d6 100644 --- a/packages/agents/test/prepare-content.test.ts +++ b/packages/agents/test/prepare-content.test.ts @@ -12,7 +12,7 @@ import { MemoryAttachmentStore } from './helpers/memory-store'; import { rawIngress } from './helpers/raw-ingress'; const img = (n: number): Uint8Array[] => - Array.from({ length: n }, (_, i) => new Uint8Array([i, i + 1, i + 2, 0x89])); + Array.from({ length: n }, (_, i) => new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, i])); describe('prepareBatch', () => { it('overlaps ingests instead of running them one after another', async () => { diff --git a/packages/agents/test/source.test.ts b/packages/agents/test/source.test.ts index c25f6ed0..213ff796 100644 --- a/packages/agents/test/source.test.ts +++ b/packages/agents/test/source.test.ts @@ -129,10 +129,6 @@ describe('NULL_SCORER', () => { expect(results).toEqual([0, 0]); }); - it('scoreRelevanceBatch returns 1.0 for all texts', async () => { - const results = await NULL_SCORER.scoreRelevanceBatch(['a', 'b'], 'any query'); - expect(results).toEqual([1, 1]); - }); }); describe('scoreSimilarityBatch', () => { @@ -170,78 +166,6 @@ describe('scoreSimilarityBatch', () => { }); }); -describe('scoreRelevanceBatch', () => { - it('returns min(localScore, originalScore) per text', async () => { - // scoreBatch returns different scores depending on the query - const scoreBatch = vi.fn(async (query: string, texts: string[]) => - texts.map((t) => { - if (query === 'original query') return t === 'bridging content' ? 0.2 : 0.8; - // local query scores everything high - return 0.9; - }), - ); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - - const scorer = source.createScorer('original query'); - const results = await scorer.scoreRelevanceBatch( - ['relevant content', 'bridging content'], - 'local tool query', - ); - - // min(0.9, 0.8) = 0.8, min(0.9, 0.2) = 0.2 - expect(results[0]).toBe(0.8); - expect(results[1]).toBe(0.2); - }); - - it('calls scoreBatch sequentially with originalQuery then localQuery', async () => { - const callOrder: string[] = []; - const scoreBatch = vi.fn(async (query: string, _texts: string[]) => { - callOrder.push(query); - return [0.5]; - }); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - - const scorer = source.createScorer('original'); - await scorer.scoreRelevanceBatch(['t'], 'local'); - - expect(callOrder).toEqual(['original', 'local']); - expect(scoreBatch).toHaveBeenCalledTimes(2); - }); - - it('when local > orig, result equals orig (min)', async () => { - const scoreBatch = vi.fn(async (q: string, _t: string[]) => - [q === 'orig' ? 0.3 : 0.9], - ); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - const scorer = source.createScorer('orig'); - const result = await scorer.scoreRelevanceBatch(['t'], 'local'); - expect(result[0]).toBe(0.3); - }); - - it('when local < orig, result equals local (min)', async () => { - const scoreBatch = vi.fn(async (q: string, _t: string[]) => - [q === 'orig' ? 0.9 : 0.3], - ); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - const scorer = source.createScorer('orig'); - const result = await scorer.scoreRelevanceBatch(['t'], 'local'); - expect(result[0]).toBe(0.3); - }); - - it('when local == orig, result equals both (min)', async () => { - const scoreBatch = vi.fn(async (_q: string, _t: string[]) => [0.6]); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - const scorer = source.createScorer('orig'); - const result = await scorer.scoreRelevanceBatch(['t'], 'local'); - expect(result[0]).toBe(0.6); - }); -}); - describe('ContextPressure fields', () => { it('percentAvailable: normal case', () => { // 8192 / 16384 = 50% diff --git a/packages/agents/test/spawn-agents.test.ts b/packages/agents/test/spawn-agents.test.ts index 560575cd..659f1b36 100644 --- a/packages/agents/test/spawn-agents.test.ts +++ b/packages/agents/test/spawn-agents.test.ts @@ -139,12 +139,10 @@ describe('Entailment boundary discipline', () => { // Verify EntailmentScorer interface has the right shape const scorer: EntailmentScorer = { scoreEntailmentBatch: async (texts) => texts.map(() => 0.5), - scoreRelevanceBatch: async (texts) => texts.map(() => 0.5), scoreSimilarityBatch: async (_ref, texts) => texts.map(() => 0), shouldProceed: (score) => score >= 0.25, }; expect(scorer.scoreEntailmentBatch).toBeDefined(); - expect(scorer.scoreRelevanceBatch).toBeDefined(); expect(scorer.shouldProceed).toBeDefined(); }); @@ -463,13 +461,13 @@ describe('Explore/exploit decoupled from lifecycle', () => { // ── EntailmentScorer interface shape ────────────────────── describe('EntailmentScorer interface', () => { - it('scoreRelevanceBatch exists on interface shape', () => { + it('scores against the original question, against a reference, and gates — exploit combines in admission, not here', () => { const scorer: EntailmentScorer = { scoreEntailmentBatch: async (texts) => texts.map(() => 0.5), - scoreRelevanceBatch: async (texts) => texts.map(() => 0.5), scoreSimilarityBatch: async (_ref, texts) => texts.map(() => 0), shouldProceed: (score) => score >= 0.25, }; - expect(scorer.scoreRelevanceBatch).toBeDefined(); + expect(scorer.scoreEntailmentBatch).toBeDefined(); + expect('scoreRelevanceBatch' in scorer).toBe(false); }); }); diff --git a/packages/agents/test/spine-multimodal.test.ts b/packages/agents/test/spine-multimodal.test.ts index e5bea146..04168fbc 100644 --- a/packages/agents/test/spine-multimodal.test.ts +++ b/packages/agents/test/spine-multimodal.test.ts @@ -25,7 +25,7 @@ import { CapturingTraceWriter } from './helpers/capturing-trace'; const SYSTEM = 'You are a research assistant.'; const img = (n: number): Uint8Array[] => - Array.from({ length: n }, (_, i) => new Uint8Array([i, i + 1, i + 2])); + Array.from({ length: n }, (_, i) => new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, i])); const markerCount = (s: string): number => (s.match(/<__media__>/g) ?? []).length; diff --git a/packages/agents/test/tool-media.test.ts b/packages/agents/test/tool-media.test.ts index 1b70be11..7c8e4482 100644 --- a/packages/agents/test/tool-media.test.ts +++ b/packages/agents/test/tool-media.test.ts @@ -7,12 +7,13 @@ * and the tool's own object left alone. */ import { describe, it, expect } from 'vitest'; -import { takeToolMedia, TOOL_MEDIA_KEY } from '../src/Tool'; +import { takeToolMedia, TOOL_ATTACHMENTS_KEY } from '../src/Tool'; +import { MANIFEST_TYPE } from '@lloyal-labs/media'; import { PNG_BYTES } from './helpers/media'; describe('takeToolMedia', () => { it('splits the images out from what the model is told', () => { - const { media, result } = takeToolMedia({ page: 'p1', [TOOL_MEDIA_KEY]: [PNG_BYTES] }); + const { media, result } = takeToolMedia({ page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES] }); expect(media).toEqual([PNG_BYTES]); expect(result).toEqual({ page: 'p1' }); @@ -22,20 +23,32 @@ describe('takeToolMedia', () => { // The bytes must reach neither the model's JSON nor the trace. Deleting // them in place made that a property of call order; this makes it a // property of the function. - const returned = { page: 'p1', [TOOL_MEDIA_KEY]: [PNG_BYTES] }; + const returned = { page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES] }; takeToolMedia(returned); - expect(returned[TOOL_MEDIA_KEY]).toEqual([PNG_BYTES]); + expect(returned[TOOL_ATTACHMENTS_KEY]).toEqual([PNG_BYTES]); }); it('drops entries that are not bytes, so markers and bitmaps stay in step', () => { const { media } = takeToolMedia({ - [TOOL_MEDIA_KEY]: [PNG_BYTES, 'not-an-image', null, PNG_BYTES], + [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES, 'not-an-image', null, PNG_BYTES], }); expect(media).toHaveLength(2); }); + it('keeps a descriptor the store would recognise, in order with bytes, and drops one it would not', () => { + // A tool that reads the content store returns ROOTS, not bytes: no + // ingress, no normalizer permit, the ingest-time digest. Anything that is + // neither bytes nor a root descriptor is not media and never was. + const root = { digest: 'sha256:' + 'a'.repeat(64), mediaType: MANIFEST_TYPE, size: 9 }; + const { media, result } = takeToolMedia({ + page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES, root, { digest: 'nope' }, 'x'], + }); + expect(media).toEqual([PNG_BYTES, root]); + expect(result).toEqual({ page: 'p1' }); + }); + it('returns a text-only result as-is, copying nothing', () => { const returned = { page: 'p1' }; const { media, result } = takeToolMedia(returned); @@ -51,15 +64,15 @@ describe('takeToolMedia', () => { }); it('strips a MALFORMED channel rather than serializing it', () => { - // `_images: Uint8Array` (not an array of them) used to return the + // `_attachments: Uint8Array` (not an array of them) used to return the // original object — JSON-encoding every byte index onto the token rail, // the exact failure this helper exists to prevent. The reserved key // never survives; an invalid value is zero media entries. for (const bad of [PNG_BYTES, 'nope', 42, { 0: 1 }]) { - const { media, result } = takeToolMedia({ page: 'p1', [TOOL_MEDIA_KEY]: bad }); + const { media, result } = takeToolMedia({ page: 'p1', [TOOL_ATTACHMENTS_KEY]: bad }); expect(media).toEqual([]); expect(result).toEqual({ page: 'p1' }); - expect(Object.keys(result as object)).not.toContain(TOOL_MEDIA_KEY); + expect(Object.keys(result as object)).not.toContain(TOOL_ATTACHMENTS_KEY); } }); }); diff --git a/packages/media/README.md b/packages/media/README.md index 71b1063d..d4c55eab 100644 --- a/packages/media/README.md +++ b/packages/media/README.md @@ -17,23 +17,26 @@ writes nothing new. ```bash npm i @lloyal-labs/media -npm i sharp # only in a process that accepts image uploads — see below +npm i sharp # only in a process that admits image uploads — see below +npm i @embedpdf/pdfium # only in a process that admits PDF uploads ``` ```ts import { materialize } from '@lloyal-labs/media'; -import { FileAttachmentStore, createImageIngress } from '@lloyal-labs/media/node'; +import { FileAttachmentStore, createContentIngress } from '@lloyal-labs/media/node'; const store = new FileAttachmentStore('media'); // a valid OCI Image Layout -const ingress = createImageIngress(store); // normalize → address → commit +const ingress = createContentIngress(store); // sniff → admit → address → commit const attachment = await ingress.ingest(bytes); // one root descriptor -const { bitmaps } = materialize(store, [attachment]); // the exact admitted pixels +const { bitmaps } = materialize(store, [attachment]); // the exact admitted pixels — none for a document ``` -`ingest` admits bytes — converting, downscaling and stripping as the admission -policy below requires — commits them, and returns a root descriptor small -enough to ride any wire. `materialize` is the inverse: given roots, it returns +`ingest` admits bytes and returns a root descriptor small enough to ride any +wire. The bytes decide the door: an image goes through the normalizer +(`createImageIngress` — converting, downscaling and stripping as the admission +policy below requires), a PDF through the document ingress +(`createDocumentIngress`, below); each is also exported on its own. `materialize` is the inverse: given roots, it returns the exact bytes to hand the projector (the model's vision encoder). Replay is `materialize` called later. @@ -46,13 +49,13 @@ trace and replay. This package is the format and the gate. | entry | holds | needs | |---|---|---| | `@lloyal-labs/media` | the OCI shapes, the store and ingress contracts, `materialize` | nothing — browser-safe, zero dependencies | -| `@lloyal-labs/media/node` | `FileAttachmentStore` (the layout on disk), `createImageIngress` (sharp) | `node:fs`; `sharp` as an optional peer | +| `@lloyal-labs/media/node` | `FileAttachmentStore` (the layout on disk), `createContentIngress` — `createImageIngress` (sharp) and `createDocumentIngress` (PDFium) | `node:fs`; `sharp` and `@embedpdf/pdfium` as optional peers | -The root entry never reaches `node:` or `sharp` — enforced by +The root entry never reaches `node:`, `sharp` or the codec — enforced by `npm run verify:packed`, which walks the packed artifact's require graph on -every push. `sharp` is required at call time, not module load, so a process -that never accepts an image pays nothing, and one that does gets an error -naming the install. +every push. Both are required at call time, not module load, so a process +that never accepts an image or a document pays nothing, and one that does +gets an error naming the install. Where things live: the **format** — how bytes are addressed and laid out — is this package. The **policy** — where a project keeps its store — is @@ -133,12 +136,72 @@ and nothing above it. ``` The `config` slot is empty for an image, which has nothing to say beyond its -layers. Timed media will use it: a timeline — timestamps, track descriptors, -sampling policy — is typed structured data, and +layers. A document fills it (next section), and timed media will: a timeline — +timestamps, track descriptors, sampling policy — is typed structured data, and [annotations are `map<string,string>`](https://github.com/opencontainers/image-spec/blob/main/annotations.md), -so the config blob is where it belongs. `putAttachment({ config })` already -accepts one, and a reader branches on `config.mediaType`, so introducing a -typed config later leaves every existing manifest valid. +so the config blob is where such things belong. A reader branches on +`config.mediaType`, so a manifest with the empty config stays valid beside one +with a sidecar. + +## A document is the same graph + +A PDF admitted through `createDocumentIngress` becomes one manifest whose +representation is what the model reads, whose source is the file as supplied, +and whose config is a **sidecar** of facts about the document: + +```jsonc +{ + "artifactType": "application/vnd.lloyal.attachment.v1", + "config": { "mediaType": "application/vnd.lloyal.document.v1+json", "digest": "sha256:…", "size": 2210 }, + "layers": [ + { "mediaType": "text/markdown", "digest": "sha256:…", "size": 18342, + "annotations": { "ai.lloyal.role": "representation" } }, // what the model reads + { "mediaType": "application/pdf", "digest": "sha256:…", "size": 812044, + "annotations": { "ai.lloyal.role": "source" } } // what the user supplied + ] +} +``` + +The sidecar (`DocumentMeta`, guarded by `asDocumentMeta` on the root entry) +records the title; the sections with their heading path, line span and pages; +a page map with per-page counts — characters, image objects, path objects, +tagged tables and figures; the figures with their captions; the tagged tables; +and the parameters the derivation ran under. Facts, not verdicts: whether a +page is worth spending a model's attention on is a rule the reader applies +over the counts. Lines are 1-based over the markdown's `\n` split, and +`pagesOf(meta, startLine, endLine)` is the one function that turns a line +span into pages. + +Every page within the render bound, and every figure crop, is its own +single-image manifest — a **page root** — named from the sidecar by +descriptor. A page root is an ordinary image attachment carrying `pdf.v1` +derive annotations, so the normalizer never runs on it and `materialize`, +replay and any OCI tool read it exactly as they read a photo: + +| key | value | +|---|---| +| `ai.lloyal.derive.profile` | `pdf.v1` | +| `ai.lloyal.derive.page` | the 1-based page it renders | +| `ai.lloyal.derive.dpi`, `.width`, `.height`, `.format` | how it was rendered | +| `ai.lloyal.derive.source` | the digest of the PDF blob it was rendered from | +| `ai.lloyal.derive.bbox` | a figure crop's box on the page, in PDF user space | + +`materialize` projects only representations in a format the projector +decodes. A document root therefore yields no bitmaps — its text is for +retrieval, not the vision encoder — while a page root yields one. That single +rule is what lets a document ride the same wire, trace and replay as an image +without anything upstream learning a new kind. + +Bounds are declared constants on the node entry: `MAX_DOCUMENT_BYTES` (32 MiB, +refused before the codec sees a byte), `MAX_RENDERED_PAGES` (200 — page one +first, then graphics-bearing pages, then the rest, so the archive under the +bound is a deterministic function of the input; a page past it simply has no +render descriptor), `MAX_TEXT_PAGES` (400), `MAX_FIGURES` (16), and +`DOCUMENT_TIMEOUT_MS` (120 s — a failure bound that publishes nothing, never +a coverage knob). The same bytes under the same constants yield the same root +on any machine. A document holds one permit of the shared admission gate for +its whole duration, and one codec instance lives exactly as long as its +document. ### Annotations we define @@ -159,7 +222,9 @@ omitting it is a legitimate choice for a large original. ## Admission -`ingest` (and `normalizeImage` underneath it) applies one policy. +`createContentIngress` sniffs the bytes: an image goes to the policy below, a +PDF to the document ingress above, anything else is refused at the door. +For images, `ingest` (and `normalizeImage` underneath it) applies one policy. Byte-identical pass-through happens only when ALL of these hold: | | | @@ -219,6 +284,7 @@ pointing at content that is not there. | **Video derivation** | The manifest already has the slot: source + N frame representations. What is missing is a decoder, and that decision carries real licensing and codec-patent weight. | | **Live capture** | The *locator* is not addressable; every bounded frame that reaches the model still is. Attachments are per-prefill rather than per-run, so a live run is many prefills — no growing manifest. | | **Reachability GC** | Nothing here deletes. Deletion needs refcounting across runs that may share a digest. | +| **Rendering past the bound** | A page beyond `MAX_RENDERED_PAGES` has no render descriptor. Rendering on demand needs the source and a codec at read time; today a document is archived at admission only. | ## Known limitations diff --git a/packages/media/package.json b/packages/media/package.json index 9185d105..cd20c64c 100644 --- a/packages/media/package.json +++ b/packages/media/package.json @@ -1,7 +1,7 @@ { "name": "@lloyal-labs/media", "version": "0.2.0-alpha.3", - "description": "Content addressing for a harness — an OCI layout, and the image normalizer that feeds it", + "description": "Content addressing for a harness \u2014 an OCI layout, and the image and document ingresses that feed it", "main": "dist/index.js", "types": "dist/index.d.ts", "publishConfig": { @@ -22,7 +22,10 @@ "multimodal", "vision", "image", - "sharp" + "sharp", + "pdf", + "pdfium", + "document" ], "license": "SEE LICENSE IN LICENSE", "type": "commonjs", @@ -46,14 +49,20 @@ "LICENSE-FAQ.md" ], "devDependencies": { + "@embedpdf/pdfium": "^2.15.0", + "@types/emscripten": "^1.41.6", "sharp": "^0.35.4" }, "peerDependencies": { - "sharp": "^0.35.4" + "sharp": "^0.35.4", + "@embedpdf/pdfium": "^2.15.0" }, "peerDependenciesMeta": { "sharp": { "optional": true + }, + "@embedpdf/pdfium": { + "optional": true } } } diff --git a/packages/media/src/content-ingress.ts b/packages/media/src/content-ingress.ts new file mode 100644 index 00000000..027dfbf0 --- /dev/null +++ b/packages/media/src/content-ingress.ts @@ -0,0 +1,45 @@ +/** + * @file The one loading door: bytes in, an attachment out, the bytes deciding + * what they are. + * + * `ContentIngress` stays one port. A host installs this on its content routes + * and on the agents `Ingress` context, and an upload or a tool result that + * carries bytes goes through the same dispatch: an image to the normalizer, a + * PDF to the document ingress. A caller never declares a type; the signature + * does. + */ +import type { AttachmentStore } from './store'; +import type { ContentIngress } from './ingress'; +import { sniffMediaType } from './media-type'; +import { createImageIngress } from './image'; +import type { NormalizeOpts } from './image'; +import { createDocumentIngress } from './pdf'; +import type { DocumentOpts } from './pdf'; + +/** + * How long a host should allow an upload that may be a document, end to end: + * transfer plus the document's own time bound. The route's default is sized + * for images; a host that installs this ingress passes this beside + * `MAX_DOCUMENT_BYTES`, so every host that mounts the plane admits the same. + * + * @category Media + */ +export const DOCUMENT_UPLOAD_TIMEOUT_MS = 180_000; + +/** + * @category Media + */ +export function createContentIngress( + store: AttachmentStore, + opts: { image?: NormalizeOpts; document?: DocumentOpts } = {}, +): ContentIngress { + const image = createImageIngress(store, opts.image); + const document = createDocumentIngress(store, opts.document); + return { + ingest(bytes: Uint8Array, signal?: AbortSignal) { + return sniffMediaType(bytes) === 'application/pdf' + ? document.ingest(bytes, signal) + : image.ingest(bytes, signal); + }, + }; +} diff --git a/packages/media/src/document.ts b/packages/media/src/document.ts new file mode 100644 index 00000000..c27868c5 --- /dev/null +++ b/packages/media/src/document.ts @@ -0,0 +1,148 @@ +/** + * @file The document sidecar — facts about a document, kept in its manifest's + * config slot. + * + * Pure data and a guard, browser-safe like `attachment.ts`. A document + * attachment has ONE representation, `text/markdown`; its page renders and + * figure crops are ordinary single-image attachments named here by root + * descriptor. This file records facts only — counts, spans, roots, the + * derivation parameters — and no policy: which pages are worth projecting is + * a rule whoever consumes the sidecar applies over the counts. + */ +import type { Descriptor } from './attachment'; + +/** The config media type a document manifest carries; a reader branches on it. */ +export const DOCUMENT_CONFIG_TYPE = 'application/vnd.lloyal.document.v1+json' as const; + +/** + * Where a section came from: the PDF's structure tree, its bookmarks, or the + * font-size heuristics applied when neither covered the page. + */ +export type SectionOrigin = 'struct' | 'bookmark' | 'heuristic'; + +/** + * The sidecar. Lines are 1-based over the markdown representation's `\n` + * split — the `Chunk.startLine` convention — and the page map is the + * authority for line → page; the markdown carries no page anchors. Descriptors + * are complete so a consumer can hand them straight to `asAttachment`. + * + * @category Media + */ +export interface DocumentMeta { + title: string; + pageCount: number; + sections: { + heading: string; + /** Hierarchical path, `A > B > C`. */ + path: string; + origin: SectionOrigin; + startLine: number; + endLine: number; + pageStart: number; + pageEnd: number; + }[]; + pages: { + /** 1-based. */ + page: number; + startLine: number; + endLine: number; + /** Extracted characters; 0 for a scanned page. */ + chars: number; + imageObjects: number; + pathObjects: number; + taggedTables: number; + taggedFigures: number; + /** Root of the page's render, present for every page within the render bound. */ + render?: Descriptor; + }[]; + figures: { + page: number; + index: number; + /** PDF user-space points, `[x0, y0, x1, y1]`. */ + bbox: [number, number, number, number]; + caption?: string; + root: Descriptor; + }[]; + /** Tagged tables emitted into the markdown as pipe tables. */ + tables: { page: number; startLine: number; endLine: number }[]; + /** What derived this document — so a replay under other settings sees the difference. */ + derive: { + profile: 'pdf.v1'; + pdfium: string; + dpi: number; + maxSide: number; + maxPixels: number; + format: 'image/png'; + renderedPages: number; + maxFigures: number; + maxTextPages: number; + tagged: boolean; + structCoverage: number; + /** A data cap bit (pages, figures, text pages). Never a timeout: a timeout publishes nothing. */ + truncated: boolean; + }; +} + +const isInt = (v: unknown): v is number => typeof v === 'number' && Number.isSafeInteger(v); +const isNum = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v); +const isStr = (v: unknown): v is string => typeof v === 'string'; +const isObj = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null; +const isDescriptor = (v: unknown): v is Descriptor => + isObj(v) && isStr(v.mediaType) && isStr(v.digest) && isInt(v.size); + +/** + * Narrow parsed JSON to a {@link DocumentMeta}, or refuse it. Never throws: + * a sidecar arrives as bytes from a store, and a malformed one is a normal + * answer, not a fault. + * + * @category Media + */ +export function asDocumentMeta(json: unknown): DocumentMeta | null { + if (!isObj(json)) return null; + const { title, pageCount, sections, pages, figures, tables, derive } = json; + if (!isStr(title) || !isInt(pageCount)) return null; + if (!Array.isArray(sections) || !Array.isArray(pages) || !Array.isArray(figures) || !Array.isArray(tables)) return null; + const sectionOk = (s: unknown): boolean => + isObj(s) && isStr(s.heading) && isStr(s.path) && + (s.origin === 'struct' || s.origin === 'bookmark' || s.origin === 'heuristic') && + isInt(s.startLine) && isInt(s.endLine) && isInt(s.pageStart) && isInt(s.pageEnd); + const pageOk = (p: unknown): boolean => + isObj(p) && isInt(p.page) && isInt(p.startLine) && isInt(p.endLine) && isInt(p.chars) && + isInt(p.imageObjects) && isInt(p.pathObjects) && isInt(p.taggedTables) && isInt(p.taggedFigures) && + (p.render === undefined || isDescriptor(p.render)); + const figureOk = (f: unknown): boolean => + isObj(f) && isInt(f.page) && isInt(f.index) && Array.isArray(f.bbox) && f.bbox.length === 4 && + f.bbox.every(isNum) && (f.caption === undefined || isStr(f.caption)) && isDescriptor(f.root); + const tableOk = (t: unknown): boolean => isObj(t) && isInt(t.page) && isInt(t.startLine) && isInt(t.endLine); + const deriveOk = (d: unknown): boolean => + isObj(d) && d.profile === 'pdf.v1' && isStr(d.pdfium) && isNum(d.dpi) && isInt(d.maxSide) && isInt(d.maxPixels) && + d.format === 'image/png' && isInt(d.renderedPages) && isInt(d.maxFigures) && isInt(d.maxTextPages) && + typeof d.tagged === 'boolean' && isNum(d.structCoverage) && typeof d.truncated === 'boolean'; + if (!sections.every(sectionOk) || !pages.every(pageOk) || !figures.every(figureOk) || !tables.every(tableOk) || !deriveOk(derive)) return null; + return json as unknown as DocumentMeta; +} + +/** + * The pages a line span touches, by the sidecar's page map. A span past the + * map — lines the extractor emitted after the last page's text — belongs to + * the last page. + * + * @category Media + */ +export function pagesOf(meta: DocumentMeta, startLine: number, endLine: number): { pageStart: number; pageEnd: number } { + return pagesOfSpan(meta.pages, startLine, endLine); +} + +/** The same rule over a bare page map — what the layout uses before a sidecar exists. */ +export function pagesOfSpan( + pages: readonly { page: number; startLine: number; endLine: number }[], + startLine: number, + endLine: number, +): { pageStart: number; pageEnd: number } { + const touched = pages.filter((p) => p.endLine >= startLine && p.startLine <= endLine); + if (touched.length > 0) { + return { pageStart: touched[0].page, pageEnd: touched[touched.length - 1].page }; + } + const last = pages[pages.length - 1]?.page ?? 1; + return { pageStart: last, pageEnd: last }; +} diff --git a/packages/media/src/gate.ts b/packages/media/src/gate.ts new file mode 100644 index 00000000..4231b374 --- /dev/null +++ b/packages/media/src/gate.ts @@ -0,0 +1,151 @@ +/** + * @file The permit gate — the process-wide bound on decoded content in memory. + * + * Each concurrent normalization holds a fully decoded bitmap, and a document + * ingest holds a page bitmap and a codec heap, so N simultaneous admissions + * cost N such allocations however small the encoded bytes were. A served host + * takes uploads from anyone who can reach it. The bound therefore belongs to + * the PROCESS, not the request, and there is ONE gate: images and documents + * take permits from the same pool, so together they never exceed it. + * + * Pure Node, no `sharp`: safe for any module behind `./node` to import. + */ + +/** + * How many admissions may hold decoded content at once, process-wide. + * + * @category Media + */ +export const MAX_CONCURRENT_NORMALIZATIONS = 4; + +/** + * How long a caller may WAIT for a permit before the host refuses it. + * + * A queue with no deadline turns a burst into a pile of held request bodies + * that never drains; a caller that has waited this long is told so and can + * retry, and the bytes it held are released. + * + * @category Media + */ +export const PERMIT_WAIT_TIMEOUT_MS = 60_000; + +/** + * How many callers may wait for a permit. Past this depth the answer is an + * immediate busy, so memory is bounded by (permits + depth) admissions and + * nothing more. + * + * @category Media + */ +export const MAX_QUEUED_NORMALIZATIONS = MAX_CONCURRENT_NORMALIZATIONS * 4; + +/** The shape a caller can recognise without knowing this module. Matches what + * `fetch` throws on abort, because that is what callers already handle. */ +export function abortError(label: string): Error { + const e = new Error(`${label}: aborted`); + e.name = 'AbortError'; + return e; +} + +/** What {@link createGate} returns: one method, one permit per successful call. */ +export interface Gate { + /** + * Take one permit; resolve to its release. + * + * `signal` is how a caller that has GIVEN UP stops occupying the queue — + * an abandoned request holding a slot is a slot a live request cannot have. + * Callers inside an Effection scope get this for free: the scope's own + * signal aborts on halt. The signal covers the WAIT and the moment before + * work starts; work already running is bounded by its own timeout. + * + * Release is idempotent: a double release would MANUFACTURE a permit, the + * same bug as leaking one with the sign flipped. + * + * @throws `AbortError` when the signal is or becomes aborted before the + * permit is used; an error with `code: 'EBUSY'` when the queue is + * full; an error naming the wait when it outlasts `waitMs`. + */ + acquire(signal?: AbortSignal): Promise<() => void>; +} + +/** + * Build a gate. The shared, process-wide instance is {@link gate}; a separate + * one exists only for tests, which need small numbers. + * + * @category Media + */ +export function createGate(opts: { label: string; permits: number; maxQueued: number; waitMs: number }): Gate { + const { label, maxQueued, waitMs } = opts; + let permits = opts.permits; + const waiting: { grant: () => void; refuse: (e: Error) => void }[] = []; + + const busy = (): Error => { + const e = new Error(`${label}: ${opts.permits} in flight and ${maxQueued} queued — busy, retry later.`); + (e as Error & { code: string }).code = 'EBUSY'; + return e; + }; + const hand = (): void => { + const next = waiting.shift(); + if (next) next.grant(); else permits++; + }; + + return { + async acquire(signal?: AbortSignal): Promise<() => void> { + if (signal?.aborted) throw abortError(label); + + if (permits > 0) { + permits--; + } else { + if (waiting.length >= maxQueued) throw busy(); + await new Promise<void>((resolve, reject) => { + const leave = (): void => { + const at = waiting.indexOf(entry); + if (at >= 0) waiting.splice(at, 1); + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = (): void => { leave(); reject(abortError(label)); }; + const entry = { + grant: () => { leave(); resolve(); }, + refuse: (e: Error) => { leave(); reject(e); }, + }; + const timer = setTimeout(() => entry.refuse(new Error( + `${label}: waited ${waitMs}ms for one of ${opts.permits} slots and none came free.`, + )), waitMs); + // Do not hold the process open on account of a queued caller. + timer.unref?.(); + signal?.addEventListener('abort', onAbort, { once: true }); + waiting.push(entry); + }); + } + + // Granted, but the caller may have given up while it waited. Handing the + // permit straight to the next in line beats spending it on work nobody + // is waiting for. + if (signal?.aborted) { + hand(); + throw abortError(label); + } + + let released = false; + return () => { + if (released) return; + released = true; + hand(); + }; + }, + }; +} + +/** + * The one process-wide gate. Module-level on purpose: the resource it protects + * is host memory, shared by every session and every request, so a per-call or + * per-store limiter would not bound anything. + * + * @category Media + */ +export const gate: Gate = createGate({ + label: 'normalizeImage', + permits: MAX_CONCURRENT_NORMALIZATIONS, + maxQueued: MAX_QUEUED_NORMALIZATIONS, + waitMs: PERMIT_WAIT_TIMEOUT_MS, +}); diff --git a/packages/media/src/image.ts b/packages/media/src/image.ts index b340b7d3..0ceb3ad8 100644 --- a/packages/media/src/image.ts +++ b/packages/media/src/image.ts @@ -17,6 +17,7 @@ import { DERIVE_PREFIX } from './attachment'; import type { AttachmentStore } from './store'; import type { ContentIngress } from './ingress'; import { PROJECTOR_FORMATS, sniffMediaType, UNKNOWN_MEDIA_TYPE } from './media-type'; +import { gate, abortError } from './gate'; /** * The default pixel ceiling — mtmd's own (`image_max_pixels`, 2048²). @@ -54,46 +55,6 @@ export const MAX_INPUT_PIXELS = 100_000_000; */ export const NORMALIZE_TIMEOUT_SECONDS = 20; -/** - * How many images may be normalized at once, process-wide. - * - * The cap belongs to the PROCESS, not the request: each concurrent - * normalization holds a fully decoded bitmap in memory, so N simultaneous - * uploads cost N bitmaps regardless of how small the encoded bytes were. A - * served host takes uploads from anyone who can reach it. - * - * @category Media - */ -export const MAX_CONCURRENT_NORMALIZATIONS = 4; - -/** - * How long an image may WAIT for a permit before the host refuses it. - * - * An unbounded queue is not a bound: it is a memory leak with a politer name, - * and it turns any permit accounting bug into a process that hangs forever - * instead of failing. A busy host should say it is busy. - * - * Still earns its keep now that {@link NormalizeOpts.signal} exists: a caller - * inside a scope gives up the moment that scope halts, but the HTTP ingress - * route runs in a plain request handler and passes no signal, so this is the - * only bound it has. - * - * Sized well above the worst legitimate wait — a full queue of - * {@link NORMALIZE_TIMEOUT_SECONDS} decodes — so reaching it means genuine - * overload or a bug, never ordinary contention. - * - * @category Media - */ -export const PERMIT_WAIT_TIMEOUT_MS = 60_000; - -/** - * How many callers may WAIT for a permit. Permits bound the decoded bitmaps; - * this bounds the encoded ones parked behind them. Past it the answer is an - * immediate busy, so a burst costs at most (permits + queue) images of memory - * instead of one image per arrival for {@link PERMIT_WAIT_TIMEOUT_MS}. - */ -export const MAX_QUEUED_NORMALIZATIONS = MAX_CONCURRENT_NORMALIZATIONS * 4; - /** * @category Media */ @@ -180,100 +141,6 @@ export type NormalizeImage = ( opts?: NormalizeOpts, ) => Promise<NormalizedImage>; -/** - * A process-wide gate of {@link MAX_CONCURRENT_NORMALIZATIONS} permits. - * - * Module-level on purpose: the resource being protected is host memory, which - * is shared by every session and every request, so a per-call or per-store - * limiter would not bound anything. - */ -const gate = { - permits: MAX_CONCURRENT_NORMALIZATIONS, - waiting: [] as { grant: () => void; refuse: (e: Error) => void }[], -}; - -/** The shape a caller can recognise without knowing this module. Matches what - * `fetch` throws on abort, because that is what callers already handle. */ -const aborted = (): Error => { - const e = new Error('normalizeImage: aborted'); - e.name = 'AbortError'; - return e; -}; - -/** Thrown when the queue is full. `code` is Node's own errno name for the - * condition, so an HTTP layer can answer 503 without importing this module. */ -function busy(): Error { - const e = new Error( - `normalizeImage: ${MAX_CONCURRENT_NORMALIZATIONS} normalizations in flight and ` + - `${MAX_QUEUED_NORMALIZATIONS} queued — busy, retry later.`, - ); - (e as Error & { code: string }).code = 'EBUSY'; - return e; -} - -/** - * Take one of {@link MAX_CONCURRENT_NORMALIZATIONS} permits. - * - * `signal` is how a caller that has GIVEN UP stops occupying the queue. That - * matters more than it sounds: an abandoned request holding a slot is a slot a - * live request cannot have, so under load the queue fills with work whose - * results nobody will read. Callers inside an Effection scope get this for - * free — the scope's own signal aborts on halt. - * - * An in-flight sharp decode cannot be interrupted (sharp exposes no abort; - * {@link NORMALIZE_TIMEOUT_SECONDS} is the only bound on it), so the signal - * covers the WAIT and the moment before work starts, which is where a - * cancelled caller's cost actually accumulates. - */ -async function acquire(signal?: AbortSignal): Promise<() => void> { - if (signal?.aborted) throw aborted(); - - if (gate.permits > 0) { - gate.permits--; - } else { - if (gate.waiting.length >= MAX_QUEUED_NORMALIZATIONS) throw busy(); - await new Promise<void>((resolve, reject) => { - const leave = () => { - const at = gate.waiting.indexOf(entry); - if (at >= 0) gate.waiting.splice(at, 1); - clearTimeout(timer); - signal?.removeEventListener('abort', onAbort); - }; - const onAbort = () => { leave(); reject(aborted()); }; - const entry = { - grant: () => { leave(); resolve(); }, - refuse: (e: Error) => { leave(); reject(e); }, - }; - const timer = setTimeout(() => entry.refuse(new Error( - `normalizeImage: waited ${PERMIT_WAIT_TIMEOUT_MS}ms for one of ` + - `${MAX_CONCURRENT_NORMALIZATIONS} normalization slots and none came free.`, - )), PERMIT_WAIT_TIMEOUT_MS); - // Do not hold the process open on account of a queued upload. - timer.unref?.(); - signal?.addEventListener('abort', onAbort, { once: true }); - gate.waiting.push(entry); - }); - } - - // Granted, but the caller may have given up while it waited. Releasing here - // hands the permit straight to the next in line instead of spending it on - // work nobody is waiting for. - if (signal?.aborted) { - const next = gate.waiting.shift(); - if (next) next.grant(); else gate.permits++; - throw aborted(); - } - - let released = false; - return () => { - // Idempotent: a double release would MANUFACTURE a permit, which is the - // same bug as leaking one with the sign flipped. - if (released) return; - released = true; - const next = gate.waiting.shift(); - if (next) next.grant(); else gate.permits++; - }; -} /** * Does this ICC profile describe sRGB? @@ -396,7 +263,7 @@ export const normalizeImage: NormalizeImage = async (bytes, opts = {}) => { // upload is the failure mode that matters — a handful of bad files would // exhaust the gate permanently and wedge the host, and bad files are exactly // what arrives in volume. - const release = await acquire(opts.signal); + const release = await gate.acquire(opts.signal); try { const input = sharp(bytes, { animated: false, limitInputPixels: MAX_INPUT_PIXELS }) .timeout({ seconds: NORMALIZE_TIMEOUT_SECONDS }); @@ -579,7 +446,7 @@ export function createImageIngress( // have fired while it ran. The caller gave up; the route has answered // 408. Nothing may be committed on its behalf now — the decode was // discarded work, and it stays that way. - if (signal?.aborted) throw aborted(); + if (signal?.aborted) throw abortError('normalizeImage'); // A derivation record describes a derivation that HAPPENED. Writing it // on a pass-through would annotate bytes nobody re-encoded with a diff --git a/packages/media/src/index.ts b/packages/media/src/index.ts index ff1af849..0f407360 100644 --- a/packages/media/src/index.ts +++ b/packages/media/src/index.ts @@ -28,3 +28,8 @@ export { materialize, NoContentIngress } from './ingress'; export type { ContentIngress, PreparedContent } from './ingress'; export { PROJECTOR_FORMATS, sniffMediaType, UNKNOWN_MEDIA_TYPE } from './media-type'; + +// The document sidecar — facts about a document, kept in its manifest's config +// slot. Pure data and a guard; a reader branches on DOCUMENT_CONFIG_TYPE. +export { DOCUMENT_CONFIG_TYPE, asDocumentMeta, pagesOf } from './document'; +export type { DocumentMeta, SectionOrigin } from './document'; diff --git a/packages/media/src/ingress.ts b/packages/media/src/ingress.ts index 48171511..b2871239 100644 --- a/packages/media/src/ingress.ts +++ b/packages/media/src/ingress.ts @@ -6,6 +6,7 @@ * batch is an orchestration concern and lives with the orchestrator. */ import { representationsOf } from './attachment'; +import { PROJECTOR_FORMATS } from './media-type'; import type { Attachment } from './attachment'; import type { AttachmentStore } from './store'; @@ -23,9 +24,11 @@ import type { AttachmentStore } from './store'; export interface PreparedContent { /** Roots, in ingest order — what the trace and the fold carry. */ attachments: readonly Attachment[]; - /** Every root's representations, flattened in order — the EXACT bytes to - * hand a builder and then the projector. One image contributes one; a video - * contributes its sampled frames. */ + /** Every root's PROJECTOR-DECODABLE representations, flattened in order — + * the EXACT bytes to hand a builder and then the projector. One image + * contributes one; a video contributes its sampled frames; a document, + * whose one representation is text, contributes none: its text reaches the + * model through retrieval and the spine outline, never the decoder. */ bitmaps: readonly Uint8Array[]; } @@ -59,6 +62,10 @@ export function materialize( ); } for (const rep of representationsOf(manifest)) { + // Only what the projector decodes is a bitmap. A text representation is + // real content — retrieval reads it — but handing it to the image decoder + // would poison the branch it was prefilled on. + if (!PROJECTOR_FORMATS.includes(rep.mediaType)) continue; // Replay rebuilds KV from these bytes under this digest. The store // refuses bytes that drifted from their name, so null here means absent // or drifted — either must refuse, not decode as something else. diff --git a/packages/media/src/media-type.ts b/packages/media/src/media-type.ts index 3a0bfc09..695897b7 100644 --- a/packages/media/src/media-type.ts +++ b/packages/media/src/media-type.ts @@ -1,13 +1,14 @@ /** - * @file Identify an image format from its leading bytes. + * @file Identify a content format from its leading bytes. * - * Its own file because it depends on nothing else in the content surface, and - * because `spine.ts` and `agent-pool.ts` already import it on its own — the - * callers treated it as a separate module before it was one. + * Its own file because it depends on nothing else in the content surface. The + * image normalizer consults it on a decode error, and the content ingress + * dispatches on it: the bytes say what they are, a caller never does. */ /** - * The image formats a vision projector decodes, by their leading bytes. + * Formats this package identifies by their leading bytes: the four image + * formats the projector decodes, and PDF. * * A table rather than a chain of ifs. Anything unmatched is still stored — * validating pixels belongs to the normalizer that runs before ingress, and to @@ -18,6 +19,10 @@ const SIGNATURES: ReadonlyArray<{ mediaType: string; magic: readonly number[] }> { mediaType: 'image/png', magic: [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] }, { mediaType: 'image/gif', magic: [0x47, 0x49, 0x46, 0x38] }, { mediaType: 'image/bmp', magic: [0x42, 0x4d] }, + // `%PDF-`. Not a projector format (see below): the content ingress routes it + // to the document ingress; the image normalizer refuses it as it refuses any + // format it cannot decode. + { mediaType: 'application/pdf', magic: [0x25, 0x50, 0x44, 0x46, 0x2d] }, ]; /** @@ -41,7 +46,7 @@ export function sniffMediaType(bytes: Uint8Array): string { * **Its own list, deliberately NOT derived from {@link SIGNATURES}.** These are * two different questions and they have different answers: * - * - `SIGNATURES` — "what can I identify from leading bytes?" Four formats. + * - `SIGNATURES` — "what can I identify from leading bytes?" Four image formats and PDF. * - `PROJECTOR_FORMATS` — "what will mtmd decode?" Nine. * * Ground truth for this list is the kernel, not an assumption: diff --git a/packages/media/src/node.ts b/packages/media/src/node.ts index 66d57c51..de7b7834 100644 --- a/packages/media/src/node.ts +++ b/packages/media/src/node.ts @@ -13,3 +13,11 @@ export { createImageIngress, DEFAULT_MAX_PIXELS, normalizeImage } from './image' export type { NormalizedImage, NormalizeImage, NormalizeOpts } from './image'; export { FileAttachmentStore } from './file-store'; + +// The document codec and the dispatching ingress. `@embedpdf/pdfium` is an +// OPTIONAL peer like sharp: required at call time, never at import. +export { createDocumentIngress, createCodec, openDocument, PdfError, MAX_DOCUMENT_BYTES, DOCUMENT_TIMEOUT_MS, MAX_RENDERED_PAGES, MAX_TEXT_PAGES, MAX_FIGURES, RENDER_PROFILE } from './pdf'; +export type { DocumentOpts } from './pdf'; +export { createContentIngress, DOCUMENT_UPLOAD_TIMEOUT_MS } from './content-ingress'; +export { gate, createGate, MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS, PERMIT_WAIT_TIMEOUT_MS } from './gate'; +export type { Gate } from './gate'; diff --git a/packages/media/src/pdf-layout.ts b/packages/media/src/pdf-layout.ts new file mode 100644 index 0000000000000000000000000000000000000000..fba177d0caafc6c57e1469d6dac3b553eaa1cb9c GIT binary patch literal 23984 zcmd6v+j1PodB=0jr)b(LW0wRLgjiM*P$W%B)S*m@ERiW)KyWzN9RNe@$vXo9Fv%)i zrOLO7tMUR#UL<d#C&};secdxXyC5xBZmhD2-I+do{hd#}ySHxL2)DxTj;m=IzFaM4 zVSHMwig8n}!m1oER+Bn3i%^8gVthU;=goeYR`asn4oB0)_>8}&WihGdC-t`0X^Nw% zULIE`=PP|E!m>Ci!>m|_pMUyGSn_nlK#hHQT88mrQjWvm)pBw?YU)=*_AJ=(7y#<9 zIB&w@m?fTn@#TIfe=Jv58FUv_a~dj!9W5?x@4EU^xg9Rb>g2Qu#eBl1tFv;lGhWOa z01wsVUTD};0BzNwnhCJQ(d$xsY5cNa#qr{N-rTdZFt%=1=i>&TL$fM1Yqj8fHrkKs zOCBv2XWFgyud}i#CPh<(rfQ~kUJhGvEFqcI=HT>nQJ3+Y)f@~9GA%J#T+G4366^tV z(Nv4MZ+BV>sJa4S#mZ-bUcj!JFc)&5)A{%m=(Lm}c01IIh_}@_TOXCv#YL!UTWDF1 ztK+Ji1U9Iz=1gj;aZB@R9$qzN-Rwe{ooR6eEk^n-DkTDbzJNWddBO8gfFj5g&T9av zm&Lf;4~tcG!a7HbrdiB(rsZ+79j?Opay#&Jx!~BfmfF2h&6bN*6W$nU>SxC<mc=~0 z3&&!|t<i2`y{&s`WOKDF8U97&$*16ZJG_99VC$LhdhPTZySHwIXLHcgGI>!8?&^Fd zqyq(v(Tu|9@C>Bx#?=9=!+-OI_-!Pp%S(frW0Wfxmh;0<pR;gyb0dVNyhK*uA7le| z3YT~G!~A@9RIc>xs`u@3ulKI^O<bM7)d^lg;FIaq@-(6tzPvK=66KGIx|E2H!V?K< zI9V;um*MD&bKOIheBOoF4>;f`od=!>i^(Jm3Z(m_Forkm7ckm%dQ^<h!Z9cra-eED zg$pH~b3@e^<#BO7ZD1-}4OV`2b?22~h_xdy5A!3qh~`gpsHf}Fy3Ogr&R)1UE$6M^ z3Q`2LI5{aNagW)!nskf3yJ19r1aO&v3s@mXvcC`5wSC%g>4zOB<)z8e5z;@Y-%s9e zY)#SE=5*U~RPeYT4xGl@k=E;f4@LY$!VTq8OkpL-lW`BaES)^TiEQU5XZ%ap4CE$0 zi>Lqov}o+S{VhUWRFhC}NEBpT$`JAX1HUi@XdHblk;2da_V1aNlzK#+f4%qW=b!%P z)z_cA+77S2)<4hn&&x0PmlwlGIgS48>Z#H1X|en?;m%L<pv9FLm4KwK78mf{aTu4= zDaNHN#wct)+gUf%y$uzuC5v%O*V_9i53x<12zYRK&lcAUTl|_4;oZSu`hNoLTa##k z^y*79d9Ok_EoCQ7J`&5t&7)OJo8G$-$@N&KtvN+g*g;&~qSQ-Fp;K#t&JypwpHN?# z$rNWgk9WG}qM9_P-JSu9EfbkPkvaqyNr8b4akzLrJ~h7g$Fh>o+-l}ah&#S4rWmzl zQ9JVq?FJ3HbBQ^KruL%5R+yPBC)-AwZ~<-Si^i;fA{3J!UpD57Al_KmX<5t#m-94C zFL0ed4U-%C__Ju0T_4L7cY?BsrYSab+MNi}Y%KW1?BF-^CGsOoCqc%5po05W-?Z2K zd^%0%$@dR$(GSz9P9thXBpV4G$*n`NAJQrO#jLoL$m{4W^J<yNvg%-W$R0&64AAOR zESn57-h)}+N%>=i#adRE<rE%k!YdX)INd|IIIR#YY_O|PR4z`(Vr9-qJkG3|JHx+V z$>%7(j^BipY;Qb+Lb8h~5i#Y@ATCKVwh-gLebYy*`JKS&_I*FSzAJ0&)AdY{Jg%HE zYWE{RZ|+|lY^CR0CLNJ(&t_HgX@8u(**c7t;BY%^tx62cYU_cMFsY#aEPv{2w|c@o zqV|w$M(3BB9EqRs-r5OiF)@uBlWT<Q^933S7HfCb;MMtjjBUz=QTl5){QQ@n_@BoG z@rytDd;a&vjm~kCqvZU7XyvTXciPt6w4Muh^^2UgX|Y0Xo8-K?Pa<c{ytfNk?Rtj8 zC5j=nrRmXXrLR?ka)kks{1ulk);z;aw<?;r7iAMotC`fO?32JkLn0)Anw66ZnZF+f zm#u0!I2<M(2=^Ztj;dn~7){Ih3E|TH`}e~gg>4FJ!X0syuU50r!DuwPtPe*u+GjAp zML!y9l_Ko$Nkikb`rs)>Wpq3xpcu4E?}m?uu<5&d3BamOZaFZHF=_560AaKaS{%hC zn6`eHZ-uvSyRWPM*QWRN9|$evHYzZZ2XFCjZN81DgG2ZTb9`Ye&aQ>R1ZY+0to=Bl zbK104a(}}Nfg)19;)7zzQ}BFebT|C(f0`n6pWNeDeBRdWTjKbFB9xun>svt~RDY4? zViB21vuio<7pr2qB&;*GGBvweOwPw>|BKZE*HUpa4xrX6!Vznq;f_5DCq(FY3JN0y z9oo7&hQ3i%YjyPWcrjC~=K-F_Zpj9=M@=C{j1i1qoh8zcXUAf)lg{7h@z}{IjSCMS zhLh11Ol6y-qsp8q+*GWB<Rok%CTSudo)E6vQ3*A1b#We6#e8fbbg0M06a+yHq36k? z@4kBcT{@@`sze26lpyYuGj@p)(d-;4k(;s*VSIa`QFARIx^U@?0#MA#CjyYO7TsA9 zg6>O?b&&@XgW{+jq?0^59uz}~ON^uVel#40PZ2uUM?n*EgHJFhM$SG*_7~wYD?BGw zD(A`tOu_}hy3-p%HW#IWst%>H%D^g&W)#z$uV4(0Ktkfj4)E<T?!NR+5yqp-J3y@Q zqszVUAbkJqyD$Frt4GfPcWlrhYtgkj#-x}>pnVDak!o{9;mXNkKE293QZ^AbalzlT zI>u(T&(Sw#)uyOTlc?xcNuZgQzwRF<dSi@8PMMa_;x8oXJkRvn@$|e##e@L>{kR~U zifR#m8O7-PCgu1*s!`zTKMdt}o+=}*m7L*LEf+`$mYFaSwnCki#mwLC!(YoJxU=Fa zlsFpV*KRct$3mP$IIIIs+ZA@>kLOUGK&%L71eA&R#j$)qaq3(V0Ab)tS-S*X5y+gJ zHai6juQ_fY&;zTwK}9?>pb?r&oOT7KMO~LO(tYRw`_I|l&c<xl{*I4L3*bC*#2>Q* zn(x!A`59B=A|n!JfIJwSnBBO!`$N5JS*XDYe=eJ0J6X7A>a~au=uJ?^cvjv*(>u8S zcuKI+aq=c7_}F8#^1uN+k+etnI^t@WVAzDPrN6JFK?cfieiLr$!wBE7{vMq?*a};S zw!OR!pdo`)bJ@PB=I3Pxe+ci|v-tKpO<?6Hw*!6O4q)$Z+${m&XwG-E9qtE9Y59pd z7;M&kkG4ssP05J~A-8u=@PRqph-BDo(9+C7)}%fj9na>wKrorz%z??L2Xaz<y1)WM zm&1|fhX>}I*ixfnGJ$RMIfDQukmeaJ5$ZvbR6myhOVZ&_<PEyQt>yz<8!;S-?ZloZ z;^yIYkek@*<!guSF)X8;rvpE@@|CSDC-Z|VUu~?6)_Eb@tweE3jhhGJy57Q8xqT=S zrI4i@q7Lf;DC4ppJwy|<-nsP_*2UcdB?os9cOLXlh>a2)e+7PDs6R8gweFs7Ztm}U z+xw&bGR6e`N72}HdLw`RF5{BK;{MnFbu3BEnb9GT$KZ)yV_c8h+T+i$^oH%2xZQMC za%l(SsL@D3nTdhXJCToq4l%z!tw^^kzm8;xVnpV6ZF0)Q$;xe_#`VDJBXV#eqy>3k zs}{Jv?rW=V_egfVj*JjJeKTh9k%`N=#ZX^0Ievh(^O><`7`RiwO^%GEM~d+Jd`50s z>1Fc9M<wzs?-<Ld!|>pwm>WQ>Hqcq-(&F|GY|8zfp3f*>xJPd8G0*V`(=sxR_O_f1 zyK@_<P>F+^b#&6qy}_w5CN>Y|Q`^QJfSvGh>%72di9QC|CZb|UBq)>rpcP$<UWdTQ zIb)3w^0M_~fTXo*PZze~wpXJ5o<r;$wi`FBX7hW}<_O7~aK?xIuoa7kTlcWhdHM|t ztJ={%s=r^<dE3oBT1>9CC=+FmEQ!Km_O}gOdqe88YRRlNsQmizqc6Vs_N#yW?%9*a zK(_mbuLlQvJD(g5AAYp^K`&Y!nu4a}Oc#?M6=A5Hg7ld-HBL@huM{bWjmz06JdS?3 z2{Yv-LMvDwAw41WhWK`Z9tC)XG^{3+AK?T=9kx0d8LRmvUs2URBGcqpnkV48KuLf0 zX6N7I6Drq?O|XU90LQ8y#i{~+{8H=%nFw@-*#*^(EQE07gNPR4kLS3@yd_A&5G%CJ zVu#l4SFG~!y_jS;E$Sy_!zASg26*V##qhg7EZ;nNw|nlsUX7k{pJ2XN5tLy@*5~>n za>eUWvv{%~#r!!D(_lFIpdaF^x*xp;O>};3#d<ld8uw&|$t5K#t9<k@dYqm<u)>^` zK-%hxBo`w)S&%R-C_;OY!ZbTY0unnbp_`m6>TG^x=xc)_twc<uB|ONXo@?b8cUWI| zY38kw=$JthLXwDy>|vX<x#^Q-rnRKGLbaLdwJeTEZtJa?FsOW-O-**pvABXJzQL#~ z>x@pyW-yC(LYi{bri!c#Ds2~4T()U1t0-}q#5(WaV{)<08&Mpvjs?Wz6vDMYV|KjV z!3zL7&w8bR9K2*7mn`N7MIV(cl}fg_eDL$1|0~?R<Kism@fLL^CjIJP2nr}{Oxo>! zH40zHjtojJmi(}ip{-?+TO^a38q9{6HZkjbQL!XRA$4|$*0!lp9BD^!>-)zM2%VDA z^4Ju~{`}|v4tt}$-^P@d0F-{V_uH=AW6s59kc>H#MOT)_G=Gibj1FP*#dPBJ3fq`8 zTW>9?M#B^`POM{dN8vN;MTyxBJ2b8l?F_}#RJ1AMz0oHCr^TEEmh3b{dp&FvUOJ*Y z6OUdumAtHIHSkBH;3W@`0q(3eqUBOui%Q*(PDVSTWiyfNn0$vsG2t{XnGWo+Vi>-C zwL?OSl${ncfaJOn%PT#T0jPp!WE65;)P6nEVwNWUPPzG$1uY6smB-~#+o8}Z6fZ2M zx&y8>g)B5nSJhII#lB@V6BqB;n?d#}wYp$=@8j>Y1j^6S8)@Mk<q=`>qE-)(+%8{( ze427G()?B<A>msz)O`norN#G4bAcbHqF;CzmLv0+lhPgFpP`X&wu#`*Yo_vkXnMxC z5(Pe2CyzE$2$`3cUAI5*P%;Z4%|%Z@_O+Xm5*B%x6V0|p;#*{>18HjMN$QgAo+C2% zgR^byTlYwskl>V*COM(oVb6e-{!?<oL?xz~(su)&ZFlz8)bIf-&krS^-SQsw&dIiE zDVxdx(z<8&y2rxDJ##Z1Q=-HZ)J;bf3+>)V5%HrEb&)c|u!eZO|Jv%yX+c}uNR2tR zwWmhK>F8>YJRyHP2jt**l&3kq3gE`TFI(BXbi*08Y!A=6WihX4G07RM#;NMKE84%C zlm}XCV`LD@$5IlOw=z#XV~eFwZR;hTkR`jh9Qk@fGv-}pCYxyt)ndwKZLqFVGvh8> zp|5G>+>qio!F86s(1r%<?c3rE@la%Xa|lxH4kRRA&eCECRXXYJ#`}FGI$4fr>!qv; zD)Aa?WeN8p_5JObB#1aYgB2Yc!ckvuU`Ek(yjmkX{UFSC6$`Uu!<c?g4lT};!9pIg z#4@OPP4F$qWy-jzrEX3Tr&hR7oKxCZF{9j>71h)YN>k2AMK$um6d($9-ZzIC@U=u+ zu(Db$j#T7E6i9`Z*bczNPhU6AR!n!roSA5;IpAtX_ienKy9*oyGO|=3=1Q;Hu8>tN zWkR1Up!B=1bDd%5jT72ZvXYUWjFEVedhvWjk|vrEqm~{SdLPCloV-BKg?1x<UtTe| zb+QLKg2@NxdK3@5K`$A9WsL}8kE=_eT;J;RV|`O4#nTisgo}JWc1c*WK!ON^y_=#U z`6RrY9~p$mJ3h_AA{6n7jfGESwSB-@SyCOXNJ_UDSyts3Um5TRXUPdxDGo<0eZ{tQ zWkM!~(4~v-hr1HI0csxQ;Qkv3vJZ&4>V1}?JwtOVlPT=^%RGC_%QH_&J5Do}QqU`O z1}UGdEp65jk@~3AR8os3P-HI6or-Rhw1OXT(9vh=DWh#$8RGfgpjT2W_JZXNWBZ0Y z>i3d2!B&pXn+j*e_|EFgayK#uR4_QHESyZzjH0yA4RRgH2f{ip6#&L)Nd;?jVXe5K zl16NSwR9(dy{73C*7pLN6El@7C6}$o19e?V*qDyYh|Z*<v(^|tLtv^CI0`MG8$|4= zR)*sY0=v*Ny~`Yi<ayu6UrT!sJgRW|sU$5{1|AFsNBC}dv}VHwH}%aTD)$JmzdZ2I z55pAesP9O&(t@q6QETOMDV<c0kZFhzBpc%_WW2|)?5x&!sIbsu1zR*J29cPIWPBA9 zlV*@);ws+xg7_cz`1)zSuBZ8UeEW8SAuT)YL2yvfEa%|G%oW=n@c(3HcEVoV+Ee`I zIH{^Rb;QKtpm=PE$8u+M=eIbpTFc)fT=wNkbwqO$H0Yn!$K1SzpdJFaKSpQ?OqQy2 z)Teq$hJKCc`6O137>Q4<9+M2L(C7RLgLIZ+f;ly@-nV9D(Nr_c(%W$AIWnZ+&xT3q zZbWSAJ#d?wG#qlPE^w1hmt9{4&00>6-Fiueq9xYSUAwt803<6Zp{kW9#+tdXXC}+! z(0VBK3i&5-WO*G^HPRIooGziR=1Cc}iX{n*s3<={!Jt$^tERoCnQjNPd8AI5?M&-7 z$FU|PI)}w=p3!Ya84^4iGpXWzXFE__$)vSMM`Nx`zHMCzata8jDGfTQV!sgDFk@X4 z-x>Y0ICKq`59{P~y5Z2*&Y;`S;JMML>X+gefy!yER(P!i!@GA3DKzO;H2$TR)%$VW zo+VJEr#d{0<WindWl%2E_%G%SEUe(N0;XKVJT1>x6>Z<#4IpSIV>i(^>h^J59{I~B z-=-u{kFkrS7&~G|K6nAQhyv;iij}>AER|6v1_{^Ef{?F8ORK#%5IdkkOy+RfrVf)0 zKozS?Lc61qT@59|B->#eV<|+9>f7I(hV3}xM2`pc3<mLNa5&tkcORk1+?9%4tz<X% z95hXdNB*X4HEv3mdwgR@?CAWyEhR5WVl1_CKe^}HqNY+wgF7|@a@)wr9yavTa0e}R z7reM=q514NHex&6wF%B44hCuD#E)51g}+N3lKm_~-Z?4Kn{Dt3o>y5P?oYS6mJJ`= zc17kXii55Acw@NON*77s*!~2)s5gdV7{|DdZ#<=xh;T3AF`d~Oz|$dQ71Rr)X_3Lx zxO+HY>bfRI)CCay;%n*|_uL$d`O}70&yceqskTxg1oQKcr|8&N4-wbEDVq%_Hw8~h zvO2NNdyk}%NyI7i>qb-Qn6~I9Nw=F^ZV0rFr`l9=BSGyqJmkI8lwC3RghZjzc<6%Z zIUPoJ+bFRBiUzYM{gGO1LwQMC-SkR2fn-T+vW=B@sdDW|2sEEKK3YW&$3@SA@R2d@ zkZTZrl~Gv^I<c#Nd#u9JosjCcn$hAC4?s9^e5Op+a|k7Tu2IleOLPI*PRh1L#JeHW zVrOfyQ(Ua)oyq@7o7Nbc`Q#hPB`&2olbsSj@>q?hL<?FOQ*$)ZQSFI7^cH^2!j&~U zAjy@S&XlD4TXu;{%6WN>Ye<?Zs)<z6PG`mIS82}i;Yy(0@Zj~K18-6udu*ib^0RA@ zWp158TL)F9>@}(q=k&~v!@+H!+(aJk>~yZF;Ow?Niz@=SO<<g&6Z&Og_)}UL`(w=v zV*7mYj4TIfA<vN|moZ|(SB&1_0|G%EAY&;pTPC3!9A(=jmQph08l|wocYPtxwLofy zp;oACH>Sdp6Ar?}HumhG_eSoyqhC0V*zys}#WJlKA6Q?E4{EP^wz3uVGSSRu>8eGB z${do1ElWz;lBoGnl|$@UEXwN}(-OL?Vmc{xX1G$71K;jQ%EMtb534p&sz*uwk}EG_ zr9~3j`^$u_#7$cO<YUqdHdr;TmO5>uEUphQ4&WbP)@utnUi~_J^v2li9rr4>oK@1{ z(y?&v<?IO$G8XYmf-fdBABHW@Gzu;%=FuJ7QZ=vzYcINS7O0@)ty3Ry;`&}~F^OK7 zzx#p$xcY-F-3-KL4oAq~V{TKCECra#kmw|E8rx1x>7U-k*GFIw*3wZ+;zXmO50Gd2 zc}oBP(!74;AwzA9tJbOnokD0YC;%s)+eSHy#`2@n!J%NHL&VZ_s{{Mai!7RyL5v<o zA38>IBHubNt^Cm&U*a8{%#B;GVCdGGxAQ4hOKCqV8y(*2cmf6DTc*{Oz+s87BzE(1 zadw57tJU($ZS8nbO=TkcsMJO5jV5sx1&Gyttm^X^|Cx=ZZ-&agl0(|Yd%7hHmbp{l z=?w=Sz*$xZ+HP4Pr*ktk*YECu?)3_bn}%3n(VTQPHwcRE|E8wPej>eF%voYHr%s>~ zbLwWtC|+gFSu+3LNg3O!1Ym31&Q-|n4rdkYn0QxqHSZtw-Mg00jV3BzIQm~pt`#(k z6>TH+nMYU3Zdx~miiFk&AnrLS$p$v7=qLFSzS9`kQ0X`MdtKM4Q^RUc4lv6y!!wrm zB0KncEe5iusw1TT#DchP>W;!=jE<a3I2XLt`C?gF+FxYE_$mg_dnFCTF=8-x^x6%H z-qkKcX-2f*9G2rzMZ=t$NH3lbv^HJmy`HSiPOa#!`1%>aRD0{6f1rJ@MI7!D>0GH* za=N$S8LiV>I5Lj)X8BeLPw6kXQdbkzSwjFtUj0pnkGfCt`jn%l?8DRSwrj5(sXbEt zhg`kp;VLHOj33^(a&BV#M*C`JI@>1LM#cv-8{f4_J2dLOA}SKp=DitV^E6l|U{W@* z@!7jSeA`YH3f_3eN`liKwAcEfQO?6nWhwBk{+67~wk6D#z_{68*+h4+e2Ikgw5c`Q z+_ydkr9`n^sVdhoyi~cyT?e!r|GWciR9=>2AKfkBke{~xa0_a5G5|I@c5MXxt=$VA zoF431O=gcs<u3nEmc2FT<~jJ$+D@2ud1X_qRV@wZPH#FTZjYE=QE`#hplg6@H8DBA zo|IRS1qW6M`&QIh$A^(rG*2LYrJn3Ja#aFrM?0at{zaJtF;!02WJYOrT0@Ys&8%#P zNs+>*Gcaa@*D@^|{IG#!!?h<*Vn#W~Z08^4e8V|kv={3<rP|)9Q`R!8T-fy|=`b~E zn7z~&FG9DNV7=4WcAKbR=~E3&nNhQ%YGt|5{W0zm*`+>Vi^5`xgGJgNR^FGwGK8fZ z*8dY{ahbo4wjN{~^xpee^q!7lUpYX1a>rPpqpTI_ca-Aa`vu~2H<OU-=@{d%_lbyt z{zxO|Ro$B5%IZtgqsV&9Ii~B7DcEthL<%gn=>~G^7E`H6;`-6+Be@cZ+LT$QQB@ad z8AO;n8|k@Ow=LId`D~~A%^rlFlXbOEqw$+|Ik8E9q^>iZ#gW~%8W+_uU?pXEGa8bE zI3a^Gn*d{a1ywc~`{0VfZh*$r2fa7;&TU1sIISNTHNJaE!xVUN?c;+zm;xaE=aAcv zJYY&_e?W39s-ta?y;mc1bN!LG0!f-s{BG~-VQ1-lNm0R8$$QxQSWWn9V=3@<U6hkx zOyliW6&Ty5smNe$0vws9=Tc}TKk2ei7&gEsOu3Xc%_PJ8z|!P&E3ovV)vCA}$x9z7 zMX9^D`?{}5>S_nX#PhwQ=g3fnKeI;UJZgDMXv*G(8oCBrT2Y-E(kxAE@=8NKuxGc? z&dv@bjwz9}z{YtqKVl^x(J)m{`3-^rS5~s9xdrX7uW2HcuI`xe+K1y6k7bc@mQc*7 znv#PZQQNpZM(W+0&giCOkiVl&n#&y&UAvyuO<Y$7$)xoqyl2yUW$|M2nhPJa3!5fN zweh$L%p*75tYXSn2tt;#7crAzZh3osg5Zy)ZT<33>@}P%@XsL(5u{$|hoU;W@9NRB zj4B;xYALG8>`5Sn*Vmb{60RZ1`*kWYVWBsdp@TG5y`F3Ncv4R>b!Dm1_<3_ua<pPH z?wKXpG}&KxVl{qQ-`|jYV0p+xn$uK+R&GN{OFGu<>aA+g?L$>>vNeX`b)PdJ#7o?Q zHo0TnPoP?TGKtK#7)f@|xZ6<gl9=i$tl$2`j^!jtDnxhDN#hB`kxNvy9bUJM>anfl z+5%cmNbx8kDnH5feqHHXng^o}I}Ogtn$Z(Xy1+ADog8JkQOm4sTmKE2VSPwU>him$ zK3ds)_0gNHUvJ5ta3db;XFRVo99oNH8@3?JwWXaqF~s76sn_|bKP0s#kLJKwL-f8W z?bJDA&kgBRwyAaCK3g<Q=a`zm@w4Y@-BxUD9PbZ0`eoGXv_H_L+hl@XDLWIo4&o?o zzfi&~faoh3k@+>*bfU;N8Pg_<<PmygZDry&C7=UFFls$bdO*9<CrHJ5|K_mYk)bJR zFNN0M30YnI=5l8HUK`4Flh*>jD4_41WZa$hLtWT({`OAlzKz6L<p%R@li+W>>fugc z-Bd|CE_dirvdX|}>~crxnyplUpJDz0q(g=p#GYG+n4i(VUxUY-v0WIw*CoL$dFV+} zPcL<|EoIDD=#Xc(GLx;<WgKjmKH~_nH*F_Wb*33MFE0v7&Mz94C9v=xgn`{Z927f$ zI*b>*wH26&E7}ZKZdCbph$U_tFgY)JZ@kXox2HFTbSD<jS~7Dy^tDaXM^JNR;gh+U zqIYP9coTs8mf5e&(Ky=P0Z8rtG(PfEb>6B&bjTeMx^bq((p7=00sBD;cC&8!5x>Ml zQ~5u}uJWWPPRd7^0tPKd?)K3!MU_K0_8kj8@TfYmF$R{3jDCT>avn?~?_&=xLFN_z zzogQCMWzA>W}_p#f`nHecfpGRKjpB)j~EV-aRawM#0sd^&rLV?`j8m3waJkSDoHec zrSq=+l$3ZO-)+z?yE#KBvEPAWOF4e}?tmo^&rje)@Raj6B3RbC(7UBuS(XbVc3kc& za2pRS_&Qcpjbh#EfrvRZ;eHw%iw7Sc>M-dI--w?KzZ#Xj0rl;iyS+DjYv=48w$dyH z2BHo~3T6CkrAP;q{%bm*Rb$eAI2Y~S1pX~C+97-VQdJtlR%r;GR_&0FvmqZ}KjdyU z<nCb}glLucgN1~M#%l3vVx?FIiAXkO_C~Q!MH*8_<Envuvm+hKJA4i)ot|47KvPGe zyUS~)!<@3dc89kp2rdR66awAa0k@{bTcWq}<sD<Yf1su8WWcSL;lm$B58qmZ@)py- zYTnjd)0~yVAL?5_Om-`hW_6QtXWm^T#ge;Iiu35D%Gz>_X6mDC8zU2yyt&dI>n8xf zLbhho0hbmTUXCV~tGmutcjsFF#U>{4BQ&GQ5YA2Ha@@u1O`t{?dN^_~P)PuoX^p_K zRVVd)NMgH?jZ{eqiJ4`e63HV=S`a))n>|!@-+$DM>KW}f?{wxU`L3h2xV8Vp-QwAL zd&XyN=r2zUTBRuJ!>uS@62(Po>&hZDc@vqE&-F2_W4gHg$;2i63~b5IPJ>hd&c<Y4 z(+DpIdn!w4^P+HDB#3T1h`-aH(VD7mN+g7+6bC!@^e;#;`<RNzNm!iR@%KN3#@cG1 z=^W;?J@h6Omdz`yPM6M$_o8Wy%bt&V_w!Hx@zEQt|Bf<Pld7oty*cfM<dMtE3#StC z=nrS1ee~RTa*#chC+mqgoDG8{bvkVJQ^a_U{v3*?`pJ@5SJh7_rY?DcWh!@be(zbo zYoJ~z{YH=drZLd+R*7KqS^zTBLhW55V@<!cwUjUP1g{NM-+?@Y!s&UQkrFbq1?d## z9$0Lq^7ZO=UVpvrd4D|?<j65nrjVc?UpD&23yWMfG_B{qZSApX+|f>L6Lem`H<2#u z%E5U*pl^7a0s+5U_`*cbcBX~PZWkI9{&UdQC2B}(b?(yiS%)Ru193P4EM%e@iHFlS zeTmqNYJ$X(G=v7AyV}prtNv18_$xl?g}Hxi3OWY#slSb$5qdv75yp?tY5zzTq7sRV k^Rb;FR=21bMoH8I{iLxnSNg^Abcu`SNe)%ccz5If0NT{WCIA2c literal 0 HcmV?d00001 diff --git a/packages/media/src/pdf.ts b/packages/media/src/pdf.ts new file mode 100644 index 00000000..7bd45708 --- /dev/null +++ b/packages/media/src/pdf.ts @@ -0,0 +1,719 @@ +/** + * @file The PDF codec at its seam: PDFium as WebAssembly, one instance per + * document, everything bounded. + * + * Why WASM and not a native build: a hostile PDF that traps a wasm instance + * takes that instance down, not the process the resident model lives in; and + * one artifact serves every platform the runtime ships for. The codec is an + * OPTIONAL peer like `sharp`, required at call time, so a harness that never + * accepts a document never loads it. + * + * Ownership is the whole design here. The module is compiled once per process + * (that is the expensive step); each document gets its OWN instance from that + * compiled module, created when its ingest starts and discarded when it ends. + * Nothing is shared between concurrent ingests, so there is no replacement + * rule, no heap threshold and no owner to hand over: a trap kills only its own + * document, and the shared permit gate bounds how many instances exist at once. + * + * Isolation is cooperative on the main thread: every FPDF call is synchronous. + * A Worker over a built entry, with `terminate()` as the hard bound, is the + * named upgrade path — not runnable from `src/*.ts` on Node 24 today. + */ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import type { WrappedPdfiumModule } from '@embedpdf/pdfium'; +import type { Attachment, Descriptor } from './attachment'; +import { DERIVE_PREFIX } from './attachment'; +import type { AttachmentStore } from './store'; +import type { ContentIngress } from './ingress'; +import { DOCUMENT_CONFIG_TYPE } from './document'; +import type { DocumentMeta } from './document'; +import { gate, abortError } from './gate'; +import { DEFAULT_MAX_PIXELS, MAX_INPUT_PIXELS } from './image'; +import { layoutDocument } from './pdf-layout'; +import type { Bookmark, LayoutResult, PageChar, PageFacts, PageImage, PageStruct } from './pdf-layout'; + +/** The failure a caller can name: which document, and why PDFium refused it. */ +export class PdfError extends Error { + readonly name = 'PdfError'; +} + +/** PDFium's `FPDF_GetLastError` codes, in its words. */ +const LOAD_ERRORS: Record<number, string> = { + 1: 'unknown error', + 2: 'file cannot be opened', + 3: 'malformed: not a PDF or a corrupt one', + 4: 'password protected', + 5: 'unsupported security scheme', + 6: 'page not found or content error', +}; + +/** One compiled module per process — the expensive step, paid once. */ +let compiled: Promise<WebAssembly.Module> | null = null; + +function compileOnce(): Promise<WebAssembly.Module> { + compiled ??= (async () => { + // Required at call time, not imported at module load: the codec is an + // optional peer, and a harness that never accepts a document should not + // pay 4.6 MB of wasm. The message names the package because the failure is + // a missing install, not a bad document. + let wasmPath: string; + try { + wasmPath = require.resolve('@embedpdf/pdfium/pdfium.wasm'); + } catch { + throw new Error( + 'createCodec: `@embedpdf/pdfium` is not installed. Add it to the harness ' + + 'that accepts document uploads (npm i @embedpdf/pdfium).', + ); + } + return WebAssembly.compile(readFileSync(wasmPath)); + })(); + return compiled; +} + +/** + * A live codec: one wasm instance, owned by one document's ingest. + * + * @category Media + */ +export interface Codec { + /** The wrapped module: FPDF calls as methods, plus `pdfium` for memory. */ + readonly pdfium: WrappedPdfiumModule; + /** Reserve `size` bytes on the wasm heap; `free` them with {@link Codec.free}. */ + malloc(size: number): number; + free(ptr: number): void; + /** Drop the instance. Idempotent. Every handle from it is dead afterwards. */ + dispose(): void; +} + +/** + * Instantiate the compiled module for one document. + * + * @category Media + */ +export async function createCodec(): Promise<Codec> { + const module = await compileOnce(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { init } = require('@embedpdf/pdfium') as typeof import('@embedpdf/pdfium'); + const wrapped = await init({ + // The glue honours this Emscripten hook: instantiate our compiled module + // instead of fetching and compiling the wasm again. Returning `{}` tells the + // glue the instantiation is asynchronous and will arrive via the callback. + instantiateWasm(imports, onInstantiated) { + WebAssembly.instantiate(module, imports).then((instance) => onInstantiated(instance)); + return {}; + }, + }); + wrapped.PDFiumExt_Init(); + + let disposed = false; + return { + pdfium: wrapped, + malloc(size: number): number { + const ptr = wrapped.pdfium.wasmExports.malloc(size); + if (ptr === 0) throw new PdfError(`codec: could not reserve ${size} bytes on the wasm heap`); + return ptr; + }, + free(ptr: number): void { + wrapped.pdfium.wasmExports.free(ptr); + }, + dispose(): void { + if (disposed) return; + disposed = true; + // Nothing to call: an instance with no references is collected, heap and + // all. Per-document lifetime IS the memory bound. + }, + }; +} + +/** + * An open PDF: its handle in one codec instance, and the heap it occupies. + * + * @category Media + */ +export interface OpenDocument { + readonly handle: number; + readonly pageCount: number; + /** Release the document and the bytes copied onto the heap. Idempotent. */ + close(): void; +} + +/** + * Open a PDF from bytes in `codec`. The bytes are copied onto the wasm heap + * and stay there until {@link OpenDocument.close}: PDFium reads lazily. + * + * @throws {PdfError} naming PDFium's reason — password, malformed, unsupported. + * + * @category Media + */ +export function openDocument(codec: Codec, bytes: Uint8Array): OpenDocument { + const { pdfium } = codec; + const ptr = codec.malloc(bytes.byteLength); + pdfium.pdfium.HEAPU8.set(bytes, ptr); + const handle = pdfium.FPDF_LoadMemDocument(ptr, bytes.byteLength, ''); + if (handle === 0) { + codec.free(ptr); + const code = pdfium.FPDF_GetLastError(); + throw new PdfError(`openDocument: ${LOAD_ERRORS[code] ?? `PDFium error ${code}`}`); + } + let closed = false; + return { + handle, + pageCount: pdfium.FPDF_GetPageCount(handle), + close(): void { + if (closed) return; + closed = true; + pdfium.FPDF_CloseDocument(handle); + codec.free(ptr); + }, + }; +} + +// ── bounds ─────────────────────────────────────────────────────── + +/** Largest PDF admitted, refused before a byte reaches the wasm heap. */ +export const MAX_DOCUMENT_BYTES = 32 * 1024 * 1024; +/** Time one document may take, end to end. A failure bound, never a coverage knob: past it nothing is published. */ +export const DOCUMENT_TIMEOUT_MS = 120_000; +/** Pages rendered and archived; the rest are text only. Graphics-bearing pages are chosen first. */ +export const MAX_RENDERED_PAGES = 200; +/** Pages whose text is extracted; past it the markdown says so. */ +export const MAX_TEXT_PAGES = 400; +/** Figure crops committed per document. */ +export const MAX_FIGURES = 16; +/** The render profile: what the derive annotations name. */ +export const RENDER_PROFILE = 'pdf.v1'; +export const RENDER_DPI = 150; +export const RENDER_MAX_SIDE = 2048; +/** A page is "graphics-bearing" at this many path objects — a chart, a rule-drawn table. */ +const GRAPHICS_PATH_FLOOR = 20; +const MIN_FIGURE_SIDE_PX = 64; +const MIN_FIGURE_AREA_RATIO = 0.02; + +// PDFium constants (fpdfview.h, fpdf_edit.h, fpdf_progressive.h). +const PAGEOBJ_PATH = 2; +const PAGEOBJ_IMAGE = 3; +const BITMAP_BGRA = 4; +const FLAG_ANNOT = 0x01; +const FLAG_REVERSE_BYTE_ORDER = 0x10; +// fpdf_progressive.h: READY 0, TOBECONTINUED 1, DONE 2, FAILED 3. +const RENDER_TOBECONTINUED = 1; +const RENDER_DONE = 2; + +/** The failure the route can map: ETIMEDOUT names the class, the message the document. */ +function timeoutError(ms: number): Error { + const e = new PdfError(`createDocumentIngress: document exceeded ${ms}ms`); + (e as Error & { code: string }).code = 'ETIMEDOUT'; + return e; +} + +// ── reading strings and structs off the heap ───────────────────── + +/** The two-call pattern PDFium uses for UTF-16 strings: length, then fill. */ +function utf16(codec: Codec, call: (buf: number, len: number) => number): string { + const { pdfium } = codec; + const len = call(0, 0); + if (len <= 2) return ''; + const buf = codec.malloc(len); + try { + call(buf, len); + return pdfium.pdfium.UTF16ToString(buf); + } finally { + codec.free(buf); + } +} + +// ── page facts ─────────────────────────────────────────────────── + +function readStruct(codec: Codec, page: number): { struct: PageStruct; altByMcid: Map<number, string> } | undefined { + const { pdfium } = codec; + const tree = pdfium.FPDF_StructTree_GetForPage(page); + if (!tree) return undefined; + const roleByMcid = new Map<number, string>(); + const altByMcid = new Map<number, string>(); + const tables: PageStruct['tables'] = []; + let figures = 0; + try { + const typeOf = (el: number): string => utf16(codec, (b, l) => pdfium.FPDF_StructElement_GetType(el, b, l)); + const ownMcids = (el: number): number[] => { + const n = pdfium.FPDF_StructElement_GetMarkedContentIdCount(el); + const ids: number[] = []; + for (let i = 0; i < n; i++) { + const id = pdfium.FPDF_StructElement_GetMarkedContentIdAtIndex(el, i); + if (id >= 0) ids.push(id); + } + if (ids.length === 0) { + const one = pdfium.FPDF_StructElement_GetMarkedContentID(el); + if (one >= 0) ids.push(one); + } + return ids; + }; + const children = (el: number): number[] => { + const n = pdfium.FPDF_StructElement_CountChildren(el); + const out: number[] = []; + for (let i = 0; i < n; i++) { + const c = pdfium.FPDF_StructElement_GetChildAtIndex(el, i); + if (c) out.push(c); + } + return out; + }; + const descendantMcids = (el: number): number[] => [...ownMcids(el), ...children(el).flatMap(descendantMcids)]; + // Producers wrap marked content in structural wrappers (`NonStruct`, `Span`, + // `Div`, …) that carry no meaning of their own: a wrapper's content takes the + // role of the nearest semantic ancestor, or a heading under `H1 > NonStruct` + // would read as body text. + const TRANSPARENT = new Set(['NonStruct', 'Span', 'Div', 'Sect', 'Part', 'Art', 'Document', 'Link', 'Private']); + const readTable = (table: number): PageStruct['tables'][number] => { + const rows: { cells: { mcids: number[] }[] }[] = []; + const visitRows = (el: number): void => { + for (const c of children(el)) { + const t = typeOf(c); + if (t === 'TR') { + const cells = children(c).filter((k) => { const kt = typeOf(k); return kt === 'TD' || kt === 'TH'; }) + .map((k) => ({ mcids: descendantMcids(k) })); + rows.push({ cells }); + } else if (t !== 'Caption') visitRows(c); + } + }; + visitRows(table); + // The table's content is its cells; a caption or a border path is not. + return { mcids: rows.flatMap((r) => r.cells.flatMap((c) => c.mcids)), rows }; + }; + const visit = (el: number, cell: string | null, inherited: string | null): void => { + const type = typeOf(el); + const semantic = TRANSPARENT.has(type) ? inherited : type; + const effective = cell ?? semantic ?? type; + for (const m of ownMcids(el)) roleByMcid.set(m, effective); + if (type === 'Figure') { + figures++; + const alt = utf16(codec, (b, l) => pdfium.FPDF_StructElement_GetAltText(el, b, l)); + if (alt) for (const m of descendantMcids(el)) altByMcid.set(m, alt); + } + if (type === 'Table') tables.push(readTable(el)); + const nextCell = type === 'TD' || type === 'TH' ? type : cell; + for (const c of children(el)) visit(c, nextCell, semantic); + }; + const n = pdfium.FPDF_StructTree_CountChildren(tree); + for (let i = 0; i < n; i++) { + const c = pdfium.FPDF_StructTree_GetChildAtIndex(tree, i); + if (c) visit(c, null, null); + } + } finally { + pdfium.FPDF_StructTree_Close(tree); + } + return { struct: { roleByMcid, tables, figures }, altByMcid }; +} + +/** Everything the layout needs about one page, read in one pass over the codec. */ +function readPage(codec: Codec, doc: OpenDocument, index: number, opts: { text: boolean }): PageFacts { + const { pdfium } = codec; + const page = pdfium.FPDF_LoadPage(doc.handle, index); + if (!page) throw new PdfError(`readPage: page ${index + 1} cannot be loaded`); + const scratch = codec.malloc(96); + try { + const width = pdfium.FPDF_GetPageWidthF(page); + const height = pdfium.FPDF_GetPageHeightF(page); + const read = readStruct(codec, page); + + // Objects: image bounds (and a decompression-bomb check on their pixel size + // before anything could decode them), and the path count. + const images: PageImage[] = []; + let pathObjects = 0; + const count = pdfium.FPDFPage_CountObjects(page); + for (let i = 0; i < count; i++) { + const obj = pdfium.FPDFPage_GetObject(page, i); + const type = pdfium.FPDFPageObj_GetType(obj); + if (type === PAGEOBJ_PATH) pathObjects++; + if (type !== PAGEOBJ_IMAGE) continue; + if (pdfium.FPDFImageObj_GetImagePixelSize(obj, scratch, scratch + 4)) { + const w = pdfium.pdfium.getValue(scratch, 'i32'); + const h = pdfium.pdfium.getValue(scratch + 4, 'i32'); + if (w * h > MAX_INPUT_PIXELS) { + throw new PdfError(`readPage: page ${index + 1} embeds a ${w}×${h} image, over the ${MAX_INPUT_PIXELS}-pixel ceiling`); + } + } + if (!pdfium.FPDFPageObj_GetBounds(obj, scratch, scratch + 4, scratch + 8, scratch + 12)) continue; + const bbox: PageImage['bbox'] = [ + pdfium.pdfium.getValue(scratch, 'float'), pdfium.pdfium.getValue(scratch + 4, 'float'), + pdfium.pdfium.getValue(scratch + 8, 'float'), pdfium.pdfium.getValue(scratch + 12, 'float'), + ]; + const mcid = pdfium.FPDFPageObj_GetMarkedContentID(obj); + const altText = mcid >= 0 ? read?.altByMcid.get(mcid) : undefined; + images.push({ index: i, bbox, ...(altText ? { altText } : {}) }); + } + + // Characters, with box, size, weight and marked-content id. + const chars: PageChar[] = []; + if (opts.text) { + const tp = pdfium.FPDFText_LoadPage(page); + if (tp) { + try { + const n = pdfium.FPDFText_CountChars(tp); + const mcidOfObject = new Map<number, number>(); + for (let i = 0; i < n; i++) { + const code = pdfium.FPDFText_GetUnicode(tp, i); + const text = code > 0 ? String.fromCodePoint(code) : ' '; + if (!pdfium.FPDFText_GetCharBox(tp, i, scratch, scratch + 8, scratch + 16, scratch + 24)) { + chars.push({ text: '\n', x0: 0, y0: 0, x1: 0, y1: 0, size: 0, weight: 400, mcid: -1 }); + continue; + } + // Rotated text — a journal's margin stamp, a landscape label — is + // not part of the page's reading flow; it would otherwise land as + // one "line" per glyph. Horizontal means the glyph's up is the + // page's up (a 180° flip counts as rotated too). + const angle = pdfium.FPDFText_GetCharAngle(tp, i); + if (Math.abs(Math.sin(angle)) > 0.1 || Math.cos(angle) < 0) continue; + // The LOOSE box is the glyph's advance box in page units — its width + // is the advance, so adjacent glyphs of one word abut and a word gap + // is a real gap. The tight box is the fallback when PDFium has none. + let left: number; let right: number; let bottom: number; let top: number; + const loose = pdfium.FPDFText_GetLooseCharBox(tp, i, scratch + 32); + if (loose) { + // FS_RECTF: left, top, right, bottom. + left = pdfium.pdfium.getValue(scratch + 32, 'float'); + top = pdfium.pdfium.getValue(scratch + 36, 'float'); + right = pdfium.pdfium.getValue(scratch + 40, 'float'); + bottom = pdfium.pdfium.getValue(scratch + 44, 'float'); + } else { + left = pdfium.pdfium.getValue(scratch, 'double'); + right = pdfium.pdfium.getValue(scratch + 8, 'double'); + bottom = pdfium.pdfium.getValue(scratch + 16, 'double'); + top = pdfium.pdfium.getValue(scratch + 24, 'double'); + } + // Size: the nominal font size × the text matrix's scale — the size + // the page was set in, whatever font drew the glyph (an em box + // varies by font: a math font's is taller than a text font's at + // one size) and however the producer arrived at it (a font set at + // size 1 under a ×9 matrix). The loose height is the fallback. + let size = pdfium.FPDFText_GetFontSize(tp, i); + if (pdfium.FPDFText_GetMatrix(tp, i, scratch + 64)) { + // FS_MATRIX: a, b, c, d, e, f. + const a = pdfium.pdfium.getValue(scratch + 64, 'float'); + const b = pdfium.pdfium.getValue(scratch + 68, 'float'); + const scale = Math.hypot(a, b); + if (scale > 0) size *= scale; + } + if (!(size > 0) && loose) size = top - bottom; + // The glyph origin's y is the baseline — the one thing a ligature + // drawn from a fallback font (a smaller em box) still shares with + // its neighbours, so lines group by it rather than by box edges. + let baseline = bottom; + if (pdfium.FPDFText_GetCharOrigin(tp, i, scratch + 48, scratch + 56)) { + baseline = pdfium.pdfium.getValue(scratch + 56, 'double'); + } + // A space PDFium synthesises between text runs has no box at all. + // It is a separator, not a glyph: give it the previous glyph's + // geometry so nothing downstream reads a zero-size box as a new + // baseline or a size-one font. + const last = chars[chars.length - 1]; + if (/\s/.test(text) && !(top - bottom > 0) && last && last.text !== '\n') { + left = last.x1; right = last.x1; bottom = last.y0; top = last.y1; size = last.size; + baseline = last.baseline ?? last.y0; + } + const obj = pdfium.FPDFText_GetTextObject(tp, i); + let mcid = -1; + if (obj) { + const cached = mcidOfObject.get(obj); + mcid = cached ?? pdfium.FPDFPageObj_GetMarkedContentID(obj); + mcidOfObject.set(obj, mcid); + } + const weight = pdfium.FPDFText_GetFontWeight(tp, i); + chars.push({ + text, x0: left, y0: bottom, x1: right, y1: top, + size, baseline, + weight: weight > 0 ? weight : 400, + mcid, + }); + } + } finally { + pdfium.FPDFText_ClosePage(tp); + } + } + } + + return { + page: index + 1, width, height, chars, images, pathObjects, textExtracted: opts.text, + ...(read ? { struct: read.struct } : {}), + }; + } finally { + codec.free(scratch); + pdfium.FPDF_ClosePage(page); + } +} + +function readBookmarks(codec: Codec, doc: OpenDocument): Bookmark[] { + const { pdfium } = codec; + const out: Bookmark[] = []; + const visit = (first: number, level: number): void => { + let bm = first; + while (bm) { + const title = utf16(codec, (b, l) => pdfium.FPDFBookmark_GetTitle(bm, b, l)); + const dest = pdfium.FPDFBookmark_GetDest(doc.handle, bm); + const page = dest ? pdfium.FPDFDest_GetDestPageIndex(doc.handle, dest) + 1 : 0; + out.push({ title, page, level }); + const child = pdfium.FPDFBookmark_GetFirstChild(doc.handle, bm); + if (child) visit(child, level + 1); + bm = pdfium.FPDFBookmark_GetNextSibling(doc.handle, bm); + } + }; + visit(pdfium.FPDFBookmark_GetFirstChild(doc.handle, 0), 0); + return out; +} + +// ── rendering ──────────────────────────────────────────────────── + +/** Pixel size for a page at `dpi`, held under the side and area ceilings. */ +function renderSize(widthPt: number, heightPt: number, dpi: number): { width: number; height: number; scale: number } { + let scale = dpi / 72; + const longest = Math.max(widthPt, heightPt) * scale; + if (longest > RENDER_MAX_SIDE) scale *= RENDER_MAX_SIDE / longest; + const area = widthPt * heightPt * scale * scale; + if (area > DEFAULT_MAX_PIXELS) scale *= Math.sqrt(DEFAULT_MAX_PIXELS / area); + return { width: Math.max(1, Math.floor(widthPt * scale)), height: Math.max(1, Math.floor(heightPt * scale)), scale }; +} + +/** Copy a bitmap's RGBA rows off the heap (the stride may exceed the row). */ +function bitmapPixels(codec: Codec, bmp: number, width: number, height: number): Uint8Array { + const { pdfium } = codec; + const buf = pdfium.FPDFBitmap_GetBuffer(bmp); + const stride = pdfium.FPDFBitmap_GetStride(bmp); + const row = width * 4; + const out = new Uint8Array(row * height); + const heap = pdfium.pdfium.HEAPU8; + for (let y = 0; y < height; y++) out.set(heap.subarray(buf + y * stride, buf + y * stride + row), y * row); + return out; +} + +/** Render a whole page progressively, stopping at the deadline or an abort. */ +function renderPageRgba(codec: Codec, page: number, width: number, height: number, stop: () => boolean): Uint8Array { + const { pdfium } = codec; + const bmp = pdfium.FPDFBitmap_CreateEx(width, height, BITMAP_BGRA, 0, 0); + if (!bmp) throw new PdfError(`render: could not allocate a ${width}×${height} bitmap`); + const pause = codec.malloc(12); + const fn = pdfium.pdfium.addFunction(() => (stop() ? 1 : 0), 'ip'); + try { + pdfium.FPDFBitmap_FillRect(bmp, 0, 0, width, height, 0xffffffff); + pdfium.pdfium.setValue(pause, 1, 'i32'); + pdfium.pdfium.setValue(pause + 4, fn, 'i32'); + pdfium.pdfium.setValue(pause + 8, 0, 'i32'); + let status = pdfium.FPDF_RenderPageBitmap_Start(bmp, page, 0, 0, width, height, 0, FLAG_ANNOT | FLAG_REVERSE_BYTE_ORDER, pause); + while (status === RENDER_TOBECONTINUED && !stop()) status = pdfium.FPDF_RenderPage_Continue(page, pause); + pdfium.FPDF_RenderPage_Close(page); + if (status !== RENDER_DONE) throw new PdfError(status === RENDER_TOBECONTINUED ? 'render: stopped' : 'render: PDFium failed'); + return bitmapPixels(codec, bmp, width, height); + } finally { + pdfium.pdfium.removeFunction(fn); + codec.free(pause); + pdfium.FPDFBitmap_Destroy(bmp); + } +} + +/** Render a region of a page — a figure's bounds — at `scale` pixels per point. */ +function renderRegionRgba(codec: Codec, page: number, pageHeightPt: number, bbox: PageImage['bbox'], scale: number): { rgba: Uint8Array; width: number; height: number } { + const { pdfium } = codec; + const [x0, y0, x1, y1] = bbox; + const width = Math.max(1, Math.round((x1 - x0) * scale)); + const height = Math.max(1, Math.round((y1 - y0) * scale)); + const bmp = pdfium.FPDFBitmap_CreateEx(width, height, BITMAP_BGRA, 0, 0); + if (!bmp) throw new PdfError(`render: could not allocate a ${width}×${height} bitmap`); + const matrix = codec.malloc(24); + const clip = codec.malloc(16); + try { + pdfium.FPDFBitmap_FillRect(bmp, 0, 0, width, height, 0xffffffff); + // Device space is top-left, y down; user space is bottom-left, y up. The + // display transform already flips y, so the region's top edge lands at the + // device origin when translated by the distance from the page top. + const m = [scale, 0, 0, scale, -x0 * scale, -(pageHeightPt - y1) * scale]; + m.forEach((v, i) => pdfium.pdfium.setValue(matrix + i * 4, v, 'float')); + [0, 0, width, height].forEach((v, i) => pdfium.pdfium.setValue(clip + i * 4, v, 'float')); + pdfium.FPDF_RenderPageBitmapWithMatrix(bmp, page, matrix, clip, FLAG_ANNOT | FLAG_REVERSE_BYTE_ORDER); + return { rgba: bitmapPixels(codec, bmp, width, height), width, height }; + } finally { + codec.free(matrix); + codec.free(clip); + pdfium.FPDFBitmap_Destroy(bmp); + } +} + +/** RGBA pixels → PNG, through sharp with its defaults, which are deterministic for a fixed libvips. */ +async function toPng(rgba: Uint8Array, width: number, height: number): Promise<Uint8Array> { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const sharp = require('sharp') as typeof import('sharp').default; + const png = await sharp(Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), { raw: { width, height, channels: 4 } }) + .removeAlpha() + .png() + .toBuffer(); + return new Uint8Array(png); +} + +/** Let the event loop breathe between pages: the socket, the model loop, an abort. */ +const breathe = (): Promise<void> => new Promise((r) => setImmediate(r)); + +function codecVersion(): string { + try { + const entry = require.resolve('@embedpdf/pdfium'); + const pkg = JSON.parse(readFileSync(join(dirname(entry), '..', 'package.json'), 'utf8')) as { version?: string }; + return pkg.version ?? 'unknown'; + } catch { + return 'unknown'; + } +} + +// ── the ingress ────────────────────────────────────────────────── + +/** + * What a host may size differently. Every default is the shipped bound above; + * tests use small numbers to reach the caps with small fixtures. + * + * @category Media + */ +export interface DocumentOpts { + maxRenderedPages?: number; + maxTextPages?: number; + maxFigures?: number; + timeoutMs?: number; + dpi?: number; +} + +/** + * The document ingress: a PDF in, a document attachment out. + * + * One permit of the shared gate for the whole document, one codec instance + * for the whole document. Extraction and rendering complete in memory first; + * the store is written only after the last check, so an abort or a timeout + * commits nothing. Coverage — which pages are rendered, how many figures — + * is a function of the input and the declared caps, never of time, so the + * same bytes yield the same root on any machine. + * + * @category Media + */ +export function createDocumentIngress(store: AttachmentStore, opts: DocumentOpts = {}): ContentIngress { + const maxRenderedPages = opts.maxRenderedPages ?? MAX_RENDERED_PAGES; + const maxTextPages = opts.maxTextPages ?? MAX_TEXT_PAGES; + const maxFigures = opts.maxFigures ?? MAX_FIGURES; + const timeoutMs = opts.timeoutMs ?? DOCUMENT_TIMEOUT_MS; + const dpi = opts.dpi ?? RENDER_DPI; + const label = 'createDocumentIngress'; + + return { + async ingest(bytes: Uint8Array, signal?: AbortSignal): Promise<Attachment> { + if (signal?.aborted) throw abortError(label); + if (bytes.byteLength > MAX_DOCUMENT_BYTES) { + throw new PdfError(`${label}: document is ${bytes.byteLength} bytes, over the ${MAX_DOCUMENT_BYTES}-byte ceiling`); + } + const release = await gate.acquire(signal); + try { + const deadline = Date.now() + timeoutMs; + const stop = (): boolean => Date.now() > deadline || !!signal?.aborted; + const check = (): void => { + if (signal?.aborted) throw abortError(label); + if (Date.now() > deadline) throw timeoutError(timeoutMs); + }; + + const codec = await createCodec(); + try { + const doc = openDocument(codec, bytes); + try { + const title = utf16(codec, (b, l) => codec.pdfium.FPDF_GetMetaText(doc.handle, 'Title', b, l)) || null; + const bookmarks = readBookmarks(codec, doc); + + const pages: PageFacts[] = []; + for (let i = 0; i < doc.pageCount; i++) { + check(); + pages.push(readPage(codec, doc, i, { text: i < maxTextPages })); + await breathe(); + } + const layout = layoutDocument({ + title, pages, bookmarks, maxTextPages, dpi, + minFigureSidePx: MIN_FIGURE_SIDE_PX, minFigureAreaRatio: MIN_FIGURE_AREA_RATIO, + }); + + // Which pages to archive under the cap: page one, then every + // graphics-bearing page, then the rest — a deterministic order. + const graphic = (p: LayoutResult['pages'][number]): boolean => + p.page === 1 || p.chars === 0 || p.imageObjects > 0 || p.pathObjects >= GRAPHICS_PATH_FLOOR || p.taggedTables > 0 || p.taggedFigures > 0; + const order = [...layout.pages] + .sort((a, b) => Number(graphic(b)) - Number(graphic(a)) || a.page - b.page) + .slice(0, maxRenderedPages) + .sort((a, b) => a.page - b.page); + + const renders = new Map<number, { png: Uint8Array; width: number; height: number }>(); + const crops: { figure: LayoutResult['figures'][number]; png: Uint8Array; width: number; height: number }[] = []; + const figures = layout.figures.slice(0, maxFigures); + for (const p of order) { + check(); + const facts = pages[p.page - 1]; + const size = renderSize(facts.width, facts.height, dpi); + const page = codec.pdfium.FPDF_LoadPage(doc.handle, p.page - 1); + if (!page) throw new PdfError(`render: page ${p.page} cannot be loaded`); + try { + const rgba = renderPageRgba(codec, page, size.width, size.height, stop); + check(); + renders.set(p.page, { png: await toPng(rgba, size.width, size.height), width: size.width, height: size.height }); + for (const f of figures.filter((x) => x.page === p.page)) { + check(); + const region = renderRegionRgba(codec, page, facts.height, f.bbox, size.scale); + crops.push({ figure: f, png: await toPng(region.rgba, region.width, region.height), width: region.width, height: region.height }); + } + } finally { + codec.pdfium.FPDF_ClosePage(page); + } + await breathe(); + } + // A figure on a page that fell outside the render cap is not cropped + // either: crops come from rendered pages, so the archive is consistent. + const truncated = layout.truncated || renders.size < doc.pageCount || layout.figures.length > figures.length; + + // The last check before the first write: from here nothing waits and + // nothing aborts, so a document is committed whole or not at all. + check(); + const source = store.putBlob(bytes, 'application/pdf'); + const derive = (page: number, width: number, height: number, extra: Record<string, string> = {}): Record<string, string> => ({ + [`${DERIVE_PREFIX}profile`]: RENDER_PROFILE, + [`${DERIVE_PREFIX}page`]: String(page), + [`${DERIVE_PREFIX}dpi`]: String(dpi), + [`${DERIVE_PREFIX}width`]: String(width), + [`${DERIVE_PREFIX}height`]: String(height), + [`${DERIVE_PREFIX}format`]: 'image/png', + [`${DERIVE_PREFIX}source`]: source.digest, + ...extra, + }); + const pageRoots = new Map<number, Descriptor>(); + for (const [page, r] of renders) { + const rep = store.putBlob(r.png, 'image/png', derive(page, r.width, r.height)); + pageRoots.set(page, store.putAttachment({ representations: [rep] })); + } + const figureRoots: DocumentMeta['figures'] = crops.map((c) => { + const rep = store.putBlob(c.png, 'image/png', derive(c.figure.page, c.width, c.height, { [`${DERIVE_PREFIX}bbox`]: c.figure.bbox.join(',') })); + return { ...c.figure, root: store.putAttachment({ representations: [rep] }) }; + }); + const markdown = store.putBlob(new TextEncoder().encode(layout.markdown), 'text/markdown'); + const meta: DocumentMeta = { + title: layout.title, + pageCount: doc.pageCount, + sections: layout.sections, + pages: layout.pages.map((p) => { const render = pageRoots.get(p.page); return render ? { ...p, render } : p; }), + figures: figureRoots, + tables: layout.tables, + derive: { + profile: RENDER_PROFILE, pdfium: codecVersion(), dpi, maxSide: RENDER_MAX_SIDE, maxPixels: DEFAULT_MAX_PIXELS, format: 'image/png', + renderedPages: renders.size, maxFigures, maxTextPages, tagged: layout.tagged, structCoverage: layout.structCoverage, truncated, + }, + }; + return store.putAttachment({ + representations: [markdown], + source, + config: { bytes: new TextEncoder().encode(JSON.stringify(meta)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); + } finally { + doc.close(); + } + } finally { + codec.dispose(); + } + } finally { + release(); + } + }, + }; +} diff --git a/packages/media/test/content-ingress.test.ts b/packages/media/test/content-ingress.test.ts new file mode 100644 index 00000000..0a04e94a --- /dev/null +++ b/packages/media/test/content-ingress.test.ts @@ -0,0 +1,43 @@ +/** + * One door, the bytes deciding: an image to the normalizer, a PDF to the + * document ingress, junk refused. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { readFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import sharp from 'sharp'; +import { FileAttachmentStore } from '../src/node'; +import { createContentIngress, DOCUMENT_UPLOAD_TIMEOUT_MS } from '../src/content-ingress'; +import { DOCUMENT_CONFIG_TYPE } from '../src/document'; +import { EMPTY_DESCRIPTOR } from '../src/attachment'; + +const fixture = (name: string): Uint8Array => new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); + +describe('createContentIngress', () => { + it('admits an image as an image manifest', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'content-ingress-'))); + const png = new Uint8Array(await sharp({ create: { width: 32, height: 32, channels: 3, background: '#0a7' } }).png().toBuffer()); + const root = await createContentIngress(store).ingest(png); + const manifest = store.getManifest(root.digest)!; + expect(manifest.config.digest).toBe(EMPTY_DESCRIPTOR.digest); + expect(manifest.layers[0].mediaType).toBe('image/png'); + }); + + it('admits a PDF as a document manifest', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'content-ingress-'))); + const root = await createContentIngress(store).ingest(fixture('untagged.pdf')); + expect(store.getManifest(root.digest)!.config.mediaType).toBe(DOCUMENT_CONFIG_TYPE); + }, 60_000); + + it('refuses bytes that are neither', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'content-ingress-'))); + await expect(createContentIngress(store).ingest(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).rejects.toThrow(); + }); + + it('states the upload allowance a host passes beside the byte ceiling', () => { + expect(DOCUMENT_UPLOAD_TIMEOUT_MS).toBe(180_000); + }); +}); diff --git a/packages/media/test/document.test.ts b/packages/media/test/document.test.ts new file mode 100644 index 00000000..dece9b95 --- /dev/null +++ b/packages/media/test/document.test.ts @@ -0,0 +1,62 @@ +/** + * The document sidecar — facts about a document, as the content plane stores + * them in an attachment's config slot. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { DOCUMENT_CONFIG_TYPE, asDocumentMeta, pagesOf } from '../src/document'; +import type { DocumentMeta } from '../src/document'; + +const page = (n: number, startLine: number, endLine: number, extra: Partial<DocumentMeta['pages'][number]> = {}) => ({ + page: n, startLine, endLine, chars: 100, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0, ...extra, +}); + +const meta: DocumentMeta = { + title: 'A paper', + pageCount: 3, + sections: [{ heading: 'Intro', path: 'Intro', origin: 'heuristic', startLine: 1, endLine: 12, pageStart: 1, pageEnd: 2 }], + pages: [page(1, 1, 8), page(2, 9, 15, { imageObjects: 1 }), page(3, 16, 20, { chars: 0 })], + figures: [], + tables: [], + derive: { + profile: 'pdf.v1', pdfium: '2.15.0', dpi: 150, maxSide: 2048, maxPixels: 4_194_304, format: 'image/png', + renderedPages: 3, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false, + }, +}; + +describe('DOCUMENT_CONFIG_TYPE', () => { + it('is a versioned +json type, so a reader can branch on it', () => { + expect(DOCUMENT_CONFIG_TYPE).toBe('application/vnd.lloyal.document.v1+json'); + }); +}); + +describe('asDocumentMeta', () => { + it('accepts the shape and returns it typed', () => { + expect(asDocumentMeta(JSON.parse(JSON.stringify(meta)))).toEqual(meta); + }); + + it('refuses junk without throwing', () => { + expect(asDocumentMeta(null)).toBeNull(); + expect(asDocumentMeta('x')).toBeNull(); + expect(asDocumentMeta({})).toBeNull(); + expect(asDocumentMeta({ ...meta, title: 7 })).toBeNull(); + expect(asDocumentMeta({ ...meta, pages: 'no' })).toBeNull(); + expect(asDocumentMeta({ ...meta, pages: [{ page: 1 }] })).toBeNull(); + expect(asDocumentMeta({ ...meta, sections: [{ heading: 'x' }] })).toBeNull(); + }); +}); + +describe('pagesOf', () => { + it('maps a line span inside one page to that page', () => { + expect(pagesOf(meta, 2, 5)).toEqual({ pageStart: 1, pageEnd: 1 }); + }); + + it('maps a span across a page break to its first and last page', () => { + expect(pagesOf(meta, 7, 17)).toEqual({ pageStart: 1, pageEnd: 3 }); + }); + + it('maps a span past the page map to the last page', () => { + expect(pagesOf(meta, 40, 45)).toEqual({ pageStart: 3, pageEnd: 3 }); + }); +}); diff --git a/packages/media/test/fixtures/pdf/encrypted.pdf b/packages/media/test/fixtures/pdf/encrypted.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fe2d6729cfbc570be90fea8f964611711e3fc71f GIT binary patch literal 4095 zcmcgvdt6gxA6G%tLHeo*`VxA$IT5#W8EkB_Auz@r=EhCI%VFmX4s1JPJBv$Fi5i%_ z-YIWoWiL{xkcjp{6D31MMa!2`K}E5QBr(kd!+f8!F(LJRv(KmZ-9LNI^Zc&g-{tu| zzDl`*hw?=n-=kft-skxGMjKwFH8?}j1__j9hIn}pj|dA*3XcN+35m+kWOW3@Gs`1V zqQfJkMnao!q8JkFB<34PlM*+Z^Jx<y4FNmJ=HG7+5eAK^(8!8bNd-V$ELj*w8nT#d zDT)QLA}oy2!YC*m3HU=efe-@GnK_)`VAw_m3BisE1_TUgf?$OXXGl4zp$SqZ<Cqx} ziR(G%<i4kFv@HqR*JQc4PWV#Rn(jz_wz2cJ+V$F|QQfC=JG?`L8L~7}d!jl0kaEwY zqC?Z0uFig^vE(cFqdMmfwc>AeSKb{SJ6W_jU>C15Y4+t%_uUV@BIA}8FDt%1S0VFu zZdrBl`Q0<u@BH@QRX>-UI~R|rPgL!m(dZ}{M=94SM~FHmHeJ-*T~92pnpA)GE2mkT zx^iO1><NCMu&sRNy`8Vr-s}i!qv!OV5C|u$5)XfTcVy)eNy?GdGd*j6y7SVlseK_s z=4YQhoawGkJmXh(N>sjSg8bAK(m?b(2v#g8DE0(8sNj#`nfry)Jg!Gu*gr459BQnY zaTNWU``oAAyJot*?(Oe!OVH`)c4%9;lIRTjQXN}SU!}8j@k@nq%bIVmDBr3Wv7#mS z#(Z5%P+82n*HV<5j&9iSft#Q^c$K2HczLYl4OO=3o}>Ty<`(oAx^(}q8Jm1Aj2E8g zJz;ymfaytvMiN%yStJ}!5)>Xr7eMI<I2Ho`L}F|n2UgJx2s4BLDidj7AghyLn>5oF zlZG^d>`<C$jhbZAVUWuZoJ1BdfKNdKVw9Do;(+vk!@)P;H$!Y%S-0#+qW?d0SkE6b z#~R4N_OK4x(jNE%;DN08m?1cpBFs=a$0h@1tyXd7LBKH!@DyX6w0rbOLcf4;oWXT; zmR(Xmgz+SU6W|0wMRv370tS_yt=a|`o4pz_kw7Nv&&u)pV)o5mvB<-Dl(r;0ao^X8 zMDM+KI<|W5{o#nG<(BWX$nP3U8h<c+F#OJp7UJwr$Gg%;x5Fjn|Joe1=X%+wqUsgj z^G|NiEl-ceV=DKj-sgKo#WtKB^3%%2S5~_;Z0ufezx|arLz>RM*RZwhveDB=N~cl3 z)g@G{jqq4BF|OypSoJ)I<ng6B#FeGh^V(~I_Qt0qJ2ZXX66MUzYH!w7oJn_IzxIti zi-y2HRd3+uS}QosTlO!S4pHr8hjwjv*X_mOrH+j&%XoLIH~&3mU-4@Gl!_5AEkXKz zQ7=DN^GzF+w_LFB^%kG)BaPnT_kWZ%elTr%`J3D4cCBv;6}xQxXjW|Ydy_k(njSo8 zqjqN=e${{bc{Uc0RV>@R`uo{`233FtJpxth8>5%5^7B+N+X5fb9ho_P%f^H~WU^lN zE%l1S=DwJ7tn<ctB>&XgZ?(7L@42qI>*p}_UR`GycJbg4$4O%h2svVOdh4?5-K{0v z!S_Fu%s{oNm0uk=xI@0IqGM&9i{j<aOgmo9>Ly%W>xO!ssg9~2+9K*b>3B{4^YyWH z!?$s28qMQ^L#L+rZIT}9^B5sP-FlncRJET;K9SyZ8U5|b`S|R>acjq({sj?yTG_nJ z=XgX_t#3*~u|ctMF1&6_jsFdFWA!Hs3U98K?eP8n$Hh52{p58qqJ?QYdKx^@Tb_~L zBM-!o=R=f=(Ng($6Y&W<ELXoh9Aj{<y!A`x?)%56EC(HRW}MT4`tSElHi(*?713R( zA7@!kp70Re-ap~QzEgZh(Q|hk`f83htR9-wSNh@1o+!^+s@OfCzWON<VDr2`0@lW3 zuK*6Lwq!DFD@u!xv!^#25IB(E!65}kT$4*OP$rp08Km6q{q<akBBb2Zz<4Cy7*1wW zQAH+FU6hcdDazG|32umt6C8qrW=--6a3fD^y|v5`Y1T-+!c}n!FP5M{QlK~LK_Nl~ zdY!>6DUfolEE2G0w_z@1C1G-<+_ZQl#L%=ZmjdO^7xPgbDwqLjO}L%}cWy3(@`YR= z(nM$_D$odR0>PJ*o6Rsr2@L1w=kxOgeA<)+qhhfbMlcw|cz}XuE;KNB0ncFe2Kuaq zP-&0Y3}(L7CccK&!&DlK@)6jsNDkaFQPw*hVin_=w1ts+dG$*H^TE@_fCC^$0NevB z325MiMMIjT+(=~{6q!w%8Jk$p!1+iZALYXRj{Si?0Or&6fe7^59SmqB@iakc3+3Qq zkzoiX<RM}nB1%Fqi9jep0{u}$f*`^0W0W9TfHfS98DQ*q2G&88HdcGOJqz>iXa{Mg zO-UdHWXXnH1D$UOH5zr4ZCbOLYt`9L6AZHv8HCs~V;CR*-%-@-*}T*n<KrcAT4T|J zdo@}v<ytHhAra#QCI~_Uc_fArJQUSxcnJ9Mv_ybbE7l4KOo;r6aQ+EXRG{Tj>~tBf z17oO^OORUJqGSFy%2G&^8K`32e&*~XG>>34T1+}?Arcyx)R8QLW{{vLz?INQw6sZ& zGoTd=gck;N!j%PsLxy5>q|938p*qI4&{1r`Np!d&OUf<a**ynoVx=Z*si31|wuO}s z9+35vM6qsxHUe@6e$=F79Oy6l{dIx{Ix`5rl$+1?-6!r|@j&B$^6KSSrzAKrGe^zn zIdB3b0@EX9GJ{z&$bK1M82ad1mer4Kxb~pv>Je9IV1s$y3iTNm&bD78ruVo>Un}2R z^**NWTSp&17V!M>4Iz289xoKnN*cA~bYo&VXZuoQZ%;;0WXu-tojYR#-?_d7-gS*r zb|E!YaWpCv|GIRy_B!{(iuTJl+7kcq+OatyUp^RC+&qkN-8*)>jF8W|+E(KP!Mzho zM}~c)oLP3$@$(CN&nKVywW>L9(aP*S?kPnQg^%N<^kES{`~Eh?^@lv3q~)WGx-GSx z29f)PuKm;bAypYJX|>MZyuDbMzI<EF>>1};wY$DNdF9f_T`uakQZH6Ki5(jBICv1O zW^e<6n+OO_fiTDp68nmSRbWDgl@Kogf)k->&fAtODC>17yPlAg7V6vlynp*8pT_F5 z`HFVSr)w<QJU<Sc3{A^0BvfwbE4lo|lqHuNuU$TPAY)p6!u$lw>!HMugp`N(hmQ0c zG~Esa(-t`hPE(OuaEDVxB!nUClYs`KKd@Q_!O;dS4NPQbWjI<6r3WJzA`e3Z3Sqb+ zEDXaU1gJtJSD*nw0pTKH04A3GF$H*C^gm@GD>zm)1ZjP`U<d?`Q=xPeXl36owtV~D zYt@dINX!oH{ZnGW7R|ziJrB38>E$>$jCbHVxO?C6adJ{rjDrp%?p{t)(`GsPd!H7J zd~3~`jiV};=8af-!qIEPq&f1%nLFmpD*5rb)$QtGV>eHnIAl!iD90z`Xl8H|Q(yu} P2}D8+y!0ay6`X$qbSc8W literal 0 HcmV?d00001 diff --git a/packages/media/test/fixtures/pdf/make.sh b/packages/media/test/fixtures/pdf/make.sh new file mode 100755 index 00000000..a89c2b3b --- /dev/null +++ b/packages/media/test/fixtures/pdf/make.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Produces the PDF fixtures the ingress tests read. Run once; commit the outputs. +# Every producer and flag is recorded here so a regeneration is reproducible. +# +# tagged.pdf Chrome's print-to-pdf with a document outline: a structure tree +# (H1/H2/P/Table/Figure), bookmarks, three pages — text, table, image. +# scanned.pdf assembled by hand below: one page whose only content is a JPEG raster, +# no characters — what a scanner produces. +# untagged.pdf Ghostscript from PostScript: two pages, headings only by font size. +# encrypted.pdf untagged.pdf re-written by Ghostscript with a user password. +set -euo pipefail +cd "$(dirname "$0")" +CHROME="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" +PNG="$(node -e "require('sharp')({create:{width:96,height:64,channels:3,background:'#0a7'}}).png().toBuffer().then(b=>process.stdout.write(b.toString('base64')))")" + +cat > tagged.html <<HTML +<!doctype html><html lang="en"><head><meta charset="utf-8"><title>A Tagged Paper + + +

A Tagged Paper

+

Introduction

+

This introduction paragraph carries enough prose to survive any minimum-length floor a chunker may apply, and it mentions the retrieval budget explicitly so a search for the word budget lands here.

+

A second paragraph follows the first, so the section holds more than one block of text and the paragraph splitter has something to do.

+

Method

+

The method section describes the procedure in ordinary prose without any table or figure, so its page should count no image objects.

+
+

Results

+

Table 1 reports the measured values for each configuration.

+ + + + +
Table 1. Measured values.
ConfigurationCellsSeconds
baseline8034.2
four images15917.9
+
+

Discussion

+
A solid green rectangle
Figure 1. A solid green rectangle used as the chart stand-in.
+

The discussion refers to Figure 1 above and closes the document.

+ +HTML + + +# One fresh profile per call (a shared one blocks the second launch on its lock) +# and a hard alarm, so a wedged renderer fails the script instead of hanging it. +for name in tagged; do + PROFILE="$(mktemp -d)" + # Chrome sometimes lingers after the file is written; the alarm ends it and + # the file on disk is the verdict, not the exit code. + perl -e 'alarm shift; exec @ARGV' 45 "$CHROME" --headless=new --disable-gpu --no-first-run --no-default-browser-check \ + --user-data-dir="$PROFILE" --no-pdf-header-footer --generate-pdf-document-outline --virtual-time-budget=2000 \ + --print-to-pdf="$PWD/$name.pdf" "file://$PWD/$name.html" >/dev/null 2>&1 || true + rm -rf "$PROFILE" + [ -s "$PWD/$name.pdf" ] || { echo "chrome produced no $name.pdf" >&2; exit 1; } +done + + +# scanned.pdf — a minimal PDF assembled by hand: catalog, pages, one page, one +# DCTDecode image XObject painted full-bleed. No producer, no dates: byte-stable. +node - <<'JS' +const sharp = require('sharp'); const fs = require('fs'); +(async () => { + const W = 850, H = 1100; + const svg = `` + + `` + + ``; + const jpg = await sharp(Buffer.from(svg)).jpeg({ quality: 75 }).toBuffer(); + const w = 612, h = 792; + const parts = [Buffer.from('%PDF-1.4\n%\xE2\xE3\xCF\xD3\n', 'latin1')]; + let pos = parts[0].length; const offsets = []; + const add = (n, head, stream) => { + offsets[n] = pos; + let b = Buffer.from(`${n} 0 obj\n${head}\n`, 'latin1'); + if (stream) b = Buffer.concat([b, Buffer.from('stream\n', 'latin1'), stream, Buffer.from('\nendstream\n', 'latin1')]); + b = Buffer.concat([b, Buffer.from('endobj\n', 'latin1')]); + parts.push(b); pos += b.length; + }; + const content = Buffer.from(`q ${w} 0 0 ${h} 0 0 cm /Im1 Do Q`, 'latin1'); + add(1, '<< /Type /Catalog /Pages 2 0 R >>'); + add(2, '<< /Type /Pages /Kids [3 0 R] /Count 1 >>'); + add(3, `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 ${w} ${h}] /Resources << /XObject << /Im1 4 0 R >> >> /Contents 5 0 R >>`); + add(4, `<< /Type /XObject /Subtype /Image /Width ${W} /Height ${H} /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /DCTDecode /Length ${jpg.length} >>`, jpg); + add(5, `<< /Length ${content.length} >>`, content); + let x = `xref\n0 6\n0000000000 65535 f \n`; + for (let i = 1; i <= 5; i++) x += String(offsets[i]).padStart(10, '0') + ' 00000 n \n'; + x += `trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n${pos}\n%%EOF\n`; + parts.push(Buffer.from(x, 'latin1')); + fs.writeFileSync('scanned.pdf', Buffer.concat(parts)); +})(); +JS + +cat > untagged.ps <<'PS' +%!PS-Adobe-3.0 +/Helvetica-Bold findfont 18 scalefont setfont 72 720 moveto (An Untagged Report) show +/Helvetica-Bold findfont 14 scalefont setfont 72 688 moveto (First Section) show +/Helvetica findfont 10 scalefont setfont +72 670 moveto (Body text line one of the first section, set in the body size.) show +72 656 moveto (Body text line two of the first section continues the paragraph.) show +/Helvetica-Bold findfont 14 scalefont setfont 72 620 moveto (Second Section) show +/Helvetica findfont 10 scalefont setfont +72 602 moveto (Body text of the second section, also in the body size.) show +showpage +/Helvetica findfont 10 scalefont setfont +72 720 moveto (Page two carries a single body line and no heading.) show +showpage +PS +gs -q -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOutputFile=untagged.pdf untagged.ps +gs -q -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOwnerPassword=owner -sUserPassword=user -dEncryptionR=3 -dKeyLength=128 \ + -sOutputFile=encrypted.pdf untagged.pdf + +rm -f tagged.html untagged.ps +ls -la ./*.pdf + +# matrix.pdf — text set at font size 1 under scaled text matrices (22× title, +# 11× body), the way some publisher pipelines emit it. PDFium then reports a +# nominal font size of 1 for every glyph; the reader must measure size and +# spacing from the loose (advance) boxes instead. Written by hand: Ghostscript +# would fold the scale back into the font size and hide the case. +python3 - matrix.pdf <<'PY' +import io, sys +content = b"""BT +/F1 1 Tf +22 0 0 22 72 700 Tm +(Scaled Title Line) Tj +11 0 0 11 72 660 Tm +(Body text set at size one under a scaled matrix.) Tj +11 0 0 11 72 644 Tm +(Second line of body text with several words.) Tj +ET +""" +objs = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length " + str(len(content)).encode() + b" >>\nstream\n" + content + b"endstream", +] +out = io.BytesIO(); out.write(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n"); offsets = [] +for i, o in enumerate(objs, 1): + offsets.append(out.tell()); out.write(f"{i} 0 obj\n".encode() + o + b"\nendobj\n") +xref = out.tell() +out.write(f"xref\n0 {len(objs)+1}\n".encode() + b"0000000000 65535 f \n") +for off in offsets: out.write(f"{off:010d} 00000 n \n".encode()) +out.write(f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) +open(sys.argv[1], 'wb').write(out.getvalue()) +PY diff --git a/packages/media/test/fixtures/pdf/matrix.pdf b/packages/media/test/fixtures/pdf/matrix.pdf new file mode 100644 index 0000000000000000000000000000000000000000..63f85eeed7bc7df20b8043a671180ec6b0245af1 GIT binary patch literal 752 zcmZuvL2lbH5WMphb1{$}8i}$bBS4UY+ZY9!qCu=%&;xC)9R@O4kaXholRfkY?MJ#R zIj)nGAV6t3JF~Og(e=mKdwp7}(eFP$e<_U|esQnP&yh7x-wD|VlT7E=$ga(rA_@lF z;Nn7&YXvC(FB1k__Sv=(w_|~SV-~-43Hm5eZr&z}Ody61I@uR$t(p2AZga-hy1?YD zV7?oQejAotkW@PJoY8-1l%mKwvRT$8W{ary(dV)S^-{g($mZKc>TQd8jcjTn4L6^t zd!S^O=E#GvKC@hrTc;h#r@nncqFq9y1jC2@LGX^Sbu9%L#DQ#MQn0(z|1g#1AWBQ` zT68Q2zrx}`{Ak%hqz4M7!=n#v99S+J>D%7Z{^56lR*IiQ<%fm3s>T&oIK(t}NEY0` zSk1kE71IS9eibXq$(z_%AH`BIwxiIG>Reh;^H_KwZaAJfUBiUKxYjP#l%P? zZax-e_+krj=r*?qI4o-BjGKzGISoTZoH%6&Obya@=iXZxkNL-z{ejy(X_|A-`M&S> zJ73Oi|M`3p6wz}e{&IoO-L5Ow1rmrsV$E7XXebmfDO!g^ag<7{$|16#I6;+#=R+w1 zB*CyS0iKsZ0?FsG1R5ufRcGYGR2d0RLn=f;o)$_*5y_p2M6!$qM^L(scrl)#Rz(tp zFcm}iK#3Fvg-DTJ5}r>KXwp$bWT(sH*Q~|UwbXoJt^~@-s;CzEL8bGwNHHGzB#Bq{1EKk4| zu{a`@={#@&EOLNR!-{9gMLRNE7GQA!n++^1ID87>%&}a0q&2e(zsq8C?JPw05-SHU zN2lr1z@Ws^&m$1YX}lN!EH=3=8dn4WRQ?taa#*MrjtGQ<25@my6U=mSIUsEJQx+Xq z5o-7TFZV=k2Z!{r6@hu7|uF4!>!p|{*MwIQi#;LR@zeP&&e)y0(9 zc*?X#i|!X>_bo~}n6<3Z!c~#kLz8JGp1k6_R!Uw^*RoZMK0Hd3X(ia|6=WvUN)b#O zl$DGdh5zVoxV4uK)jw%I{`RU`W6e-G@vF`;b*TF5+Q-A;;sN8*@SyJD@XqI(J+^fk zny2Vn;kbA5o0fCjBKOJ@B~6tvH`_#=Dryp;j5++sx6)eZc&|9DMbJObL!~HOzPjG) z+Ed$q?x*%(RZk?3-!*ni5# z7?sh<1a4Qh>YnMI4)^N2Zgte?8kcqSRUgsqGXa@#)iV>Qdl8=1Yv`yR*i1YgHUW=r z6A0=znt*ZM*7@UZ=D+=D;i+i&Z2#bC@7OxO`q-+u4JS7`G`h_FRJ~8ZqNN;o%oq? z)iz^zU+Im%xMowgsGnqC=ERqyN#9}2Cd?s|Rok-Tto(W6Udw8M3|#MwUhTTcchW%8 zOyFGeko{nXq1m9D`>_DPiKptm7n|Z&pHihbrP=>`Q6#tIH_sKP7Z*2G`PA+HvF?(s zZQ57>+q!A{+U|gd{R=+$G+9~iJ3nH>t~o1@ol!2dYrFJLTQ`sA-%nA$rFCjp+@EP} zZ>rF42pca_RBu|R%)+^dqTZu*61FfcoKe0WF4%;zC*G)$l{b`gPp#kRSnSo9;(Rit zHi5He=i`;i3(AV#whNDz_S^v7UDVDY8dd;83aOn3H0;29M&SSH!N?6_CX54&1B?TV z1B?TV1B?TV1B?TV1B?TV1B?TV1B?TV1C!r@aMMp@2+kZhGbj1Tc-+f4oH>~`CYQFJ zir*pryyYBM7=@{o+&_?Jpnc{U|>LiEC6Q0ku?dbN)DL!=UN~N z=8apENKr6x+?q6C{F(%lqJ-WEPODL=b8ronWlUBV<1i5A1_^?UA(}Ic`C64mOZ6$i OWHL zgG9i3rk22o&%iqRW?)-eK_eSWb1*GEGXo9X)jvp1$3Vx%_{YTnG14}c`u2KYfEqbd zV;x$68IU<}_D?d6EiInv*a2KTm*@gGdqAZT46JQhNoaXOMNf`?Q^h& zp`8(kjgFp&2V`pp2-6(FDKS)G+GLvVp4)uEL`S)_^AU{~y>bx2w{QVm{$!s7-zXAu zS-5YfU_-~FM^p+>!U=UfiR^|nq4ETm)(efitZfbXk>|S~-^Sg`C>pZ*CPr$eS0iZy z%aX5YF>x$)9>~o&3U551sw|;JM2NGnS7A$tN$Y54pMNq-RQ~PT=EmofuP@lX9=@=r zcV%B{WKFm<)4uqq=+gUUd5QV~WZpv3%3R0E$1w<5%afBx(LwTIiZ%Wa-V;AAF4uU9t&hR<*gRes8F5U(1TD*DVIopdTFD~fj&bhy$R-h7{8no2GSnd|sEqlCF z)kJ!d9mX3u_;~Ew%O2uM8C6V~b$pDXu%0g$?$10-6C{laDPy|pSN#wms;vADS=OFr zac>xqzk2rI!F*+4pGFVrSJU@>G*}N$+w7B5`S4iu5MN_dcgHh#y^e};$eyib6Bf3% zQ|h+9zs!Kx;IjLKI_4K#i;wy898{O2qR-8nV8aH!4vm0sg*kp zbul;7qNXPIwO%Vszu3k$SJC#Ry9r*vG}YwEVH5O9>;~r=%4=Dse$f3b1BpPoS_ag8k|^) z?@_(w`S70`M9UtDQM%z4P`r>q;-TMzWOM-QFX#3le4$lpB&})nyoAK$(0g@dfj?~J zhb_|6{m<+Su=(3|Rwn<`&SE>t?-?6dPQmIFzL_{^@+0x)smV82)vd55SYL~&TaPv; z9hTMG-Ey+)i%XsCtSr2QhKFZW`J$|hGEMAa@RV{`oqxX@cH_BPH-AdlpmPULz0U&| zgGa>qT&geQGc76=9+q$yhX|hVM;nO4pKpW z`q9f1ucPt{1BsVFP^z0o+g(918MSo!tRZw+Sj%kRg$6MCI)e~xi+Q$=GzODZf)Tc& zGxE%1Tv=fS5}YIxYw)0myoFethx|V4@Y(0C5HxJ_%66=J8lN8 z5Y;CU(yqm9r@L*U&wqPwa4HKKBmD)(kDb)&eBDMw@ezU_Ir96VmtCMP()W}z&UX%# z9F)EX`W}lig-eRsiOV|CMH;7Hyc|z{_cZFhOm^c2Qwkpmj=Mr?cNECrdO`+%-M_e zjMh?z1={d=pt25~^|qAFTqLO;q5^c_GbWuX?6O1{1An{*_JR0@lDFiopNO^+w*-&Q z3Pp+56^Sw@AigJONWm+l3b4l^2~CVZS24^HTlS1_4=v$O>91n=8Ijwua(<1cQVZ#S zb*#bSL#%LoH0_`L(cp1x54qfYW%Mh|m&+Me*@&EBvp8e?SUQEG5_}Id7p33f2E@P7 zd10Cwl=IAg5YbULC_2A9hx9mR6ePRyiNbpiJiDLUk}T4nedp}$tjoEUW6?J6Kj8G% zq#+4nM4qGo#}(gY$wg%F zZ%I9qJAkyawX&z8srOvhBZo)%v_Skar6i4CHr#2Uxnr@h?wBthB0MIA_hqy2>^(I_ zwL}jQK5fcr0$6=<&o40~XSIh?G~$Q^mdj8?eCXx5Ebs){y;3Jn49FI{W2I6-P&)T* zod)RmE1_8~a2yOKkek`Q-`P5F(POPkAhoobfKhGau=hvW+gOxI>%7}Bfj&qlGnp!U z7Y75!j}mufVlFH)>-*i~q$Bd>{cJppao!4b_$M}9uS=b?vT?h??sd0#mV)u|6;pv$G)3J_8#Y8rgwZn1Jxc z(#+CE&PqoQ3{s;N6%-WEu?2!3`YSccY5-Hdv7N0n*hbLO+{)4dY+(mt`@Kq;m~Lxy zyO|#i0s;vNq&D3dWm<%VuacVm*S|}iF$|pOe_)L17h}Hz5feSz?IF`6r%mwa@A-c@ zM7P!0-Qi7b$#2BgFxZ8{AjRV~*o`yk|P#fl;m*^JMl zB~2gZGOpz`_1)$oGvJK&RY&Hgh$_ulcW(!^mtxML!_rvfrsHdpfrA$7!S1LBcf*aU zZN>TSfXqB&J;OcOFsH|=Y5A5hBzR{}m-AGLr5dh<@tnUTyUqa#o u{f6M zNQjirmp!w5QtDO+^}VAd`@Hk%>ht^7FPePa9xdvaN!u9nBr1T=;E5qntsr7YS;+6s z#&#@-KuOGt`uGsgRWmK8*-Que|zH)yc5bG=z{3>V-1hG{>BS*q(>YYax5`ro@B#bF%*)My$G5 zTTE|u!IVc$_Z`aCc;5bl9#CS+cD9z^#p8wS#Syr-w2E%dM{*iCJ@dBst15q2^xpt1 zeT`OGu&t%NjULz*1O(?m+^Z)CwgVt~74p)`gPrU^wAZi}`0*+D<5Ls>t&o$Qh@9P3 zAPih8!U(zsGZ3=U3R?mQ2960cfv(Uc3mjnvG5`3+0%E!P#smBTVKP|XSVzFp38Z#4 z@v_h}fLPfXfUX6B9so121zp9x!1Z!=HuidU(mFQR2k8)K<#lvzL9}u}sD6D6M1S4= zuXvdaxbEkQvnT zuWNPQQv2To@jp~H{SUPKS=$W%1NN`9|2K8pUg5|dfFdwIiW}?Og4C`_1zll77IZB# zS&d(ip}*49pE|jrL&ywl4j}MqO=fX(1V9~#{<;_7n<41BS6~Pb-L2042Ee~`_6*P} z(8~{ei-WF52dv`CaIc79_?t%m(!rmoVYo6M064d)0ak*4bapGOy_voK(%CaW$hWDv z9{n~ox8jX|?BLJT{E*0PY8Y-&bJhGiK}-LIn4ga2r)d7B?Z0sIXJUSU>NYVzJoJZX zZq0xH$k|_s`HQ1rxV19>$Ls%0%(d_Pvz~yd{10NTntvymzv<~3KesygPZ(zS2Mjac z>g?Yz{8!QZaCJYqVf;@JX8Z-=A7lYW@r#)2=Kms^>(OsPnDIYBnDG~c|Dfg;gs+=_ zN6lYEb3OQNV*X9he?kr8HRi5S^9M1vAbj2YJ7NTNtn7e2t1GKw{9z5^AYhm5#&7?y zD#rf=9mXGo09N$}Nw7FRI&a|3Yt&juiv5Pb?Fw*Yp@=&t`57$|b*Q5X6FwJHlO2O#fgBEVm^1Z_fO$nr6E8=Km*}X1c{R|H*`z{=tM~GBea4~;%4*$q3BgOxg zS>}I>mj2A_^{fYI=awe~HV6M;_TSvg|b z&HgRBVE#|uh4~-63;nG)`nvtMGg?gDpTdzz(a9g`pYvDk%$M`4xZtBK4N}u>2?2!}81Z{Oau%Jg?h-r>$FI^-s#g zLDw|giuA8re%H%S?tdv=9He$FGg;6LiMLeC@}Jxd%P)8Hi^E$BhwJv=(e{^B%=PHE zc)Q-c_}%qSE%ofpfh{{AKMiORL=LvN)+7B7#{e*TvyB3}*}niWT&o1ca7{Oe;rcYt z%_Ackx23A$dw z%YqngX0o3XH_K07r*^Yu0^KYqK{rdeAIm4u&AJ726Ul>aVq4Hna0X(!wk*)ig#Ke< z0$ooSvLNPbSEljDU_Xsa?Z$e5tfIAHnkNsFRW6 zXY73YC=+n>TFbY|xaA>kHtMbkyRFG<$ovD;u2`0}v;+_a>=yi(9)1Z!64<~N2RmPF z0RB1%V!2lI)w=NNY7pzSsDJq@)+-eK9O|Z81t>kat_ZnWMFR+a3bqCI$ZxPptE{35 z)S>`YB#iU`o>-WeXjs^pudo8h>c{3Z>yIUmoX`)T15&)&T4%k+->s2vTldY#5@0(c zOZ|TulbxN0osE@|4%lp^XQW|gU|<8T{^v1&n47c>*a39S9WZa*5OCdci-4c@0xYwC z6zTx^0edq$+dn9{!Sr9aW(BzZ9R+{v?OaoE&Gk~EX}pp|PX{>u98x#Rz%M(H2*PR~lu1}Kt+jg^Luo$*Sg zx0LW>iUAb=bH7;F*aiU4Pc1yxx%%+~w>O*Bw{}rjuP*SNeS3pi49cx{@bq-|$z!Dh(jM9mLA8;PRFyuI zu)Q0Tac(XfB>jv{a@fDT)YrdMbeO5mm@rphh@@hNIA#d0=x_}`ZkODAqoKO`l3m=z zpnvQ1sE3u&#nqJ$;t4iXKt3L(|KnLs7gzKY5lFkk+|9X0!o{zU8IXnLhE|blcw`MH zwtOWx3vM|qGo|@i25q!nBV2rEmlCFy8ixbXvq<;mV4rtPJu5k`y@O47=cNDe z(4heTB4rVs&lFPr5coAcj_ZLs@EpR~ZIi{ECg+_cL?T?uQQ4+THQv#9670kshb&hsI;A6 zA(~K7(&#<;n~WM?adkoN>c=u^n4MUO_E1VN0@8R-5e!51V{KJBx5QEAODLzs7@KK-qIc5GOj8 zHHd{mu7*f!K|sNEl78i^>}zrG>9JX3`%{5!+aiaRwyD;*oaBc1?``sXv@e9e&j}_x zemj4j7eF9GXu)%gf1J2FF^W3halDraX^wo31=n}k?t7UO$ZKuP_1$bJ(d!r?x9!78 zCcRbGt5pMetK$^A_dExvXLE`8oQ|mHu0+MBJn*hW>N5{3y;^hlK81Y6(~8Gl8gJsf zbp6DfkDsFbHFGCv^yEp`-Xpd|s8Ej%s{lh!RCAy9u_QEkhe>V+%Zzvxc?W;U z06hS1hu<`*Xy^I1~xEMg&RPa8*netQ9dt z%MGvZJ@_Kfi1ehfcfy~5%{TIdeXbGb9L?hqV%0|ps5dgO3bItx>FIW!DD+|GV7Li> zmadJe@5}WCqBy*fPK?%sn#-x!D<4{snBLzlK~un)c^Nw41AT%CR9LYE$~nOnEZNtF z&WHOCedNdNdgN6bB_cE;Ks_QBoR!2f3Ip~6f6jDU2?pvb;-K$vd-N3vkE>DvE zU^I}3bPg7dAn~IxX@*X9wGF|#AkmmMF^^B)L|DHL7GYD2HOzd6=s9LX2OL`f;z>B$ zCs7g1Vp*0z`DYI)Qz^6Gn?#Tc6TAh;C#W-l`=J(Y00o#)Yd-^0Dt*N;m)HDWh<0V5!#1M@imJL&? zir}qdP6?KOd*ex~fLS@m8s22) z<>E6#;&6bYxpPUFV9O$?+iDn`|!=xN4 zSbEn(=Y{IC3Ta%)Pl{#&>#1_{qK=JST@!0rzHt*Vr-~M+4vUEJ?{RsX@QnCij`i+q z3NaEG2Jg|>+&2u5#`dGy<7o(-hiR{P6kzEWJnd5&$i|95GDl2rn_@2B1_m?OES_rMT1b9dGCAcXhXw{X(zslKld1f&8IT%@8EVb~?p+N*Q@& zyvO}^?j`+ z&BZPV?1PvLZO=Z7BK0z?f;Ue*xX+_HjcqnEYkF;#P8UFryQiB}y+EMw zoOCj#=%uS38p?-e-4|Iil=CE6+esnnLA@F4N<(CPMjG6$(t?88>!(XaSruF0w_tU)pbtjeGE zn5w?va?^kQ%G4GQcjli^jq5|o^oXM7iLy5upSLJvn2zvqG>;MzBYJQ~teTlw5I88* z1aHmqvnEagYeQWYefMVQ9dTWzR1tMFpNV_o)Nj}G5B6KxzI!{+e@2czpjztai+b~^ zE2-EQLqja9CY8J0UPLcfd{2iejJ>TrcE7wrBW5HxJ)?%!G=$#6*71v#?eou5?2K6s z6spIwj6S4!h};C*^ySP`gsFLh%b<}WGQ&YC77AvK9Bz7_3D6O8!OrKS*1{C-o#+_} zT}M48f9LJJ=M-`Z-lW9&oF;V{+MUk|d6Wy5%QVLcI?t7oMaRA`$BLG5x=*1$G1l0n zr%iLZPfVXhwYnjb^pbux`Tff|3hbS@h!q9dC~s1U3;LFje4Wh^?YnJG4GITH-y!Kg z-xu+414|q~4IQ(*fR1`!oJjQLCAPg(6c|0Kw2Q9Cl=pcl6{VbRPDpzq8g?VK7G{*p z2eW|7c5HiCF#4O1I8zOI$^ zTfRbOHfT$0H;NwLG!7z27$@W$i(Ls7_F?cQdNs0SJT>$s9^2M9FhmVw&5e-=248Hr+!Enp8KqwOiWVdxcBQ+#QfGV>IR=!meP}_ z=BC*%wynI8xje>sYscOaDX#xq7AOA%i(cKC8%aq6t7UZDz)37H#Kdng-h-$`u0Xr; z=nEr$ed3}qDMK@5%kXz0q;U<@mLYq3bq{lYJTd(dDy06CWBM5Klw zw=&o^(qBBtNs+_$j0?;8YOU}&?ErE=3vz!LdKDUa^&O<`wr7<|qq=2Uj5IPMfg-qo za-USL>@;U|waKJJIWP7cK_=EXnzBS;BWfshwM1s$@%^gE`_=pp&ekJ|Sj^*`M%aFI z!-n$CF_e~a;ABdy4-6?{3Hnq_-)jeID+bQf1@?mk_M-&$St&3?v>&JHv+MHMwyImT zCc3n`l60RlUEIsk56o<)6W}H3MzggTBZG;%#Q$P-mmx{qfT)$z1}4rO@5?SC!^Pvd z*R&&sL!9da4`5#45tL#>UEDuo3c9?Lrz3XBktcIFeh;Z|_+aXt-qW*|YGCguRal z{EyiQ*{MF-F~xDCq2^=g2X**3BJ@im@gK3l;08JBBQUse%)O@mSV}r4L;BIqrB*LU znkY@JP>NtFBanS2#O>f73~nQjy#-#OxVhFV4U;o6qtNVvRA!Si6V+6kK~&Lg#|Ka* za}Q8^?(3^!qDH|p2ra#~;m=iFh9SfG{+Lb_8#xNG0R`Ky!sq?&=X^^_;->E9 z7%rUVZpeh)$d?*UO&*GDbFYbA3-k)Co)ii=u@uU)k#yol$olzsAuMjbMqk|fsJdv1 zmQEw##HNcy7_^1S^M#jc=7BO5(plu8qo~DV-?L|x>3Ri)^s&@t@LdN3a?Rhg(+g=x zh6Sy4LbV=$muoipZt#>&5mPFKXtpE7s)>gzGd5 zMokZ%qq>36>32{}M2`K)1v{o;syeUfE;((s+3BY(et zilHN4zeT{?**o0RNZfe7Z;{C)dEXrn>a=Vf5s&HbIXFx|T`@PCerk+2%_TF{b7wT7 z=T5S;YnR#X?h|rWr;r_M_B)&E_D37#@vH5oFZ1uJxKdPIDh8NNYAU8BPC(}&PV2o| zd8!u~CGd!B3YSV&6HQndBge=DU+0Y&ra95O(A;ceFIg!(ywt(53A0h8Gi9*c5@hFKpaKd^oz_yvjojsp=O|bR+fyZir%-t* z4_VqB#W+VZYBb^DN2 zj`Q@uWu7b^?@l*8M0G!ZWj8!#H-uZ3@klWA-4t8!Bmb90^GE;~~9pb9g(*_nbH$KT9z{G&#@O^yNhb znmQfh;^T_vTHy>KcO;ietfOAig9xx!5v?PClHnc z`l_^e=TEE82omZZ`*Muz9m=Oe8Uv$H!BTEzu;Um&SgX zv7oTW*vviOTk5bMTk3cf6C*>$5%_>TJ3Bu+qcci;6lbn$LtaHDoaj-3VrK6%M~*}s z%}?hY=EO@i2C%l->rauB5*cfpb3MRGqeeN{1@Q4DCT**G;H>~Ve@(o8qU3%l+6iD{ zwEEoXt|;@)TB*9rHprr(*F#WKv^tRTWLvvSNuzh)l_=!io=_H}!1l+Z=(_DPwxC`sTT3h|KDbjeP!;GinGVpr$)q^xSsM{!HKOB4}> z8OpKUjHjeuyuf|n_99=eZeL|hO|p-oB;oA=p3QN(s($&`LS9 zG6{7CbR!$MoTeq?Plr%MV2$^w1jGL-l_cBu!!MtD7AqIVn&12@whFNUO_1GhU%C#CyNB5 z0@u;fT#Lf#*j~+@cxH?)0y)UtJN2lNt}5{(eaHJg8!ki4oDBNCw(;W(Og$UELY5`# zTW;Eza?$KUMBOA(tF=A(J#KOF&*KiCu>^L1<>@AGjiySLiSzF?%Zqv^?kw8Se;%=| z!s)lwy}h{|u!URGGe-CtUqv&sI9>hS{_spH&2d0d(&Avz;4bh#Lb^oG@aI;_Uh!NO!;VDSxkB(0k9}m+ zp6MjT?&HwoTKWoQ+rHh@JVF9LR5xkLa(H4 zgkso?e!_SV5w)#Gd!hV7{1ZuB9vfHcL%YweYf|%tVJiLhC?QF1GI1B4RL-l-!;)3P zn=}@<$B3O0!zwW|49=qpqb!H(Y>#8YJ#^2~IAOZdL+tX_y9;tZ#B+6Se3V{KkXP6q z`CR(G0Mx+OyH6E(SuwGqL09LYdU?{@6|==om92L04L#VrZ+Xhka_4(H87n(Ah4F>} zn=Hp^yJz=Ec}Bg(pvS;RGTZG_`9w{8oY=vG;S0oitEGrX58F%%CxQ~?HskT`1Q;^$ zFk9He`yYH-HFE)5m?x|&GdAKhMU@ROeRV5L!L~M59Kfa2#c>r@75;`l^Rlh_gL4tH zDmrrR&~avN@e$S`oCVcXFfFEAo8yRM^Ec@EoO(ysFIX3Yj_=63KR*tL-6XtUBJ^;a zI8vw!%g=1ZqZfDZbxhK`hnl9=6Eg!e#T!q85fFtUPbfq{RB8sZQZS*iMa>?CBzB+A zKODL`ZJCq9m0FZaK2!-c>%-j)F1*w^5TwlLyW5u&l}hm~0@9RW3!$NJ*;oNJ%g^r3 zgM;GL%k+{l4E?mTvyN`}G7onResd?B&?sgPw-o}pZi5Qm-PH(wEVu9~-Tw3bFG-HP zY65Y7`5SiK_zea_ehRWxEczvZ(=F`jO9d1)l}y(Dn~Rdo$h5ANLfN)yXA}!2f_T)c z?^i$MaF@{TCpbg`t09BVujSYw!r~VB$s145GCMQ(PiOpdWqfFvLs8N%SVGe~bJse# z3i`>bFbKc44rJ}?8Ro5V!u5qdcGS|a+4O7H({YhNR*|AauQdKj5-_}Ir|(1{*|97- zidCWTwpUB+y;P1kRh?BOt7f#}U}(~~bBl|C7_6LNcSX;adB#gJ#dgZozMRpSmU{8? z&tPikHd(7Z?pW;j4w4i1uvMX6I9hNga1RaVxE5h`_lK7P|d6>nxV>h!wS5K+-Rsy-Ec?#RH; z!p^#k-0?v;)0c{)VrG#0ILgFnI3lqY{}r(^>lIs8pXfufbdFB;sV&n_=W^Xha%-^2 zmnk%(ytHK!<70{nphC~FNCz!k@q}5f8k4{yWhD#+>MY)gN8g}S9BHzp*vo0}Ar>}9 zw7py6-Nff+HQ;7tTa7ohzbGAMCUf@Gu(X>RNnjA&j_uXdE4UkK;XG{bP+C$_vj0iK z#BprG)B;t)wt>!?k(^1Z7C%<3>ivfp19t^`u)CK{Uk;VHDEAIZCaa)4lxOwbHhovC z)ggWR!J$aBln5i^7Z%frS;Gds1=ZlQfp!&6%XPe0&QIb*9*MW8fL#oZkdxdURt;!X zP|z`nwyTw%iFY4m5CLXZ?JJxm5wnpgPu!(v{McOVB=?O24z#UTxC8!wydW*_wG^9jUjkanPJw_lQuShKF*`R`-AjO;1AG5D~g3e)|&zSo* zTxO*_8!s;huZ(?EO+6H&t<5AVMG}ep-RURuw>z<}8v^|wvK6DriIcLd*#uRvS9HJg z^ZS>+e^+4mZQ>Pfm-l^hRP?@?U9E!Ji}m+o&AN{?hR=(zrXBh{=#GXSA6Jq(o;tmu zkuO+GPth4VGWarr>V7?93p+we~zFSJFoK35U-Z zF}Ary@?Z?c0yD#A1d>Mm!}(yIzBS~e&s>>S`(XmlXN@w&dcd{HEf4leUT};K!e9-H zrVGqwh-Lsmi-$Rd9_e9enY)pUh9$6d_qFuBU)i>vW%MT0A|0Yw`Zv+ zFJeM1RhU(@!oDNI=)S4Fsx`2+yM8jjkCJb;Tj~Fsl-$*wTR@gm&&Jpaco@UYEj?EW ze-JJI^&geIF_8VI7O*ta2O7i39iJ=ve){DUy|$hLsISnFD!vAQx$@$A4Ak0|G9(qCw#Lo-k?#238toM&R)Z zAZjK$RvHEd7NDkzo|To2hK-I5_{P9~m3O9RV*^sISC;{aKk(HpWLLElzcPhvx5^8C zv3YaFFJ^&fCtO_y+=~g^nt6Q(+duLA$IYICqED|*y6TSs;0vfFuoMw}DxqTq+#sh9 zNW<99`TDS&vn?QmpGj3&u%R)a181Pl3AoGWs;WuK2B_CHzABFSc^Pm}&i?9=7FU%s z*Y_0BDqYocsIjuL(y%cy&|Rf@>44%DI*qF;6g5@`1{!)6R(4j6t5PtabchZ(i=FMN zis9;?g@uKN9S{<`##N<*8Z$Es4e%%npr(b1iIIkliIsH{I7yMgXj6Cl{9Nf6mj^@`EtSKe1qtmIdMH<*YDW&<8N=4) zIynlC%SUv1w$O3x7TJkJyJ>1J0dct~G74;6D{oul8W!B!nZWCgKrfY96x~x4+{6Y> zDR~YVFy%npZ3}T3SQqE5rR_MlL~tC^;1W7FLFiCXJjpoM)91G2$>OOk1c}Sp-aS56 zR|XCBsz@+z@XmwimOQgde63)?XT)S9p}5SF`3mc5)5P6=_uhNf0yoiW8y)gqJkN5; zB+PrYqX$`aFN<7J3``RDlp`YgwF4Jg=d*}J+T|#$_i||=He_9392wj*=laC+qoOkE ztXeKP(o?(0dUt(-6Bbftmsc#Ud99tfjP#flZB>JLS%>Ekq*SkL{%)DKOHi&8_E%WD z!o`nUjjvz_n96UVDr5mX8srLDe*lsixar~w@PPU20LB7z2^8{NTgTNs$nus77RFak z69HXo__hJvs*U&~)y@FitqIuBZyrX%%FYNBo83;eBfHACRUPm5up9Qf>$4g%%*P*+xQ4tOS~#ZPUb7GXrx^-aQfrysW+kI%c!vo|Eg8Lq~C?>S!sJ`-VK zARd@O-r>lPC(3fwp50A`+@*(DUsoNj3ajl>hvrj;P@WyF8QizLn7>4Za6yGwQ>)*p zX6_wK_kj2~3!iejTRoZ(!)r%(^u*H;%9B0Qa56q&Y_5nMDaGZ!r%P65jqT(DStv9_ zB`$=C%ZqAG>YPJE2rCYV6PGcw+N|_&GbrtIh)T1=sr)|YRX<0laC-{YDF)=RrPeEKBjqOM!)gf*(L?Y$!NmF7#~l#ARTWIq!yn8EB**=YRj6If>;7?_$uCyO?wqfQ!9d_gsYpRwEvO??2?qg|*FAEmjYFlk)}`QrqV&WC0? z^hT(Z+}w{yg7`1qp*l1kgOF@>L{>+8I3c7VsdT7kkWsbKYR_z3WE`s}9BoJd?z;Dd z71Esre8okBjMvWaPDZ}MC#2`N5)-j;$?OpuaA;x=5fDL;$4zd#pe?VC1!HVqosb1} zu2_{fli|*|P=(kp7EmhT99kM0r4|dYo9VJhOgkS!kbA;EmEZC!geLSCS%j$#e9_b7 z=frK%o_c~KO4q@#$>8F}2SND|0d65+m+PQ~b^&@Ml@PHIZG1u0CcKEDCbmwU_Sh1K zVxQ+F9mZ-CvK8wLVn7O!JG5mB#R<}-_hY;Fh(h?UMwNXSt?6mnFPvV|=~GQgp+xWt zcb_BfLnSb}QtsFEsO$>R`=&sdXj3xrGnNVnfUTg-JZB4u+~Ik>A_q{RcTJrT=e!*k zr9QsL*ftVi4O$oCjn+XdUwE$m(!3Iqs7++=_yiIX$~QdqdE#UzRLz^5RTe|nxX;fu zGHCX{#2tG<65>lkFw@~=e4rXf;U0j_(S~*HFQ}MjSS{ zwb%S8&P?w-VaatUgsr&7bg9SQg;*yBG8SR1eS%W=HLt`jXaiTmI)36^96n*Xgir2> zJb^oh;BB(~OwO(gdP5?)8gaU;Se&r+*!U||hVQ8_wXR@pPkraxRdscOirBNqj(n+0 z$++K)P8N0tkrIr1w?1z5o>T5WTyKI>pu>x9lwJ2KD}-ce#&m*kXrh@wu31EzK>dyY zw}{P!+~2fsbzic%dzS&J@w<1$!m$;CWQ+01JRQ7`)tyUk{Lh0tUk5Pkv#4;}C?}U2+t+V1IzfE{8pC%}oo+TqfrBS`b%;1P_Uhl28rU8$>mqy=Fk;^1e zQNx1ggQ^X6tctkl_6tVGmv{^AZEh)hrYK)gcWX{i3(>LO`wr4pF;JB@^EhF?ct*<} z)GH11B-dB45{-vUHStx#0K!Ef4vWBYutsw0G#_ZGleI7r`=eu3T#v;B%_26vj`PQ#}^n!_~r`U+S)W{aEECfKv}frvv5z=JfJ z-9=?AU=-3~8{pL8f*B3C0|_x@&vDs>Uz#oa%ATOts9wF-<_ScHG)5S-wu(RWg;KMD zNYVEUv>Y8sCd%^1!FiBAnl9jpDH*d1SYxvV@=x&PcRW)k*ze4~(b*I(LdHrHFLXxn zG|WPfDq@d@(+j(#(p^)>R1-uKDRhQFZezhIxTmJD1kvckUk#&{138dAp^A|(ERf%6 z1)j)`7j{F#e$#&E?V~VlQ%ELUNkjpQzBf8+A-Zdtgb7m#l5JzgkWR>%&?VaEK@-gY z1p~6JXl4Wu{k9y1u@XxDc_?1-WvAYDcUj87+#)eg3!n$S;&LlfS1I|MaYhWf>eX<0e5WbAOnq3J zcD0(d?`f%^UxH*F@vf`rcau3q%WK&0mD~eNxBfXu?jbpYfhJXCj|ah@hA06w{9?pdU4kQD5fn|;!v+Qc9X@22?e>c>tOt33`v2LQ2*y+q!3!gQ&8XtO*KLyzB&; zVoI*M*H8*;E??mcBXt{0#-RQFf%CUd-QT!|xwrwbn(;}jm5I)}1->MM-Je!;lC=Wj z3sSNftm4_g*49F(he0jbY;Je^nfklXeBV$CXgy*;S4^pO%^+(-?>A1Ej-DD?mh6K=+e#!G#f)r$1;N;#YN1+-M6 zb0fBHZaX~HsKv$JyDL!6X?&0vo_Cq3D#2l@WkO%8jlGuN)ZVRN9Yj~uc0KJmP3_L% zYxC&0T2c1T>B9IRkCT51N3U*6iZrf?)?y!H=p;}DWgoS;{vE&N0j-YH5OAv5la|PC zjF(wRQm4SF9;8U!O08BcD>xQUlv2DHxDGwS7BlFPpvmbd()fg)c|{D!p_&jb2TDAg z9;ViPEAVZI`n|%#;|#T@!2&y?6)z3^%)~2RdJF7GR=li2n2fhTlR&7VntgQ7&mPWIVec&}%*z&$ugAtEw*8IHnt?-?AtKE*vJ zUpq(v|0mGCuSQQ46qeS~@b0-k=YU)c$2DXol3ps)KgHrudQDRqq<#z=^uZ)}xthsj zb*K0D9+P*)+_MzQiQr+ZSBFow`YcE+mR9rmtfmkSSqdFUc@JM$?YP+0BD5*%O zkVvS=Y`fvEVR&$c9nE^!QvZP6i1Kp1ru=TfhC}9ug*daXqSbZM^1DlRE`0L4QX5CQ zMGuMQeH^I0Uw#qbq=7;E4soVT*$|ooFEQV&kbV9bh92V4Vghf9VMlJWOCWz9>w7O1Fe%Ey4=&r6H zHfs})eF&b&sulEBO8{T%VWCvgG{?ldu>wB=lZ(Sju{Yxjlx6Ou`Lb2Yki}Cw9Pru3 zN*rv0@>Zf8$~2U*&pN1{!cGtEIqUkMVezTM*5H2n48JC*Zq7tTzypk>z~QRuifI@c z(!fmME@U>a`1~cv93#Lpo12?pf{Id^zL@g$>rwbuJs=p)ky0dmdj&Dj2M1iwlx#Ba zcvB3@&V7u73?J(E-f!&+pUW50)j;$;**~&N>}wO7bi;&&&)G({(bO(Dajqf1ESsgh z1kW4qmAGZLF?M93rL}!WqQnj-_>}CE*~SY?)z+l#cOoNATaOpr%Ia|L83yC-2RW(t zzypIRxFJNXRS7Dy?ZLL+C#mV+e9J^@t!knJYjMOX5+@rZS?uX&J9$U$xPMjY|ej=@^ zjv-BF^;{iig7gRmojRi}p8$b>5~TTnDL8m(<&!tto5M#1cY~1b%JSL^jun}I{8o6V z(os&cSvF?iHW86qqZ)B&Ic}ga89_{~EB^GM)AK~!s)#?US!o^a&}Q5i{0;pb`QjGC0@MjP=b zWa`ho-XO9K&qOm4;N1fi`nHptk=^H~_QFAy9D-A|K{e`}O(Avkik`WvS`1sd%rX#B z0Q^8wwRvD9N^2ylwVFg+dT2c)Qtu%D(V#)tf{>y%ZiYLeCSistq9*Vlo89k#@SPiF z+VL(@3Yina7sn}=&7{Le|4yk{HPga&zUA4iM^-lEtHk$>gtasmt-NHzW#2`aefq6p2Bt9WbkW=Emc?S;B zR;o$itGz6fW7)D6&Ld`4g1HJQg~g*r6a_F`g@^)sxDY~wY7ogV$)JOJYCuXh=E!1bN`&RrjF#eDla;kkl-YhcUu_lWYNoV612cAPgejB(f?li6<^j+T%c_@k?I&^+ z3%Y>Ukr&n`@#N~9SzQIiYxs-r=+$@)Cc(th^Hs<6--5;;GCMjx>m|tAuG6Y*vrzfG zhhKl3^xCoEQ&_qNH`Df(WF4_RYx>Tcc&>m7UWd5P9Y%)^SsQd*W3Kkoiyg~583zu| zHQx-J^p8^WIB7RF=Mde>cwXQXiclz1ZJQqTb<2NA8!a)`;=F#9tW26{^-`^}&e1)d zskGK(v316^fH<6Ah45@|J^b)JmgysGqA&^k6IzCpHRabjFU2z#g?wG}Bp)t?ITa)< zA^9032lnCy^C;f)_#(1CfRFz6hn%^C%LPu)Qx?7gG!& z;zUjG598?91n<9hl-Y|Cy>wt0SbSw~horDnY@&Op-g174JJ_YDRDxM_QR}$aYGQ#0 z(s<~8cO05D{B<|t;m36wo0CU%Mx#|oua(m&^F)jZ=Tc8{#l%0c`-jyaAJy-~(ro6GP-|Ss&l(U;o zH1Hu@Z2CO({%1X=f%LF8-*>|@w&=Z>B2lv}G*QbI7?LDq)9vhnOjuAQq$Yv!1cR@K zAtH$l=wVHJMx&nR3KaDEMo9(qstIxW zw}>4sn`7U~Yp=46^U5(T9bHq4OC6o+Oz5*zL1y?ZBj6ZJRJ_U*5Qqz}dGhl_@S6gTyQ zhvSmJl(Wq$26UP@uBZuc=NrV-rIEZRCLA<&E&9e^LOO+KjUuckS>nflRrWM>b)ALgK5k6zC%q_!5>i}ye@|RO5^G=T z(Id(YR%y)^Jv0upIhpvBa^YDC+=R*LVO$zjeup}nvq`n++{S?3HrSTV#s5d$JqF1Z zY>T39+jg(EZQHhO+qP}nw(ZrnZTITc=Iwn>ytDUy=e!&5-;EnlRZ%q~=8TzRR#s)z z_;RF3%c2^5OX641&e@HJ%Chz%*vXEJZ&@v8^BMg21_O`XSvymjay-qDmD;Rv)dX}^ z?(UMek%sXZ(=UXn5AdeM^DqFEAzKfD2AU&^zf926Q9MyS&?O;XDY+>-W;V0#y5pD6 z-n7R@sMHU-L^RJgvxkn|>Qh}?t~Cnm!vtgNpacke*ADaY+S1-QWn{38lkW}K+fe76 zs+K9MZ*yhwC1FxhgurG=L4)hXRl6=H*Jno%$CN;iL6klT#Uzp%RdpEfmr2P4^_%z1 ziFL*8rEfr}9;ubTJ2yDPLRZCk6_sFBkk(&^GLWU9F2#E%y{@x)+@@mQD5KK0w;4}E zW)+^F*h{}#Ycto;W|uiB>R9P+(KmCkd2icf;#<|a<{$4bF~ES9GnA~6SAj_(1T=FI z(Fje3N7t)cyEW`#sgICHLj(({5w&Y|5}LHt6Q_`bujgviQfMJTzGtB{8euWsh#X;BNnX1Lq6!`R{xje4NRnhIb@%dOAxGO~bMUg# zYWHcYP>0Mnq1Wkk!}c_@E`es?Tj2y{n~ADVHA|C5#f;dbv4KW~!}dY?3Gb+Vg0eXB z5Cq-57bN4@!$apUa0;aE51kRrux~wiJMkSbqTirdP1@EbQns2E83`23Hi8R=fBUz? z`9>gzZZ$Hj8ec0yg+NE{wTHLos*wAoWW0a9U~zE$=#42z!w9n_bxnQ>DW-5vj>6tR zrDWT{ZOulvYlwGL@Z+SPJ=q&;hx2A@|K`EHF9M%)(#xo;Y?1_-Xmy9vZRBRZFaA%l z=To-C)#s7Nkt(f)hMtoBjUPxw-7`?Rt0L6w1luMID?Gy* zEL>sJB~GTHj6^Tbt^r+ZTJriNPSkZ2F4Ywk4$lY=$!@FP=AYnC$T5d^hcKPFNLDvU zBRU5ORWSy77kOpQiCIaJ4T;2#+HqAX51qs#myE|~QEHPI&AV{&(HL)wi0x9huE0PI zf}F8H&ODT2-ikx}dTE{-IVGN-3Z zF3SivK^YI=9ko6mgzX~HMfJ-gD4XLsr<%XFKHltg9LY)w z2IRSsez~N}bS*GFEG=vD44#RX?Kb-UK<%=*zt)7=7)~HuI7C2Oe4B*JuHz%OE=j^P z^sPZ3HKWZHsF0g2ptd|^pD^b3kRwqRJeF=CGD8lFfuWZFUN+m4O4w06J_bz?=Qxt1 zAn6uq4Jk^Urj%v!B=KC(T|wN^OpJyaHx@8-w6oTcvxDB!)L^?h4j4B!Fl1PBqpxag zEsh^h7ziD|HeC?g8^iI$L#s`eVDW+r%tew4s?oQp>I`T~=&2c3raVv&9W8w4KRVaM zUOWhR9=Q7F$G-4zZX{?yqyQjW%HBvAip+1E$8rOczhs-_RkRJRhUXt>vQ zQkrA_O};t&AViTo&~!ho3XgRy3a&Jje1r{k1qx>w93>6uHQ2D#ywSVyhX$Eti0oR9 zGq;8B@~rKdE1Tm92fW}(T=FXM(k@y-2X*3<@lqp{4!1$DMzm#!vA23^O1pk^PQ(fZ zlnD5=f(33Y)R4$4HPiT0e<52Z8<~t&{oqZi7Phs|D8joHqzBnjFveYNLEefusw{d{^(J%PV>ZCR$%(>m9>XlfI40p79BeS-TmLP?m!6-uJzve1Vh8|v zZ_@ed85h6t)^cb0JB6o;qv|y6l%~y&)dtP;ujh59$XiM&D$pXB%@rZ65i|lXTRuy> zds-9|b#uPR$ytP2{P>vnI{&s>02iaZDLq9tuC%VR?vfr!l7zHoekGE%$FJA?&=TFM zR@31Kt)cNeZ_Essk9Vn|Jfk(*M`sy}_Es@R?=YPH!u>W~ACs3h@~^j;5H0W>ujiDS zh}AooUp{08o#{z?0q;kziS!I+efHAs3hp}Ya_m&*9XKc=147)KqJf07#C&9^=cE!h zJA{8QDRym*rff&AC)n(xiR935&cd_Yb4H0SrfKtbPT>~Vg5We(_csr`md9SY69S4a zoOiI-y@E`|mT;zXFJobuw4a*=gW)d^TB81FnC={-)Gt?u*b&m5tYFL)L& zGr8SO=gtee3_jBjrx8)L@jGT43qo((7=wxY;uB5DGGm)~e(XBhv8#lpym!7t+IQH?$D{5h8I30$fPqpIY zw)0GRJ$NOv{iR)>JkN(nohU2-u%TgX5)iEeT(NM$>gczMJ;mmB%w1hEl&9C5N${v! z!Jr1|=+_e7F2dlr^_*Zqz>e%7?K^PJx$&8NOuPhHg74IIT!x=p08FXgfgl;yRFcx$ zyQrK}VV%e>yU9;5>ML#^y>dxg5l*aG=KHK*u+z4v=^E5qcYS}@z%vzK-c?iR`54{* zj0$@Sh8mieJY8myaC_`Aj=NA}DqG8t=6fLk7(2G;^|t;2z2Z|COo+I3)3c|4|B|83 zkW1fDy@b7xy_mfiJ)vLMZSFRmLXi&0c_->pGV&046chT@B(q7xSBrRD@XQfhX2-zA zDa~E7_dzlv!gaU{1PmeU0Rsv+zddy7*Yd+hBf)qIUr0+uE*I(Wtjoo5E@}+wH}awt zsot;5YG0__Ey|0he`Gp62&PLLrp9+}bWZ94FE0T2vaPaI5gyQ1Wrlc-%DpSBX1=^s z)tz%9L?gy>Fr(xts*_|&Ni=DXD8ofO+?z4Hw=Q>$Za?7;&L570euMUK7XbbD4F7-l zjDMTk|Brn9KRL>O@|6FyH-i87*qa|o^6&QMUrSp2i)8rs#{T+`X#AVK`5)T+U;5$y zq`mnuHyrkEauDb$cfY6DJC-vND8&qsX?T-=(7i8jW5Ltw{8ScZG0KP0#EERot? zEt|pu_)>dM-hH`|Zw92}_e5b?!8aZyjgjR^5F zaZH6&gc6c@UB#3Ux}Gk+Oo`|D1B@m&g?FT&hP5y80c)?evOOypT9%@ZcrJn#fTW_}}W@){C&Vs?(?FxC+S#|GnS;fAoj{dT;+X+4Dbig#W*?=RXRq z{JZS=e^otU|Ce$r|BvkXp<>McV0ix<*~7q0&+${e@}IQl+6&TMd3pWLYi9e_+U@Cj zCU>3DiZLTYno)w3Dxkg&5<^Bk`6t&PBH{q5KxODq&}vwv+K}om2;L%z+9ug&$d{IC z$u{4*;=LX3OJjWhypy3{-z?Yn{knPT_jSrAbkp&i?KI1C+nWvi3qXFz^q0Q=Mz!!Y zdLkw+fF}$97+MXEwv0W|H=w@;K)dOD=;I9?jnxDoJs7}($Da!+Ro?n9I4phzaDJPf zqlFms@6=os@I5cs{F9`TC^cOUqd|N}fRkx}-!~;vVCdg!Lt0h)yS_h@60!J?t@q@N)}4il<{~^}rM$iN~KA z0Rlqk55PKz3&Cj?K$rGgtHWZ;9nLGf_K*`3c(s}#tmKLo6&2xsT*rCt+u1Yp7Ju*G zTDHUY0`CgYoK3c4@ky@5Bvl#dP=g6|LpJ zwA_1DE?44LfV0p}LmA|vS+HFQcE;a0d>+5K!CFP@fAl|8z8-wD`KAURqX0Zox{pEx zegihlsL}U#xx-730o+Q=68~fYUi?AK2?5{40hl1?G|!f>RIo z z!}vla8Svr+NZf&B0&MD29_f=<22wr%kMomxgXV%WSqH1w!*uS^-t{luO`S#Qhr)0J zim-#?u;)X18t`txJ;&h%Z2t_NoF5A2;|M_6gb;V|K0%4~nYQ3CtoOoX)#0iXVCwL< zfu`wEZ-UJB$i|sBU_{GQPh&P3)1kz;{F2_H~H=7{MgYf#}e~+Xi**>tY0| z#Y5}F^h!GxNr|NmX{W=f9fG$? zAc$^=+CB(Ok%l?mKGB7!N>NKunV6UWFd<4lIIVw5JyLBz);_O0=8iwxUfFnvAbHS9 z{<1mKMyT$HO>nMpc|{Q$001nki zuY`F75J%7kz32AX&ituZA3TVBekjfZKfHA4qdkdrh?W8Pm4DD3;Rk{*R9_I{@GroO z1ZbX^9kIRv{}%RWw!>Sbs&JcOU8HgF@#hLtOR4kug@}16QMcG$hPPdRKww~B2m!Q4 zd3X-<08D!5H+=@`5L8CLEqE^je4Id|J64>qX9Ez)@d!rpsEs*qXVf<=&cN70K7(k` zt3d;tQLVe67Xfb!{1H23)baI6Et1H1!)1BQEBONN%vH^P>FsF{9<17mLWcxVy@xMKw-OQGv>U>RW^bF9pmlYEbIblAdUbL396?O;59_l`z7v}x)Hbi zZhK;XbU&DBk@%UBb%uX2h{GU;MYKh@MdTs*k=h)2-do;t9Vy(idjx&}dqg6Y6SIV6 zN@<)c@he9_*79@azhgy4l@ za09{f<1P(&-Ql6+1F*;gbKltm6Bl4r0zwV=_x(P7Y}b(uGjG$NF-@A!U$p>@(JWEE z!g4jdDY$ic*%D#)6Rn%&aTW{vd1R*El}os7n|!W>)5{w} zj_TabNiD&>AzcU{N(6dX7(uW-FTAOva1>PDN z%TR$?_!TtIYqN|#7|eSpFME?w-J;-guqcP;j4AVU|~x*XHoZ=9bP) z$xn#I`?P=JNVC3@W)DA5y<_sz!tF~;%7^vY9oq9tt8i2T!fB!3f2}W_+&a?KOx0;; z>ucy3kf;QiHxBU#ffl{Y9^hfup2->&fDx}47L9(D?ba9!o}^r|5mny6g5xY%f# zlvi-#&6RP0^0r)Zb4DQx9dN9zvlWqVa2=Fxr5VbdS(u)B+>uVcFtd3CYNO zfSCBWY*?=^wV#;yhz#BInW*&+<_$qcfC=7Qo32)&D_4le9EutQy>i`el6m#-nSy-~ z)Ha2l9eMe7WnR0=*mfn?cID+(<>|Z)J+AVZ;%g+-#d#&ZtXhklT8p$Ai?kX~6czyp zr7#K%!E&2WrJ?xaWBf&CyrqyhDs)Az*tU;}03m5YyOaIR^IXkuMwOYzTxx7k=cFm}>IB>;UapptH zLapc7*v`na(a)wl`xdPEM+8#D_Bm4sql#WI&LdAE$={DDL1WZJU@s0e9XX}5AY_NWSe!#qA+aS z{+6h_bz4s1)5T63sIthtiX4t2_Lx)Q=^8zjL1myE)^?&bpfx<9*C3rH&Hy+x%Z?6W zsOMDGU#?C()<`YUNF1TA4?Q@Wg+iprG0q3AD3myt{QV4+THBs&GZ;QX4@o@D9(+S@ zW8Tnwmf6x#8~axqjB<7SHPYxTH&>%a$P33usAXZgyr|BMoCfnv=q&AQXeL(ZjHu$` zk|&t^EG>uW*i}wUFR#SR{QhBw@U7fQ1M#!*NASxFTzo+IEO97fT%BEBVrEj!0vkLi zn2iyn)2R6LgxP})E^s4^VtTCA92-DBNLi?dDN78=@(bdn_LChM>?Va?Z({xY!HEtx zDOApGof2kUaTlZE^*Uzn}?zWr#YnNPV&)lm93+iu(QNz3Yneb(4bJgFw3*G1~oqm!%TMyuG) zMRNBd`DepE+N&0lb`|@j|w9OibMj42qK6G{el&UfINy!4%3;QGI``F2F7HOl1Nn;ge$c=|OfWuY;-5K4Qj{gDftnt87;Ccw`8aKskKZQrciE~%Z#Xc_ zfH;#5IN%GryMy^0@G#V4j?@zl#3Koo=Pa28?4kqz2nAS>tbC4Dop@$4njanprKAeE zT4Y~9z|Y>+dRpTA4! z$zeFBzY)N6Iuv8`!VWodi`;Vy$Mp5j{qSK#KRxv2$3-ef{}HS&Z~C9)kQfyT^UtJ_ zbC%E&)NWA+xP~tgkqUauU?-ovzk~UhZI(kG`g^F|fqdX#v8v?XGW6urVAmb=^3x% zkxmBs$&#Oa;9xBJ7Pq;bX%~P%80LOhMG+`E!cNeH@l(P;?vdHz2y3JBQ^FvssqZmm zR;46J8J>)MCd7Oy>?wJK(MJU&e2M6?Ct~+36Ui$@u$DZU@GS-IZ*D5pn176*OQ+iE^nG(S_!#@xHfLEsA* z{pwTR_&POoHf1+-yS zYECq0J$XT5Qx)dPJqWNSe!BRsbdr18>#5njJEQ=j0m3O|uZo?|WgqN~i=_uE)O<(brV?_lY+E|4Rk*07wg~M!lmk z!8;S(pVN@3S?|!iipaCZ&$p0J!p_!w6$(h{M$t%Jh(i8JW)ax3H3#I(xMf8M(m#zmXlXjK)rP{CEk_X2{Jo8 zkIqIVQsZq@Q8i17-^gzsjwqIk&x)D;M${2o;Re1`H$RaA9Kc#KtQDxIU#%8M?K3t@ z%TbR$o6B0rVv0)&VMk;21w_|_!2>j^dw?q5f&9$aFkIH8fx=U04$v(qjSda70liyG z8CJb^S}|jQh6DJyDB%%BWWh>gVFe>A&P#@qaYtIABsXH3IK7_2zN_R8;}~{;?CA|(L1lKOY+$KIQf03|?E)shvw@KZrc4eXKY3avMi!IH zMUZ{;dr{Q@Y3|f6?3;M{DlJ0GqGt@P$_}Q;5>dsR%VQ|9WIFH6mwt$R!-xY8HI+un z3M#8s+ToaUd|$Etey5D`cSFCu=(Bd^zJ|i>`}g%-} ziP=VIZPk8ykz!>gOs9Xt^|EFX#B|VM@47BDo94^du#uX~$Z863pmHJDpC6b7T}uTi zTw$I#6byx+S*BD0Q4B;PaPJSH`ADq^4?{lR1MVJpkm8NV0gb~wMKY0>kWO7V)wbmv zup@)~(3B1iOU}?HSCpzz_HFFvdj!(l3_F+%j3{**3qgF_)Y0jK&+onwrSc#A#5oIm=w~nxDSL>z?JX#DM&4A&u z4@%3B=RMoQctmWH-Z&u)ruTz}-v=g{CpKc%L>@J?Z4?Ht!WgN`-)Jt}f)kf%1@oox zv}Gw3AD%J`3%v~p2vra(*A5=pcwu41830s@7Q)I37Q$Zb%t+V8@MVC6g1V}n%d2{E zl+gq)z`S<9)}k9dHPA3v?mdWO>A4c-0eJ1?Pa^SgDs`S?7Ra7=jTP(D`^V;!Z1c#a zNvfhtWSsDFyDeOwix(#Q^wRZkB7832*m}dmTWCPWD9Im?lazeIi=HA^E;5jxXgvEK zgnFzxnMfsM&7ITZs~eD3(1xpCX|7f=3^AK>w@0^z6R5{1OU_~lAq&F|L7U)=gptzs z;;|4Ucwi(G`z7P=%E92A-j^sL;~Kw`u3R*1(n&U?$`?~Q1Vk|(ihPZQd5)@-o0vW9Wvl9zO-eku{X8PPTkIR^yUJi8)H&W58d5Ytpz8UdM4-G{8M<(|a=3V& zMQ-0V13?M=iqGO%U~i|a`lI5yM%F&S*>hbv=0$su;U@%8oX<| zG}QtLF(K8>^GX1kFllrf43(mv%Jvu?M^>Ip*0t~b=2}VdO~`&Y?1AUuQ}45w-R9$b zY0;Y%-#y#ru6LZQ$*_~|d#n<6vi|)$%2t=}axjeajt2PBj?&j@`}p&G#wxGJ_B{bu zuiLi1L~TJxrf4j@XNgc=2m}tb)X(^J{H@|xiE5o>#zotpPjJg{m+%JZrr{Rl8DciL z51W@32fpgCdfBicbPI%#JY?A}!B+jdIciCqN5RIxO^iG$asx`UIE6bU-Ox&W(4JcJ zN%<|Tpu|KyZ8QF?4d!wx@#VMr%JFYro%hi|HhCH7N`FGw>|g~^p^^l~a#>(})5UN! z^>sA?MnjU(5NTD#gZAjLWLQjC_1}IB1JbG&za`-rDP1VGe9`3=+Uzvi9j4LYziqF7 zU+QYUlJ_#fF)3eq^ejv1L>N)iA6wUls*l| z$te0;+wBF%H|EXdz}1n`6Vj#NQuHP|9W!`cySd#Ju5+|q*|qFub%W;G;o4Efsf=qm zRZk6TngnSO1!@qr94XHA$3?T=O4-#V{JH(gHfGa!1Py4)(wW4NgH38tA!j*~Js18` znp9W|5_z@(Pco*OG?$ddb{Hr&lSYKOEdK}!FP3Bri)G+VlVd6fs|NLEMu#&u-$>Cee%~arK1qM1^!GU38}~R_iD$+}h2FBTAV}AA_WLh(x?I$+i_@ zsg&rIMlcug{*@+>Q!h!_>Ma4Nz?tcHA&O&Rq?;Z=ah^(oG#7C+nWq zFJL4$9pk%^Jef5A$!ALfdO-< zq>)6Vub^L$#Y*HQ*q8Z^k0!dWvQCf`hwN*ur;MXTgXP}m@5mA)VUH_GGibW12Aaig zqlogxq_A5OQzR{p#~S6VcGlKfQ?1P$M<3;qeNkD9=3=%>jTg3jgwDfqF2!^osx)m$ zYt`3hz$C|R&k%=7u9m1I1E@-qp{z97{VdNu%hM+@C@I4TQ9H2CtDW?q86O^1I zoJOREZylj>4^bn_XKc|uVxcdyWy2O{ko(Iqx)cW*FuY9%_1Pj~Dfvu{p$o?hU?_w% zmBv+39UBPz!uQ-mgo>?cyJ)D*qtdw%oI49mgh7Fg!V}$P7_alZ3B3tnp!2*JlIwd+ zoA{$R_zY7q8X5f!gCQu6FeSqwtb4C6eM0HrJfba`cP_?7Z$CS@5Fz75ZFB9sm+ zb#NpxF*%vX?Xj}pcXBr1<&hC1@8{>&_sbz(_aqwvD-2=+{wZWU3zbEs>ZvZL+2SYy z8_Mcg`?|8kQxcx9`|P2vcHF_2Z`YLx5*k0jRo(MvfVL&c4GTriPo;=P`^9e0>7SvG z9+T3m^HIBX_;DCHrabGgd%Ppwo!}kkMd#6df^5rPkufG}RXJTaxfhr$8pMojb(oBlh>zhT^UQ&mhcrT9S+_Zw?+g+iS{5#qV}ay=KGbdxo8uNFFfXHO>~a?GP| z9#M+-^%0%xvzpS{^=

Ei{(Y(9v{Ys$;cKrkxYvR;%e3q3rJ(j43_F(*yagAIo;K z=9yI4P*3Qi~ZLhK5nj=3i8_-NV4|>e=*ITt6 z%Flc@Nlz?GS@==kdZ*Lz?kA*F`gjanQqS<7Jy3jZ-E{Y?SA(bUC){hUR6TP(M@9>g zGQ(O!j<#$)Q*G1E$jxy*lkVTe5Dli#L(rzstXdB+b%n|ld1)P5aSt_xgJGNAv{HzN zWgmiJQU}U@D(ip?$HZW`H+B5U(TJ%t8(7ANZD&rD9g_E<9VHC)eu&Za?XF^ui0j`L$P|%?qWo8Bs!V5{0J_`h8 zVnk=anKI+*Rg9@50Xe}o!Pkj|KF%((PwgTGFHS^F=BQRo!KBH77FnFP25>B}24GC+ z1gL~(i#B3jrvh)9;7$cg)E;%*G9!jTMO@C;*A%5n?H1Q5O=L=xk3Ezj=;J9_>M$KW z@dD+GK^(o24UrJoeKINK<#y1Y^^5hk+U~cpS>5UV8#T<^HXoOnb&!6Ck-|z+GCJvq zjXmq#*8*n%FU#tv#Ot@`m`1)l>5Bv{l(NK5?Wk0)&39|!e0 zxpw?prBf>O9K-*#?I>_a`n zsWD%Ur#oA@7*T82v;vzvYyyGhR7+j!I}lcdC?W&WTZoV~63$16!_p`T%t)rNSd8Sc zVe-fl?{8O&qDD*Ibc-4OrPh5}8K3_6%SOGqa|HeUVKwV>+o$^?x>F+>M|s6^X*isl zmixA>l}4kW`z%C^=ZUK#qn()1(O&9Sh@u0G0Mr5qWG_OHb&NQv@k%c@2UrL?+%H$j zO;h+y#ycnk`al}xT1HaBPfK~~OwWva_cwSESFnJP6gqojUxFSI2r+Dq90a-Xk>}0A zPd5JAPFHpj)@ss|4oLuQKinjn?LeyRqS1)LK*IA-oDl(z*hJ5`2uE}0YZy5pgYTw? z=G;Y@tNFs;tx?D6ZB}MA?nAVgQ*9r6qp$IPx>i&!ht86T(^If!=$dG?)?#1y7lD!` zkuLF5AYP{d?NpFa$Uub6_H|p!_D1iRtO5b$WXtA;)AeM8)FnO(x8=U(qhL1|RzpIm z|7t97Qpk#CEf*HSpi>8CON#+F+wwJ(9CRN7=^%1x$;tYbDj3`8(%%zM=_E62_$#PHvxehKISSRdJ>sUnJ$@PiAFe|o`@nM$8;Y@b@sQFa1=O6 zOs)w!|GoCXLn}8unk{8|%^}FR3<8k?x0a8M=#tQH%*Nv>v9j!v&b5 z9G=HXoVxGV+bgPLB07IOgK68_9`+XD@ns*IDGpEMPc!8{Uk6yAKi^={8%y}mR;F#) zD(DI66_sd0VBl`B+CRW^hNCUjgtj-(+e2vELfwdaA5u#cRTdaj4TdWzJ?zhcj$Ut&Kv0sUm!63ai3p(Xo1vo=LY= zTG8gupF}i=gE!eqorikGiLaV0AUV^CL5siwr$tFK>^WX3^#(1IP^@bZqKU-?3uSF$ zt{n2>d2Sr*-BbsBO?!{4Mft`9X{-|MZvm{9^o-DV{ zXl)uSMoRJ>6#!;uWbrP$RPkn#&_Ta_~3hf&y}y#yh=v2c@pn>Lb*TQY^v z;KAddBpxI(Oo0xnf(Q&U+8A+Yv9~gKz#RfPKHlI}97#apPBS!QKN39=+m0jW%Q8mw z4jd>@4|#<{0C|Zcp;hx5q(<+$dQDElA`Tok%_m;kGn>Um0O`dnXE!fye{WhO04)iM zKuZv}Y07~n@_6FxK~Gu}y{?dPdK%GUF7Mm0C-99w+2M@m8;s9{)YfFuGqy+E zYStoVH*e9vDUG|?IX4TKk<3Cm^U@Nrk8a$2KS;L8pLzd9+DK;OL1F&tSuhWym{lWU zoL*wM6;80gte~(U;d%n9iGo4-q;fZ^SbWl-AsH;kgcgT`A+evJd)0UNRdKqLOhm9q zA~l6A?nGNQ4sQ`n8z~Z6D5^DLR=kvK6D{r=ucqdg_4&EMpon$-3jA zy08=+Gl`L>0@ruq4Y?5V_!;vk)|^mPn{U~lIrP?9@|O2z!r7GGEJs;t58+Z*v_F4> z0Y(c)Z0F^-GVF4iyHZR^2qL9td#7$Au`#p46BKLasOEi2=$tc`#~~p;8l823P1BX_ zu>;J(=dWrRJ6wJr8V3m~+pbzj1Tt48tPg9E;4NJsfly2&l}rVlT2HHhP;lwGGp=ti z%vaplDPd$T;a#SlViNipNT76VQ6EgRTCvD@Ro+Vuon!u0j3yu1!vpAyb%!Y()Pph{ z)jS>5d|q94*L2s!gN_+QSye71!L~vlnk`z~3js zwLNPSuZU~Na=eg{?Ug|!c33{<-2|&1x8{Ib9#!!3)XCMJS9n)=okVBhh)rffr}-d1y3E+L#7t-PptqrUXXc2Wynw!p z$xW5XI=tXhk{vV8bz)$tm|g6RrUG8K!`RnEHMgO(?0s0gT`~Yku|nFIp<#Zj?`=Aq zNiz_=9z0!o)sCs*%0g{D8{N&<0l1#;tr_s*|3X@v57(=5*0fQ&cE@uew!Ajl$t_qz zm2%hYPVN_V=HNwQD&0{v3^srLG=e>>84+w!yEJU%t00FYEm;Q1mKB6Ws&eV@*F{>V zKd+qV+7NN5jj>aoTL7V$cq*g$C)2jhEJ*=IG4w0p@Ti`(47Ujw>8--I(HbL&Ad1Eq zl09J=tvQsBDf-iD#^e@_SmF25Gc$|wvbxk3>chF~i-z)fCrYnR@I`2Li5~g^aJ1mbz9=4jqWF(q) zgtB@Rt}2ky(0i`-e2zR`FcSHyRWiJEa>77Rs@I#80LnB)FM=769(fdC!7&ry4ATgp zVgQMsbODJUs1DjcUr!3lmQY1Z~!)D!Q@GtQw~;GK2;8#2+p*F98x=^8G(WV^g{~3 z=*~MIzQEIDIx8OzLMZ$s3s!Z6aSO^@H(t?gydK-YFNQ*-H!BleecrpmqQKFGrMAYA zoyUX^?v5ZxxoMXM=CwAq9{V`nG|w?x8hV}0C9)qSGSh(j-@|(lLTf@M#IngRMQZ62 zjTx?fx_Mf1<)BRKU7=yvo(o5WxJpVI@p%0}ft#J{uQsCd{@5Hi0&~+r0^cUsb;37& zm309hPdgBiN@THj6B3VNJ;D8CWU;W-2Q^l|SrG!P#h}xd0@PyV);E&0;sbxv7`&Un z?!M6HGe@;dt3?=v_QH6|r59i&>E5Wa=bzQRT1`1UE#`3|A22F8dP30gZnU;g;8)#h zCekubT8-j{^`ykY{#*|8mIr7Gqi}o95h24>IaZeZHmQt>jQL$7$vZ_llF0@w{s4e7 zuOqkItr3ozP?rBkRZI9X3$*L3p~HHd^igA%8l$8)T&xTz_YusSGPhW}LK9c0q!P6S z3;V6cW+~K0$J&a9IEY;Rl%|jm4FsZgW94PUB4u2HNiN(s2ko z_MxspIav8UgT|uK$FL#iqaGhprqv+o`lx=(xGDPs0kXB!AnVvzqj*Gq?HaL}Stv|{ z%+abxR$nj~nLBVYV$_i;ad5Fr`UEOWvfyT&ohEDHee__sVb6g*#`#cBl< z_qf8!y$64EG8k7W0c;lKS0u6BrXEwTy7O=m3=YSor%W05v8PN}O;OfvlcsJ{D-KG0 znQ{rjGci$og^BauVPM5UKntzWDAue*8Q1h~C{@%tDi(Fq%5A(&1smOoh3ryhi&&^4 zR~L93+#GA?k6GC|u#=8bcOf7$f1reH2Yre!P?TpPdt|NxAcXfD2AA+k$5`?*ZZ#*W zoo$SJhrK68I+STyTqcM(< zGtHOi%!3I-fgSiE{rDGD7BHiKKSiO$L*G**8cbYCG`w(FN}kpX5+nZV8Henpyei86 zmDMHAArtXC;>Ma(YbQLxL?#9!^AA{Ne715D0tx7Y1)t@@JGx8r8}RFEd5@`PYwg5} z)PC$62l{jtLCZxYN|peaI*BA zXn3y3v#9{cO`mbK6!#LpASV?NYuu6hRRNBTh>;vo_>vPl|=b+TGXk~nmzEzqwGH%#dnnTHxVas)by%+4*Z z&zos5qox-;dhqP1GiqBFVXW@LMN>pB;g^Fl7b0N=0$epvJbaA1pDLP-Pq* z;jR9)z|r)kBeC}pP}9B`SN8J$V5Yu(^~~XjgzsXE%wfJ=3I#=i};`} zI?`>WTIu?vOy|{fG@q-1%r>STyf%zjL(S{PrPBsRmVy?|6aha*A(yMbyWey^u4KLtXfTtdCu{kEnOz zL@AMQf;7mtvL+8ptO%k}lk)JoSb2oH8PRYjWp4=d#$2p5qOc^n98mqa*X{VL=Sx4vL0w zk)j0hvU>F(#9Wz|h8b~UVV*f%zp!#y#)*brVl)3~FynF;3Gl3vqg}2~v0y0B{1THR z@yXK>(FKKS`=2G&9`K5~H$7C;+xwWxU0J>ZyCzI&@FS=^u>@^=2I9HKHey_`X=hgz4Y<#TFMPjj*$ZR#Q9u7Ut&Ya{ z=IFe>mX)$24kNphJa5haX-#n_U@0-oTee7>0nm3Sq#zQOP0FXSB$G5l&m2a=e1RD>-2DJsP)cgBsj7!a8&#&ey*@Y(5j@WI^+ghJqN_L^ z_x0rNbH1?fa0lBf8YIe9Od3J6YciwOLyi%|V(7;fw^`e&; z>j@g_3$?y$JZxoSk0Z6A`KFj2lSx`DE%mu&uWBYKyCjVEeks8twGDt2u~qciN_S{m zxU7%o=OXK4i<3Ls_q1FGhbjb%A19PVI>Aorsc1HH_yZ_r-`#kX4_)su} z7xr7ReK0Q<7{JQ|H~yDm`#eCd9}@PQd;loC)II?GqtrfJ*#3v4I=s|AgqMpC$_M_>J!SH=qCIME?Ka`~EDd4|g8)quTxt zjdR_fl-j@5-{5Fk< zYiqBZbpWl9^tIMJQwu!$G3F@m?cXJ(egfcRm=|!M?;m5NShOHZw_sbINO7ST9GF67 za&fjTa}KCDy!qH0>wJhAn5Nor^r`icg%N` zkk4+LE(D3xa+Z(8%W1O?wt1#}=!)9{G<{&zZ)${>;XieWsKuHWPt3ht_m&-P{#!kF zx9i@3J88u;-9wdLc@1nc`2S3XWrn*+>&mz=%+M3V+-e)EuRL!E@;d=}h0mqRUT1D$ z`s!Mj;xSm?J!e^qFQ93EVN8q~u2wswG#>wmJ8Fxx$i9N58xU${cut!p7r98tMT}k_ zIqp%jFC=gDgmZy=g*Kh;b{hGGsr~Er`gfF3UW16Jw53dhaU_IE? z>UtoE&^0TCnv=)m0J789GXeiV`Ue)i*<~XxNC97;ldyD9_YjIAzhsscLHL{~d&rqA z8yOq$8BZok0$SEbY@?}}*Vh4b-D|q4PPBEcmkcs@*t6=EdKxzfa^e>5?N2iM${D-YQBRSmkik z^hMV4reHp9E%*H$d-PXdWLXOALtiUETfy{Ok=X=}#!%)b`P-~quG~|@L_kxao_6O9 zNy|ntK*D-~j4R@g!e;POnz=;+37aykGt0X>Is-5sPTZ;mX1OT4GAoP z(944`%|rOYADe{nAq|rEFn-#S7)A@QNtL`{*y}4Q9YYI$UTUQE!ecln94P{_JN$`xKw^!kMBtF zgpD~mILSDvAMqZDqMx)yIkMXZQ@52ZI&5?yurl`b2qScd(4eeVS+tth;JdXdKSc&5 zhrBz$(tD16SjHfGU-qr^9s&{-OYof?|5xp*JP6$F=!Igq<|g!9?Ixz&31V{(cC)TF zUNK#+T(z`5ts#0Fr2CkDClK+1vjLTPsqBD!Ya{}^Qcl)afizo=EZ_*)qbJflAEcK8jZ54HF)`%cKK_P8JF2+#&Jhqkl> zu4_!{Z712HV{9g6kbgzfp9*5STZ1sUlyM>Cg+twLxTyLR z=er;EBJ)!zDn^C(x$%7C1tkD7FHw3J z$dAod1WfO1Kb(m3>^7)O5>&`!^39W1V+F<4Me4}e1Qzd5YHG0 z6-9h*MEy2oC>*O``BO3?ayf~H#KC^Ca0moJE}~~nOZxpvm&?5m#NhUGS_Jijdrz?` zXp!8-S=0SU1ZB-zuU^IpVv_fWl{1OtYK|5A58J(zsP<2o!6MJ`vA;uexg<^Rrv^DX zUfGUc-oN)*K?z6r5|LJ9D6X$w|IuJfoL#jUoN%ma9x~FBzwK(J6Q$F^VukwP4p!P_ zBA10=8T;GrWoCH{X|twT@}X^E#bIK1H@HGK>DI>#{mRs1afR$za>J)F`7cL_nuaAO z$U_{nhNt@lmbv*39}x)Te>fhz*j4ZvC#Q}-j$6GZftA^O$>Gw>8s4<6Is?O&r&euW zRG*xlLEJ&8Scl~<0TU=#OFVuEIz>y+$o;0KNs7?JSSjbvoe)shy9=h;7e{>V++m6t z6Z4-<-D_X_w?SXL8X9_zyg;US9^J&%xE`sRDK~pr$gWf)O(9|{tJ}go#el_ACfDog zUe#5!XPNFI<4tz`1bcy~TJQMnY2TMxy<@v|_0I5uy!wJ>MlKh-)R$~-GI@KmOek2h z`2qs7%ckD%gJx~2I zMSLeaO9`cHi1<#c=7w#!Y9>jK{h*J9uc=t`yN>DEMw+(=-J5y4UW&ea8!G$-^Z58=+?%!$U{*d!>b~Bn^H~slTT`i94VQO4FbV(KtTj$QAhN9iEndR-& zohR9>*6y|G1Bdjgv&#=tcivj^(TFcTkKxZ4nB=Z}@uId+DaenTMx5tU@5$K(zo%Z< z*?EC%1G`Ud^TIrLX`q_6WQ`V0B$njMds%`5bZ^txG)tHD-DKcJF*L{bBEYDUEL^r} zK7$Q(@m@mhdFwzu4HogDe#gt^u%(C!9z7USNO4o@(vx61H5r`HCZUjG?#DH2+vIq! zh8*p(F^p1{>U;^>Pfd)p$OzkU4F?=evN3S)wl6meg`^FdkckFSG-ELO5FFh^+bi1} zg~>3a-7VG2m;Opvsuv;+X3~M>NZTvCcS4J?m45G3B5g18-f5Fg`<3PD30*C0q<2&D zXikzni76J|EnR~F0`CwdJ;A60`ykyEXSu3IS4%t6%S%@)Jkrbhz)AmgLe2vxyVnV- zVJoSU><`0D;eXT9X~SFcs3XOc5XhH)-lxAXM#&I%(O61kvG3^+pa{stzN0mY%8`PO zvd4UkNGn5v(AMR29b0lpw}}{J7;7&&#oWa@Ah7ZjrhVa_A=m9y6QuX;Tm!0|MxVs2 z#K>nU1EUH}$ZF4VYIyj9oYpyS??QHN8BPG#+?psI|5P%r{&pN>d00q zB8T}ZiSRHuXlNk(!i&uVyyWaiC!7%mg%P}xmppnnix-{3$tA3tTR{Qk%#qF@79;&jW=juT za3>z4p#@Y}fg8ZmNQ@X=B1F41-5Rb1uufPk1R>>MT$gJ07hxUucl(!oKGg;ad`auN8J%q1C6q+P+Wc%_2s@W%h=QliULm3wQ}WN= z1oYKHPNeSD>qPGL4y=>=GCdA=Z9hlF(73y<vu3@wRerua#bggu2tgmMOa$#97n7WCKm@TxHXv0>D&xanXdF{ zfus(qCfR|Lo@D#L=b?V{Hz`>8>8!>_WTbKJroa@ooCx(zbt|?g**ujuL~O5-)bo?Q z9(E@*%?_$ns=QW{t_YCkmGRzGP7n-g|n_dpgM#6O}0nYb2FZG~Sc z#zyqPgkQvof5Zv%0*t27JNYDT8?Ulx$8{xDc?qDia(pIg7u{rZVkS#L%U#Fk-3e64D6h%NNoEd{SJpQl+KQdi&5gEI3!G8x5r z0R?%?CUXV2H*A(ky%lCoiKBL7Dbg5DL92shg-xn&GU-lT3&tliR18&vSe*<9p65dh zJPc~APq)<8imhq@r~25~2oFEu0Q2`lxF=NflwOO8vHKNnG!(uu@o;4Lazg1pid+j*F>G8J#Utp zx4#&pNmdv5VY7#lSe=Fd)4=l&YNQEd#K>=yao7;@j&wh^sObTYRHKi4Dk=SqA{pOZG@W^;aRfFg zSe5J9Kp+}mbgV1agLf2rG89cshuAV#UjkDnD$>YydnXN@)3kCYneRJ0j&tG@=F;ypOYxZKOK?8DA&k%nN4*z7TNnJf*EogGSnEN%4>s17 z!e(pi-tIOO!Zx0c3q5`|a-F`qz(K7us1s@aZrW@~R~~nbWEQ87V|#T%cuc2{vko^O zcMUgjE`M%qE)nr&ETrH*@}5{uhjHF+FI7T@ktBx#i)UDvRBMcl(P$}&jg$^gQm6sE0{<3Lmi;gX$MvNX;;qmO4=a|g10#yd=^YwMqXnfqxsj&)GG!qsKcxC0&zz`*t zFfi< z9TnLX$`9eQNvskwJHSr18_k?!c4}OEzs!J=cAuHAtdc(Zwg-J?^)BStq z2&KpFzyN*0Y@bH_{9Oa51F}c$tlYwkH&%Cb;+ToS&wxQ$6yNApv;}QO-;R`ia^SHC z2`9UWUuA!b?i`aUop~A+gR#7mP4)f?K~nrKZML{%M$6ZTAddP{=Gnb>vvxuO+IOoi zUEmbDk(v}NwBor9qv3t2vWErAEH#hO@-Rq>ii@_aC+uHcd84I7Yxxd0e^$J-Te=f( z>z;F!LsOs5Xse;1CF#`i;fZJ_b45>J1+7)SbjmU2haeu3=N&kmO446NOLythTsNSV zAF5ij#fovYw4GcW)r{XI5}Du_lbhjju$Pvw3=Lr$8~7MJsI4d&7qkwfTbYwp&F|>X z*8Zyfs^+tYSbQE%bSWsWdBI87P`}l6`&9~f_v0+Pp^W>i+K21_()~UCdlcs@XMHxB z-uy>0bM9LW&(dbB)F&4W4|d%R>y~2&w;Eg{&dCQjimgfi7x)Ma;rY|ovR9xR;q7T`n*wYwEyzCsC2mUSJT*Q8}7gDOYn=qYjAT9Vd;{w&Ny?jI}a8Eeg-n zRo6Cy=P=d$+LAj$#H${FNprmWmc436dkqmsy_G7@r%k)#cWKKI+hmJR15qklV)M0>>%jm$>%{E#>7~?;pgL`7H`B`+NKQ zlOUAjurwQrE(f;ioZZVwUIfY1lnACe(v!u|ARgWpH0-c&uvs#Ix!sKCdH9xj#YjM* zL$>pebV*9(z<*OB`$ z-`J%xCh@_2!lczSfY5WPi0?HvI^I4CPw}X@>^)3daX)p$&Fp)4y1gP6=}E12otYEJ zpO-N=Nu1OP)l@uVksab7l8qM&^k29$$V%ZM@fDoU;t`vwi0g-|w-{0!{LDEY-FqU1 zhtNVFlQp!sDF4?ztlQPYTxlGMozTWdwA9Hs-oRMYxiepdwmzsK+`bO zZ#lJG)l%g$f-6Vx%@C?sQDDLziZ*l{)2YNGNC^%RPoA-})zYx8lFl43t4bb9qpd|# z?Cmil%9vh=V;TGeY%d_!e7cQ7+sqF=|7!KT>V=n7gv@1p5h!A|M^&0TbF8kC?rP>m zD3TlPdeh9c2yNrZ2xt9%7jBl{Vz=!Uu{FYI3m)(>)3w4>hW(^P!K&p=FI#f|^!@2+ zm|JsgL`FlC-sG2z)eNDEuG#g*4qv$?JN0{AbGvi9-=AEztK&xtX!+D^0oUT3Oj6!W zrBbD|z=${#4^LgyoDGE4y_-?DImzBXu0!Pv%~RwV27iF4vu5Z`|U3q)X6lhc5HUThH|*!igHC=GkrEl5IRL`t~KK`um&5 zJKLI%nzjc%9db5CFUkdBy4uzheGkEfWbURsb}`gYHUicOzDPE3sQAjNGaTb*MXKTtY;`f7RR}Gel@L`hiVtUK`{p@>}B(?dZpX-*2W|Xw?a$8q$CG0 zwp836HUEZyefN0>#XXaXP9VXDk$G|cEu@*zcLiPRR&odrc~8Xo`|mW!GU58GKY1pu z?oZzc&}GuEIw^c=N}N*p$AUQq1HO7uQu zxw4jdqJH0iM_kjag|saVyDLgd&gC1^1F1kw&1dShGtMe3|vs#*_u*)yvi}Yd*}v(z(6z^7#f?!f2WU*a|B-014Tw`ceyXiMDxW z;EE_-y6)`CyLH*3y}e*6=D_jk_*hVTr3Vh{GNlk(hOwn|Hhm5Zyb83MIrle0`KjMX>w3F{r^fX0?* z07k}{hQo_>k|rOgZwG0y_lpp-XkK2sTZ;;dnEB&=4VpkYQKHHhEqyw5l>J42wLZ0( z^%1+GO>%Gw_JGZ)&2&@0@H?j}^9mmVxepTc##ONZ4~)1yWh=YL0Vx{`@)4HdQQ?gH z18p2)?65i8s|}bh&sknyf`&H-NqaIuuWV~PX2U%;B13*eDXU%#8s{C4+}ML{i-*Ep zekuDIhLb$iN!8b3J7QhkY21xG@G_SM0=zaihv)O&M&2jtlt}S=+T&hZWY8tB5&w`z z!LB)e#ZYUEIfYy0&V`R2)di`DwG2Q0fas55ly`<0av&4c__hS36=Qn2J24_Q*tPM}EZn{Gr zo{zubFX_U@;oz3(nakVRDq^L}Ek~S$l%y#yj8Rx8rz!I^=6hf4-t8pCi}O>;WxE;a zM;*%#D$pAPTC98xHe>TZjb~Q8yGg_`r}79(IBtX_Ij@m<_wf=ic%Rrur*M>9+Q@%| zP^-{eloSX#7==% zx%cKEyQa%K1ZRvItsyo6UGR+88S-e_8hu(UEg?bHTf)0a=aP=DG$eskPfu(1VO4f3 zS!DkA4Fi?S=zYe!9ftFAy%I0b&Gi8zHeAzH`3=VB=l7Z{`T@NDN9k#^X`KR2`K242 znP1~x5y_XW0>>laEP_@Rjrk8%AL`f^Of$5`J`1jIHTtM_b~|`FB&#K-ZTFOM;67@! z{xpWBN+GfHXiI3b`kCgDer2FWATkL?I{2PKDN~?^XcCD=;kYdAtXY_;Rfc$e-}oD1 zUO8z(vLxB{K1ABt)t8^?Nd;{@C)@-DDXTs;J^qlfnP%R6Xe9ao|jUU3TMSVaS>o^(QhP6<-=bF+hpowp52`YK9w9OtgK8f zDM?KahbfyAyvy(i^GJu8`gC5`T?#q#7h5=or%fK<-7Gi6xvyv(rGPkIEp^8$mXV0r z*}aFBmPjeAZ;6rT#=J1K(0xzUxoiY0*3CJ~dp8;q6D`cl3{$;K3pglu@xgsnKNG_I zF0kUOqgE=TgRn&$e)+4fH@3UQ6V6>zS~#jf(xh+71RHu)@V`Olz_kpP+$?F4vk|&J zyuRAGPdm+y&oA17^3QM|ZaMWr1C#i$&D1t|nD1*xp`C|3(;dMTp?m-S{xn0q1u_fW z$+N=HeautPr(D6D2ES@nw&yl)#tKVLToi=>+VVc|8YFMz&010c537X)%$LA0sBuc>U1a8w4!RpFJoaZ zdsFhJ9dM*KR3(*EU|m^6JGIW+Ou3g*D?j7exjM@p&8sG}7uGbtvbX(idL{^+f8!Ky zA%d0i9#_@4j{fs4pYaF0TkqfPSFIFO(N%slq1+?cF*lRJY@eCwK6^p^-hfFhq0JWo zv^+q48#)5V_WTRn`H3C*3Cj8vfhPGY0!`e(*7W}YfCh$g!T3Nx-rw+N5D*t!h5!Ki zulO?ngbM_O@qpn7G#&^Xfd&TsK%hb3fHoc|{4cot02d6v3;i8|_74Q*S4hea0g79? z8n+N=|E$Ju2(-WW{4cEL{|5l=Cy)yW2a5fI(gMK+Cw>us_zgGq6S_t7ugb*#K*nId zK*r#L5WkM_f0d7bW67X!U>M-%=zmd>fbsoY`2Pm}xve5$ANWH>!UKzXRw6;vs$K@8 zNG|lFkuTo_y5GlgX?jUop@TC*-tFA`cQ@CkzBNthX(aWgB3#c!zv)ECsF+#in{N=e ztbimkvRdY}$sTG*ER!XD%F8V9lQ64~S%o2dZ<&{G9V&-yS9EYLI~e3ZszL|UCv-cv z)0`KeS-3P>%6^?Mo?MRBOqz=>R@iDAWja8Ecj>|EcuGun+=tDMu>1H$q)@ieWy zBTn=(30@8or}W#o&+9i;s)kX9m18D|wRfhQv@8`}XnHH9yMi|4UFc#Zqa_(*A{lZ5 znPVi;izmKmUfT2N=1~_W^5J8`q**WITai&&kK^BZ7mXeDn311jzJaSqNPN~xp8;^2 z95$&z>hi2%BDP`)j(^$AKiB%_UdH17H(+n!!2S~5|4SGc91jRrbom_y_TyFmvge*U znVDn3RZl=zJpW!aU=RohqA{oW*9J#X{&Sbc{y#PV3=ALq$Mw_31B8G2-3HGA062X1 zw{`&Vk3{jijSm34)sBY;2!v}~{MMHT2;hMO*M75s06bv0SHIbKcmX_r&dUSjQ-L>ALP~?Krrmqx(J5rwwx34ejmO0fD#Hj}OH6m%eZW z?4RSoU;1qu?=9c>pupRGA#nKZALBvz_-^|FgZyPY*sXRT_`&q&cp!Md+_LfT-5L)B z1mAiV5a4#afZ%B1TjRm8lYiR4AowBhhi@Pl@b)u9LATZ!%zN8kFwbp&!F;!$1q_E5 z|2Z$@FTQ~xfZOweLAPQT3P9pu9Y{`a zSSSRJT>oP{*kAg>fIklKKjwwrUIQ2p@2z$)-rF$_1Kf@)82B%Jp|{q67Yx0X+u=-v zTX`HFxwmozFZ><-=`R2Ty_GirU|ztV@dy9Hc`Kg+Ah+`$JWbw;dp=&iTX~lcE~NBl zyzs%#fIsI2!HF5SZ9u@E?-#@ihW)Y^T%3%oY|Wgo1O>TOt)7|vIDWZR92{I|ek3Lu m_;*_wdvgaG_`v@h)y^(PPA>l()F1%-tixhte4r?W^?v}HbTZ`t literal 0 HcmV?d00001 diff --git a/packages/media/test/fixtures/pdf/untagged.pdf b/packages/media/test/fixtures/pdf/untagged.pdf new file mode 100644 index 0000000000000000000000000000000000000000..16dee38f36571e67ffb298224db98d0814b92c6b GIT binary patch literal 3847 zcmcgvc~}!?8dr|0BMNFot5!!8gJLq72@sMci6I2ZYJ?&PNOc)zCNPp@LS|xu$5U`| zwIGVOtOsa4YfhnDeV*>lAIUf0yx)8L-t!BL zi;5PaA}JJjxPI0OC@@fG9?uXsM>A$QoWa7vY*SdY(^ zjV&wcrdTv}FrH!9AACJXUBCWrN>%#2`sz6k3pY?*7c40n&-NeaI5;|cO5^#Z-M>ly zY3d>M+@N74gdpq8>D!VOGQ8+!bk$;;J~?P1`wk>uXqTA2ciM*;5-ZSVddO% zBZ>b1)Zs0ELmj6hTk69b=n8va3xEf9;={toFq&lHRLI2x<+YB7#=yWZ8?Y4PjdWYo z%%O>a2Ip`ili}vn1fibda1wlhQK?%kH-i?b=gPL0$>z>ROe#?-o6~O0Y}8yS_R-px zMoKp=IiK#?eRNHrrf|hQ?_MQPM3n!B8F%iz)CoS8Px3hR^M}f#w;dA{w{o_f7&_*9 zw~b*Za=x6DdOvit;xTn{_2mon%)5WxdgyNQaCTD9W6xso_MU5st{e`QmYO#H+qTAwjJ6HYGom>kzyvFVEUe0J>t zuzWTJ{7fu6EE!wc&@gY>ck3?{Ua+gZ>%G4Fe%n8?5bv(VUcPGYh_=?a)+u7|u6`2V zd^$!DP%^!EVTeR$i1e!I-XCq-DTXMk-nXo9cJ0v)%Yt`28XNq5qVK9{#h%AYUYS!C z1dQ>=_AgJFzK-|B8yU-2ucmtTpFx$N;VnU(qc&Fz@zFk$Y0bqtm?#cl(kfRZ);{&X zua^yuS>fluck+JKX>GSFHSM=QnXx8($;>s`$&arSB{d7nb5t`fr|8iqA4(v8g6u}i;w>y4_-1H4Qx z#!7nb`An7g2+h7|}G zCLJ@uN{yT_B7vBYMaW2jN(n`%a8Rr%p#!%F4bHB`!W2s=0yOc%qXT#XhumSZm_Q=J z4wKQ$${h-U^N1X*`E5i1J4v`Kg&-1}ko&?2I)FMnz!) zz|u+@ox)_2tg&>pTEOG>~+wCH|M8sG#5L70UAz}={Fd?82vN>iBcL>dFKOoPk z2%X$~%*=|MI*ACzgwV+dDiR}ZK~caQD{bL>J|k z8jBJWV_`zEOemHnh%vb&OfC)$M#XZmI0AWt5_k(7)qpVrj2-WBv<0Q>s5{)==kTv+ zTS#WC2_OWNiRLUbV>hcU79;H%wOr~tWj4`7AiPIfAokuE>h=E}MU#n-OOr*fmq#&# z%>??YE=nP=*=SNO!%0jMj)n>;3?qdoY9NGS@CdO`5|iQ~5}AZZ{~hl99je5Gl1uZE zs4?O!s}PWs0k;{s|BbLwl$8adc(dO-I!U4#R*TJQbS5H6Ae52f5oAGtq5xNtkQ*4Q z3Fkm5cRs zGeO7*SPG^`+RB1iGu+(_K$U3GmWyVop$f4dXvM>Qww&-6_56fyOcY% zCZ?x1!~7U6E!fleO4WOYhgW3VJP*GfLHW5uG(mkm0=zwfI>RxW!*8!MTzot*wx9pM zhRzZD?ptRcBW>nzot0){L%RFlYb0; zvAQv2t;%QGxdl5awpO0N+bk};_i57B+S1a(kET`dtKu8lv^i#lGwb+Yd)a` zLzegI($TL??@N~&1hf0c)(cWH-v$Z|Z$`=m3ydVkQwGo==?N5!%J_Ml0dqB=iiZ)M z*}%a3MI;_VbWw0>gjB5!!B8nCQA@)^)M02CD%NVXs0PEdvT$*Q- zKxHs^O@dQVAcemSxSaQ^tDoCLQYk+HH;^HY> zaoY6U>3Qw^{SHov{tQFc4|BP YZIoCJw{i|Ez(o=&!N84I8ygM%0yPe0h5!Hn literal 0 HcmV?d00001 diff --git a/packages/media/test/gate.test.ts b/packages/media/test/gate.test.ts new file mode 100644 index 00000000..c6a42f6b --- /dev/null +++ b/packages/media/test/gate.test.ts @@ -0,0 +1,81 @@ +/** + * The permit gate — the process-wide bound on decoded content in memory. + * + * One construct, extracted from the image normalizer so a document ingest can + * hold a permit of the SAME gate for its whole duration: images and documents + * together never exceed the permit count. These cases pin the gate's own + * contract; the normalizer's suite pins that the normalizer still stands + * behind it. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { createGate, gate, MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS } from '../src/gate'; + +const tick = () => new Promise((r) => setTimeout(r, 0)); + +describe('createGate', () => { + it('bounds concurrency to its permits and hands a freed permit to the next waiter', async () => { + const g = createGate({ label: 'test', permits: 2, maxQueued: 4, waitMs: 1_000 }); + const r1 = await g.acquire(); + const r2 = await g.acquire(); + let third = false; + const p3 = g.acquire().then((r) => { third = true; return r; }); + await tick(); + expect(third).toBe(false); + r1(); + const r3 = await p3; + expect(third).toBe(true); + r2(); r3(); + }); + + it('refuses immediately with EBUSY once the queue is full', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 1, waitMs: 1_000 }); + const r1 = await g.acquire(); + const queued = g.acquire(); + await expect(g.acquire()).rejects.toMatchObject({ code: 'EBUSY' }); + r1(); + (await queued)(); + }); + + it('lets a queued caller give up before any slot frees, and refuses an already-aborted one', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 4, waitMs: 1_000 }); + const r1 = await g.acquire(); + const ctrl = new AbortController(); + const queued = g.acquire(ctrl.signal); + ctrl.abort(); + await expect(queued).rejects.toMatchObject({ name: 'AbortError' }); + await expect(g.acquire(AbortSignal.abort())).rejects.toMatchObject({ name: 'AbortError' }); + r1(); + // The gate is intact afterwards. + (await g.acquire())(); + }); + + it('times out a waiter and names the wait', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 4, waitMs: 20 }); + const r1 = await g.acquire(); + await expect(g.acquire()).rejects.toThrow(/waited 20ms/); + r1(); + }); + + it('makes release idempotent so a double release cannot manufacture a permit', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 4, waitMs: 1_000 }); + const r1 = await g.acquire(); + r1(); r1(); + const r2 = await g.acquire(); + let third = false; + const p3 = g.acquire().then((r) => { third = true; return r; }); + await tick(); + expect(third).toBe(false); + r2(); + (await p3)(); + }); +}); + +describe('the shared gate', () => { + it('is one instance sized by the normalizer constants', () => { + expect(typeof gate.acquire).toBe('function'); + expect(MAX_CONCURRENT_NORMALIZATIONS).toBe(4); + expect(MAX_QUEUED_NORMALIZATIONS).toBe(16); + }); +}); diff --git a/packages/media/test/ingress.test.ts b/packages/media/test/ingress.test.ts index 2f5d4e52..f4ef30e6 100644 --- a/packages/media/test/ingress.test.ts +++ b/packages/media/test/ingress.test.ts @@ -11,6 +11,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { FileAttachmentStore } from '../src/node'; import { createImageIngress } from '../src/image'; +import { materialize } from '../src/ingress'; const png = () => sharp({ create: { width: 64, height: 64, channels: 3, background: '#0a7' } }).png().toBuffer() @@ -47,3 +48,34 @@ describe('createImageIngress', () => { expect(store.getManifest(root.digest)).toBeTruthy(); }); }); + +describe('materialize — bitmaps are the projector-decodable representations', () => { + // A document's one representation is text. It reaches the model through + // retrieval and the spine outline, never through the projector, so a + // document root contributes NO bitmaps: feeding its bytes to the image + // decoder would poison the branch it was prefilled on. + it('yields no bitmaps for a root whose only representation is text/markdown', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'materialize-'))); + const md = store.putBlob(new TextEncoder().encode('# Title\n\nBody.\n'), 'text/markdown'); + const root = store.putAttachment({ representations: [md] }); + + const prepared = materialize(store, [root]); + + expect(prepared.attachments).toEqual([root]); + expect(prepared.bitmaps).toEqual([]); + }); + + it('yields exactly the image bitmaps for a mixed batch, in root order', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'materialize-'))); + const bytes = await png(); + const image = await createImageIngress(store).ingest(bytes); + const md = store.putBlob(new TextEncoder().encode('text'), 'text/markdown'); + const doc = store.putAttachment({ representations: [md] }); + + const prepared = materialize(store, [doc, image]); + + expect(prepared.attachments).toEqual([doc, image]); + expect(prepared.bitmaps).toHaveLength(1); + expect(prepared.bitmaps[0]).toEqual(bytes); + }); +}); diff --git a/packages/media/test/normalize.test.ts b/packages/media/test/normalize.test.ts index 29c5d5c6..6f133325 100644 --- a/packages/media/test/normalize.test.ts +++ b/packages/media/test/normalize.test.ts @@ -9,12 +9,13 @@ */ import { describe, it, expect } from 'vitest'; import sharp from 'sharp'; -import { normalizeImage, DEFAULT_MAX_PIXELS, MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS, MAX_INPUT_PIXELS } from '../src/image'; +import { normalizeImage, DEFAULT_MAX_PIXELS, MAX_INPUT_PIXELS } from '../src/image'; +import { MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS } from '../src/gate'; import type { NormalizedImage } from '../src/image'; // Its own list of nine, sourced from stb_image — NOT derived from the sniff // table, which knows four. Deriving it was a defect: the pass-through gate read // as covering six formats and covered one. -import { PROJECTOR_FORMATS } from '../src/index'; +import { PROJECTOR_FORMATS, sniffMediaType } from '../src/index'; const solid = (width: number, height: number, format: 'jpeg' | 'png' | 'webp' | 'tiff' = 'jpeg') => sharp({ create: { width, height, channels: 3, background: '#0a7' } })[format]().toBuffer() @@ -181,6 +182,22 @@ describe('the two format lists are different questions', () => { }); }); +describe('a PDF is identified, and is not an image', () => { + const pdf = new TextEncoder().encode('%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n%%EOF\n'); + + it('sniffs the PDF signature — the content ingress dispatches on it', () => { + expect(sniffMediaType(pdf)).toBe('application/pdf'); + }); + + it('is not something the projector decodes', () => { + expect(PROJECTOR_FORMATS).not.toContain('application/pdf'); + }); + + it('is refused by the image normalizer, exactly as an unknown format is', async () => { + await expect(normalizeImage(pdf, {})).rejects.toThrow(); + }); +}); + describe('KNOWN DEFECT — pinned, not endorsed', () => { it('DEFECT: metadata retention depends on whether the image hit the ceiling', async () => { // Re-encoding strips EXIF/ICC (sharp's default); the pass-through path diff --git a/packages/media/test/pdf-ingress.test.ts b/packages/media/test/pdf-ingress.test.ts new file mode 100644 index 00000000..6e0f3a21 --- /dev/null +++ b/packages/media/test/pdf-ingress.test.ts @@ -0,0 +1,126 @@ +/** + * The document ingress against real fixtures: the shape it commits, identity, + * the bounds, and that nothing is written on abort or timeout. + * + * @category Testing + */ +import { describe, it, expect, vi } from 'vitest'; +import { readFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { FileAttachmentStore } from '../src/node'; +import { createDocumentIngress, MAX_DOCUMENT_BYTES, PdfError } from '../src/pdf'; +import { DOCUMENT_CONFIG_TYPE, asDocumentMeta } from '../src/document'; +import { representationsOf, sourceOf } from '../src/attachment'; +import { materialize } from '../src/ingress'; +import type { DocumentMeta } from '../src/document'; + +const fixture = (name: string): Uint8Array => new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); +const freshStore = (): FileAttachmentStore => new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'pdf-ingress-'))); + +/** The committed sidecar of a document root. */ +function metaOf(store: FileAttachmentStore, digest: string): DocumentMeta { + const manifest = store.getManifest(digest)!; + expect(manifest.config.mediaType).toBe(DOCUMENT_CONFIG_TYPE); + const meta = asDocumentMeta(JSON.parse(new TextDecoder().decode(store.get(manifest.config.digest)!))); + if (!meta) throw new Error('sidecar did not validate'); + return meta; +} + +describe('createDocumentIngress — the shape', () => { + it('commits one text/markdown representation, the PDF as source, and the sidecar as config', async () => { + const store = freshStore(); + const root = await createDocumentIngress(store).ingest(fixture('untagged.pdf')); + const manifest = store.getManifest(root.digest)!; + const reps = representationsOf(manifest); + expect(reps).toHaveLength(1); + expect(reps[0].mediaType).toBe('text/markdown'); + expect(sourceOf(manifest)?.mediaType).toBe('application/pdf'); + const meta = metaOf(store, root.digest); + expect(meta.pageCount).toBe(2); + expect(meta.title).toBe('An Untagged Report'); + // A document materializes to no bitmaps: its text is for retrieval, never the projector. + expect(materialize(store, [root]).bitmaps).toEqual([]); + }, 60_000); + + it('reads headings by size on an untagged page and maps sections to pages', async () => { + const store = freshStore(); + const root = await createDocumentIngress(store).ingest(fixture('untagged.pdf')); + const meta = metaOf(store, root.digest); + const md = new TextDecoder().decode(store.get(representationsOf(store.getManifest(root.digest)!)[0].digest)!); + expect(md).toContain('# An Untagged Report'); + expect(md).toContain('\n## First Section\n'); + expect(md).toContain('\n## Second Section\n'); + expect(md).toContain('Body text line one of the first section, set in the body size. Body text line two of the first section continues the paragraph.'); + const first = meta.sections.find((s) => s.heading === 'First Section')!; + expect(first.origin).toBe('heuristic'); + expect([first.pageStart, first.pageEnd]).toEqual([1, 1]); + expect(meta.pages.map((p) => p.page)).toEqual([1, 2]); + expect(meta.pages[1].chars).toBeGreaterThan(0); + }, 60_000); + + it('yields the same root for the same bytes, twice, in two stores', async () => { + const a = await createDocumentIngress(freshStore()).ingest(fixture('untagged.pdf')); + const b = await createDocumentIngress(freshStore()).ingest(fixture('untagged.pdf')); + expect(a.digest).toBe(b.digest); + }, 60_000); +}); + +describe('createDocumentIngress — text at size one under a scaled matrix', () => { + it('measures size and spacing from the advance boxes, so words stay whole and the title is the big line', async () => { + // PDFium reports font size 1 for every glyph of matrix.pdf; a gap rule + // scaled by that size calls every glyph gap a space. The loose box carries + // the real advance and em height. + const store = freshStore(); + const root = await createDocumentIngress(store).ingest(fixture('matrix.pdf')); + const meta = metaOf(store, root.digest); + expect(meta.title).toBe('Scaled Title Line'); + const md = new TextDecoder().decode(store.get(representationsOf(store.getManifest(root.digest)!)[0].digest)!); + expect(md).toContain('# Scaled Title Line'); + expect(md).toContain('Body text set at size one under a scaled matrix. Second line of body text with several words.'); + expect(md).not.toMatch(/B o d y/); + }, 60_000); +}); + +describe('createDocumentIngress — the bounds', () => { + it('refuses a document over the byte ceiling before touching the codec', async () => { + const store = freshStore(); + const big = new Uint8Array(MAX_DOCUMENT_BYTES + 1); + big.set(new TextEncoder().encode('%PDF-1.4')); + await expect(createDocumentIngress(store).ingest(big)).rejects.toThrow(/ceiling/); + }); + + it('names a password-protected document and a malformed one', async () => { + const store = freshStore(); + await expect(createDocumentIngress(store).ingest(fixture('encrypted.pdf'))).rejects.toThrow(/password/i); + await expect(createDocumentIngress(store).ingest(new TextEncoder().encode('%PDF-1.4 not really'))).rejects.toThrow(PdfError); + }); + + it('commits nothing when the caller gives up while the document is being read', async () => { + const store = freshStore(); + const putBlob = vi.spyOn(store, 'putBlob'); + const putAttachment = vi.spyOn(store, 'putAttachment'); + const ctrl = new AbortController(); + const pending = createDocumentIngress(store).ingest(fixture('tagged.pdf'), ctrl.signal); + // The ingress yields to the loop between pages; the abort lands there. + setImmediate(() => ctrl.abort()); + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }); + expect(putBlob).not.toHaveBeenCalled(); + expect(putAttachment).not.toHaveBeenCalled(); + }, 60_000); + + it('publishes nothing when the time bound bites, and says it timed out', async () => { + const store = freshStore(); + const putAttachment = vi.spyOn(store, 'putAttachment'); + await expect(createDocumentIngress(store, { timeoutMs: 1 }).ingest(fixture('tagged.pdf'))) + .rejects.toMatchObject({ code: 'ETIMEDOUT' }); + expect(putAttachment).not.toHaveBeenCalled(); + }, 60_000); + + it('serves five concurrent ingests of one document through the shared gate to one root', async () => { + const store = freshStore(); + const ingress = createDocumentIngress(store); + const roots = await Promise.all(Array.from({ length: 5 }, () => ingress.ingest(fixture('untagged.pdf')))); + expect(new Set(roots.map((r) => r.digest)).size).toBe(1); + }, 120_000); +}); diff --git a/packages/media/test/pdf-layout.test.ts b/packages/media/test/pdf-layout.test.ts new file mode 100644 index 00000000..ee0e7678 --- /dev/null +++ b/packages/media/test/pdf-layout.test.ts @@ -0,0 +1,340 @@ +/** + * The layout heuristics, specified on synthetic character records. + * + * Nothing here touches the codec: a page is a list of characters with boxes, + * sizes and weights in PDF user space (origin bottom-left, y up), plus what + * the page's objects and structure tree said. That is the whole input, so the + * rules are stated where they can be read. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { layoutDocument } from '../src/pdf-layout'; +import type { PageChar, PageFacts } from '../src/pdf-layout'; + +/** Lay a string of characters on one baseline. Each glyph is `size * 0.5` wide. */ +function line(text: string, y: number, opts: { x?: number; size?: number; weight?: number; mcid?: number } = {}): PageChar[] { + const size = opts.size ?? 10; + const w = size * 0.5; + let x = opts.x ?? 72; + const out: PageChar[] = []; + for (const ch of text) { + out.push({ text: ch, x0: x, y0: y, x1: x + w, y1: y + size, size, weight: opts.weight ?? 400, mcid: opts.mcid ?? -1 }); + x += w; + } + return out; +} + +function page(n: number, chars: PageChar[], extra: Partial = {}): PageFacts { + return { page: n, width: 612, height: 792, chars, images: [], pathObjects: 0, textExtracted: true, ...extra }; +} + +const OPTS = { title: null, bookmarks: [], maxTextPages: 400, dpi: 150, minFigureSidePx: 64, minFigureAreaRatio: 0.02 }; + +describe('lines and paragraphs', () => { + it('groups characters by baseline into lines and joins lines of one block into a paragraph', () => { + const chars = [ + ...line('First line of the paragraph.', 700), + ...line('Second line of the same paragraph.', 686), + ...line('A new paragraph after a gap.', 640), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('First line of the paragraph. Second line of the same paragraph.'); + expect(r.markdown).toContain('\n\nA new paragraph after a gap.'); + }); + + it('joins a hyphenated line break without the hyphen when the next line starts lowercase', () => { + const chars = [...line('This is a hyphen-', 700), ...line('ated word here.', 686)]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('hyphenated word here.'); + }); + + it('inserts a space where glyphs leave a gap on one baseline', () => { + const chars = [...line('left', 700, { x: 72 }), ...line('right', 700, { x: 200 })]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('left right'); + }); +}); + +describe('headings and the title, by size and weight', () => { + const doc = () => [ + ...line('A Report Title', 720, { size: 18 }), + ...line('First Section', 688, { size: 14 }), + ...line('Body text in the first section, long enough to be body.', 670), + ...line('Body text continues for a second line here.', 656), + ...line('Subheading', 630, { size: 12 }), + ...line('More body text under the subheading.', 612), + ...line('Bold lead', 590, { weight: 700 }), + ...line('Body after a bold short line.', 576), + ]; + + it('takes the largest line on page one as the title when metadata has none', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, doc())] }); + expect(r.title).toBe('A Report Title'); + expect(r.markdown.startsWith('# A Report Title\n')).toBe(true); + }); + + it('trusts a metadata title only when a heading-size line on page one carries it', () => { + // A producer's Title field is often junk — here an e-mail that also runs + // in the footer at body size. The page's largest line is the title. + const chars = [...doc(), ...line('someone@example.com', 40, { size: 8 })]; + const r = layoutDocument({ ...OPTS, title: 'someone@example.com', pages: [page(1, chars)] }); + expect(r.title).toBe('A Report Title'); + expect(r.markdown.startsWith('# A Report Title\n')).toBe(true); + }); + + it('takes a metadata title the page carries, and does not repeat it as a heading', () => { + const r = layoutDocument({ ...OPTS, title: 'A Report Title', pages: [page(1, doc())] }); + expect(r.title).toBe('A Report Title'); + expect(r.markdown.match(/^# /gm)).toHaveLength(1); + expect(r.markdown).not.toMatch(/^## A Report Title$/m); + }); + + it('keeps a metadata title only for a page with no text to corroborate it', () => { + const r = layoutDocument({ ...OPTS, title: 'From Metadata', pages: [page(1, [], { images: [{ index: 0, bbox: [0, 0, 612, 792] }] })] }); + expect(r.title).toBe('From Metadata'); + }); + + it('folds a metadata title that wraps over two lines into the title, not into headings', () => { + // Chrome prints a long as two 22pt lines; each is a piece of the + // metadata title, neither equals it. They are the title, so the real + // sections stay top-level and the table of contents keeps its topics. + const chars = [ + ...line('Continuous Tree Batching: A', 720, { size: 22 }), + ...line('Measurement Note', 692, { size: 22 }), + ...line('Abstract', 660, { size: 14 }), + ...line('We measure how the cost of a run scales.', 642), + ...line('Method', 610, { size: 14 }), + ...line('Each configuration runs the same question.', 592), + ]; + const r = layoutDocument({ ...OPTS, title: 'Continuous Tree Batching: A Measurement Note', pages: [page(1, chars)] }); + expect(r.markdown.match(/^# /gm)).toHaveLength(1); + expect(r.markdown).not.toMatch(/^#+ Continuous Tree Batching: A$/m); + expect(r.markdown).not.toMatch(/^#+ Measurement Note$/m); + expect(r.sections.map((s) => [s.heading, s.path])).toEqual([ + ['Continuous Tree Batching: A Measurement Note', 'Continuous Tree Batching: A Measurement Note'], + ['Abstract', 'Abstract'], + ['Method', 'Method'], + ]); + }); + + it('ranks heading levels by size and treats a bold short body-size line as the lowest level', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, doc())] }); + expect(r.markdown).toContain('\n## First Section\n'); + expect(r.markdown).toContain('\n### Subheading\n'); + expect(r.markdown).toContain('\n#### Bold lead\n'); + expect(r.sections.map((s) => [s.heading, s.path, s.origin])).toEqual([ + ['A Report Title', 'A Report Title', 'heuristic'], + ['First Section', 'First Section', 'heuristic'], + ['Subheading', 'First Section > Subheading', 'heuristic'], + ['Bold lead', 'First Section > Subheading > Bold lead', 'heuristic'], + ]); + }); + + it('gives every section a contiguous line span that tiles the markdown', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, doc())] }); + const lines = r.markdown.split('\n'); + let next = 1; + for (const s of r.sections) { + expect(s.startLine).toBe(next); + expect(lines[s.startLine - 1]).toMatch(new RegExp(`^#+ ${s.heading}$`)); + expect(s.endLine).toBeGreaterThanOrEqual(s.startLine); + next = s.endLine + 1; + } + expect(next - 1).toBe(lines.length); + }); +}); + +describe('separators and wrapped titles', () => { + it('a synthesised space with no box is a separator, never a line break', () => { + // PDFium inserts a space between two text runs on one baseline and gives it + // an empty box. The reader hands it the previous glyph's geometry; the + // layout treats any whitespace as a separator — either way one line. + const chars = [ + ...line('2.1', 700, { size: 13 }), + { text: ' ', x0: 61, y0: 702.7, x1: 61, y1: 702.7, size: 1, weight: 400, mcid: -1 }, + ...line('Structure-Activity Relationship', 700, { x: 96, size: 13 }), + ...line('Body text under the heading, long enough to be body text.', 680), + ...line('And a second body line to settle the body size.', 666), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('2.1 Structure-Activity Relationship'); + expect(r.markdown).not.toMatch(/^#+ 2\.1$/m); + }); + + it('a ligature drawn from a fallback font with a small em box stays in its word and its line', () => { + // Chrome sets "fi" from another font: an 8pt-tall advance box on a 20pt + // line, same baseline. Grouping by baseline keeps "Configuration" whole. + const big = line('Con', 700, { size: 20 }); + const fi = { text: 'fi', x0: big[big.length - 1].x1 + 0.2, y0: 702, x1: big[big.length - 1].x1 + 3.4, y1: 710.4, size: 8.4, baseline: 700, weight: 400, mcid: -1 }; + const rest = line('guration matters here.', 700, { x: fi.x1 + 0.2, size: 20 }); + const body = [...line('Body text sets the body size for the page here.', 670), ...line('A second body line, so the heading rank has a body.', 656)]; + const r = layoutDocument({ ...OPTS, pages: [page(1, [...big.map((c) => ({ ...c, baseline: 700 })), fi, ...rest.map((c) => ({ ...c, baseline: 700 })), ...body])] }); + expect(r.markdown).toContain('Configuration matters here.'); + expect(r.markdown).not.toContain('Con fi'); + }); + + it('the title is the run of largest lettered lines, not a bare chapter number set larger', () => { + const chars = [ + ...line('2', 730, { size: 30 }), + ...line('Topical Corticosteroids:', 700, { size: 20 }), + ...line('Pharmacology', 676, { size: 20 }), + ...line('Gagandeep Kwatra and Sandip Mukhopadhyay', 650, { size: 12 }), + ...line('Abstract Topical corticosteroids are widely used for inflammatory disorders.', 630), + ...line('They are available in a number of formulations for the skin.', 616), + ]; + const r = layoutDocument({ ...OPTS, title: 'kwatragagandeep@gmail.com', pages: [page(1, chars)] }); + expect(r.title).toBe('Topical Corticosteroids: Pharmacology'); + expect(r.markdown.match(/^# /gm)).toHaveLength(1); + expect(r.markdown).not.toMatch(/^#+ Pharmacology$/m); + // The bare chapter number is not a heading either. + expect(r.markdown).not.toMatch(/^#+ 2$/m); + expect(r.sections.map((s) => s.heading)).toEqual(['Topical Corticosteroids: Pharmacology', 'Gagandeep Kwatra and Sandip Mukhopadhyay']); + }); + + it('orders a line by position when the producer emits its words out of sequence', () => { + // "DELTA" is drawn after "RULE" though it sits between "WITH" and "RULE"; + // and the title contains "DELTA", which must not swallow the subtitle's. + // 20 glyphs of 7pt from x=72 end at 212; DELTA sits at 220..255, RULE after it. + const a = line('IMPROVING MAMBA WITH', 700, { size: 14 }); + const rule = line('RULE', 700, { x: 262, size: 14 }); + const delta = line('DELTA', 700, { x: 220, size: 14 }); + const chars = [ + ...line('GATED DELTA NETWORKS', 730, { size: 17 }), + ...a, ...rule, ...delta, + ...line('Body text of the abstract, long enough to be body text here.', 670), + ...line('And a second body line to settle the body size of the page.', 656), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.title).toBe('GATED DELTA NETWORKS'); + expect(r.markdown).toContain('\n## IMPROVING MAMBA WITH DELTA RULE\n'); + }); + + it('a small-caps heading is a heading: its capitals are set larger than the rest', () => { + const chars = [ + ...line('A Report Title', 730, { size: 18 }), + ...line('1 I', 700, { size: 12 }), + ...line('NTRODUCTION', 700, { x: 72 + 3 * 6, size: 9.6 }), + ...line('Body text under the small-caps heading, long enough to be body.', 680), + ...line('And a second body line to settle the body size of the page.', 666), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.sections.map((s) => s.heading)).toEqual(['A Report Title', '1 INTRODUCTION']); + }); + + it('a heading wrapped over two adjacent lines is one heading', () => { + const chars = [ + ...line('A Report Title', 730, { size: 18 }), + ...line('2.1 Structure-Activity', 700, { size: 13 }), + ...line('Relationship', 685, { size: 13 }), + ...line('Body text under the wrapped heading, long enough to be body text.', 665), + ...line('And a second body line to settle the body size of the page.', 651), + ...line('2.2 Potency', 620, { size: 13 }), + ...line('More body text under the second heading of the page here.', 602), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.sections.map((s) => s.heading)).toEqual(['A Report Title', '2.1 Structure-Activity Relationship', '2.2 Potency']); + expect(r.markdown).toContain('\n## 2.1 Structure-Activity Relationship\n'); + expect(r.markdown).not.toMatch(/^## Relationship$/m); + }); +}); + +describe('pages, counts and the page map', () => { + it('records one span per page, contiguous across pages, and the per-page counts', () => { + const p1 = page(1, [...line('Page one body text.', 700)], { images: [{ index: 0, bbox: [72, 300, 372, 500] }], pathObjects: 3 }); + const p2 = page(2, [...line('Page two body text.', 700)]); + const r = layoutDocument({ ...OPTS, pages: [p1, p2] }); + expect(r.pages.map((p) => [p.page, p.chars, p.imageObjects, p.pathObjects])).toEqual([[1, 19, 1, 3], [2, 19, 0, 0]]); + expect(r.pages[1].startLine).toBe(r.pages[0].endLine + 1); + const total = r.markdown.split('\n').length; + expect(r.pages[1].endLine).toBe(total); + }); + + it('emits a note for a page with no characters and counts zero chars', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, [], { images: [{ index: 0, bbox: [0, 0, 612, 792] }] })] }); + expect(r.markdown).toContain('*[page 1: no extractable text — scanned image]*'); + expect(r.pages[0].chars).toBe(0); + expect(r.pages[0].imageObjects).toBe(1); + }); + + it('stops extracting text past the page cap, says so, and marks the result truncated', () => { + const pages = [1, 2, 3].map((n) => page(n, line(`Text of page ${n}.`, 700), { textExtracted: n <= 2 })); + const r = layoutDocument({ ...OPTS, maxTextPages: 2, pages }); + expect(r.truncated).toBe(true); + expect(r.markdown).toContain('*[pages 3–3 not extracted: page limit]*'); + expect(r.markdown).not.toContain('Text of page 3.'); + expect(r.pages).toHaveLength(3); + }); +}); + +describe('figures and captions', () => { + it('keeps an image object as a figure when it is large enough, anchored to a nearby caption line', () => { + const chars = [ + ...line('Some text above the figure.', 700), + ...line('Figure 1. The green rectangle.', 280), + ]; + const p = page(1, chars, { images: [{ index: 3, bbox: [72, 300, 372, 500] }, { index: 4, bbox: [500, 700, 520, 720] }] }); + const r = layoutDocument({ ...OPTS, pages: [p] }); + expect(r.figures).toEqual([{ page: 1, index: 3, bbox: [72, 300, 372, 500], caption: 'Figure 1. The green rectangle.' }]); + expect(r.pages[0].imageObjects).toBe(2); + }); + + it('falls back to the structure tree alt text when no caption line is near', () => { + const p = page(1, line('Unrelated text far away.', 750), { + images: [{ index: 0, bbox: [72, 100, 372, 300], altText: 'A chart of cells per image' }], + }); + const r = layoutDocument({ ...OPTS, pages: [p] }); + expect(r.figures[0].caption).toBe('A chart of cells per image'); + }); +}); + +describe('a tagged page', () => { + it('takes heading levels from structure roles, emits a tagged table as a pipe table, and reports coverage', () => { + const roles = new Map<number, string>([[0, 'H1'], [1, 'H2'], [2, 'P'], [3, 'TH'], [4, 'TH'], [5, 'TD'], [6, 'TD'], [7, 'P']]); + const chars = [ + ...line('A Tagged Paper', 720, { size: 18, mcid: 0 }), + ...line('Results', 690, { size: 12, mcid: 1 }), + ...line('Table 1 reports the values.', 672, { mcid: 2 }), + ...line('Config', 640, { x: 72, mcid: 3 }), ...line('Cells', 640, { x: 200, mcid: 4 }), + ...line('baseline', 624, { x: 72, mcid: 5 }), ...line('803', 624, { x: 200, mcid: 6 }), + ...line('Closing paragraph.', 590, { mcid: 7 }), + ]; + const p = page(1, chars, { + struct: { + roleByMcid: roles, + tables: [{ mcids: [3, 4, 5, 6], rows: [{ cells: [{ mcids: [3] }, { mcids: [4] }] }, { cells: [{ mcids: [5] }, { mcids: [6] }] }] }], + figures: 0, + }, + }); + const r = layoutDocument({ ...OPTS, title: 'A Tagged Paper', pages: [p] }); + expect(r.tagged).toBe(true); + expect(r.structCoverage).toBeGreaterThanOrEqual(0.9); + // H1 equals the title: emitted once as the title, not again as a heading. + expect(r.markdown.match(/A Tagged Paper/g)).toHaveLength(1); + expect(r.markdown).toContain('\n### Results\n'); + expect(r.markdown).toContain('| Config | Cells |\n| --- | --- |\n| baseline | 803 |'); + expect(r.tables).toHaveLength(1); + expect(r.pages[0].taggedTables).toBe(1); + expect(r.sections.find((s) => s.heading === 'Results')?.origin).toBe('struct'); + }); +}); + +describe('bookmarks', () => { + it('takes a heading\'s path from the bookmark chain when a bookmark matches it', () => { + const chars = [ + ...line('Doc', 720, { size: 18 }), + ...line('Methods', 690, { size: 14 }), + ...line('Body of methods.', 672), + ...line('Sampling', 640, { size: 12 }), + ...line('Body of sampling.', 622), + ]; + const bookmarks = [ + { title: 'Methods', page: 1, level: 0 }, + { title: 'Sampling', page: 1, level: 1 }, + ]; + const r = layoutDocument({ ...OPTS, bookmarks, pages: [page(1, chars)] }); + const s = r.sections.find((x) => x.heading === 'Sampling')!; + expect(s.path).toBe('Methods > Sampling'); + expect(s.origin).toBe('bookmark'); + }); +}); diff --git a/packages/media/test/pdf-render.test.ts b/packages/media/test/pdf-render.test.ts new file mode 100644 index 00000000..7e3c42f0 --- /dev/null +++ b/packages/media/test/pdf-render.test.ts @@ -0,0 +1,112 @@ +/** + * Page renders and figure crops: the archive, the projection facts, the tagged + * structure, and the scanned page — against real fixtures. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { readFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import sharp from 'sharp'; +import { FileAttachmentStore } from '../src/node'; +import { createDocumentIngress } from '../src/pdf'; +import { normalizeImage } from '../src/image'; +import { asDocumentMeta } from '../src/document'; +import { representationsOf } from '../src/attachment'; +import { materialize } from '../src/ingress'; +import type { DocumentMeta } from '../src/document'; + +const fixture = (name: string): Uint8Array => new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); +const freshStore = (): FileAttachmentStore => new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'pdf-render-'))); + +async function ingest(name: string, opts: Parameters<typeof createDocumentIngress>[1] = {}) { + const store = freshStore(); + const root = await createDocumentIngress(store, opts).ingest(fixture(name)); + const manifest = store.getManifest(root.digest)!; + const meta = asDocumentMeta(JSON.parse(new TextDecoder().decode(store.get(manifest.config.digest)!)))!; + const markdown = new TextDecoder().decode(store.get(representationsOf(manifest)[0].digest)!); + return { store, root, meta, markdown }; +} + +describe('the archive', () => { + it('renders every page within the bound as its own image root the projector can read, byte-stable under the normalizer', async () => { + const { store, meta } = await ingest('tagged.pdf'); + expect(meta.pages.map((p) => !!p.render)).toEqual([true, true, true]); + for (const p of meta.pages) { + const prepared = materialize(store, [p.render!] as never); + expect(prepared.bitmaps).toHaveLength(1); + const png = prepared.bitmaps[0]; + const norm = await normalizeImage(png, {}); + expect(norm.derived).toBe(false); + expect(norm.bytes).toBe(png); + expect(norm.mime).toBe('image/png'); + } + expect(meta.derive.renderedPages).toBe(3); + expect(meta.derive.truncated).toBe(false); + }, 90_000); + + it('records per-page facts: the image page and the table page carry objects, the text page does not', async () => { + const { meta } = await ingest('tagged.pdf'); + const [p1, p2, p3] = meta.pages; + expect(p1.chars).toBeGreaterThan(100); + expect(p1.imageObjects).toBe(0); + expect(p2.taggedTables + p2.pathObjects).toBeGreaterThan(0); + expect(p3.imageObjects).toBeGreaterThan(0); + }, 90_000); + + it('chooses graphics-bearing pages first under a page cap, marks the result truncated, and stays deterministic', async () => { + const a = await ingest('tagged.pdf', { maxRenderedPages: 2 }); + const rendered = a.meta.pages.filter((p) => p.render).map((p) => p.page); + // Page one always. Pages two (a tagged, rule-drawn table) and three (an + // image) are both graphics-bearing, so the tie falls to page order and + // the cap keeps page two; the text-only page would lose to either. + expect(rendered).toEqual([1, 2]); + expect(a.meta.derive.truncated).toBe(true); + const b = await ingest('tagged.pdf', { maxRenderedPages: 2 }); + expect(b.root.digest).toBe(a.root.digest); + const one = await ingest('tagged.pdf', { maxRenderedPages: 1 }); + expect(one.meta.pages.filter((p) => p.render).map((p) => p.page)).toEqual([1]); + }, 120_000); +}); + +describe('the tagged path', () => { + it('takes sections from the structure tree, emits the table as pipe rows, paths from bookmarks, and reports coverage', async () => { + const { meta, markdown } = await ingest('tagged.pdf'); + expect(meta.derive.tagged).toBe(true); + expect(meta.derive.structCoverage).toBeGreaterThanOrEqual(0.9); + expect(markdown).toContain('# A Tagged Paper'); + expect(markdown).toMatch(/\n### Introduction\n|\n## Introduction\n/); + expect(markdown).toContain('| Configuration | Cells | Seconds |'); + expect(markdown).toContain('| baseline | 803 | 4.2 |'); + expect(meta.tables).toHaveLength(1); + const results = meta.sections.find((s) => s.heading === 'Results')!; + expect(['struct', 'bookmark']).toContain(results.origin); + expect(results.pageStart).toBe(2); + }, 90_000); + + it('crops the figure with its caption and the crop is the green rectangle, not the page', async () => { + const { store, meta } = await ingest('tagged.pdf'); + expect(meta.figures).toHaveLength(1); + const fig = meta.figures[0]; + expect(fig.page).toBe(3); + expect(fig.caption).toMatch(/^Figure 1\./); + const png = materialize(store, [fig.root] as never).bitmaps[0]; + const stats = await sharp(Buffer.from(png)).stats(); + const [r, g, b] = stats.channels.map((c) => c.mean); + // #0a7 is rgb(0, 170, 119): green dominant, red near zero. + expect(g).toBeGreaterThan(120); + expect(r).toBeLessThan(60); + expect(b).toBeGreaterThan(60); + }, 90_000); +}); + +describe('the scanned page', () => { + it('says the page has no extractable text, counts zero chars, and still archives the render', async () => { + const { meta, markdown } = await ingest('scanned.pdf'); + expect(markdown).toContain('*[page 1: no extractable text — scanned image]*'); + expect(meta.pages[0].chars).toBe(0); + expect(meta.pages[0].imageObjects).toBe(1); + expect(meta.pages[0].render).toBeTruthy(); + }, 60_000); +}); diff --git a/packages/media/test/pdfium-load.test.ts b/packages/media/test/pdfium-load.test.ts new file mode 100644 index 00000000..3c4b9517 --- /dev/null +++ b/packages/media/test/pdfium-load.test.ts @@ -0,0 +1,63 @@ +/** + * The codec loads offline, once compiled, one instance per document. + * + * This is the step that proves the dependency: the wasm ships in the package + * and is read from disk (no CDN), the module compiles once and instantiates + * per document through the glue's `instantiateWasm` hook, and a document + * opened in one instance is invisible to another. Everything above this in + * the ingress rests on these three facts. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { createCodec, openDocument, PdfError } from '../src/pdf'; + +const fixture = (name: string): Uint8Array => + new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); + +describe('createCodec', () => { + it('loads the wasm from the package on disk and answers a page count', async () => { + const codec = await createCodec(); + try { + const doc = openDocument(codec, fixture('untagged.pdf')); + try { + expect(doc.pageCount).toBe(2); + } finally { + doc.close(); + } + } finally { + codec.dispose(); + } + }); + + it('gives each document its own instance: a handle from one is meaningless in another', async () => { + const a = await createCodec(); + const b = await createCodec(); + try { + expect(a.pdfium).not.toBe(b.pdfium); + const doc = openDocument(a, fixture('untagged.pdf')); + try { + // The same handle number in the other instance is not a document. + expect(b.pdfium.FPDF_GetPageCount(doc.handle)).not.toBe(2); + } finally { + doc.close(); + } + } finally { + a.dispose(); + b.dispose(); + } + }, 30_000); + + it('names the reason a document cannot be opened', async () => { + const codec = await createCodec(); + try { + expect(() => openDocument(codec, fixture('encrypted.pdf'))).toThrow(PdfError); + expect(() => openDocument(codec, fixture('encrypted.pdf'))).toThrow(/password/i); + expect(() => openDocument(codec, new TextEncoder().encode('%PDF-1.4 not really'))).toThrow(/format|malformed/i); + } finally { + codec.dispose(); + } + }); +}); diff --git a/packages/media/tsconfig.json b/packages/media/tsconfig.json index 152ea8e0..dc561f10 100644 --- a/packages/media/tsconfig.json +++ b/packages/media/tsconfig.json @@ -2,7 +2,11 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "outDir": "dist", - "rootDir": "src" + "rootDir": "src", + "lib": [ + "ES2022", + "DOM" + ] }, "include": [ "src/**/*.ts" diff --git a/packages/abilities/corpus/src/bm25.ts b/packages/rig/src/bm25.ts similarity index 100% rename from packages/abilities/corpus/src/bm25.ts rename to packages/rig/src/bm25.ts diff --git a/packages/rig/src/content-routes.ts b/packages/rig/src/content-routes.ts index 681aafc3..0ba2af3a 100644 --- a/packages/rig/src/content-routes.ts +++ b/packages/rig/src/content-routes.ts @@ -1,5 +1,5 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; -import { representationsOf, DIGEST_PATTERN, type Attachment } from '@lloyal-labs/media'; +import { representationsOf, sourceOf, DIGEST_PATTERN, MANIFEST_TYPE, type Attachment } from '@lloyal-labs/media'; import type { AttachmentStore, Descriptor } from '@lloyal-labs/media'; /** Thrown when a body exceeds the cap, so the caller can answer 413 rather @@ -48,7 +48,10 @@ export interface ContentRoutesOpts { * that question, and the ingress is where they are decoded. */ ingest?: (bytes: Uint8Array, signal?: AbortSignal) => Promise<Attachment>; - /** Ceiling on a single upload body. @default 8 MiB */ + /** Ceiling on a single upload body. @default 8 MiB — sized for an image. A + * host that installs the content ingress passes `MAX_DOCUMENT_BYTES` from + * `@lloyal-labs/media/node` beside `DOCUMENT_UPLOAD_TIMEOUT_MS`, so every + * host that mounts the plane admits the same. */ maxUploadBytes?: number; /** * Ceiling on how long one upload may take, end to end. @@ -86,7 +89,10 @@ export interface ContentRoutesOpts { * else it serves. * * ``` - * POST /v1/media/ingress upload → normalize → root descriptor + * POST /v1/media/ingress upload → admit → root descriptor + * GET /v1/media/<manifest> the manifest itself, by digest + * GET /v1/media/<manifest>/config its typed config blob (a document's sidecar; OCI's empty blob for an image) + * GET /v1/media/<manifest>/source the original as supplied, when the ingest retained it * GET /v1/media/<manifest>/representations/<i> the bytes the model actually saw * HEAD /v1/content/<digest> existence, for pre-flight dedupe * ``` @@ -344,7 +350,12 @@ export function createContentRoutes( // A full normalization queue is overload: retryable, not a client // fault and not ours. `EBUSY` is the errno the ingress sets for it. const busy = typeof e === 'object' && e !== null && (e as { code?: unknown }).code === 'EBUSY'; - const code = tooLarge ? 413 : tooSlow ? 408 : busy ? 503 : 400; + // The ingress's own time bound is the same class as a slow upload. + const timedOut = typeof e === 'object' && e !== null && (e as { code?: unknown }).code === 'ETIMEDOUT'; + // An error carrying a syscall came from the filesystem under the + // store — disk full, read-only volume. That is ours, not the client's. + const ours = typeof e === 'object' && e !== null && typeof (e as { syscall?: unknown }).syscall === 'string'; + const code = tooLarge ? 413 : (tooSlow || timedOut) ? 408 : busy ? 503 : ours ? 500 : 400; fail(res, code, e instanceof Error ? e.message : 'ingress failed'); // Now that the status is on the wire, stop the upload. A stalled // client will not close on its own — that is the whole problem — @@ -355,6 +366,48 @@ export function createContentRoutes( return true; } + // GET /v1/media/<manifest> — the manifest itself, by digest. A manifest is + // a blob, and serving it by digest is what any OCI reader expects; it is + // how a UI learns what an attachment IS (its config media type) and what + // roots it names, without a "kind" field anywhere on the wire. + const man = /^\/v1\/media\/([^/]+)$/.exec(path); + if (man && (method === 'GET' || method === 'HEAD')) { + const digest = decodeSegment(man[1]); + if (digest === null || !DIGEST_PATTERN.test(digest)) { fail(res, 400, 'malformed digest'); return true; } + if (!opts.store.getManifest(digest)) { fail(res, 404, 'no such attachment manifest'); return true; } + serveBlob(req, res, { mediaType: MANIFEST_TYPE, digest, size: 0 }, method === 'HEAD'); + return true; + } + + // GET /v1/media/<manifest>/config — the typed config blob, resolved + // THROUGH the manifest the way representations are. A document's + // sidecar lives here; an image's config is OCI's canonical empty blob. + const cfg = /^\/v1\/media\/([^/]+)\/config$/.exec(path); + if (cfg && (method === 'GET' || method === 'HEAD')) { + const digest = decodeSegment(cfg[1]); + if (digest === null || !DIGEST_PATTERN.test(digest)) { fail(res, 400, 'malformed digest'); return true; } + const manifest = opts.store.getManifest(digest); + if (!manifest) { fail(res, 404, 'no such attachment manifest'); return true; } + serveBlob(req, res, manifest.config, method === 'HEAD'); + return true; + } + + // GET /v1/media/<manifest>/source — the original as supplied, when the + // ingest retained it: a document's PDF, an image before normalization. + // Resolved THROUGH the manifest by ROLE, never by a blob digest a + // client typed — raw blobs stay HEAD-only. + const src = /^\/v1\/media\/([^/]+)\/source$/.exec(path); + if (src && (method === 'GET' || method === 'HEAD')) { + const digest = decodeSegment(src[1]); + if (digest === null || !DIGEST_PATTERN.test(digest)) { fail(res, 400, 'malformed digest'); return true; } + const manifest = opts.store.getManifest(digest); + if (!manifest) { fail(res, 404, 'no such attachment manifest'); return true; } + const source = sourceOf(manifest); + if (!source) { fail(res, 404, 'no source retained for this attachment'); return true; } + serveBlob(req, res, source, method === 'HEAD'); + return true; + } + fail(res, 405, 'unsupported method or path'); return true; } catch (e) { diff --git a/packages/rig/src/index.ts b/packages/rig/src/index.ts index 4f520ee6..7ec14ff9 100644 --- a/packages/rig/src/index.ts +++ b/packages/rig/src/index.ts @@ -38,6 +38,15 @@ export type { SourceContext } from './sources/types'; export { chunkFetchedPages, chunkHtml } from './sources/chunking'; export type { FetchedPage } from './sources/chunking'; +// Retrieval primitives shared by every source ability (pure TS — RN-safe). +export { BM25Index } from './bm25'; +export type { Bm25Opts, Bm25Hit } from './bm25'; +export { fitChunks, splitParagraphs, DEFAULT_CHUNK_TOKENS } from './resources/fit'; +export type { FitOpts } from './resources/fit'; +export { mergeRanges, subtractRanges } from './ranges'; +export { loadDocuments } from './resources/documents'; +export type { Document } from './resources/documents'; + // Resource types (pure TS — RN-safe) export type { Resource, Chunk } from './resources/types'; diff --git a/packages/rig/src/ranges.ts b/packages/rig/src/ranges.ts new file mode 100644 index 00000000..9db21d87 --- /dev/null +++ b/packages/rig/src/ranges.ts @@ -0,0 +1,46 @@ +/** + * @file Half-open line-range arithmetic. + * + * Shared by the read tools that remember which lines each agent has already + * been shown, so a second read returns only what is new. Pure; ranges are + * `[start, end)`. + */ + +/** + * The parts of `target` not covered by any range in `covered`, in order. + * + * @category Rig + */ +export function subtractRanges( + [s, e]: [number, number], + covered: readonly [number, number][], +): [number, number][] { + let ranges: [number, number][] = [[s, e]]; + for (const [cs, ce] of covered) { + ranges = ranges.flatMap(([a, b]): [number, number][] => { + if (ce <= a || cs >= b) return [[a, b]]; + const result: [number, number][] = []; + if (a < cs) result.push([a, cs]); + if (ce < b) result.push([ce, b]); + return result; + }); + } + return ranges; +} + +/** + * Overlapping or touching ranges collapsed into the minimal set, sorted by start. + * + * @category Rig + */ +export function mergeRanges(ranges: readonly [number, number][]): [number, number][] { + if (ranges.length === 0) return []; + const sorted = ranges.map(([a, b]): [number, number] => [a, b]).sort((x, y) => x[0] - y[0]); + const merged: [number, number][] = [sorted[0]]; + for (let i = 1; i < sorted.length; i++) { + const last = merged[merged.length - 1]; + if (sorted[i][0] <= last[1]) last[1] = Math.max(last[1], sorted[i][1]); + else merged.push(sorted[i]); + } + return merged; +} diff --git a/packages/rig/src/registry.ts b/packages/rig/src/registry.ts index bbaaffe3..603469a2 100644 --- a/packages/rig/src/registry.ts +++ b/packages/rig/src/registry.ts @@ -37,7 +37,7 @@ import { AbilityRegistryCtx, AbilityConfigStoreCtx, GrantStoreCtx, - RerankerCtx, + RerankerCtx, Attachments, } from '@lloyal-labs/lloyal-agents'; import type { Ability, @@ -125,6 +125,10 @@ export function* createAbilityRegistry( reranker = undefined; } + // The content store the harness installed (the null store when none was): + // an ability that reads documents resolves them through it. + const attachments = yield* Attachments.expect(); + const [scope, destroy] = createScope(); let added = false; return yield* scoped(function* () { @@ -154,6 +158,7 @@ export function* createAbilityRegistry( yield* AbilityConfigStoreCtx.set(configStore); yield* AbilityRegistryCtx.set(registry); if (reranker !== undefined) yield* RerankerCtx.set(reranker); + yield* Attachments.set(attachments); const constructed = yield* factory(); resolve(constructed); yield* suspend(); diff --git a/packages/rig/src/reranker.ts b/packages/rig/src/reranker.ts index a16d4e61..f184ac67 100644 --- a/packages/rig/src/reranker.ts +++ b/packages/rig/src/reranker.ts @@ -19,12 +19,14 @@ export interface RerankerLoadOpts { /** Decode batch size (default floor(nCtx / nSeqMax)). */ nBatch?: number; /** - * KV cache types for the reranker context. Both default to `q4_0` in this - * version. + * KV cache types for the reranker context. Both default to `q8_0`. * - * The score is a logit difference, so KV precision bounds the smallest - * score difference that is meaningful. Set these explicitly if you need a - * known resolution. + * The score is a logit difference read back through these cells, so KV + * precision bounds the smallest score difference that is meaningful. + * Measured on real document windows (2026-09-07): at `q4_0` ten identical + * passages spread 4–6 logits across the leaves and the verdict changed + * sign; at `q8_0` they spread 0.05–0.12 at the same pass time. + * `test/reranker-resolution.test.ts` holds the default to that floor. */ typeK?: KvCacheType; typeV?: KvCacheType; @@ -85,8 +87,8 @@ export function createReranker( nCtx, nSeqMax, nBatch, - typeK: opts?.typeK ?? 'q4_0', - typeV: opts?.typeV ?? 'q4_0', + typeK: opts?.typeK ?? 'q8_0', + typeV: opts?.typeV ?? 'q8_0', })); const rerank = yield* call(() => Rerank.create(ctx as unknown as SessionContext, { diff --git a/packages/rig/src/resources/documents.ts b/packages/rig/src/resources/documents.ts new file mode 100644 index 00000000..6771dbd7 --- /dev/null +++ b/packages/rig/src/resources/documents.ts @@ -0,0 +1,80 @@ +/** + * @file From document attachments in the content store to the resources and + * chunks the retrieval stack already understands. + * + * A document attachment is a manifest whose one representation is + * `text/markdown` and whose config is the sidecar (`DocumentMeta`). Nothing + * here parses markdown: the sidecar's sections already say where each one + * starts and ends, so a chunk is a line slice. No codec, no native addon — + * this module is pure and sits on the platform-agnostic barrel. + */ +import { representationsOf, asDocumentMeta, DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, AttachmentManifest, AttachmentStore, DocumentMeta } from '@lloyal-labs/media'; +import type { Resource, Chunk } from '@lloyal-labs/lloyal-agents'; + +/** + * A document as retrieval sees it: the root it came from, the sidecar, and + * the resource and section chunks cut from its markdown. + * + * @category Rig + */ +export interface Document { + attachment: Attachment; + manifest: AttachmentManifest; + meta: DocumentMeta; + resource: Resource; + /** One chunk per sidecar section, `tokens` empty — size them with `fitChunks`. */ + chunks: Chunk[]; +} + +/** + * Load the documents among `roots`. A root whose config is not a document + * sidecar (an image, say) is skipped — it is legitimately something else. + * A root whose manifest or blobs are missing or drifted THROWS, naming the + * digest: silent degradation here would index an empty document behind a + * digest that promises content. + * + * Resource names are the documents' titles, deduplicated in order + * (`Title`, `Title (2)`), so two uploads with one title stay addressable. + * + * @category Rig + */ +export function loadDocuments(store: AttachmentStore, roots: readonly Attachment[]): Document[] { + const out: Document[] = []; + const seen = new Map<string, number>(); + const short = (digest: string): string => `${digest.slice(0, 19)}…`; + for (const attachment of roots) { + const manifest = store.getManifest(attachment.digest); + if (!manifest) { + throw new Error(`loadDocuments: attachment manifest ${short(attachment.digest)} is not in the content store, or its bytes no longer hash to its digest.`); + } + if (manifest.config.mediaType !== DOCUMENT_CONFIG_TYPE) continue; + const configBytes = store.get(manifest.config.digest); + if (!configBytes) throw new Error(`loadDocuments: sidecar ${short(manifest.config.digest)} is missing from the content store or has drifted.`); + const meta = asDocumentMeta(JSON.parse(new TextDecoder().decode(configBytes))); + if (!meta) throw new Error(`loadDocuments: sidecar ${short(manifest.config.digest)} is not a ${DOCUMENT_CONFIG_TYPE} record.`); + const markdown = representationsOf(manifest).find((r) => r.mediaType === 'text/markdown'); + if (!markdown) throw new Error(`loadDocuments: document ${short(attachment.digest)} carries no text/markdown representation.`); + const bytes = store.get(markdown.digest); + if (!bytes) throw new Error(`loadDocuments: markdown ${short(markdown.digest)} is missing from the content store or has drifted.`); + const content = new TextDecoder().decode(bytes); + + const n = (seen.get(meta.title) ?? 0) + 1; + seen.set(meta.title, n); + const name = n === 1 ? meta.title : `${meta.title} (${n})`; + const lines = content.split('\n'); + const chunks: Chunk[] = meta.sections + .map((s) => ({ + resource: name, + heading: s.heading, + section: s.path, + text: lines.slice(s.startLine - 1, s.endLine).join('\n'), + tokens: [], + startLine: s.startLine, + endLine: s.endLine, + })) + .filter((c) => c.text.trim().length > 0); + out.push({ attachment, manifest, meta, resource: { name, content }, chunks }); + } + return out; +} diff --git a/packages/rig/src/resources/files.ts b/packages/rig/src/resources/files.ts index 71613b6b..49a390d6 100644 --- a/packages/rig/src/resources/files.ts +++ b/packages/rig/src/resources/files.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import ignoreFactory = require("ignore"); import { loadBinary } from "@lloyal-labs/lloyal.node"; import type { Resource, Chunk } from "./types"; +import { splitParagraphs } from "./fit"; interface Section { heading: string; @@ -126,26 +127,20 @@ export function loadResources(input: string): Resource[] { function chunkByParagraph(res: Resource): Chunk[] { const lines = res.content.split("\n"); const chunks: Chunk[] = []; - let start = 0; - for (let i = 0; i <= lines.length; i++) { - const blank = i === lines.length || !lines[i].trim(); - if (blank && i > start) { - const text = lines.slice(start, i).join("\n").trim(); - if (text) { - chunks.push({ - resource: res.name, - heading: - text.slice(0, 60).replace(/\n/g, " ") + - (text.length > 60 ? "\u2026" : ""), - section: '', - text, - tokens: [], - startLine: start + 1, - endLine: i, - }); - } - } - if (blank) start = i + 1; + for (const [start, end] of splitParagraphs(lines)) { + const text = lines.slice(start, end).join("\n").trim(); + if (!text) continue; + chunks.push({ + resource: res.name, + heading: + text.slice(0, 60).replace(/\n/g, " ") + + (text.length > 60 ? "\u2026" : ""), + section: '', + text, + tokens: [], + startLine: start + 1, + endLine: end, + }); } return chunks; } diff --git a/packages/rig/src/resources/fit.ts b/packages/rig/src/resources/fit.ts new file mode 100644 index 00000000..2c013d0c --- /dev/null +++ b/packages/rig/src/resources/fit.ts @@ -0,0 +1,129 @@ +/** + * @file Passages sized for a reranker leaf, by lines. + * + * Pure: it knows nothing about rerankers beyond the `tokenize` it is handed, + * and nothing about where chunks came from. The size is a retrieval choice the + * caller makes; the reranker's capacity is the reranker's own affair. Kept + * free of `./files` on purpose — that module statically imports the native + * package, and nothing on the platform-agnostic barrel may depend on it — so + * the paragraph splitter lives here and `./files` imports it. + */ +import type { Chunk } from '@lloyal-labs/lloyal-agents'; + +/** + * Default passage size, in tokens of the scoring reranker's vocabulary. + * + * A granularity, not a capacity: passages of a few hundred tokens are what + * first-stage and cross-encoder retrieval work best over, whatever the + * hardware. It is sized so that a passage of this length plus a 64-token + * query fits the smallest reranker sizing rig ships (nSeqMax 10 · nCtx 4096), + * which `test/reranker-capacity.test.ts` pins against real weights. + * + * @category Rig + */ +export const DEFAULT_CHUNK_TOKENS = 256; + +/** + * Half-open line ranges `[start, end)` of the blank-separated paragraphs in + * `lines`. A whitespace-only line is blank; runs of blank lines separate + * exactly as one does. + * + * @category Rig + */ +export function splitParagraphs(lines: readonly string[]): [number, number][] { + const out: [number, number][] = []; + let start = -1; + for (let i = 0; i <= lines.length; i++) { + const blank = i === lines.length || lines[i].trim() === ''; + if (!blank && start < 0) start = i; + if (blank && start >= 0) { + out.push([start, i]); + start = -1; + } + } + return out; +} + +/** What {@link fitChunks} needs: a size, and the tokenizer whose vocabulary the size is in. */ +export interface FitOpts { + /** Longest window, in tokens. Positive integer. */ + maxTokens: number; + /** The scoring reranker's tokenizer — chunk tokens must be in ITS vocabulary. */ + tokenize: (text: string) => Promise<number[]>; +} + +/** + * Cut chunks into windows of at most `maxTokens`, returning them tokenized. + * + * A chunk within the size is returned whole. A longer one is split on its + * own line structure: whole paragraphs are packed greedily into windows, a + * paragraph over the size is packed by lines, and a single line over the size + * stays one window — the reranker head-truncates it. **Never inside a line**: + * admission joins scored chunks back to their source by `(resource, + * startLine)`, so two windows must never share a start line. Windows tile + * their parent's non-blank lines, carry real `startLine`/`endLine`, inherit + * `resource`, `heading` and `section`, and never overlap (BM25 would count + * the overlap twice). + * + * Every window is measured once more after packing, so the tokens it carries + * are exactly what the reranker will score — not a sum of its parts. + * + * @throws When `maxTokens` is not a positive integer: an uncapped chunker is + * a configuration error, never a silent fallback. + * + * @category Rig + */ +export async function fitChunks(chunks: readonly Chunk[], opts: FitOpts): Promise<Chunk[]> { + const { maxTokens, tokenize } = opts; + if (!Number.isInteger(maxTokens) || maxTokens < 1) { + throw new Error(`fitChunks: maxTokens must be a positive integer, got ${String(maxTokens)}`); + } + + const out: Chunk[] = []; + for (const chunk of chunks) { + const tokens = await tokenize(chunk.text); + if (tokens.length <= maxTokens) { + out.push({ ...chunk, tokens }); + continue; + } + + const lines = chunk.text.split('\n'); + const slice = (s: number, e: number): string => lines.slice(s, e).join('\n'); + + // Units to pack: paragraphs, or the lines of a paragraph too long to be one. + const units: [number, number][] = []; + for (const [s, e] of splitParagraphs(lines)) { + const n = (await tokenize(slice(s, e))).length; + if (n <= maxTokens) units.push([s, e]); + else for (let i = s; i < e; i++) units.push([i, i + 1]); + } + + // Greedy packing of consecutive units; a candidate window is measured + // exactly rather than summed, because tokenization is not additive across + // the joins. + let ws = -1; + let we = -1; + const flush = async (): Promise<void> => { + if (ws < 0) return; + const text = slice(ws, we); + out.push({ + ...chunk, + text, + tokens: await tokenize(text), + startLine: chunk.startLine + ws, + endLine: chunk.startLine + we - 1, + }); + ws = -1; + we = -1; + }; + for (const [s, e] of units) { + if (ws < 0) { ws = s; we = e; continue; } + if ((await tokenize(slice(ws, e))).length <= maxTokens) { we = e; continue; } + await flush(); + ws = s; + we = e; + } + await flush(); + } + return out; +} diff --git a/packages/rig/src/sources/chunking.ts b/packages/rig/src/sources/chunking.ts index ea64eaef..0ac9660d 100644 --- a/packages/rig/src/sources/chunking.ts +++ b/packages/rig/src/sources/chunking.ts @@ -12,6 +12,7 @@ */ import type { Chunk } from '../resources/types'; +import { splitParagraphs } from '../resources/fit'; /** * Raw page content buffered during web research for post-research reranking @@ -46,35 +47,32 @@ export interface FetchedPage { export function chunkFetchedPages(pages: FetchedPage[]): Chunk[] { const chunks: Chunk[] = []; for (const page of pages) { - const paragraphs = page.text - .split(/\n\s*\n/) - .map((p) => p.trim()) - .filter((p) => p.length > 40); - - if (paragraphs.length === 0) { - if (page.text.trim().length > 40) { - chunks.push({ - resource: page.url, - heading: page.title || page.url, - section: '', - text: page.text.trim(), - tokens: [], - startLine: 1, - endLine: 1, - }); - } - continue; + // Real line ranges over the page text: a consumer holding the same text can + // resolve any chunk, and windows over these chunks keep distinct start lines. + const lines = page.text.split('\n'); + const before = chunks.length; + for (const [start, end] of splitParagraphs(lines)) { + const text = lines.slice(start, end).join('\n').trim(); + if (text.length <= 40) continue; + chunks.push({ + resource: page.url, + heading: page.title || page.url, + section: '', + text, + tokens: [], + startLine: start + 1, + endLine: end, + }); } - - for (let i = 0; i < paragraphs.length; i++) { + if (chunks.length === before && page.text.trim().length > 40) { chunks.push({ resource: page.url, heading: page.title || page.url, section: '', - text: paragraphs[i], + text: page.text.trim(), tokens: [], - startLine: i + 1, - endLine: i + 1, + startLine: 1, + endLine: lines.length, }); } } @@ -107,21 +105,27 @@ export async function chunkHtml(html: string, url: string, title: string): Promi const chunks: Chunk[] = []; let currentHeading = title; let currentText = ''; - let chunkIndex = 0; + // Real line ranges over the text this function builds: the emitted sections + // joined by one blank line. Distinct start lines per chunk, contiguous + // ranges, and a line count that matches each chunk's text. + let cursor = 0; function flushSection() { const text = currentText.trim(); if (text.length > 40) { + if (chunks.length > 0) cursor += 1; + const startLine = cursor + 1; + const endLine = cursor + text.split('\n').length; + cursor = endLine; chunks.push({ resource: url, heading: currentHeading || title || url, section: '', text, tokens: [], - startLine: chunkIndex + 1, - endLine: chunkIndex + 1, + startLine, + endLine, }); - chunkIndex++; } currentText = ''; } diff --git a/packages/abilities/corpus/test/bm25.test.ts b/packages/rig/test/bm25.test.ts similarity index 100% rename from packages/abilities/corpus/test/bm25.test.ts rename to packages/rig/test/bm25.test.ts diff --git a/packages/rig/test/chunking.test.ts b/packages/rig/test/chunking.test.ts new file mode 100644 index 00000000..8cdaac1d --- /dev/null +++ b/packages/rig/test/chunking.test.ts @@ -0,0 +1,45 @@ +/** + * The HTML chunkers address the text they build by real, contiguous line + * ranges — not ordinals — so a consumer holding the same built text can + * resolve any chunk, and windows over these chunks keep distinct start lines. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { chunkFetchedPages, chunkHtml } from '../src/sources/chunking'; + +const LONG = (tag: string) => `${tag} paragraph with enough words in it to clear the forty character floor.`; + +describe('chunkFetchedPages — real line ranges over the page text', () => { + it('numbers each paragraph by its lines in the page text and keeps the ranges contiguous', () => { + const text = `${LONG('First')}\nsecond line of the first\n\n${LONG('Second')}\n\n\n${LONG('Third')}`; + const chunks = chunkFetchedPages([{ url: 'https://x/y', title: 'T', text }]); + expect(chunks.map((c) => [c.startLine, c.endLine])).toEqual([[1, 2], [4, 4], [7, 7]]); + for (const c of chunks) { + expect(c.text.split('\n')).toHaveLength(c.endLine - c.startLine + 1); + } + }); + + it('gives a page with no paragraph over the floor one chunk spanning its lines', () => { + const text = `${LONG('Only')}\nand a tail line`; + const chunks = chunkFetchedPages([{ url: 'u', title: 't', text }]); + expect(chunks).toHaveLength(1); + expect([chunks[0].startLine, chunks[0].endLine]).toEqual([1, 2]); + }); +}); + +describe('chunkHtml — real line ranges over the built text', () => { + it('assigns contiguous, distinct line ranges across sections of one page', async () => { + const html = `<article><h1>Intro</h1><p>${LONG('Intro')}</p><p>${LONG('More intro')}</p>` + + `<h2>Body</h2><p>${LONG('Body')}</p></article>`; + const chunks = await chunkHtml(html, 'https://x/y', 'T'); + expect(chunks.length).toBeGreaterThanOrEqual(2); + let expectedStart = 1; + for (const c of chunks) { + expect(c.startLine).toBeGreaterThanOrEqual(expectedStart); + expect(c.text.split('\n')).toHaveLength(c.endLine - c.startLine + 1); + expectedStart = c.endLine + 1; + } + expect(new Set(chunks.map((c) => c.startLine)).size).toBe(chunks.length); + }); +}); diff --git a/packages/rig/test/content-routes.test.ts b/packages/rig/test/content-routes.test.ts index e52ddac5..8ab8cfbf 100644 --- a/packages/rig/test/content-routes.test.ts +++ b/packages/rig/test/content-routes.test.ts @@ -343,3 +343,101 @@ describe('content routes', () => { }); }); }); + +describe('content routes — the manifest and its config, two plain doors', () => { + // A document attachment's page renders are their own roots named in its + // sidecar, which lives in the manifest's config blob. The UI resolves a + // citation through those two records, so both need a door — OCI-shaped, by + // digest, through the manifest — while raw blobs stay HEAD-only. + it('serves a manifest by digest with the caching headers a blob gets, and 304 on a match', async () => { + const { store, root, rep } = fixture(); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/vnd.oci.image.manifest.v1+json'); + expect(res.headers.get('etag')).toBe(`"${root.digest}"`); + const manifest = await res.json() as { layers: { digest: string }[] }; + expect(manifest.layers.some((l) => l.digest === rep.digest)).toBe(true); + const again = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}`, { headers: { 'If-None-Match': `"${root.digest}"` } }); + expect(again.status).toBe(304); + const missing = await fetch(`${base}/v1/media/sha256:${'0'.repeat(64)}`); + expect(missing.status).toBe(404); + }); + }); + + it('serves the retained source through the manifest under its own media type, and 404 when none was kept', async () => { + // The original as supplied — a document's PDF, an image before + // normalization — resolved by ROLE through the manifest. Raw blobs stay + // HEAD-only; this door exists because a manifest names the source. + const { store, root, source } = fixture(); + const md = store.putBlob(new TextEncoder().encode('# T\n'), 'text/markdown'); + const sourceless = store.putAttachment({ representations: [md] }); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}/source`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('image/jpeg'); + expect(res.headers.get('etag')).toBe(`"${source.digest}"`); + expect(new Uint8Array(await res.arrayBuffer())).toEqual(JPEG); + const head = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}/source`, { method: 'HEAD' }); + expect(head.status).toBe(200); + const none = await fetch(`${base}/v1/media/${encodeURIComponent(sourceless.digest)}/source`); + expect(none.status).toBe(404); + const missing = await fetch(`${base}/v1/media/sha256:${'0'.repeat(64)}/source`); + expect(missing.status).toBe(404); + }); + }); + + it('serves the config blob through the manifest under its own media type', async () => { + const dir = mkdtempSync(join(tmpdir(), 'lloyal-routes-')); + const store = new FileAttachmentStore(dir); + const md = store.putBlob(new TextEncoder().encode('# T\n'), 'text/markdown'); + const sidecar = new TextEncoder().encode(JSON.stringify({ title: 'T', pages: [] })); + const root = store.putAttachment({ representations: [md], config: { bytes: sidecar, mediaType: 'application/vnd.lloyal.document.v1+json' } }); + const { root: imageRoot } = fixture(); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}/config`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/vnd.lloyal.document.v1+json'); + expect(await res.json()).toEqual({ title: 'T', pages: [] }); + // An image manifest's config is OCI's canonical empty blob, served as such. + const empty = await fetch(`${base}/v1/media/${encodeURIComponent(fixture().root.digest)}/config`).catch(() => null); + void empty; void imageRoot; + }); + }); + + it('still refuses GET on a raw blob: the manifest doors do not open one for sources', async () => { + const { store, source } = fixture(); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/content/${encodeURIComponent(source.digest)}`); + expect(res.status).toBe(405); + }); + }); +}); + +describe('content routes — whose fault an ingest failure is', () => { + it('answers 5xx when the store cannot write, not 400: disk full is not the client\'s doing', async () => { + const { store } = fixture(); + const noSpace = (): never => { + const e = new Error('ENOSPC: no space left on device, write') as Error & { code: string; syscall: string }; + e.code = 'ENOSPC'; e.syscall = 'write'; + throw e; + }; + await withServer({ store, ingest: async () => noSpace() }, async (base) => { + const res = await fetch(`${base}/v1/media/ingress`, { method: 'POST', body: PNG }); + expect(res.status).toBe(500); + }); + }); + + it('answers 408 when the ingress reports its own time bound, as it does for a slow upload', async () => { + const { store } = fixture(); + const timedOut = (): never => { + const e = new Error('document exceeded 120000ms') as Error & { code: string }; + e.code = 'ETIMEDOUT'; + throw e; + }; + await withServer({ store, ingest: async () => timedOut() }, async (base) => { + const res = await fetch(`${base}/v1/media/ingress`, { method: 'POST', body: PNG }); + expect(res.status).toBe(408); + }); + }); +}); diff --git a/packages/rig/test/documents.test.ts b/packages/rig/test/documents.test.ts new file mode 100644 index 00000000..a5f359ae --- /dev/null +++ b/packages/rig/test/documents.test.ts @@ -0,0 +1,79 @@ +/** + * `loadDocuments` — from document attachments in the content store to the + * resources and chunks the retrieval stack already understands. No codec, no + * addon: a document root is a manifest whose config is the sidecar. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { FileAttachmentStore } from '@lloyal-labs/media/node'; +import { DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, DocumentMeta } from '@lloyal-labs/media'; +import { loadDocuments } from '../src/resources/documents'; + +const MARKDOWN = ['# A paper', '', '## Intro', '', 'First line of the intro.', 'Second line of the intro.', '', '## Method', '', 'Method body.'].join('\n'); + +const meta = (title: string): DocumentMeta => ({ + title, pageCount: 2, + sections: [ + { heading: title, path: title, origin: 'heuristic', startLine: 1, endLine: 2, pageStart: 1, pageEnd: 1 }, + { heading: 'Intro', path: 'Intro', origin: 'heuristic', startLine: 3, endLine: 7, pageStart: 1, pageEnd: 1 }, + { heading: 'Method', path: 'Method', origin: 'heuristic', startLine: 8, endLine: 10, pageStart: 2, pageEnd: 2 }, + ], + pages: [ + { page: 1, startLine: 1, endLine: 7, chars: 60, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }, + { page: 2, startLine: 8, endLine: 10, chars: 12, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }, + ], + figures: [], tables: [], + derive: { profile: 'pdf.v1', pdfium: 'test', dpi: 150, maxSide: 2048, maxPixels: 4_194_304, format: 'image/png', renderedPages: 2, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false }, +}); + +function commitDocument(store: FileAttachmentStore, title: string, markdown = MARKDOWN): Attachment { + const md = store.putBlob(new TextEncoder().encode(markdown), 'text/markdown'); + return store.putAttachment({ + representations: [md], + config: { bytes: new TextEncoder().encode(JSON.stringify(meta(title))), mediaType: DOCUMENT_CONFIG_TYPE }, + }); +} + +describe('loadDocuments', () => { + it('turns a document root into a resource and one chunk per section, with real lines and the section path', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const root = commitDocument(store, 'A paper'); + const [doc] = loadDocuments(store, [root]); + expect(doc.attachment).toEqual(root); + expect(doc.meta.title).toBe('A paper'); + expect(doc.resource).toEqual({ name: 'A paper', content: MARKDOWN }); + expect(doc.chunks.map((c) => [c.heading, c.section, c.startLine, c.endLine])).toEqual([ + ['A paper', 'A paper', 1, 2], ['Intro', 'Intro', 3, 7], ['Method', 'Method', 8, 10], + ]); + expect(doc.chunks[1].text).toBe('## Intro\n\nFirst line of the intro.\nSecond line of the intro.\n'); + expect(doc.chunks.every((c) => c.resource === 'A paper' && c.tokens.length === 0)).toBe(true); + }); + + it('skips roots that are not documents — an image root is legitimately something else', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const image = store.putAttachment({ representations: [store.putBlob(new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2]), 'image/png')] }); + const doc = commitDocument(store, 'Only one'); + expect(loadDocuments(store, [image, doc]).map((d) => d.meta.title)).toEqual(['Only one']); + }); + + it('dedupes resource names so two documents with one title stay addressable', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const a = commitDocument(store, 'Same'); + const b = commitDocument(store, 'Same', MARKDOWN + '\nextra'); + expect(loadDocuments(store, [a, b]).map((d) => d.resource.name)).toEqual(['Same', 'Same (2)']); + }); + + it('throws naming the digest when the markdown blob is missing or drifted', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const root = commitDocument(store, 'Drifted'); + const md = store.getManifest(root.digest)!.layers[0]; + const path = join((store as unknown as { _dir: string })._dir, 'blobs', 'sha256', md.digest.slice(7)); + require('node:fs').writeFileSync(path, 'tampered'); + expect(() => loadDocuments(store, [root])).toThrow(new RegExp(md.digest.slice(0, 19))); + }); +}); diff --git a/packages/rig/test/fit-chunks.test.ts b/packages/rig/test/fit-chunks.test.ts new file mode 100644 index 00000000..be3d3e72 --- /dev/null +++ b/packages/rig/test/fit-chunks.test.ts @@ -0,0 +1,104 @@ +/** + * `fitChunks` — passages sized for a reranker leaf, by lines. + * + * A word-count tokenizer makes every expectation readable by hand. The size + * is a granularity choice the caller makes; the function knows nothing about + * rerankers beyond the tokenize it is handed. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import type { Chunk } from '@lloyal-labs/lloyal-agents'; +import { fitChunks, splitParagraphs, DEFAULT_CHUNK_TOKENS } from '../src/resources/fit'; + +const words = async (text: string): Promise<number[]> => + text.split(/\s+/).filter(Boolean).map((_, i) => i + 1); + +function chunk(text: string, startLine = 1, extra: Partial<Chunk> = {}): Chunk { + const lines = text.split('\n'); + return { + resource: 'doc.md', heading: 'H', section: 'A > H', text, tokens: [], + startLine, endLine: startLine + lines.length - 1, ...extra, + }; +} + +const PARA = (n: number, w = 'w') => Array.from({ length: n }, () => w).join(' '); + +describe('splitParagraphs', () => { + it('returns half-open line ranges of blank-separated paragraphs, skipping blank runs', () => { + const lines = ['a', 'b', '', '', 'c', '', 'd', 'e', 'f']; + expect(splitParagraphs(lines)).toEqual([[0, 2], [4, 5], [6, 9]]); + }); + + it('treats whitespace-only lines as blank and an empty input as no paragraphs', () => { + expect(splitParagraphs([' ', 'x', ' \t', 'y'])).toEqual([[1, 2], [3, 4]]); + expect(splitParagraphs([])).toEqual([]); + }); +}); + +describe('fitChunks', () => { + it('exports a default passage size', () => { + expect(DEFAULT_CHUNK_TOKENS).toBe(256); + }); + + it('keeps a chunk within the size as one tokenized window', async () => { + const c = chunk(`${PARA(4)}\n\n${PARA(3)}`, 10); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out).toHaveLength(1); + expect(out[0]).toMatchObject({ startLine: 10, endLine: 12, text: c.text }); + expect(out[0].tokens).toHaveLength(7); + }); + + it('packs whole paragraphs into windows that tile the parent lines, inheriting heading and section', async () => { + // 6 + 6 + 6 words: two fit in ten? no — 6+6 = 12 > 10, so one paragraph per window. + const c = chunk(`${PARA(6)}\n\n${PARA(6)}\n\n${PARA(6)}`, 1); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[1, 1], [3, 3], [5, 5]]); + for (const w of out) { + expect(w.tokens.length).toBeLessThanOrEqual(10); + expect(w.heading).toBe('H'); + expect(w.section).toBe('A > H'); + expect(w.resource).toBe('doc.md'); + expect(w.text.split('\n')).toHaveLength(w.endLine - w.startLine + 1); + } + }); + + it('packs consecutive short paragraphs together, keeping the blank lines between them', async () => { + const c = chunk(`${PARA(3)}\n\n${PARA(3)}\n\n${PARA(3)}\n\n${PARA(3)}`, 1); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + // 3+3+3 = 9 ≤ 10, then 3. + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[1, 5], [7, 7]]); + expect(out[0].tokens).toHaveLength(9); + }); + + it('splits a paragraph over the size by lines', async () => { + const c = chunk([PARA(4), PARA(4), PARA(4)].join('\n'), 20); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[20, 21], [22, 22]]); + }); + + it('keeps a single line longer than the size as one window and never splits inside a line', async () => { + const c = chunk(`${PARA(3)}\n${PARA(25)}\n${PARA(3)}`, 5); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[5, 5], [6, 6], [7, 7]]); + expect(out[1].tokens).toHaveLength(25); + }); + + it('never yields two windows with the same (resource, startLine)', async () => { + const c = chunk(`${PARA(30)}\n\n${PARA(30)}`, 1); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + const keys = out.map((w) => `${w.resource}:${w.startLine}`); + expect(new Set(keys).size).toBe(keys.length); + }); + + it('preserves input order across chunks and resources', async () => { + const a = chunk(PARA(3), 1, { resource: 'a.md' }); + const b = chunk(`${PARA(8)}\n\n${PARA(8)}`, 1, { resource: 'b.md' }); + const out = await fitChunks([a, b], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => `${w.resource}:${w.startLine}`)).toEqual(['a.md:1', 'b.md:1', 'b.md:3']); + }); + + it('throws on a size below one instead of silently not capping', async () => { + await expect(fitChunks([chunk('x')], { maxTokens: 0, tokenize: words })).rejects.toThrow(/maxTokens/); + }); +}); diff --git a/packages/rig/test/helpers/rerank-model.ts b/packages/rig/test/helpers/rerank-model.ts new file mode 100644 index 00000000..f8705ae2 --- /dev/null +++ b/packages/rig/test/helpers/rerank-model.ts @@ -0,0 +1,17 @@ +/** + * Where the weights-gated reranker tests find a Qwen3-Reranker GGUF: the + * environment first, then the usual local homes. `null` skips the suite, + * as the SDK's integration tests do without weights. + * + * @category Testing + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +export const RERANK_MODEL_PATH: string | null = [ + process.env.LLAMA_RERANK_MODEL, + path.join(os.homedir(), '.cache/lloyal/models/qwen3-reranker-0.6b-q4_k_m.gguf'), + path.join(os.homedir(), 'dev/apps/lloyal-node/models/qwen3-reranker-0.6b-q4_k_m.gguf'), + path.join(os.homedir(), '.cache/lloyal/models/qwen3-reranker-0.6b-q8_0.gguf'), +].find((p): p is string => !!p && fs.existsSync(p)) ?? null; diff --git a/packages/rig/test/ranges.test.ts b/packages/rig/test/ranges.test.ts new file mode 100644 index 00000000..9d548bc4 --- /dev/null +++ b/packages/rig/test/ranges.test.ts @@ -0,0 +1,34 @@ +/** + * Half-open line-range arithmetic shared by the read tools' per-agent dedup. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { mergeRanges, subtractRanges } from '../src/ranges'; + +describe('subtractRanges', () => { + it('returns the target untouched when nothing is covered', () => { + expect(subtractRanges([0, 10], [])).toEqual([[0, 10]]); + expect(subtractRanges([0, 10], [[10, 20]])).toEqual([[0, 10]]); + }); + + it('cuts covered spans out, splitting around a hole in the middle', () => { + expect(subtractRanges([0, 10], [[3, 5]])).toEqual([[0, 3], [5, 10]]); + expect(subtractRanges([0, 10], [[0, 4], [6, 10]])).toEqual([[4, 6]]); + }); + + it('returns nothing when the target is fully covered', () => { + expect(subtractRanges([2, 8], [[0, 10]])).toEqual([]); + }); +}); + +describe('mergeRanges', () => { + it('collapses overlapping and touching ranges, sorted by start', () => { + expect(mergeRanges([[5, 8], [0, 3], [3, 6]])).toEqual([[0, 8]]); + }); + + it('keeps disjoint ranges apart and an empty input empty', () => { + expect(mergeRanges([[0, 2], [4, 6]])).toEqual([[0, 2], [4, 6]]); + expect(mergeRanges([])).toEqual([]); + }); +}); diff --git a/packages/rig/test/registry.test.ts b/packages/rig/test/registry.test.ts index 696d9596..7eddbef5 100644 --- a/packages/rig/test/registry.test.ts +++ b/packages/rig/test/registry.test.ts @@ -33,6 +33,8 @@ import { describe, it, expect, vi } from 'vitest'; import { run, ensure } from 'effection'; import { AbilityConfigStoreCtx } from '@lloyal-labs/lloyal-agents'; +import { Attachments } from '@lloyal-labs/lloyal-agents'; +import type { AttachmentStore } from '@lloyal-labs/media'; import type { Ability, AbilityManifest, AbilityFactory } from '@lloyal-labs/lloyal-agents'; import { createAbilityRegistry } from '../src/registry'; import { createInMemoryConfigStore } from '../src/config-store'; @@ -134,6 +136,25 @@ describe('createAbilityRegistry', () => { expect(seen).toEqual({ key: 'value' }); }); + it('seeds the content store into the factory scope, beside the config store and the reranker', async () => { + // An ability that reads documents resolves them through the content store + // at enable time and in its tools; the detached scope must carry the store + // the harness installed, not the null default. + const seen = await run(function* () { + const store = { marker: 'the harness store' } as unknown as AttachmentStore; + yield* Attachments.set(store); + let inFactory: unknown; + const factory: AbilityFactory = function* () { + inFactory = yield* Attachments.expect(); + return fakeApp({ name: 'docs' }); + }; + const registry = yield* createAbilityRegistry({ configStore: createInMemoryConfigStore() }); + yield* registry.enable(factory); + return inFactory === store; + }); + expect(seen).toBe(true); + }); + it('runs the factory body (setup) when enabled', async () => { const onSetup = vi.fn(); await run(function* () { diff --git a/packages/rig/test/reranker-capacity.test.ts b/packages/rig/test/reranker-capacity.test.ts new file mode 100644 index 00000000..72834e8b --- /dev/null +++ b/packages/rig/test/reranker-capacity.test.ts @@ -0,0 +1,90 @@ +/** + * The default passage size fits the smallest reranker sizing rig ships. + * + * `DEFAULT_CHUNK_TOKENS` is a retrieval granularity choice, not a capacity + * contract: the reranker's per-leaf budget depends on the instruction and the + * chat template, which only a loaded `Rerank` knows. This test therefore + * asks the loaded reranker one question through its existing truncation + * callback — does a passage of the default size, under a query of the + * reserve we assume, score whole at nSeqMax 10 / nCtx 4096? — and, separately, + * proves the callback path is live with a passage no leaf could hold. No + * assertion names a boundary. + * + * Runs only when a Qwen3 reranker GGUF is present (same discovery as the SDK's + * integration suite): `LLAMA_RERANK_MODEL`, then the cache and sibling paths. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import * as path from 'node:path'; +import { createContext } from '@lloyal-labs/lloyal.node'; +import { Rerank } from '@lloyal-labs/sdk'; +import type { RerankTruncation, SessionContext } from '@lloyal-labs/sdk'; +import { DEFAULT_CHUNK_TOKENS } from '../src/resources/fit'; +import { RERANK_MODEL_PATH } from './helpers/rerank-model'; + +const MODEL_PATH = RERANK_MODEL_PATH; + +/** The smallest sizing `createReranker` ships when a host says nothing. */ +const N_SEQ_MAX = 10; +const N_CTX = 4096; +/** The query allowance the default chunk size is stated against. */ +const QUERY_TOKENS = 64; + +/** A text that tokenizes to exactly `n` tokens: one repeated word is monotone + * in its count, so the search converges in a few steps. */ +async function textOfTokens(rerank: Rerank, n: number): Promise<string> { + let words = n; + for (let i = 0; i < 40; i++) { + const text = Array.from({ length: words }, () => 'alpha').join(' '); + const len = (await rerank.tokenize(text)).length; + if (len === n) return text; + words += n - len; + } + throw new Error(`textOfTokens: could not build a ${n}-token text`); +} + +async function drain<T>(iter: AsyncIterable<T>): Promise<void> { + for await (const _ of iter) { /* consume */ } +} + +const describeWithModel = MODEL_PATH ? describe : describe.skip; + +describeWithModel(`DEFAULT_CHUNK_TOKENS fits the shipped reranker sizing — ${MODEL_PATH ? path.basename(MODEL_PATH) : 'no model'}`, () => { + it( + 'a default-size passage under the query reserve scores whole; a passage no leaf could hold is truncated', + async () => { + const truncations: RerankTruncation[] = []; + const ctx = (await createContext({ + modelPath: MODEL_PATH!, + nCtx: N_CTX, + nSeqMax: N_SEQ_MAX, + typeK: 'q4_0', + typeV: 'q4_0', + })) as unknown as SessionContext; + const rerank = await Rerank.create(ctx, { + nSeqMax: N_SEQ_MAX, + nCtx: N_CTX, + onTruncate: (e) => truncations.push(e), + }); + try { + const query = await textOfTokens(rerank, QUERY_TOKENS); + const filler = await rerank.tokenize('word '.repeat(2 * Math.floor(N_CTX / N_SEQ_MAX) + 16)); + + // Granularity fits: the default passage is scored whole. + await drain(rerank.score(query, [filler.slice(0, DEFAULT_CHUNK_TOKENS)], 1)); + expect(truncations).toEqual([]); + + // The callback path is live: twice the per-sequence slice cannot fit any leaf. + const oversize = filler.slice(0, 2 * Math.floor(N_CTX / N_SEQ_MAX)); + await drain(rerank.score(query, [oversize], 1)); + expect(truncations).toHaveLength(1); + expect(truncations[0]).toMatchObject({ docIndex: 0, origLen: oversize.length }); + expect(truncations[0].maxLen).toBeGreaterThanOrEqual(DEFAULT_CHUNK_TOKENS); + } finally { + rerank.dispose(); + } + }, + 120_000, + ); +}); diff --git a/packages/rig/test/reranker-resolution.test.ts b/packages/rig/test/reranker-resolution.test.ts new file mode 100644 index 00000000..65caa0de --- /dev/null +++ b/packages/rig/test/reranker-resolution.test.ts @@ -0,0 +1,43 @@ +/** + * The reranker's resolution under the KV type `createReranker` ships by + * default: ten identical passages in one call fill the ten leaves, so any + * spread between their scores is KV noise, not content. Measured 2026-09-07 + * on real document windows: at q4_0 the spread was 4–6 logits and the verdict + * changed sign between leaves; at q8_0 it was 0.05–0.12 at the same speed. + * Every admission threshold stands on this number. + * + * Weights-gated like `reranker-capacity.test.ts`. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { run, call } from 'effection'; +import * as path from 'node:path'; +import { createReranker } from '../src/reranker'; +import { RERANK_MODEL_PATH } from './helpers/rerank-model'; + +const describeWithModel = RERANK_MODEL_PATH ? describe : describe.skip; + +const QUERY = 'What is the capital of France?'; +const RELEVANT = 'Paris is the capital and largest city of France, on the river Seine.'; +const IRRELEVANT = 'The recipe calls for two eggs, a cup of flour and a pinch of salt.'; +const COPIES = 10; +/** The widest spread the default KV type may show between identical leaves. */ +const MAX_SPREAD = 0.5; + +describeWithModel(`the default KV type resolves the reranker's verdicts — ${RERANK_MODEL_PATH ? path.basename(RERANK_MODEL_PATH) : 'no model'}`, () => { + it('ten identical passages score within half a logit of each other, and the relevant passage stays above the irrelevant one', async () => { + const scores: number[] = await run(function* () { + const reranker = yield* createReranker(RERANK_MODEL_PATH!, { nSeqMax: COPIES, nCtx: 4096 }); + const texts = [...Array<string>(COPIES).fill(RELEVANT), ...Array<string>(COPIES).fill(IRRELEVANT)]; + return yield* call(() => reranker.scoreBatch(QUERY, texts)); + }); + const relevant = scores.slice(0, COPIES); + const irrelevant = scores.slice(COPIES); + const spread = (xs: number[]): number => Math.max(...xs) - Math.min(...xs); + const show = (xs: number[]): string => xs.map((s) => s.toFixed(2)).join(' '); + expect(spread(relevant), `relevant copies: ${show(relevant)}`).toBeLessThan(MAX_SPREAD); + expect(spread(irrelevant), `irrelevant copies: ${show(irrelevant)}`).toBeLessThan(MAX_SPREAD); + expect(Math.min(...relevant), `relevant ${show(relevant)} vs irrelevant ${show(irrelevant)}`).toBeGreaterThan(Math.max(...irrelevant)); + }, 120_000); +}); diff --git a/scripts/cut-alpha.test.ts b/scripts/cut-alpha.test.ts index 4def725e..7358b274 100644 --- a/scripts/cut-alpha.test.ts +++ b/scripts/cut-alpha.test.ts @@ -166,12 +166,16 @@ describe('the abilities admit the set', () => { }; const view = (name: string) => { if (name in registry) return registry[name]; throw e404; }; const stamped = planAlphas({ cut: 99, packages: arcPackages(CUTS, EXTERNAL, manifestOf), view }); - for (const dir of ['packages/abilities/web', 'packages/abilities/corpus', 'packages/abilities/wikipedia']) { + for (const dir of ['packages/abilities/web', 'packages/abilities/corpus', 'packages/abilities/wikipedia', 'packages/abilities/documents']) { const peers = manifestOf(dir).peerDependencies ?? {}; for (const [name, range] of Object.entries(peers)) { if (!(name in stamped)) continue; expect(satisfies(stamped[name], range), `${dir}: ${name} ${range} admits ${stamped[name]}`).toBe(true); - expect(satisfies(registry[name], range), `${dir}: ${name} ${range} admits stable ${registry[name]}`).toBe(true); + // A member the registry has never seen stable (media, whose first + // publish was an alpha) has no stable to admit yet. + if (registry[name] !== undefined) { + expect(satisfies(registry[name], range), `${dir}: ${name} ${range} admits stable ${registry[name]}`).toBe(true); + } } } }); diff --git a/scripts/verify-packed-install.sh b/scripts/verify-packed-install.sh index a2fc50da..89d975ec 100755 --- a/scripts/verify-packed-install.sh +++ b/scripts/verify-packed-install.sh @@ -90,14 +90,25 @@ node -e " if (opt !== true) throw new Error('sharp peer is not marked optional'); if ((p.devDependencies||{}).sharp && !peer) throw new Error('devDependency only: never installed for consumers'); if (!(p.files||[]).includes('README.md')) throw new Error('README.md missing from files — the format spec never ships'); - console.log(' ok — peer '+peer+', optional, README packaged'); + // The PDF codec sits beside sharp on the same terms: a consumer that never + // admits a document installs nothing for it. + const pdf = (p.peerDependencies||{})['@embedpdf/pdfium']; + const pdfOpt = ((p.peerDependenciesMeta||{})['@embedpdf/pdfium']||{}).optional; + if (!pdf) throw new Error('@embedpdf/pdfium is not a peerDependency — a consumer cannot discover it'); + if (pdfOpt !== true) throw new Error('@embedpdf/pdfium peer is not marked optional'); + console.log(' ok — peers sharp '+peer+' and @embedpdf/pdfium '+pdf+', both optional, README packaged'); " echo "3. './node' normalizes from the packed build once sharp is installed" npm install --silent --no-audit --no-fund sharp@^0.35.4 node -e " - const { normalizeImage, FileAttachmentStore } = require('@lloyal-labs/media/node'); + const { normalizeImage, FileAttachmentStore, createContentIngress, createDocumentIngress } = require('@lloyal-labs/media/node'); if (typeof FileAttachmentStore !== 'function') throw new Error('the node entry is incomplete'); + // The document ingress is exported, but loading the entry must not have + // loaded the codec: it is required at call time, like sharp, and it is not + // installed in this consumer. + if (typeof createContentIngress !== 'function' || typeof createDocumentIngress !== 'function') throw new Error('the content ingress is missing from the node entry'); + if (Object.keys(require.cache).some(k => k.includes('@embedpdf'))) throw new Error('the node entry loaded @embedpdf/pdfium at module load'); const fs = require('fs'); const src = new Uint8Array(fs.readFileSync('$CAT')); normalizeImage(src, { maxPixels: 65536 }).then(o => { diff --git a/tsconfig.test.json b/tsconfig.test.json index 4c24952e..55ef82a5 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -54,7 +54,8 @@ "@lloyal-labs/relay": ["packages/relay/src/index.ts"], "@lloyal-labs/corpus-ability": ["packages/abilities/corpus/src/index.ts"], "@lloyal-labs/web-ability": ["packages/abilities/web/src/index.ts"], - "@lloyal-labs/wikipedia-ability": ["packages/abilities/wikipedia/src/index.ts"] + "@lloyal-labs/wikipedia-ability": ["packages/abilities/wikipedia/src/index.ts"], + "@lloyal-labs/documents-ability": ["packages/abilities/documents/src/index.ts"] } }, "include": [ From 5d7e97bb5a4caa79ced8397051f7edb6ef376d36 Mon Sep 17 00:00:00 2001 From: lloyal-research <research@lloyal.ai> Date: Mon, 7 Sep 2026 13:31:06 +1000 Subject: [PATCH 2/2] =?UTF-8?q?fix(media,rig,abilities):=20review=20round?= =?UTF-8?q?=20on=20988d7ae=20=E2=80=94=20each=20finding=20behind=20a=20tes?= =?UTF-8?q?t=20that=20was=20red=20first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - media: image bounds are clipped to the page as they are read and a crop is sized by the same `fitScale` rule as a page, so the side and area ceilings hold for crops (an image painted far past the page no longer asks for a 20,000-pixel bitmap); `readPage` walks Form XObjects with a depth bound and a visit cap, applying the pixel-ceiling check to nested images and carrying their bounds into page space through the composed form matrices. Fixtures `clipped.pdf`, `form.pdf`, `bigimage.pdf`, `bigform.pdf` (hand-written in make.sh) pin all three. - rig: `DelegateTool` forwards the delegating call's `attachments` into the child pool — the run's staged roots plus everything admitted so far — so a delegated agent sees the documents; `delegate-assets.test.ts`. - documents: a repeated `view_page` carries the page again with a note. The tool cannot see whether its last result landed (a settle nudge may have replaced it), so suppression left the model blind; admission is the gate. - rig: `reranker-options.test.ts` states the q8_0 default contract; the earlier gate statement had misattributed this file's failure. --- .../documents/src/tools/view-page.ts | 15 ++-- .../documents/test/view-page.test.ts | 8 +- packages/media/README.md | 2 +- packages/media/src/pdf.ts | 84 +++++++++++++++--- packages/media/test/fixtures/pdf/bigform.pdf | Bin 0 -> 841 bytes packages/media/test/fixtures/pdf/bigimage.pdf | Bin 0 -> 667 bytes packages/media/test/fixtures/pdf/clipped.pdf | Bin 0 -> 667 bytes packages/media/test/fixtures/pdf/form.pdf | Bin 0 -> 884 bytes packages/media/test/fixtures/pdf/make.sh | 38 ++++++++ packages/media/test/pdf-render.test.ts | 55 ++++++++++++ packages/rig/src/tools/delegate.ts | 4 + packages/rig/test/delegate-assets.test.ts | 43 +++++++++ packages/rig/test/reranker-options.test.ts | 6 +- 13 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 packages/media/test/fixtures/pdf/bigform.pdf create mode 100644 packages/media/test/fixtures/pdf/bigimage.pdf create mode 100644 packages/media/test/fixtures/pdf/clipped.pdf create mode 100644 packages/media/test/fixtures/pdf/form.pdf create mode 100644 packages/rig/test/delegate-assets.test.ts diff --git a/packages/abilities/documents/src/tools/view-page.ts b/packages/abilities/documents/src/tools/view-page.ts index 6081233f..b387a0fd 100644 --- a/packages/abilities/documents/src/tools/view-page.ts +++ b/packages/abilities/documents/src/tools/view-page.ts @@ -35,6 +35,11 @@ export function projectable(page: PageFacts): boolean { * DESCRIPTOR: no bytes, no ingress, no normalizer permit; the pool resolves it * through the store and admits it on the media rail. A text-only page says * so instead, and a page past the render bound says it is not archived. + * + * A repeat by the same agent carries the page AGAIN, with a note. The tool + * cannot see whether its last result landed — the pool may have replaced it + * with a settle nudge — so suppressing a repeat would leave the model blind; + * admission is the only gate on what a page costs. */ export class ViewPageTool extends Tool<{ document: string; page: number; figure?: number }> { readonly name = 'view_page'; @@ -68,9 +73,9 @@ export class ViewPageTool extends Tool<{ document: string; page: number; figure? const where = { document: doc.meta.title, id: doc.id, page: page.page }; const key = `${context?.agentId ?? ''}:${doc.id}:${page.page}:${args.figure ?? ''}`; - if (this._viewed.has(key)) { - return { ...where, note: args.figure !== undefined ? `Already viewed figure ${args.figure} on page ${page.page}` : `Already viewed page ${page.page}` }; - } + const again = this._viewed.has(key) + ? { note: args.figure !== undefined ? `You viewed figure ${args.figure} on page ${page.page} before.` : `You viewed page ${page.page} before.` } + : {}; if (args.figure !== undefined) { const figures = doc.meta.figures.filter((f) => f.page === page.page); @@ -81,13 +86,13 @@ export class ViewPageTool extends Tool<{ document: string; page: number; figure? : `Page ${page.page} has ${figures.length} figure(s); figure must be between 1 and ${figures.length}.` }; } this._viewed.add(key); - return { ...where, figure: args.figure, ...(fig.caption ? { caption: fig.caption } : {}), + return { ...where, ...again, figure: args.figure, ...(fig.caption ? { caption: fig.caption } : {}), cite: pageCite(doc.attachment, page.page), [TOOL_ATTACHMENTS_KEY]: [fig.root] }; } if (!projectable(page)) return { ...where, note: `Page ${page.page} is text only — read_document gives you its text.` }; if (!page.render) return { ...where, note: `Page ${page.page} is not archived as an image.` }; this._viewed.add(key); - return { ...where, cite: pageCite(doc.attachment, page.page), [TOOL_ATTACHMENTS_KEY]: [page.render] }; + return { ...where, ...again, cite: pageCite(doc.attachment, page.page), [TOOL_ATTACHMENTS_KEY]: [page.render] }; } } diff --git a/packages/abilities/documents/test/view-page.test.ts b/packages/abilities/documents/test/view-page.test.ts index 1e891edd..2dff68dc 100644 --- a/packages/abilities/documents/test/view-page.test.ts +++ b/packages/abilities/documents/test/view-page.test.ts @@ -56,10 +56,14 @@ describe('view_page', () => { expect(TOOL_ATTACHMENTS_KEY in r).toBe(false); }); - it('page one is always projectable; a repeat by the same agent is a note, another agent sees it', async () => { + it('page one is always projectable; a repeat by the same agent carries the page again with a note — admission is the only gate', async () => { const { doc, renders, view } = setup(); expect((await view({ document: documentId(doc), page: 1 }))[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); - expect((await view({ document: documentId(doc), page: 1 })).note).toMatch(/Already viewed page 1/); + // A settle rejection can replace the first result with a nudge; the tool + // cannot see that, so a repeat must still put the page in front of the model. + const again = await view({ document: documentId(doc), page: 1 }); + expect(again[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); + expect(again.note).toMatch(/viewed page 1 before/); expect((await view({ document: documentId(doc), page: 1 }, [doc], 2))[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); expect((await view({ document: documentId(doc), page: 5 })).error).toMatch(/out of range/); }); diff --git a/packages/media/README.md b/packages/media/README.md index d4c55eab..3d70ad03 100644 --- a/packages/media/README.md +++ b/packages/media/README.md @@ -184,7 +184,7 @@ replay and any OCI tool read it exactly as they read a photo: | `ai.lloyal.derive.page` | the 1-based page it renders | | `ai.lloyal.derive.dpi`, `.width`, `.height`, `.format` | how it was rendered | | `ai.lloyal.derive.source` | the digest of the PDF blob it was rendered from | -| `ai.lloyal.derive.bbox` | a figure crop's box on the page, in PDF user space | +| `ai.lloyal.derive.bbox` | a figure crop's box on the page, in PDF user space, clipped to the page | `materialize` projects only representations in a format the projector decodes. A document root therefore yields no bitmaps — its text is for diff --git a/packages/media/src/pdf.ts b/packages/media/src/pdf.ts index 7bd45708..0a00c6de 100644 --- a/packages/media/src/pdf.ts +++ b/packages/media/src/pdf.ts @@ -193,6 +193,33 @@ const MIN_FIGURE_AREA_RATIO = 0.02; // PDFium constants (fpdfview.h, fpdf_edit.h, fpdf_progressive.h). const PAGEOBJ_PATH = 2; const PAGEOBJ_IMAGE = 3; +const PAGEOBJ_FORM = 5; +/** How deep a walk follows Form XObjects nested in Form XObjects. */ +const MAX_FORM_DEPTH = 8; +/** How many page objects one page's walk will visit, forms included. */ +const MAX_WALKED_OBJECTS = 20_000; + +/** A PDF matrix `[a b c d e f]`: (x, y) ↦ (a·x + c·y + e, b·x + d·y + f). */ +type Matrix = [number, number, number, number, number, number]; +const IDENTITY: Matrix = [1, 0, 0, 1, 0, 0]; +/** `outer ∘ inner`: apply `inner` first, then `outer`. */ +function compose(outer: Matrix, inner: Matrix): Matrix { + const [a, b, c, d, e, f] = outer; const [a2, b2, c2, d2, e2, f2] = inner; + return [a * a2 + c * b2, b * a2 + d * b2, a * c2 + c * d2, b * c2 + d * d2, a * e2 + c * f2 + e, b * e2 + d * f2 + f]; +} +/** The axis-aligned bounds of a rectangle after a matrix. */ +function transformRect(m: Matrix, [x0, y0, x1, y1]: PageImage['bbox']): PageImage['bbox'] { + const xs: number[] = []; const ys: number[] = []; + for (const [x, y] of [[x0, y0], [x1, y0], [x0, y1], [x1, y1]] as const) { + xs.push(m[0] * x + m[2] * y + m[4]); ys.push(m[1] * x + m[3] * y + m[5]); + } + return [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)]; +} +/** The part of `r` inside the page, or null when nothing of it is. */ +function clipToPage(r: PageImage['bbox'], width: number, height: number): PageImage['bbox'] | null { + const c: PageImage['bbox'] = [Math.max(0, r[0]), Math.max(0, r[1]), Math.min(width, r[2]), Math.min(height, r[3])]; + return c[2] > c[0] && c[3] > c[1] ? c : null; +} const BITMAP_BGRA = 4; const FLAG_ANNOT = 0x01; const FLAG_REVERSE_BYTE_ORDER = 0x10; @@ -316,15 +343,32 @@ function readPage(codec: Codec, doc: OpenDocument, index: number, opts: { text: const read = readStruct(codec, page); // Objects: image bounds (and a decompression-bomb check on their pixel size - // before anything could decode them), and the path count. + // before anything could decode them), and the path count. The walk follows + // Form XObjects — rendering will decode what they hold, so the check and + // the counts must see it too — with each child's bounds carried into page + // space through the form matrices, and clipped to the page: an object may + // extend far past the page and the renderer clips it, so the visible part + // is the figure. const images: PageImage[] = []; let pathObjects = 0; - const count = pdfium.FPDFPage_CountObjects(page); - for (let i = 0; i < count; i++) { - const obj = pdfium.FPDFPage_GetObject(page, i); + let walked = 0; + const matrixBuf = codec.malloc(24); + const walk = (obj: number, depth: number, m: Matrix): void => { + if (++walked > MAX_WALKED_OBJECTS) return; const type = pdfium.FPDFPageObj_GetType(obj); - if (type === PAGEOBJ_PATH) pathObjects++; - if (type !== PAGEOBJ_IMAGE) continue; + if (type === PAGEOBJ_PATH) { pathObjects++; return; } + if (type === PAGEOBJ_FORM) { + if (depth >= MAX_FORM_DEPTH) return; + let fm: Matrix = IDENTITY; + if (pdfium.FPDFPageObj_GetMatrix(obj, matrixBuf)) { + fm = [0, 1, 2, 3, 4, 5].map((k) => pdfium.pdfium.getValue(matrixBuf + k * 4, 'float')) as Matrix; + } + const inner = compose(m, fm); + const n = pdfium.FPDFFormObj_CountObjects(obj); + for (let k = 0; k < n; k++) walk(pdfium.FPDFFormObj_GetObject(obj, k), depth + 1, inner); + return; + } + if (type !== PAGEOBJ_IMAGE) return; if (pdfium.FPDFImageObj_GetImagePixelSize(obj, scratch, scratch + 4)) { const w = pdfium.pdfium.getValue(scratch, 'i32'); const h = pdfium.pdfium.getValue(scratch + 4, 'i32'); @@ -332,14 +376,22 @@ function readPage(codec: Codec, doc: OpenDocument, index: number, opts: { text: throw new PdfError(`readPage: page ${index + 1} embeds a ${w}×${h} image, over the ${MAX_INPUT_PIXELS}-pixel ceiling`); } } - if (!pdfium.FPDFPageObj_GetBounds(obj, scratch, scratch + 4, scratch + 8, scratch + 12)) continue; - const bbox: PageImage['bbox'] = [ + if (!pdfium.FPDFPageObj_GetBounds(obj, scratch, scratch + 4, scratch + 8, scratch + 12)) return; + const own: PageImage['bbox'] = [ pdfium.pdfium.getValue(scratch, 'float'), pdfium.pdfium.getValue(scratch + 4, 'float'), pdfium.pdfium.getValue(scratch + 8, 'float'), pdfium.pdfium.getValue(scratch + 12, 'float'), ]; + const bbox = clipToPage(depth === 0 ? own : transformRect(m, own), width, height); + if (!bbox) return; const mcid = pdfium.FPDFPageObj_GetMarkedContentID(obj); const altText = mcid >= 0 ? read?.altByMcid.get(mcid) : undefined; - images.push({ index: i, bbox, ...(altText ? { altText } : {}) }); + images.push({ index: images.length, bbox, ...(altText ? { altText } : {}) }); + }; + try { + const count = pdfium.FPDFPage_CountObjects(page); + for (let i = 0; i < count; i++) walk(pdfium.FPDFPage_GetObject(page, i), 0, IDENTITY); + } finally { + codec.free(matrixBuf); } // Characters, with box, size, weight and marked-content id. @@ -464,12 +516,17 @@ function readBookmarks(codec: Codec, doc: OpenDocument): Bookmark[] { /** Pixel size for a page at `dpi`, held under the side and area ceilings. */ function renderSize(widthPt: number, heightPt: number, dpi: number): { width: number; height: number; scale: number } { - let scale = dpi / 72; + const scale = fitScale(widthPt, heightPt, dpi / 72); + return { width: Math.max(1, Math.floor(widthPt * scale)), height: Math.max(1, Math.floor(heightPt * scale)), scale }; +} + +/** `scale`, lowered until a `widthPt` × `heightPt` region fits the side and area ceilings. */ +function fitScale(widthPt: number, heightPt: number, scale: number): number { const longest = Math.max(widthPt, heightPt) * scale; if (longest > RENDER_MAX_SIDE) scale *= RENDER_MAX_SIDE / longest; const area = widthPt * heightPt * scale * scale; if (area > DEFAULT_MAX_PIXELS) scale *= Math.sqrt(DEFAULT_MAX_PIXELS / area); - return { width: Math.max(1, Math.floor(widthPt * scale)), height: Math.max(1, Math.floor(heightPt * scale)), scale }; + return scale; } /** Copy a bitmap's RGBA rows off the heap (the stride may exceed the row). */ @@ -509,9 +566,12 @@ function renderPageRgba(codec: Codec, page: number, width: number, height: numbe } /** Render a region of a page — a figure's bounds — at `scale` pixels per point. */ -function renderRegionRgba(codec: Codec, page: number, pageHeightPt: number, bbox: PageImage['bbox'], scale: number): { rgba: Uint8Array; width: number; height: number } { +function renderRegionRgba(codec: Codec, page: number, pageHeightPt: number, bbox: PageImage['bbox'], pageScale: number): { rgba: Uint8Array; width: number; height: number } { const { pdfium } = codec; const [x0, y0, x1, y1] = bbox; + // The bounds were clipped to the page when read, so at the page's scale the + // crop is never larger than the page render; the ceilings hold regardless. + const scale = fitScale(x1 - x0, y1 - y0, pageScale); const width = Math.max(1, Math.round((x1 - x0) * scale)); const height = Math.max(1, Math.round((y1 - y0) * scale)); const bmp = pdfium.FPDFBitmap_CreateEx(width, height, BITMAP_BGRA, 0, 0); diff --git a/packages/media/test/fixtures/pdf/bigform.pdf b/packages/media/test/fixtures/pdf/bigform.pdf new file mode 100644 index 0000000000000000000000000000000000000000..227597ea60f0e288dc522ca9ce1b70c0b4c53b67 GIT binary patch literal 841 zcmb7DO;5r=5aqnTVlF-E?ox^z2nRrvr~$!56XKz?3zX0<ZWn|8Ne})5{Ugrw0})J& zn>OuE-^_daX6g*Py%Tv_W6sCt+dGpWKu?#f*#v&~vR1%5zV%mn4t(g(RSH!EjG)zG zDhUZl`=1Fpj$cJ#3KN&$k61--5({!Kk-i)!5{*#^&3X8Z3M0R*w=fZimm<5d-bkf- zV*(UJN*@N(r3!5J-rGv>h!wf$Lh8g~#T05qw3JT!Qo*%K=5_%xQ<z$#{Fprfr(#_V zx0F&ui*pl2nVcI{is0kT)MnQ8w26VY|0Ta4OQ`S5FO<20yxS=wCrU@?86Xj*{w|gZ z{w1~_V}F6)M<Mwjgpj~5RWx5<+i)ky$dxwZwI3j?tDd7kjV{`FzR0FSWjZ=u>x5d@ z0A7mT(_E=kj$PauHDdzK7%}7v>YnF%FoRu8qSv$_`Ht1Gc?U6B#o;}ORlVbwD{;0C qaJ9xBhc$k*QYQNc#?h;SGn_ZAX?%;>PpxlkroHYqnB$xedh83K*yBO~ literal 0 HcmV?d00001 diff --git a/packages/media/test/fixtures/pdf/bigimage.pdf b/packages/media/test/fixtures/pdf/bigimage.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9807e555e16a413073d989c9a05fba70931d1b52 GIT binary patch literal 667 zcmZWnO;5r=5aqnTVlEtYyOcH_2nRrvr~$!56XIcMm$HO*al07wPkQhl=pS*WABwPP z)9%|h^Il)4IT;Mk#Ce-FzrH^{nE(&WV$C`o;16$G1-vhv+?WLTL?$YS780h=?J||c zM5O)yf|AE?bezN7C;BsXF}uuxI4Go((?XK53aO<Izf-Z6J+p_ohkW4`H};#V-0W<G zhREsDc(GQIE8a(`fPhp{i!PMT9Cpm1U7@vg+Sd+lRhGCF_#QdTomDbrFNI#MOV20; zPW=Z4?x{;ftfC;*x?h=H?TgQL3s<B$N-;&?k2*#*{2WgJzg9X~VK}(+$nM72*;YnK z8>m+usp(Y@cu%{0qHN!!Ta!^h7r<-N2c8>^#;LKrRZHeU$Vj1F5C(xCz!H8jf%a)Z z_J@TS(GezEI3h<_D>%h`{{VAV>W#9+RL%6ef)M|pF*Fd>H04fOSNLpS;4C&T$3yl5 DA;+^Q literal 0 HcmV?d00001 diff --git a/packages/media/test/fixtures/pdf/clipped.pdf b/packages/media/test/fixtures/pdf/clipped.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0100b3d5bd9ab7301e522727a056982573abd6ab GIT binary patch literal 667 zcmZWnT~ER=6y>?U;=b^RT{lJ`Av^%0L=6Zgnh+n#R>lZhu`34slRo$l^oQuZV-tk# zBkk?E_ne;I>bO6siHka`etmy@G64?g`HHpMz#rZ=3V2Uixz-Wzv5Zs(E&?Xd=`fXs z1f=!<gguVm#9;<AkKoT(MQ>9J;wX`J&J&5oD1`2L_?-%4+0{FkIfxfd{$ag|%JkL* zD2S9kjpi#A*nB=r1o*^?T=XJ!YO!Jl^&(nIr*)~|R;7_$g4ZO4nKeo#>;+s03F*lv zwU+Nd{tFV|#M9SsPt}Tq!V+RH>+U@)_3`O;ZgW?M3AzdVF%FSc0KZmov?PO2?pmAa zMh3XuSFdrPCRbhH-PmShWqLZ<=#&g<0xv}$Y5u8H&IRp^S}+G143AR`X!yS8!vg+d z0%g;H^bl*HMJF-gVvC%_T>m`gd94G^8X2#Z$-8PAzbk0q4Qfs2ExIYQ(%4*Qy@|b8 Iy&MhLFUeuFP5=M^ literal 0 HcmV?d00001 diff --git a/packages/media/test/fixtures/pdf/form.pdf b/packages/media/test/fixtures/pdf/form.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e6364821bfe15237099882b1398a644530473058 GIT binary patch literal 884 zcmah|OHRWu5M}LC%%)3loHnIM6^l{=2oR-30tvBjQ<EAsadBKI9Er=Y;0ACJX3`|| zgTS(tcsw)jy&1dK>xSE6r@?BUU+*7GfCpyyz}ju#S1(fq{6IQ6F){F-j8z6cI`m<G zpQ&_2j<jD5SorZ%J<8y=PVRS@#mrI%VkwYLHUkO9Ds)}o@N+fNvSa3O>!H8z<qPxm zRc2;3LPEs!=3@AuBA5S%a{&RNA{I@AP90{<pix*$>a;HzoT)T+WAIzVFmqPPggqi( zLKV>PXhsQe%HB0_NjVC)q7Z~xW?NLQWW@)wq04I-8jCqPl^%h5aG{=a>8wVTC8D5% zKXslY0-A_f=`1AFy;TVak&7t4B!WVJ7eNXC6LFH@MF4)SM^rH2$4bXzY8X~}V(efl zBRuY^CmpH&Q3v&Em-UoAFv--U)Y}&Df~@eD$ZowoS97zzbF1!|2fNIx1~h}99>6_R zHi2EB0qKveiG5zP2_NgcX7hthTixH=@C$q_QW@*4)Dva%=Q7Z*3U={o8bdd|cqW;X Q*5%xrEx~HF!;6r81JR7{eE<Le literal 0 HcmV?d00001 diff --git a/packages/media/test/fixtures/pdf/make.sh b/packages/media/test/fixtures/pdf/make.sh index a89c2b3b..3962c635 100755 --- a/packages/media/test/fixtures/pdf/make.sh +++ b/packages/media/test/fixtures/pdf/make.sh @@ -142,3 +142,41 @@ for off in offsets: out.write(f"{off:010d} 00000 n \n".encode()) out.write(f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) open(sys.argv[1], 'wb').write(out.getvalue()) PY + +# Bounds fixtures, hand-written (Ghostscript would normalise them away): +# clipped.pdf — a 1×1 red image painted into a 2000-pt square on a 100-pt page: +# the visible figure is the page; the crop must not be the square. +# form.pdf — a 2×2 green image inside a Form XObject (/Matrix translate 10,10 +# under an outer cm translate 40,40): the image is nested, at +# page-space [50,50]–[150,150]. +# bigimage.pdf — a direct image whose dictionary says 10001×10000 pixels. +# bigform.pdf — the same image inside a Form XObject. +python3 - <<'PY' +import io +def pdf(path, objs): + out = io.BytesIO(); out.write(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n"); offsets = [] + for i, o in enumerate(objs, 1): + offsets.append(out.tell()); out.write(f"{i} 0 obj\n".encode() + o + b"\nendobj\n") + xref = out.tell() + out.write(f"xref\n0 {len(objs)+1}\n".encode() + b"0000000000 65535 f \n") + for off in offsets: out.write(f"{off:010d} 00000 n \n".encode()) + out.write(f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) + open(path, 'wb').write(out.getvalue()) +def stream(dict_head, data): return dict_head + b" /Length " + str(len(data)).encode() + b" >>\nstream\n" + data + b"\nendstream" +def image(w, h, rgb, data=None): + data = data if data is not None else bytes(rgb) * (w * h) + return stream(b"<< /Type /XObject /Subtype /Image /Width %d /Height %d /ColorSpace /DeviceRGB /BitsPerComponent 8" % (w, h), data) +def page(box, content, xobjects): + return b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %d %d] /Resources << /XObject << %s >> >> /Contents 4 0 R >>" % (box, box, xobjects) +CAT = b"<< /Type /Catalog /Pages 2 0 R >>"; PAGES = b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>" +pdf('clipped.pdf', [CAT, PAGES, page(100, None, b"/Im1 5 0 R"), + stream(b"<<", b"q 2000 0 0 2000 -950 -950 cm /Im1 Do Q"), image(1, 1, (255, 0, 0))]) +form = stream(b"<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] /Matrix [1 0 0 1 10 10] /Resources << /XObject << /Im1 6 0 R >> >>", b"q 100 0 0 100 0 0 cm /Im1 Do Q") +pdf('form.pdf', [CAT, PAGES, page(200, None, b"/Fx1 5 0 R"), + stream(b"<<", b"q 1 0 0 1 40 40 cm /Fx1 Do Q"), form, image(2, 2, (0, 170, 119))]) +big = image(10001, 10000, (0, 0, 0), data=b"\x00\x00\x00") +pdf('bigimage.pdf', [CAT, PAGES, page(100, None, b"/Im1 5 0 R"), stream(b"<<", b"q 100 0 0 100 0 0 cm /Im1 Do Q"), big]) +bigform = stream(b"<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] /Resources << /XObject << /Im1 6 0 R >> >>", b"q 100 0 0 100 0 0 cm /Im1 Do Q") +pdf('bigform.pdf', [CAT, PAGES, page(100, None, b"/Fx1 5 0 R"), stream(b"<<", b"q /Fx1 Do Q"), bigform, big]) +PY +ls -la clipped.pdf form.pdf bigimage.pdf bigform.pdf diff --git a/packages/media/test/pdf-render.test.ts b/packages/media/test/pdf-render.test.ts index 7e3c42f0..4834071e 100644 --- a/packages/media/test/pdf-render.test.ts +++ b/packages/media/test/pdf-render.test.ts @@ -13,6 +13,8 @@ import { FileAttachmentStore } from '../src/node'; import { createDocumentIngress } from '../src/pdf'; import { normalizeImage } from '../src/image'; import { asDocumentMeta } from '../src/document'; +import { RENDER_MAX_SIDE } from '../src/pdf'; +import { DEFAULT_MAX_PIXELS } from '../src/image'; import { representationsOf } from '../src/attachment'; import { materialize } from '../src/ingress'; import type { DocumentMeta } from '../src/document'; @@ -110,3 +112,56 @@ describe('the scanned page', () => { expect(meta.pages[0].render).toBeTruthy(); }, 60_000); }); + +describe('the bounds on figures', () => { + /** The crop's own record of its size: the derive annotations on its representation. */ + function cropSize(store: FileAttachmentStore, meta: DocumentMeta, i = 0): { width: number; height: number } { + const manifest = store.getManifest(meta.figures[i].root.digest)!; + const a = manifest.layers[0].annotations ?? {}; + return { width: Number(a['ai.lloyal.derive.width']), height: Number(a['ai.lloyal.derive.height']) }; + } + + it('an image painted far past the page is cropped to its visible part, under the same ceilings as a page', async () => { + // A 1×1 red image into a 2000-pt square on a 100-pt page: the object's + // bounds are 20× the page; only the page is visible. + const { store, meta } = await ingest('clipped.pdf'); + expect(meta.pages[0].imageObjects).toBe(1); + expect(meta.figures).toHaveLength(1); + const [x0, y0, x1, y1] = meta.figures[0].bbox; + expect([x0, y0]).toEqual([0, 0]); + expect(x1).toBeCloseTo(100, 0); + expect(y1).toBeCloseTo(100, 0); + const { width, height } = cropSize(store, meta); + expect(Math.max(width, height)).toBeLessThanOrEqual(RENDER_MAX_SIDE); + expect(width * height).toBeLessThanOrEqual(DEFAULT_MAX_PIXELS); + // The crop is the visible page, so it is no larger than the page render. + const page = store.getManifest(meta.pages[0].render!.digest)!.layers[0].annotations ?? {}; + expect(width).toBeLessThanOrEqual(Number(page['ai.lloyal.derive.width'])); + const png = materialize(store, [meta.figures[0].root] as never).bitmaps[0]; + const [r, g] = (await sharp(Buffer.from(png)).stats()).channels.map((c) => c.mean); + expect(r).toBeGreaterThan(200); + expect(g).toBeLessThan(40); + }, 90_000); + + it('an image inside a Form XObject is counted, placed on the page through the form matrices, and cropped', async () => { + // A 2×2 green image in a form with /Matrix translate (10,10), drawn under + // an outer translate (40,40): page space [50,50]–[150,150] on a 200-pt page. + const { store, meta } = await ingest('form.pdf'); + expect(meta.pages[0].imageObjects).toBe(1); + expect(meta.figures).toHaveLength(1); + const [x0, y0, x1, y1] = meta.figures[0].bbox; + expect(x0).toBeCloseTo(50, 0); expect(y0).toBeCloseTo(50, 0); + expect(x1).toBeCloseTo(150, 0); expect(y1).toBeCloseTo(150, 0); + const png = materialize(store, [meta.figures[0].root] as never).bitmaps[0]; + const [r, g, b] = (await sharp(Buffer.from(png)).stats()).channels.map((c) => c.mean); + expect(g).toBeGreaterThan(120); + expect(r).toBeLessThan(60); + expect(b).toBeGreaterThan(60); + }, 90_000); + + it('an image over the pixel ceiling is refused before anything renders — direct, and nested in a form alike', async () => { + await expect(ingest('bigimage.pdf', { maxRenderedPages: 0 })).rejects.toThrow(/pixel ceiling/); + await expect(ingest('bigform.pdf', { maxRenderedPages: 0 })).rejects.toThrow(/pixel ceiling/); + }, 90_000); +}); + diff --git a/packages/rig/src/tools/delegate.ts b/packages/rig/src/tools/delegate.ts index dc8a2bf6..2f3994c3 100644 --- a/packages/rig/src/tools/delegate.ts +++ b/packages/rig/src/tools/delegate.ts @@ -212,6 +212,10 @@ export class DelegateTool extends Tool<Record<string, unknown>> { parent: context?.branch, pruneOnReturn: opts.pruneOnReturn ?? true, scorer: context?.scorer, + // The child pool starts with the assets this call can see: the run's + // staged roots plus everything any agent admitted so far. The static + // pool options predate the run and cannot carry them. + attachments: context?.attachments, }); const result = { diff --git a/packages/rig/test/delegate-assets.test.ts b/packages/rig/test/delegate-assets.test.ts new file mode 100644 index 00000000..1d96deb0 --- /dev/null +++ b/packages/rig/test/delegate-assets.test.ts @@ -0,0 +1,43 @@ +/** + * A delegated child pool starts with the assets the delegating call can see — + * the run's staged roots plus everything admitted so far — not with the + * static pool options alone. + * + * @category Testing + */ +import { describe, it, expect, vi } from 'vitest'; +import { run } from 'effection'; + +const seen = vi.hoisted(() => ({ poolOpts: [] as Record<string, unknown>[] })); +vi.mock('@lloyal-labs/lloyal-agents', async (importOriginal) => { + const actual = await importOriginal<typeof import('@lloyal-labs/lloyal-agents')>(); + return { + ...actual, + agentPool: vi.fn(function* (opts: Record<string, unknown>) { + seen.poolOpts.push(opts); + return { agents: [], totalTokens: 0, totalToolCalls: 0 }; + }), + }; +}); + +import { Trace, NullTraceWriter } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { DelegateTool } from '../src/tools/delegate'; + +const root = (hex: string): Attachment => + ({ mediaType: 'application/vnd.oci.image.manifest.v1+json', digest: `sha256:${hex.repeat(64)}`, size: 700 }) as unknown as Attachment; + +describe('delegate — the child pool inherits the run\'s assets', () => { + it('forwards the delegating call\'s attachments into agentPool, admitted roots included', async () => { + const staged = root('a'); + const admittedMidRun = root('b'); + const tool = new DelegateTool({ poolOpts: {}, systemPrompt: 'sys', extractTasks: (a) => a.tasks as string[] }); + await run(function* () { + yield* Trace.set(new NullTraceWriter()); + return yield* tool.execute({ tasks: ['look into it'] }, { agentId: 7, attachments: [staged, admittedMidRun] } as unknown as ToolContext); + }); + expect(seen.poolOpts).toHaveLength(1); + expect(seen.poolOpts[0].attachments).toEqual([staged, admittedMidRun]); + }); +}); diff --git a/packages/rig/test/reranker-options.test.ts b/packages/rig/test/reranker-options.test.ts index 5637c4e7..6a8f5f10 100644 --- a/packages/rig/test/reranker-options.test.ts +++ b/packages/rig/test/reranker-options.test.ts @@ -67,12 +67,12 @@ describe('createReranker — KV precision', () => { yield* createReranker('/fake/reranker.gguf', opts); }); - it('requests q4_0 for both KV types when the caller specifies neither', async () => { + it('requests q8_0 for both KV types when the caller specifies neither — the resolution the verdicts need', async () => { await load(); expect(createContext).toHaveBeenCalledTimes(1); const args = createContext.mock.calls[0][0]; - expect(args.typeK).toBe('q4_0'); - expect(args.typeV).toBe('q4_0'); + expect(args.typeK).toBe('q8_0'); + expect(args.typeV).toBe('q8_0'); }); it('requests the caller\'s KV types when given', async () => {