diff --git a/package-lock.json b/package-lock.json index 34e0483d..d4dcfac4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,6 +46,13 @@ "node": ">=18" } }, + "node_modules/@embedpdf/pdfium": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/@embedpdf/pdfium/-/pdfium-2.15.0.tgz", + "integrity": "sha512-KgpRND2MYcdbhzb2EMb4WzWcJYrR0A6JXvhMv4WthEHKt6qmNo2v/MC68bpYvpveYT9GNnUnY/+TG5MpXY3pRw==", + "dev": true, + "license": "MIT" + }, "node_modules/@emnapi/runtime": { "version": "1.11.3", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", @@ -1063,6 +1070,10 @@ "resolved": "packages/dev-tools", "link": true }, + "node_modules/@lloyal-labs/documents-ability": { + "resolved": "packages/abilities/documents", + "link": true + }, "node_modules/@lloyal-labs/host": { "resolved": "packages/host", "link": true @@ -1246,12 +1257,6 @@ "win32" ] }, - "node_modules/@lloyal-labs/lloyal.node/node_modules/@lloyal-labs/lloyal.node-darwin-x64": { - "optional": true - }, - "node_modules/@lloyal-labs/lloyal.node/node_modules/@lloyal-labs/lloyal.node-linux-x64": { - "optional": true - }, "node_modules/@lloyal-labs/media": { "resolved": "packages/media", "link": true @@ -1637,6 +1642,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.6", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.6.tgz", + "integrity": "sha512-uN+9i8bFT5CUcZfyIEYDrSueACEyKGbUs5kC/72DGlZZoinh84sJfVV0i8UOJD1asdzkvLPBRrKs41kZ8MdEXg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -3842,6 +3854,17 @@ "effection": "^4.0.2" } }, + "packages/abilities/documents": { + "name": "@lloyal-labs/documents-ability", + "version": "0.1.0", + "license": "SEE LICENSE IN LICENSE", + "peerDependencies": { + "@lloyal-labs/lloyal-agents": "^5.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/media": ">=0.2.0-0 <0.3.0", + "@lloyal-labs/rig": "^5.5.0 || >=5.6.0-0 <6.0.0", + "effection": "^4.0.2" + } + }, "packages/abilities/web": { "name": "@lloyal-labs/web-ability", "version": "2.0.2", @@ -3926,12 +3949,18 @@ "version": "0.2.0-alpha.3", "license": "SEE LICENSE IN LICENSE", "devDependencies": { + "@embedpdf/pdfium": "^2.15.0", + "@types/emscripten": "^1.41.6", "sharp": "^0.35.4" }, "peerDependencies": { + "@embedpdf/pdfium": "^2.15.0", "sharp": "^0.35.4" }, "peerDependenciesMeta": { + "@embedpdf/pdfium": { + "optional": true + }, "sharp": { "optional": true } diff --git a/packages/abilities/corpus/src/index.ts b/packages/abilities/corpus/src/index.ts index 8c4ba4db..99f84641 100644 --- a/packages/abilities/corpus/src/index.ts +++ b/packages/abilities/corpus/src/index.ts @@ -1,8 +1,8 @@ /** * `@lloyal-labs/corpus-ability` — HDK reference ability: local-corpus research. * - * Requires a reranker (its `search` tool scores chunks); loads + tokenizes the - * corpus at construction, and returns a validated {@link Ability} whose + * Requires a reranker (its `search` tool scores chunks); loads the corpus and + * fits it into reranker-sized windows at construction, and returns a validated {@link Ability} whose * {@link CorpusSource} is already-bound. * * @packageDocumentation @@ -13,15 +13,13 @@ import { join } from "node:path"; import { call } from "effection"; import { AbilityConfigStoreCtx, RerankerCtx } from "@lloyal-labs/lloyal-agents"; import type { AbilityManifest, Tool } from "@lloyal-labs/lloyal-agents"; -import { defineAbility } from "@lloyal-labs/rig"; +import { defineAbility, fitChunks, DEFAULT_CHUNK_TOKENS } from "@lloyal-labs/rig"; import type { Reranker } from "@lloyal-labs/rig"; import { loadResources, chunkResources } from "@lloyal-labs/rig/node"; import { CorpusSource } from "./source"; export { CorpusSource } from "./source"; export type { CorpusSourceOpts, CorpusPromptData } from "./source"; -export { BM25Index } from "./bm25"; -export type { Bm25Opts, Bm25Hit } from "./bm25"; // The declarative manifest + skill template, read once at module load. The // manifest is handed to defineAbility, which advertises it on the factory — so the @@ -63,8 +61,11 @@ export const createCorpusAbility = defineAbility(manifest, function* () { } const resources = loadResources(corpusPath); - const chunks = chunkResources(resources); - yield* call(() => reranker.tokenizeChunks(chunks)); + // Sections become windows the reranker scores whole; the size is a retrieval + // choice (see DEFAULT_CHUNK_TOKENS), the tokens are the reranker's own. + const chunks = yield* call(() => + fitChunks(chunkResources(resources), { maxTokens: DEFAULT_CHUNK_TOKENS, tokenize: (t) => reranker.tokenize(t) }), + ); const source = new CorpusSource(resources, chunks, reranker); const tools: Record = {}; diff --git a/packages/abilities/corpus/src/tools/read-file.ts b/packages/abilities/corpus/src/tools/read-file.ts index 4f453f2c..c2cba480 100644 --- a/packages/abilities/corpus/src/tools/read-file.ts +++ b/packages/abilities/corpus/src/tools/read-file.ts @@ -2,64 +2,8 @@ import type { Operation } from 'effection'; import { Tool } from '@lloyal-labs/lloyal-agents'; import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; import type { Resource, Chunk } from '@lloyal-labs/rig'; +import { mergeRanges, subtractRanges } from '@lloyal-labs/rig'; -/** - * Subtract previously-covered ranges from a target range - * - * Given a target half-open interval `[s, e)` and an array of - * already-covered intervals, returns the sub-ranges of `[s, e)` - * that have not yet been covered. Used by {@link ReadFileTool} - * to avoid re-reading lines the agent has already seen. - * - * @param range - Target range `[start, end)` (0-indexed) - * @param covered - Array of previously-covered `[start, end)` ranges - * @returns Uncovered sub-ranges of the target - * - * @category Rig - */ -export function subtractRanges( - [s, e]: [number, number], - covered: [number, number][], -): [number, number][] { - let ranges: [number, number][] = [[s, e]]; - for (const [cs, ce] of covered) { - ranges = ranges.flatMap(([a, b]): [number, number][] => { - if (ce <= a || cs >= b) return [[a, b]]; - const result: [number, number][] = []; - if (a < cs) result.push([a, cs]); - if (ce < b) result.push([ce, b]); - return result; - }); - } - return ranges; -} - -/** - * Merge overlapping or adjacent half-open ranges into a minimal set - * - * Sorts the input ranges by start position, then collapses any - * overlapping or touching intervals. Used by {@link ReadFileTool} - * to maintain a compact record of lines already read per agent. - * - * @param ranges - Array of `[start, end)` ranges to merge - * @returns Merged non-overlapping ranges sorted by start - * - * @category Rig - */ -export function mergeRanges(ranges: [number, number][]): [number, number][] { - if (ranges.length === 0) return []; - const sorted = [...ranges].sort((a, b) => a[0] - b[0]); - const merged: [number, number][] = [sorted[0]]; - for (let i = 1; i < sorted.length; i++) { - const last = merged[merged.length - 1]; - if (sorted[i][0] <= last[1]) { - last[1] = Math.max(last[1], sorted[i][1]); - } else { - merged.push(sorted[i]); - } - } - return merged; -} /** * Read content from corpus files by line range diff --git a/packages/abilities/corpus/src/tools/search.ts b/packages/abilities/corpus/src/tools/search.ts index 851e7832..9d451f8b 100644 --- a/packages/abilities/corpus/src/tools/search.ts +++ b/packages/abilities/corpus/src/tools/search.ts @@ -4,7 +4,7 @@ import { Tool, Trace, admitChunks } from '@lloyal-labs/lloyal-agents'; import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; import type { Chunk } from '@lloyal-labs/rig'; import type { Reranker } from '@lloyal-labs/rig'; -import { BM25Index } from '../bm25'; +import { BM25Index } from '@lloyal-labs/rig'; /** * Default score floor for search hits — a useful **discrimination signal**, diff --git a/packages/abilities/corpus/test/ability.test.ts b/packages/abilities/corpus/test/ability.test.ts index f4a8cd18..a491edc7 100644 --- a/packages/abilities/corpus/test/ability.test.ts +++ b/packages/abilities/corpus/test/ability.test.ts @@ -10,9 +10,12 @@ import { createInMemoryConfigStore } from '@lloyal-labs/rig'; import { createCorpusAbility } from '../src/index'; import { SearchTool } from '../src/tools/search'; -// The factory only calls reranker.tokenizeChunks at construction; search -// scoring (which needs a real cross-encoder) isn't exercised here. -const mockReranker = { tokenizeChunks() {} } as unknown as Reranker; +// The factory fits the corpus into windows at construction, which needs only +// the reranker's tokenizer; search scoring (a real cross-encoder) isn't +// exercised here. Words stand in for tokens. +const mockReranker = { + tokenize: async (text: string) => text.split(/\s+/).filter(Boolean).map((_, i) => i + 1), +} as unknown as Reranker; let dir: string; beforeAll(() => { @@ -47,6 +50,55 @@ describe('createCorpusAbility', () => { ).rejects.toThrow(/requires a reranker/); }); + it('fits a long section into more than one window with real line ranges', async () => { + // One heading over 360 words on 60 lines: longer than DEFAULT_CHUNK_TOKENS + // under the word tokenizer, so the factory must cut it into windows the + // reranker can score whole. The source keeps its chunks private; the + // search envelope's `totalScored` is the chunk count, and each hit carries + // the window's real lines. + const longDir = mkdtempSync(join(tmpdir(), 'corpus-long-')); + const body = Array.from({ length: 60 }, (_, i) => `line ${i + 1} alpha beta gamma delta`).join('\n'); + writeFileSync(join(longDir, 'long.md'), `# Long\n\n${body}\n`); + try { + const wordy: Reranker = { + ...mkScoringReranker(new Map()), + tokenize: async (text: string) => text.split(/\s+/).filter(Boolean).map((_, i) => i + 1), + score(_query: string, chunks: Chunk[]) { + return (async function* () { + yield { + filled: chunks.length, total: chunks.length, + results: chunks.map((c) => ({ + file: c.resource, heading: c.heading, section: c.section, snippet: c.text, + score: 1, startLine: c.startLine, endLine: c.endLine, + })), + }; + })(); + }, + }; + const result = (await run(function* () { + yield* Trace.set(new NullTraceWriter()); + const store = createInMemoryConfigStore(); + yield* store.set('corpus', { corpusPath: longDir }); + yield* AbilityConfigStoreCtx.set(store); + yield* RerankerCtx.set(wordy); + const ability = yield* createCorpusAbility(); + const search = ability.tools.find((t) => t.name === 'search')!; + return yield* search.execute({ query: 'alpha' }); + })) as { hits: ScoredChunk[]; totalScored: number }; + + expect(result.totalScored).toBeGreaterThan(1); + const starts = result.hits.map((h) => h.startLine); + expect(new Set(starts).size).toBe(starts.length); + for (const h of result.hits) { + expect(h.endLine).toBeGreaterThanOrEqual(h.startLine); + expect(h.startLine).toBeGreaterThanOrEqual(1); + expect(h.endLine).toBeLessThanOrEqual(62); + } + } finally { + rmSync(longDir, { recursive: true, force: true }); + } + }); + it('throws when corpusPath config is missing', async () => { await expect( run(function* () { diff --git a/packages/abilities/documents/LICENSE b/packages/abilities/documents/LICENSE new file mode 100644 index 00000000..d19bdb84 --- /dev/null +++ b/packages/abilities/documents/LICENSE @@ -0,0 +1,107 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2026 Lloyal Labs + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publish, and distribute the +Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A "Permitted Purpose" is any purpose other than a Competing Use. A +"Competing Use" means making the Software available to others in a +commercial product or service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a Licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives +of the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, +INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO +THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL +DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software +under the Apache License, Version 2.0 that is effective on the second +anniversary of the date we make the Software available. On or after that +date, you may use the Software under the Apache License, Version 2.0, in +which case the following will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/abilities/documents/LICENSE-FAQ.md b/packages/abilities/documents/LICENSE-FAQ.md new file mode 100644 index 00000000..cd89f587 --- /dev/null +++ b/packages/abilities/documents/LICENSE-FAQ.md @@ -0,0 +1,257 @@ +# Licensing FAQ + +> Canonical version at https://docs.lloyal.ai/licensing/faq. +> This file is a synced copy. Edit the canonical source and re-run +> `scripts/sync-license-faq.sh` in lloyal-sdk to update all copies. + + +**You can build and sell commercial products using HDK.** + +> HDK is free to build products with; it is not free to become the +> replacement HDK platform. + +That single sentence is the entire restriction reduced to one line. The rest +of this page is illustration. + +## The short version + +HDK 3.0 runtime packages — `liblloyal`, `lloyal-node`, and the lloyal-sdk +packages (`agents`, `sdk`, `rig`, `abilities/corpus`, `abilities/web`) — are +**[Fair Source](https://fair.io)** under **FSL-1.1-Apache-2.0** (the Functional +Source License, Apache 2.0 future grant). + +Each version converts to Apache 2.0 two years after its release. The +restriction during those two years is narrow: **you cannot offer a competing +HDK runtime, a managed HDK service, or an alternative Ability distribution +channel.** Everything else — commercial use, redistribution, modification, +sale, embedding in shipped products — is freely permitted. + +The reason the channel restriction exists is consumer-protective: every Ability +listed on `apps.lloyal.ai` is reviewed by Lloyal Labs for tool-safety, +manifest conformance, and signature provenance before publication. Pinning +the Ability ecosystem to one verified channel keeps the AI-safety review meaningful +(consumers can rely on a single trust boundary) and prevents protocol +fragmentation (an Ability that works on one harness works on every harness). + +The `lloyal-ai` CLI is licensed under **MIT**; the `hdk-create-app` +scaffolder under **Apache 2.0**. Both are unrestricted, and neither is +part of the runtime stack. + +## Can I ship a commercial product built on HDK? + +**Yes.** This is the question everyone has and the answer is straightforward. +Concretely: + +- **Shipping a paid intelligent inbox app to consumers** — permitted ✅ +- **Selling an Excel-with-AI desktop app to enterprises** — permitted ✅ +- **Embedding HDK in a medical device sold commercially** — permitted ✅ +- **A consulting firm building a custom intelligent harness for a Fortune + 500 client, charging $500K for the engagement, deploying on client + infrastructure** — permitted ✅ +- **An indie dev shipping a paid productivity app on the Mac App Store** — + permitted ✅ +- **An OEM shipping HDK inside an infotainment system or industrial device** + — permitted ✅ +- **A startup building an end-user product on top of HDK** — including + vertical research apps, workflow tools, and agent applications — permitted + ✅, *as long as the product is not offering HDK itself as a substitute + runtime, managed HDK service, or competing Ability distribution channel*. +- **A research lab using HDK in published academic work** — permitted ✅ +- **Forking HDK on GitHub to learn, modify, demo, or contribute** — permitted ✅ +- **Running HDK internally inside your company for any business use** — + permitted ✅ + +If you are building something *with* HDK, you are almost certainly fine. + +## What is actually restricted? + +The restriction is narrow and specific: **don't become the replacement +platform vendor**. Concretely: + +- **AWS / Google / Microsoft launching "Bedrock Managed HDK" or "Vertex HDK" + as a hosted runtime service** — restricted ❌ +- **A competitor publishing "OpenHDK" as a forked harness runtime under a + different name** — restricted ❌ +- **A clean-room reimplementation of the HDK runtime in Python / Rust / Go + intended as a drop-in replacement** — restricted ❌ (Competing Use doesn't + require forking source code — a reimplementation that competes is the + same problem) +- **Launching `apps.competitor.com` as an alternative Ability distribution + channel** — restricted ❌ +- **Offering "managed HDK hosting" or "HDK-as-a-Service" as a competing + SaaS** — restricted ❌ +- **A hosted orchestration service exposing HDK-compatible APIs as a + substitute for the HDK runtime** — restricted ❌ + +Notice the pattern: every restricted scenario is "become the platform +vendor," not "build products with HDK." If your project doesn't compete +directly with the HDK runtime or its distribution channel, FSL doesn't +affect you. + +## Why does the LICENSE list four narrow Permitted Purposes then? + +You may read the FSL LICENSE and see this section: + +> Permitted Purposes specifically include using the Software: +> 1. for your internal use and access; +> 2. for non-commercial education; +> 3. for non-commercial research; and +> 4. in connection with professional services that you provide to a +> Licensee using the Software in accordance with these Terms and +> Conditions. + +A careful first reading can mistake this list for the *exclusive* set of +permitted uses — leading to the (incorrect) conclusion that commercial +product distribution is prohibited. + +It isn't. **The operative definition is broader.** Earlier in the same +section, the license states: + +> A "Permitted Purpose" is any purpose other than a Competing Use. + +The four enumerated items are **illustrative examples** added because those +specific cases are ones a careful reader might otherwise hesitate about +("is research permitted? is consulting permitted?"). The four items are +additive clarifications, not a closing of the open-ended definition. + +Sentry, who authored FSL and uses it on their own software, [confirms this +explicitly in their FAQ](https://fsl.software): + +> "You can do anything with FSL software except undermine its producer. You +> can run it for almost all purposes, study it, modify it, and distribute +> your changes…" + +If the license felt restrictive on first read, that's a documented +[FSL adoption hazard](https://fair.io) — many developers hit the same wall. +The answer is to read the "any purpose other than a Competing Use" line as +the operative definition and treat the four enumerated items as examples, +not as a closed list. + +## Will it become Apache 2.0? + +**Yes — automatically, on a per-version schedule.** Each released version of +the runtime stack converts to Apache 2.0 exactly two years after its release +date. The conversion is irrevocable and written into the license text — it's +not a promise from Lloyal Labs, it's a contractual clause. + +For example: if `lloyal-sdk @lloyal-labs/lloyal-agents` v3.0.0 is released +on 2026-06-01, that exact version becomes available under Apache 2.0 on +2028-06-01. Any consumer can elect to use that version under Apache 2.0 +from that date forward — Lloyal Labs takes no action; the grant is +automatic. + +New versions released after v3.0.0 start their own two-year clock from +their own release dates. There is no single global Change Date. + +## Is this OSI-approved open source? + +**No, and we want to be honest about that.** FSL is not OSI-approved +because the OSI definition of open source (clause 6, "No Discrimination +Against Fields of Endeavor") does not permit restrictions on specific +use cases. FSL restricts Competing Use. That restriction takes it out of +strict OSI compliance. + +FSL falls under the [Fair Source](https://fair.io) classification — +source-available licenses that are explicitly developer-friendly: +commercial use permitted, free redistribution, eventual open-source +conversion. Fair Source is a more developer-friendly framing than the +generic "source-available" label, which has been tainted by the +SSPL / Elastic / MongoDB relicensing trauma cycles. + +What this means practically: + +- **You can read the source.** ✅ +- **You can modify it.** ✅ +- **You can sell products built with it.** ✅ +- **You can redistribute it (with the same FSL terms).** ✅ +- **It will be Apache 2.0 in two years.** ✅ +- Some enterprise procurement policies that strictly require OSI-approved + licenses will require an exception for this. We're working on making + that exception easy to grant. + +## Why FSL specifically — why not stay Apache? + +HDK 3.0 introduces installable Abilities. Every Ability declares against a +specific Ability protocol — the bytes-locked intro, catalog format, +tool-selection rule, and boundary marker that the runtime renders into +the spine. Your Ability's reliability depends on every HDK runtime your users +install agreeing on the same protocol. + +Under a permissive license alone, the protocol is forkable. A +well-resourced redistributor could fork the runtime, modify the protocol +surface, and distribute a variant under a different name with captive +distribution. Ability developers then face a fragmented ecosystem: target one +protocol, target both, or pick the bigger distribution and abandon the +others. The cost of that split is paid by Ability developers in testing +burden, divergent behavior, and reliability degradation across runtimes. + +FSL's two-year Competing Use restriction is shaped to block that +fragmentation specifically. After the conversion, anyone can build +whatever they want — by which time the protocol has had enough time to +stabilize through ecosystem use and the protection is no longer the load- +bearing thing keeping it coherent. + +For the longer treatment of this argument, see +[Why FSL](./why-fsl). + +We could have used Apache 2.0 and tried to protect only the channel via +terms-of-service. We could have written a custom license. We chose +standard FSL because it's: + +- **Off-the-shelf** — no bespoke license review at every adopter +- **Recognizable** — Sentry, PowerSync, and others use it +- **Documented** — the FAQ, definitions, and edge cases have been + litigated publicly +- **Time-bounded** — the protocolual Apache 2.0 conversion is the answer + to the "is this just source-available forever?" critique +- **Pre-launch** — relicensing at HDK 3.0 launch is structurally + different from MongoDB / Elastic / HashiCorp relicensing under an + existing installed base, which is what causes the backlash cycle + +## What about the lloyal stack — what's under FSL and what's not? + +| Component | License | Why | +|---|---|---| +| `liblloyal` (C++ engine) | FSL-1.1-Apache-2.0 | Native primitives the runtime is built on | +| `lloyal-node` (N-API binding) | FSL-1.1-Apache-2.0 | The binding that lets Effection drive llama.cpp | +| `@lloyal-labs/lloyal-agents` | FSL-1.1-Apache-2.0 | Runtime framework | +| `@lloyal-labs/lloyal-sdk` | FSL-1.1-Apache-2.0 | Runtime framework | +| `@lloyal-labs/rig` | FSL-1.1-Apache-2.0 | Runtime framework — holds the Ability protocol | +| `@lloyal-labs/corpus`, `@lloyal-labs/web` | FSL-1.1-Apache-2.0 | Reference Abilities shipped in-tree | +| **`lloyal-ai`** (the CLI) | **MIT** | Scaffolder — unrestricted for scaffolding new harnesses and Abilities | +| **`hdk-create-app`** (when shipped) | **Apache 2.0** | Scaffolder — unrestricted, not part of the runtime stack | +| `llama.cpp` (vendored dependency) | MIT (unchanged) | External upstream library; we don't relicense their code | + +## Can I contribute to HDK? + +**Yes.** Contributions are welcome under the same FSL license terms. If you +submit a PR, you're granting Lloyal Labs the right to distribute your +contribution under FSL-1.1-Apache-2.0 (and automatically under Apache 2.0 +two years after each release that includes it). The CONTRIBUTING file in +each repo has the details. + +## I have a use case that's borderline — who do I ask? + +Email [legal@lloyal.ai](mailto:legal@lloyal.ai) (or open a discussion in the repo). The runtime +team will help you confirm whether your use case falls under Permitted +Purpose or Competing Use. We'd rather give you a quick yes than have you +worry about it. + +## Further reading + +- [FSL official site](https://fsl.software) — Sentry's canonical FSL + resources and FAQ +- [Fair Source](https://fair.io) — the category FSL belongs to +- [The FSL template, instantiated for each repo](./fsl-template) +- [Why we chose FSL over BSL, Apache, or a custom license](./why-fsl) + +## Is there a safe harbor for building products with the HDK? + +Yes. The [Lloyal Harness Builder Grant](https://github.com/lloyal-ai/hdk/blob/main/GRANT.md) +irrevocably guarantees that building, selling, and hosting Harnesses and Abilities +is a Permitted Purpose and never a Competing Use — even products that compete +head-on with Lloyal's own (including reasoning.run). Only three uses remain +restricted: offering the HDK itself as a developer framework, hosting the HDK +as-a-service for third-party developers, and operating a general-purpose Ability +distribution channel. Private/internal Ability distribution and your Harness's +own plugin system are explicitly permitted. diff --git a/packages/abilities/documents/README.md b/packages/abilities/documents/README.md new file mode 100644 index 00000000..e9a926c5 --- /dev/null +++ b/packages/abilities/documents/README.md @@ -0,0 +1,13 @@ +# @lloyal-labs/documents-ability + +**Give your agents the documents the user attached.** + +HDK Ability `lloyal/documents` — a PDF attached to the conversation becomes searchable passages with page numbers, readable sections, and page images an agent can look at when a table or chart matters. Retrieval is the corpus stack: BM25 first stage, cross-encoder rerank, honest scores. Evidence is cited by content address, so a page an agent quotes opens in the UI at the exact bytes the run used. + +| Tool | What agents get | +| --- | --- | +| `search_documents` | Passages ranked by the reranker, each with its document, heading, line range, pages and a `cite` URL | +| `read_document` | The exact text of a page or a line range — the verification step after search | +| `view_page` | The page (or one figure on it) as an image, for tables and charts; text-only pages say so | + +Protocol: `document_research` · Documents reach the ability as the assets available to a run (`ToolContext.attachments`) and are indexed once per digest. The table of contents is published via `Source.promptData(attachments)` for spine placement. Distributed through the signed channel. diff --git a/packages/abilities/documents/ability.json b/packages/abilities/documents/ability.json new file mode 100644 index 00000000..50e9d40d --- /dev/null +++ b/packages/abilities/documents/ability.json @@ -0,0 +1,10 @@ +{ + "name": "documents", + "abilityProtocolVersion": "3.0", + "protocol": { + "name": "document_research", + "useWhen": "reading the documents attached to this conversation — PDFs, papers, reports: finding passages, quoting sections with page numbers, and checking a table or figure on a specific page.", + "tools": ["search_documents", "read_document", "view_page"] + }, + "services": ["reranker"] +} diff --git a/packages/abilities/documents/package.json b/packages/abilities/documents/package.json new file mode 100644 index 00000000..d6550f02 --- /dev/null +++ b/packages/abilities/documents/package.json @@ -0,0 +1,33 @@ +{ + "name": "@lloyal-labs/documents-ability", + "version": "0.1.0", + "private": true, + "description": "HDK reference ability — research over the documents attached to a conversation (PDFs through the content plane). Distributed via the signed-bundle channel, never public npm.", + "license": "SEE LICENSE IN LICENSE", + "type": "commonjs", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist/", + "ability.json", + "skill.eta", + "attention-surface.json", + "LICENSE", + "LICENSE-FAQ.md" + ], + "scripts": { + "build": "tsc -b" + }, + "peerDependencies": { + "@lloyal-labs/lloyal-agents": "^5.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/media": ">=0.2.0-0 <0.3.0", + "@lloyal-labs/rig": "^5.5.0 || >=5.6.0-0 <6.0.0", + "effection": "^4.0.2" + } +} diff --git a/packages/abilities/documents/skill.eta b/packages/abilities/documents/skill.eta new file mode 100644 index 00000000..d7c62bb0 --- /dev/null +++ b/packages/abilities/documents/skill.eta @@ -0,0 +1,30 @@ +You are a thorough research assistant answering from the documents attached to this conversation. +<% if (it.agentCount > 1) { -%> + +You are one of <%= it.agentCount %> parallel agents. The other agents are covering: +<%= it.siblingTasks.map(function(q) { return '- ' + q }).join('\n') %> + +Stay focused on your own task. The other agents handle the rest. +<% } -%> + +You have <%= it.maxTurns %> tool calls. Plan your investigation within this budget. + +The attached documents are listed under `document_research` in your briefing, each with its page count and topics. + +RULES FOR TOOL OPTIMAL USE: +- search_documents returns the best-matching passages with their scores, each with its document, line range and pages — always some, ranked, never a verdict. Pass `document` (the id or title from a previous result) to search one document only. +- Read before you quote: read_document returns the exact text of a page or a line range. Search snippets are truncated. +- view_page is expensive — it puts the page image in front of you. Use it for a table, chart or figure after reading the text around it, and only on pages a result pointed you to. +- A figure or table named in the question (Fig. 2.2, Table 1): search for its caption text, then view_page the page the hit names. +- A brand or product name that returns nothing: search its generic or chemical name, then search the document again. +- Never present a different figure, table or section as the one asked for. If you did not find the one named, say so and describe what you did find. +- If a tool returns an error, name the failure in your report. +- If a query returns `Resource unavailable`, another agent already issued that exact call. Try different terms. +- Don't repeat the same query, read the same lines twice, or view the same page twice. +- When a tool returns an error about time limit, KV limit or word limit, stop and call report() with all your findings so far. + +PROCESS: +1. search_documents to find the passages that bear on your task. +2. read_document to verify every passage in context, with its page. +3. For a figure, chart or table you are asked about, view_page the page a result pointed you to, after reading around it. +4. Call report() with comprehensive findings: direct quotes with page numbers and data points. Put every page you cite in the report's sources with its `cite` URL. State what you found AND what you checked but could not find. Do not summarize — preserve the detail. diff --git a/packages/abilities/documents/src/documents-index.ts b/packages/abilities/documents/src/documents-index.ts new file mode 100644 index 00000000..a7827794 --- /dev/null +++ b/packages/abilities/documents/src/documents-index.ts @@ -0,0 +1,130 @@ +/** + * @file From the assets available to a run to a searchable value. + * + * A function from attachments to an index: the same attachments give the same + * index. Each document is fitted once per digest and remembered, so a second + * call in the same session — the next tool call, the next run of the thread — + * re-tokenizes nothing. No sync step, no state machine: what a tool sees is + * whatever is available to the run at the call, indexed. + */ +import { BM25Index, fitChunks, DEFAULT_CHUNK_TOKENS, loadDocuments } from '@lloyal-labs/rig'; +import type { Document } from '@lloyal-labs/rig'; +import type { Chunk } from '@lloyal-labs/lloyal-agents'; +import type { Attachment, AttachmentStore } from '@lloyal-labs/media'; + +/** How many hex digits of the root digest make a document's handle. */ +export const ID_LENGTH = 12; + +/** + * A document's handle: the first twelve hex digits of its root digest. It is + * the asset's own identity, copied from tool output like any id — never an + * ordinal three parties must order the same way. The UI resolves it against + * the digests it already holds and requires exactly one match. + */ +export function documentId(root: Attachment): string { + const hex = root.digest.slice(root.digest.indexOf(':') + 1); + return hex.slice(0, ID_LENGTH); +} + +/** The citation for a page: what the model copies into its report. */ +export function pageCite(root: Attachment, page: number): string { + return `attachment://${documentId(root)}/page/${page}`; +} + +/** A loaded document with its handle. Its chunks and resource are named by the handle. */ +export interface IndexedDocument extends Document { + id: string; +} + +export interface DocumentsIndex { + documents: IndexedDocument[]; + /** Every document's fitted chunks, `resource` = the document id. */ + chunks: Chunk[]; + /** Lexical first stage over `chunks`; null when there is nothing to index. */ + bm25: BM25Index | null; + /** By id, or by title (exact, case-insensitive). */ + find(document: string): IndexedDocument | undefined; +} + +export type Tokenize = (text: string) => Promise; + +/** + * Build the indexer for one session. Returns the function from attachments to + * an index. Roots that are not documents (an image) are skipped. A root whose + * content is missing or drifted throws, naming the digest — an empty index + * behind a digest that promises content would be a silent wrong answer. + */ +export function documentIndexer( + store: AttachmentStore, + tokenize: Tokenize, +): (attachments: readonly Attachment[]) => Promise { + type Fitted = { document: IndexedDocument; chunks: Chunk[] }; + const fitted = new Map>(); + const indices = new Map>(); + + const fit = (root: Attachment): Promise => { + let p = fitted.get(root.digest); + if (!p) { + p = (async () => { + const [loaded] = loadDocuments(store, [root]); + if (!loaded) return null; + const id = documentId(root); + const chunks = await fitChunks( + loaded.chunks.map((c) => ({ ...c, resource: id })), + { maxTokens: DEFAULT_CHUNK_TOKENS, tokenize }, + ); + const document: IndexedDocument = { ...loaded, id, resource: { name: id, content: loaded.resource.content } }; + return { document, chunks }; + })(); + fitted.set(root.digest, p); + // A failure is not remembered: the next call asks the store again. + p.catch(() => { fitted.delete(root.digest); }); + } + return p; + }; + + return (attachments) => { + // The same root attached twice is one document. + const roots = [...new Map(attachments.map((a) => [a.digest, a] as const)).values()]; + const key = roots.map((r) => r.digest).join('\n'); + let index = indices.get(key); + if (!index) { + index = (async () => { + const entries = (await Promise.all(roots.map(fit))).filter((e): e is Fitted => e !== null); + const documents = entries.map((e) => e.document); + const byId = new Map(); + for (const d of documents) { + const other = byId.get(d.id); + if (other) { + throw new Error( + `documents: two attached documents share the handle ${d.id} ` + + `(${other.attachment.digest} and ${d.attachment.digest}); a citation could not name one of them.`, + ); + } + byId.set(d.id, d); + } + const chunks = entries.flatMap((e) => e.chunks); + const bm25 = chunks.length > 0 ? new BM25Index(chunks.map((c) => c.tokens)) : null; + return { + documents, chunks, bm25, + find(document: string) { + const needle = document.trim().toLowerCase(); + return byId.get(needle) ?? documents.find((d) => d.meta.title.toLowerCase() === needle); + }, + }; + })(); + indices.set(key, index); + index.catch(() => { indices.delete(key); }); + } + return index; + }; +} + +/** The error a tool returns for a document it cannot find: it lists what is attached. */ +export function unknownDocument(asked: string, index: DocumentsIndex): string { + const attached = index.documents.map((d) => `${d.meta.title} (${d.id})`).join(', '); + return `Unknown document: ${asked}. Attached: ${attached || 'none'}.`; +} + +/** The message when nothing is attached — the same from every tool. */ +export const NO_DOCUMENTS = 'No documents are attached to this conversation.'; diff --git a/packages/abilities/documents/src/index.ts b/packages/abilities/documents/src/index.ts new file mode 100644 index 00000000..c4d42dd6 --- /dev/null +++ b/packages/abilities/documents/src/index.ts @@ -0,0 +1,70 @@ +/** + * `@lloyal-labs/documents-ability` — HDK reference ability: research over the + * documents attached to a conversation. + * + * Documents reach the ability as the assets available to a run: every tool + * reads `ToolContext.attachments` at the call and indexes what is there, once + * per digest. The table of contents the harness places on the spine comes + * from `Source.promptData(attachments)`. Nothing here touches the native + * addon or a node-only entry of a sibling package. + * + * @packageDocumentation + */ + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { RerankerCtx, Attachments } from '@lloyal-labs/lloyal-agents'; +import type { AbilityManifest, Tool } from '@lloyal-labs/lloyal-agents'; +import { defineAbility } from '@lloyal-labs/rig'; +import type { Reranker } from '@lloyal-labs/rig'; +import { DocumentsSource } from './source'; +import { documentIndexer } from './documents-index'; +import { SearchDocumentsTool } from './tools/search-documents'; +import { ReadDocumentTool } from './tools/read-document'; +import { ViewPageTool } from './tools/view-page'; + +export { DocumentsSource, buildToc } from './source'; +export type { DocumentsPromptData } from './source'; +export { documentIndexer, documentId, pageCite, NO_DOCUMENTS } from './documents-index'; +export type { DocumentsIndex, IndexedDocument } from './documents-index'; +export { SearchDocumentsTool, locate } from './tools/search-documents'; +export type { DocumentHit, IndexFor } from './tools/search-documents'; +export { ReadDocumentTool } from './tools/read-document'; +export { ViewPageTool, projectable } from './tools/view-page'; + +// The declarative manifest + skill template, read once at module load and +// handed to defineAbility, which advertises the manifest on the factory. +const dir = join(__dirname, '..'); +const manifest = JSON.parse(readFileSync(join(dir, 'ability.json'), 'utf8')) as AbilityManifest; +const skill = readFileSync(join(dir, 'skill.eta'), 'utf8'); + +/** + * Construct the documents ability. Reads the reranker from `RerankerCtx` and + * the content store from `Attachments`, builds the per-session indexer, and + * wires the three tools. Under `lloyal describe` the default store answers + * null and the ability is simply empty. + */ +export const createDocumentsAbility = defineAbility(manifest, function* () { + let reranker: Reranker; + try { + reranker = yield* RerankerCtx.expect(); + } catch { + throw new Error( + 'createDocumentsAbility: the documents ability requires a reranker (its `search_documents` tool ' + + 'scores passages), but RerankerCtx is unset. The harness boot normally provisions it from ' + + "the ability's `services: ['reranker']` — call provisionAbilityModels({ abilities, projectRoot }) " + + '(or otherwise set RerankerCtx) before enabling this ability.', + ); + } + const store = yield* Attachments.expect(); + const indexFor = documentIndexer(store, (text) => reranker.tokenize(text)); + const tools: Tool[] = [ + new SearchDocumentsTool(indexFor, reranker), + new ReadDocumentTool(indexFor), + new ViewPageTool(indexFor), + ]; + const source = new DocumentsSource(store, tools, reranker); + const byName: Record = {}; + for (const t of tools) byName[t.name] = t; + return { source, tools: byName, skill }; +}); diff --git a/packages/abilities/documents/src/source.ts b/packages/abilities/documents/src/source.ts new file mode 100644 index 00000000..b08e944f --- /dev/null +++ b/packages/abilities/documents/src/source.ts @@ -0,0 +1,70 @@ +import { Source } from '@lloyal-labs/lloyal-agents'; +import type { Tool, Chunk } from '@lloyal-labs/lloyal-agents'; +import type { Reranker, Document } from '@lloyal-labs/rig'; +import { loadDocuments } from '@lloyal-labs/rig'; +import type { Attachment, AttachmentStore } from '@lloyal-labs/media'; + +/** Data for the harness to place — once, in shared KV. */ +export type DocumentsPromptData = { + /** One line per attached document: title, page count, top-level topics. */ + toc: string; +}; + +/** Titles and headings are model-facing prose from a user's file: control + * characters (Unicode category Cc) go, whitespace collapses, length is capped. */ +const MAX_LABEL = 120; +const MAX_TOPICS = 8; +function label(text: string): string { + const clean = text.replace(/\p{Cc}/gu, ' ').replace(/\s+/g, ' ').trim(); + return clean.length > MAX_LABEL ? `${clean.slice(0, MAX_LABEL - 1)}…` : clean; +} + +/** The table of contents for a set of documents: ` — <n> pages (topics: …)`. */ +export function buildToc(documents: readonly Document[]): string { + return documents.map((d) => { + const title = label(d.meta.title); + const topics: string[] = []; + for (const s of d.meta.sections) { + if (s.path.includes(' > ')) continue; + const heading = label(s.heading); + if (heading === title || topics.includes(heading)) continue; + topics.push(heading); + if (topics.length >= MAX_TOPICS) break; + } + const pages = `${d.meta.pageCount} page${d.meta.pageCount === 1 ? '' : 's'}`; + return topics.length > 0 ? `${title} — ${pages} (topics: ${topics.join(', ')})` : `${title} — ${pages}`; + }).join('\n'); +} + +/** + * The documents source: `search_documents`, `read_document` and `view_page` + * over the documents available to a run. The tools read the run's assets per + * call; this class holds only what is per session — the store and the tools. + */ +export class DocumentsSource extends Source<Chunk> { + /** @inheritDoc */ + readonly name = 'documents'; + private readonly _store: AttachmentStore; + private readonly _tools: Tool[]; + + constructor(store: AttachmentStore, tools: Tool[], reranker: Reranker) { + super(); + this._store = store; + this._tools = tools; + this._reranker = reranker; + } + + /** @inheritDoc */ + get tools(): Tool[] { + return this._tools; + } + + /** + * The table of contents of the documents among `attachments` — the assets + * the run is being staged with — for the harness to place on the spine. + * Built from the sidecars alone: no tokenizing, no index. + */ + override promptData(attachments: readonly Attachment[] = []): DocumentsPromptData { + return { toc: buildToc(loadDocuments(this._store, attachments)) }; + } +} diff --git a/packages/abilities/documents/src/tools/read-document.ts b/packages/abilities/documents/src/tools/read-document.ts new file mode 100644 index 00000000..bf9218c9 --- /dev/null +++ b/packages/abilities/documents/src/tools/read-document.ts @@ -0,0 +1,83 @@ +import { call } from 'effection'; +import type { Operation } from 'effection'; +import { Tool } from '@lloyal-labs/lloyal-agents'; +import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; +import { mergeRanges, subtractRanges } from '@lloyal-labs/rig'; +import { pagesOf } from '@lloyal-labs/media'; +import { pageCite, unknownDocument, NO_DOCUMENTS } from '../documents-index'; +import type { DocumentsIndex } from '../documents-index'; +import type { IndexFor } from './search-documents'; + +/** + * The exact text of a document: a page, or a line range from a search hit. + * A page reads as the whole sections that touch it, so a table split by a + * page break stays whole. Per-agent read tracking returns only the unread + * part, as the corpus reader does. + */ +export class ReadDocumentTool extends Tool<{ document: string; page?: number; startLine?: number; endLine?: number }> { + readonly name = 'read_document'; + readonly protected = false; + readonly description = 'Read the exact text of an attached document: a whole page, or a line range from search results. Name the document by id or title.'; + readonly parameters: JsonSchema = { + type: 'object', + properties: { + document: { type: 'string', description: 'Document id or title, from search results or the briefing' }, + page: { type: 'number', description: 'Page number — returns every section that touches the page' }, + startLine: { type: 'number', description: 'Start line (1-indexed, from search results)' }, + endLine: { type: 'number', description: 'End line (1-indexed, from search results)' }, + }, + required: ['document'], + }; + + private readonly _indexFor: IndexFor; + private readonly _defaultMaxLines: number; + private readonly _read = new Map<string, [number, number][]>(); + + constructor(indexFor: IndexFor, opts?: { defaultMaxLines?: number }) { + super(); + this._indexFor = indexFor; + this._defaultMaxLines = opts?.defaultMaxLines ?? 100; + } + + *execute( + args: { document: string; page?: number; startLine?: number; endLine?: number }, + context?: ToolContext, + ): Operation<unknown> { + const index: DocumentsIndex = yield* call(() => this._indexFor(context?.attachments ?? [])); + if (index.documents.length === 0) return { error: NO_DOCUMENTS }; + const doc = index.find(args.document ?? ''); + if (!doc) return { error: unknownDocument(args.document ?? '', index) }; + const lines = doc.resource.content.split('\n'); + + // [s, e): 0-based start, exclusive end — the range helpers' convention. + let s: number; + let e: number; + if (args.page !== undefined) { + const page = doc.meta.pages.find((p) => p.page === args.page); + if (!page) return { error: `Page ${args.page} is out of range: ${doc.meta.title} has ${doc.meta.pageCount} pages.` }; + const covering = doc.meta.sections.filter((sec) => sec.pageStart <= page.page && page.page <= sec.pageEnd); + s = (covering.length > 0 ? Math.min(...covering.map((c) => c.startLine)) : page.startLine) - 1; + e = covering.length > 0 ? Math.max(...covering.map((c) => c.endLine)) : page.endLine; + } else { + s = Math.max(0, (args.startLine ?? 1) - 1); + e = Math.min(lines.length, args.endLine ?? s + this._defaultMaxLines); + } + if (e <= s) return { error: `Nothing to read: lines ${s + 1}-${e} of ${doc.meta.title}.` }; + + const key = `${context?.agentId ?? ''}:${doc.id}`; + const prev = this._read.get(key) ?? []; + const unread = subtractRanges([s, e], prev); + if (unread.length === 0) return { document: doc.meta.title, id: doc.id, note: `Lines ${s + 1}-${e} already read` }; + this._read.set(key, mergeRanges([...prev, [s, e]])); + + const content = unread.map(([a, b]) => lines.slice(a, b).join('\n')).join('\n...\n'); + const { pageStart, pageEnd } = pagesOf(doc.meta, s + 1, e); + return { + document: doc.meta.title, id: doc.id, + lines: unread.map(([a, b]) => `${a + 1}-${b}`), + pageStart, pageEnd, + cite: pageCite(doc.attachment, pageStart), + content, + }; + } +} diff --git a/packages/abilities/documents/src/tools/search-documents.ts b/packages/abilities/documents/src/tools/search-documents.ts new file mode 100644 index 00000000..f64035aa --- /dev/null +++ b/packages/abilities/documents/src/tools/search-documents.ts @@ -0,0 +1,126 @@ +import { call } from 'effection'; +import type { Operation } from 'effection'; +import { Tool, Trace, admitChunks } from '@lloyal-labs/lloyal-agents'; +import type { JsonSchema, ToolContext, Chunk } from '@lloyal-labs/lloyal-agents'; +import type { Reranker, ScoredChunk } from '@lloyal-labs/rig'; +import { pagesOf } from '@lloyal-labs/media'; +import type { Attachment } from '@lloyal-labs/media'; +import { pageCite, unknownDocument, NO_DOCUMENTS } from '../documents-index'; +import type { DocumentsIndex } from '../documents-index'; + +/** The function from a run's attachments to their index (see `documentIndexer`). */ +export type IndexFor = (attachments: readonly Attachment[]) => Promise<DocumentsIndex>; + +/** How many passages a search returns: the top K within a token budget, the + * allowance `fetch_page` gives a page. No score floor: the reranker is a + * relative judge — its scale shifts from passage to passage and its verdict + * on an attached document's own text runs within a few logits of zero — so + * the best K are returned with their scores and the agent reads them. */ +const DEFAULT_TOP_K = 5; +const DEFAULT_TOKEN_BUDGET = 2048; +/** First-stage cap: BM25 narrows to the top-K lexical matches before the + * cross-encoder, whose cost is linear in candidates. */ +const DEFAULT_FIRST_STAGE_K = 100; + +/** A hit as this ability returns it: the corpus shape plus where it is. */ +export interface DocumentHit extends ScoredChunk { + document: string; + id: string; + pageStart: number; + pageEnd: number; + /** `attachment://<id>/page/<pageStart>` — copied into report sources as-is. */ + cite: string; +} + +/** Place a scored chunk in its document: title, id, pages, citation. */ +export function locate(index: DocumentsIndex, hit: ScoredChunk): DocumentHit { + const doc = index.documents.find((d) => d.id === hit.file); + if (!doc) throw new Error(`search_documents: hit names ${hit.file}, which is not a document in the index`); + const { pageStart, pageEnd } = pagesOf(doc.meta, hit.startLine, hit.endLine); + return { ...hit, document: doc.meta.title, id: doc.id, pageStart, pageEnd, cite: pageCite(doc.attachment, pageStart) }; +} + +/** + * Semantic search over the documents available to the run: BM25 first stage, + * then the platform's admission pipeline (cross-encoder, top-K within a token + * budget, trace events). What is document-shaped lives here: the per-call + * index, the optional scope to one document, the pages on every hit — and the + * stance: a document search always scores in explore mode. The run's exploit + * stance exists to keep off-topic web pages out; an attached document is the + * on-topic universe, and a min() with the original question would veto the + * passage that answers a sub-question of it. + */ +export class SearchDocumentsTool extends Tool<{ query: string; document?: string }> { + readonly name = 'search_documents'; + readonly protected = false; + // Reranker (its own llama_context) + in-memory BM25 — no op on the MAIN + // context, so it runs off the loop fiber under concurrent dispatch. + readonly fanout = true; + readonly description = 'Search the attached documents. Returns passages ranked by relevance, each with its document, line range and pages for read_document. Pass `document` to search one document only.'; + readonly parameters: JsonSchema = { + type: 'object', + properties: { + query: { type: 'string', description: 'Search query' }, + document: { type: 'string', description: 'Document id or title, to search one document only' }, + }, + required: ['query'], + }; + + private readonly _indexFor: IndexFor; + private readonly _reranker: Reranker; + private readonly _topK: number; + private readonly _tokenBudget: number; + private readonly _firstStageK: number; + + constructor(indexFor: IndexFor, reranker: Reranker, opts?: { topK?: number; tokenBudget?: number; firstStageK?: number }) { + super(); + this._indexFor = indexFor; + this._reranker = reranker; + this._topK = opts?.topK ?? DEFAULT_TOP_K; + this._tokenBudget = opts?.tokenBudget ?? DEFAULT_TOKEN_BUDGET; + this._firstStageK = opts?.firstStageK ?? DEFAULT_FIRST_STAGE_K; + } + + *execute(args: { query: string; document?: string }, context?: ToolContext): Operation<unknown> { + const query = args.query?.trim(); + if (!query) return { error: 'query must not be empty' }; + const index: DocumentsIndex = yield* call(() => this._indexFor(context?.attachments ?? [])); + if (index.documents.length === 0) return { error: NO_DOCUMENTS }; + const scope = args.document ? index.find(args.document) : undefined; + if (args.document && !scope) return { error: unknownDocument(args.document, index) }; + const inScope = (c: Chunk): boolean => !scope || c.resource === scope.id; + const pool = index.chunks.filter(inScope); + + const tw = yield* Trace.expect(); + let candidates = pool; + if (index.bm25 && this._firstStageK < pool.length) { + const bm25Start = performance.now(); + tw.write({ + traceId: tw.nextId(), parentTraceId: null, ts: bm25Start, + type: 'bm25:start', query, candidateCount: pool.length, firstStageK: this._firstStageK, + }); + const queryTokens: number[] = yield* call(() => this._reranker.tokenize(query)); + // The index spans every document; a scoped search keeps the hits in + // scope and takes its K from those. + const hits = index.bm25.score(queryTokens, scope ? index.chunks.length : this._firstStageK); + candidates = hits.map((h) => index.chunks[h.index]).filter(inScope).slice(0, this._firstStageK); + tw.write({ + traceId: tw.nextId(), parentTraceId: null, ts: performance.now(), + type: 'bm25:end', candidateCount: pool.length, keptCount: candidates.length, durationMs: performance.now() - bm25Start, + }); + } + + const admitted = yield* admitChunks(this._reranker, candidates, query, context ? { ...context, explore: true } : context, { + tool: 'search_documents', + select: { mode: 'budget', topK: this._topK, tokenBudget: this._tokenBudget }, + }); + // The budget decides how many; the hits keep their addresses. The + // selection is a prefix of the ranking (every window has text), so the + // first `passages.length` scored chunks are the passages, with lines. + const kept = admitted.passages?.length ?? 0; + return { + hits: admitted.scored.slice(0, kept).map((h) => locate(index, h)), + totalScored: admitted.totalScored, + }; + } +} diff --git a/packages/abilities/documents/src/tools/view-page.ts b/packages/abilities/documents/src/tools/view-page.ts new file mode 100644 index 00000000..b387a0fd --- /dev/null +++ b/packages/abilities/documents/src/tools/view-page.ts @@ -0,0 +1,98 @@ +import { call } from 'effection'; +import type { Operation } from 'effection'; +import { Tool, TOOL_ATTACHMENTS_KEY } from '@lloyal-labs/lloyal-agents'; +import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { DocumentMeta } from '@lloyal-labs/media'; +import { pageCite, unknownDocument, NO_DOCUMENTS } from '../documents-index'; +import type { DocumentsIndex } from '../documents-index'; +import type { IndexFor } from './search-documents'; + +/** The per-page facts the sidecar records. */ +export type PageFacts = DocumentMeta['pages'][number]; + +/** How many drawing operations make a page "has graphics". */ +const PATH_OBJECTS_FOR_GRAPHICS = 20; + +/** + * The projection rule: which pages are worth a model's attention as an image. + * The facts are the sidecar's; the rule lives here, with the tool that spends + * the cells. Page one always (the title page carries the figure of record + * more often than not); a page with no extractable text (scanned); a page + * with images, drawings, tagged tables or tagged figures. + */ +export function projectable(page: PageFacts): boolean { + return page.page === 1 + || page.chars === 0 + || page.imageObjects > 0 + || page.pathObjects >= PATH_OBJECTS_FOR_GRAPHICS + || page.taggedTables > 0 + || page.taggedFigures > 0; +} + +/** + * Put a page — or one figure on it — in front of the model. The result names + * the page root (or figure root) under the framework's attachment key as a + * DESCRIPTOR: no bytes, no ingress, no normalizer permit; the pool resolves it + * through the store and admits it on the media rail. A text-only page says + * so instead, and a page past the render bound says it is not archived. + * + * A repeat by the same agent carries the page AGAIN, with a note. The tool + * cannot see whether its last result landed — the pool may have replaced it + * with a settle nudge — so suppressing a repeat would leave the model blind; + * admission is the only gate on what a page costs. + */ +export class ViewPageTool extends Tool<{ document: string; page: number; figure?: number }> { + readonly name = 'view_page'; + readonly protected = false; + readonly description = 'Look at a page of an attached document as an image — for a table, chart or figure, after reading the text around it. `figure` picks one figure on the page by its number.'; + readonly parameters: JsonSchema = { + type: 'object', + properties: { + document: { type: 'string', description: 'Document id or title' }, + page: { type: 'number', description: 'Page number, from search or read_document results' }, + figure: { type: 'number', description: 'Which figure on the page (1 = first), to view just the figure' }, + }, + required: ['document', 'page'], + }; + + private readonly _indexFor: IndexFor; + private readonly _viewed = new Set<string>(); + + constructor(indexFor: IndexFor) { + super(); + this._indexFor = indexFor; + } + + *execute(args: { document: string; page: number; figure?: number }, context?: ToolContext): Operation<unknown> { + const index: DocumentsIndex = yield* call(() => this._indexFor(context?.attachments ?? [])); + if (index.documents.length === 0) return { error: NO_DOCUMENTS }; + const doc = index.find(args.document ?? ''); + if (!doc) return { error: unknownDocument(args.document ?? '', index) }; + const page = doc.meta.pages.find((p) => p.page === args.page); + if (!page) return { error: `Page ${args.page} is out of range: ${doc.meta.title} has ${doc.meta.pageCount} pages.` }; + + const where = { document: doc.meta.title, id: doc.id, page: page.page }; + const key = `${context?.agentId ?? ''}:${doc.id}:${page.page}:${args.figure ?? ''}`; + const again = this._viewed.has(key) + ? { note: args.figure !== undefined ? `You viewed figure ${args.figure} on page ${page.page} before.` : `You viewed page ${page.page} before.` } + : {}; + + if (args.figure !== undefined) { + const figures = doc.meta.figures.filter((f) => f.page === page.page); + const fig = figures[args.figure - 1]; + if (!fig) { + return { error: figures.length === 0 + ? `Page ${page.page} has no figures.` + : `Page ${page.page} has ${figures.length} figure(s); figure must be between 1 and ${figures.length}.` }; + } + this._viewed.add(key); + return { ...where, ...again, figure: args.figure, ...(fig.caption ? { caption: fig.caption } : {}), + cite: pageCite(doc.attachment, page.page), [TOOL_ATTACHMENTS_KEY]: [fig.root] }; + } + + if (!projectable(page)) return { ...where, note: `Page ${page.page} is text only — read_document gives you its text.` }; + if (!page.render) return { ...where, note: `Page ${page.page} is not archived as an image.` }; + this._viewed.add(key); + return { ...where, ...again, cite: pageCite(doc.attachment, page.page), [TOOL_ATTACHMENTS_KEY]: [page.render] }; + } +} diff --git a/packages/abilities/documents/test/ability.test.ts b/packages/abilities/documents/test/ability.test.ts new file mode 100644 index 00000000..fa4fde2d --- /dev/null +++ b/packages/abilities/documents/test/ability.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { run } from 'effection'; +import { RerankerCtx, Attachments } from '@lloyal-labs/lloyal-agents'; +import type { Reranker } from '@lloyal-labs/rig'; +import { createDocumentsAbility } from '../src/index'; +import { makeFixture, wordTokenize } from './helpers/fixture'; + +const mockReranker = { tokenize: wordTokenize } as unknown as Reranker; +const pkg = join(__dirname, '..'); + +describe('createDocumentsAbility', () => { + it('builds document_research with the three tools and the documents source', async () => { + const { store } = makeFixture(); + const ability = await run(function* () { + yield* RerankerCtx.set(mockReranker); + yield* Attachments.set(store); + return yield* createDocumentsAbility(); + }); + expect(ability.manifest.protocol.name).toBe('document_research'); + expect(ability.manifest.services).toEqual(['reranker']); + expect(ability.manifest.configSchema).toBeUndefined(); + expect(ability.source.name).toBe('documents'); + expect(ability.tools.map((t) => t.name).sort()).toEqual(['read_document', 'search_documents', 'view_page']); + }); + + it('ships a skill in the corpus register: no boundary marker, no toc variable', () => { + const skill = readFileSync(join(pkg, 'skill.eta'), 'utf8'); + expect(skill).not.toContain('Apply the **'); + expect(skill).not.toContain('it.toc'); + expect(skill).toContain('view_page is expensive'); + }); + + it('throws a clear error when no reranker is set', async () => { + await expect(run(function* () { return yield* createDocumentsAbility(); })).rejects.toThrow(/requires a reranker/); + }); + + it('constructs with the default store and no attachments, and the toc is empty', async () => { + const source = await run(function* () { + yield* RerankerCtx.set(mockReranker); + return (yield* createDocumentsAbility()).source; + }); + expect(source.promptData()).toEqual({ toc: '' }); + expect(source.promptData([])).toEqual({ toc: '' }); + }); + + it('never reaches the native addon: no lloyal.node peer and no node-only sibling entry in src', () => { + const manifest = JSON.parse(readFileSync(join(pkg, 'package.json'), 'utf8')) as { peerDependencies: Record<string, string> }; + expect(Object.keys(manifest.peerDependencies)).not.toContain('@lloyal-labs/lloyal.node'); + const files: string[] = []; + const walk = (d: string): void => { + for (const n of readdirSync(d)) { + const p = join(d, n); + if (statSync(p).isDirectory()) walk(p); else if (p.endsWith('.ts')) files.push(p); + } + }; + walk(join(pkg, 'src')); + expect(files.length).toBeGreaterThan(3); + for (const f of files) { + const text = readFileSync(f, 'utf8'); + expect(text, f).not.toMatch(/lloyal\.node|@lloyal-labs\/(rig|media)\/node/); + } + }); +}); diff --git a/packages/abilities/documents/test/helpers/fixture.ts b/packages/abilities/documents/test/helpers/fixture.ts new file mode 100644 index 00000000..b47625bb --- /dev/null +++ b/packages/abilities/documents/test/helpers/fixture.ts @@ -0,0 +1,120 @@ +/** + * One document in a real file store: four pages, a tagged table on page two + * with a figure, a text-only page three, and a page four past the render + * bound. Page roots are the PNG-header idiom — nothing in these tests decodes. + */ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { FileAttachmentStore } from '@lloyal-labs/media/node'; +import { DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, AttachmentStore, DocumentMeta } from '@lloyal-labs/media'; +import type { Reranker, Chunk } from '@lloyal-labs/rig'; + +export const PNG_BYTES = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + +export const MARKDOWN = [ + '# Fixture Paper', '', // 1-2 page 1 + '## Introduction', '', 'Intro text alpha beta.', '', // 3-6 page 1 + '## Results', '', '| Config | Cells |', '| --- | --- |', '| baseline | 803 |', '', // 7-12 page 2 + '## Discussion', '', 'Closing words gamma.', '', // 13-16 page 3 + '## Appendix', '', 'Chart described here.', // 17-19 page 4 (+ line 20 blank from the trailing newline) +].join('\n') + '\n'; + +const DERIVE: DocumentMeta['derive'] = { + profile: 'pdf.v1', pdfium: 'test', dpi: 150, maxSide: 2048, maxPixels: 4194304, format: 'image/png', + renderedPages: 3, maxFigures: 16, maxTextPages: 400, tagged: true, structCoverage: 1, truncated: true, +}; + +export interface Fixture { + store: FileAttachmentStore; + doc: Attachment; + image: Attachment; + renders: Record<1 | 2 | 3, Attachment>; + figure: Attachment; + meta: DocumentMeta; +} + +export function makeFixture(): Fixture { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-ability-'))); + const png = (tag: number): Attachment => + store.putAttachment({ representations: [store.putBlob(new Uint8Array([...PNG_BYTES, tag]), 'image/png')] }); + const image = png(100); + const renders = { 1: png(1), 2: png(2), 3: png(3) } as const; + const figure = png(9); + const meta: DocumentMeta = { + title: 'Fixture Paper', + pageCount: 4, + sections: [ + { heading: 'Fixture Paper', path: 'Fixture Paper', origin: 'struct', startLine: 1, endLine: 2, pageStart: 1, pageEnd: 1 }, + { heading: 'Introduction', path: 'Introduction', origin: 'struct', startLine: 3, endLine: 6, pageStart: 1, pageEnd: 1 }, + { heading: 'Results', path: 'Results', origin: 'struct', startLine: 7, endLine: 12, pageStart: 2, pageEnd: 2 }, + { heading: 'Discussion', path: 'Discussion', origin: 'struct', startLine: 13, endLine: 16, pageStart: 3, pageEnd: 3 }, + { heading: 'Appendix', path: 'Appendix', origin: 'struct', startLine: 17, endLine: 20, pageStart: 4, pageEnd: 4 }, + ], + pages: [ + { page: 1, startLine: 1, endLine: 6, chars: 40, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0, render: renders[1] }, + { page: 2, startLine: 7, endLine: 12, chars: 30, imageObjects: 0, pathObjects: 30, taggedTables: 1, taggedFigures: 0, render: renders[2] }, + { page: 3, startLine: 13, endLine: 16, chars: 20, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0, render: renders[3] }, + { page: 4, startLine: 17, endLine: 20, chars: 21, imageObjects: 1, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }, + ], + figures: [{ page: 2, index: 0, bbox: [72, 300, 372, 500], caption: 'Figure 1. Cells per configuration.', root: figure }], + tables: [{ page: 2, startLine: 9, endLine: 11 }], + derive: DERIVE, + }; + const doc = store.putAttachment({ + representations: [store.putBlob(new TextEncoder().encode(MARKDOWN), 'text/markdown')], + source: store.putBlob(new TextEncoder().encode('%PDF-1.4 fixture'), 'application/pdf'), + config: { bytes: new TextEncoder().encode(JSON.stringify(meta)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); + return { store, doc, image, renders, figure, meta }; +} + +/** A second, one-page document in the same store. */ +export function makeSecond(store: AttachmentStore, title = 'Second Report'): Attachment { + const md = `# ${title}\n\n## Findings\n\nDelta epsilon zeta.\n`; + const meta: DocumentMeta = { + title, pageCount: 1, + sections: [ + { heading: title, path: title, origin: 'heuristic', startLine: 1, endLine: 2, pageStart: 1, pageEnd: 1 }, + { heading: 'Findings', path: 'Findings', origin: 'heuristic', startLine: 3, endLine: 6, pageStart: 1, pageEnd: 1 }, + ], + pages: [{ page: 1, startLine: 1, endLine: 6, chars: 30, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }], + figures: [], tables: [], derive: { ...DERIVE, renderedPages: 0, tagged: false, structCoverage: 0, truncated: false }, + }; + return store.putAttachment({ + representations: [store.putBlob(new TextEncoder().encode(md), 'text/markdown')], + config: { bytes: new TextEncoder().encode(JSON.stringify(meta)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); +} + +/** Words stand in for tokens: fitting needs only a tokenizer. */ +export const wordTokenize = async (text: string): Promise<number[]> => + text.split(/\s+/).filter(Boolean).map((_, i) => i + 1); + +/** A reranker whose scorer admits everything with score 1 — the funnel's + * shape is under test here, not the cross-encoder. */ +export function scoringReranker(): Reranker { + return { + tokenize: wordTokenize, + tokenizeChunks: async (chunks: Chunk[]) => chunks, + // The judge's verdict, shaped like the real one: log-odds, positive when + // a query word occurs in the passage, negative when none does — so a + // query the document never mentions is judged "no" everywhere. + score(query: string, chunks: Chunk[]) { + const words = query.toLowerCase().split(/\W+/).filter((w) => w.length > 2); + return (async function* () { + yield { + filled: chunks.length, total: chunks.length, + results: chunks.map((c) => ({ + file: c.resource, heading: c.heading, section: c.section, snippet: c.text, + score: words.some((w) => c.text.toLowerCase().includes(w)) ? 1 : -1, + startLine: c.startLine, endLine: c.endLine, + })), + }; + })(); + }, + scoreBatch: async () => [], + dispose: () => {}, + } as unknown as Reranker; +} diff --git a/packages/abilities/documents/test/index.test.ts b/packages/abilities/documents/test/index.test.ts new file mode 100644 index 00000000..95dc8dd8 --- /dev/null +++ b/packages/abilities/documents/test/index.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from 'vitest'; +import { MANIFEST_TYPE } from '@lloyal-labs/media'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId, pageCite } from '../src/documents-index'; +import { makeFixture, makeSecond, wordTokenize } from './helpers/fixture'; + +describe('documentIndexer', () => { + it('the same attachments give the same index, and a digest is fitted once', async () => { + const { store, doc, image } = makeFixture(); + let calls = 0; + const indexFor = documentIndexer(store, async (t) => { calls++; return wordTokenize(t); }); + const a = await indexFor([doc]); + const b = await indexFor([doc]); + expect(b).toBe(a); + expect(a.documents).toHaveLength(1); + expect(a.documents[0].id).toBe(documentId(doc)); + expect(a.chunks.length).toBeGreaterThan(0); + expect(a.chunks.every((c) => c.resource === a.documents[0].id)).toBe(true); + const fitted = calls; + expect(fitted).toBeGreaterThan(0); + // A different set — the image adds no document — builds a new index but re-tokenizes nothing. + const c = await indexFor([image, doc]); + expect(c).not.toBe(a); + expect(c.documents.map((d) => d.id)).toEqual([documentId(doc)]); + expect(calls).toBe(fitted); + // The same root twice is one document. + expect((await indexFor([doc, doc])).documents).toHaveLength(1); + }); + + it('a second document is searchable once it is available to the run, by id or title', async () => { + const { store, doc } = makeFixture(); + const second = makeSecond(store); + const indexFor = documentIndexer(store, wordTokenize); + expect((await indexFor([doc])).documents).toHaveLength(1); + const both = await indexFor([doc, second]); + expect(both.documents.map((d) => d.meta.title)).toEqual(['Fixture Paper', 'Second Report']); + expect(both.find('second report')?.id).toBe(documentId(second)); + expect(both.find(documentId(second))?.meta.title).toBe('Second Report'); + expect(both.find('nope')).toBeUndefined(); + }); + + it('a root that is not in the store throws naming the digest, and the failure is not remembered', async () => { + const { store, doc } = makeFixture(); + const indexFor = documentIndexer(store, wordTokenize); + const gone = { digest: 'sha256:' + 'e'.repeat(64), mediaType: MANIFEST_TYPE, size: 9 } as Attachment; + await expect(indexFor([doc, gone])).rejects.toThrow(/sha256:e{12}… is not in the content store/); + // The good document still indexes on its own afterwards. + expect((await indexFor([doc])).documents).toHaveLength(1); + }); + + it('handles and citations derive from the digest', () => { + const { doc } = makeFixture(); + const id = documentId(doc); + expect(id).toMatch(/^[0-9a-f]{12}$/); + expect(doc.digest).toContain(id); + expect(pageCite(doc, 7)).toBe(`attachment://${id}/page/7`); + }); +}); diff --git a/packages/abilities/documents/test/read-document.test.ts b/packages/abilities/documents/test/read-document.test.ts new file mode 100644 index 00000000..99996fc1 --- /dev/null +++ b/packages/abilities/documents/test/read-document.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from 'vitest'; +import { run } from 'effection'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId, NO_DOCUMENTS } from '../src/documents-index'; +import { ReadDocumentTool } from '../src/tools/read-document'; +import { makeFixture, wordTokenize } from './helpers/fixture'; + +type Read = { document?: string; id?: string; lines?: string[]; pageStart?: number; pageEnd?: number; cite?: string; content?: string; note?: string; error?: string }; + +function setup() { + const fx = makeFixture(); + const tool = new ReadDocumentTool(documentIndexer(fx.store, wordTokenize)); + const read = (args: { document: string; page?: number; startLine?: number; endLine?: number }, attachments: readonly Attachment[] = [fx.doc], agentId = 1) => + run(function* () { return (yield* tool.execute(args, { agentId, attachments } as ToolContext)) as Read; }); + return { ...fx, read }; +} + +describe('read_document', () => { + it('reads a line range by id, and by title regardless of case', async () => { + const { doc, read } = setup(); + const byId = await read({ document: documentId(doc), startLine: 3, endLine: 5 }); + expect(byId).toMatchObject({ + document: 'Fixture Paper', id: documentId(doc), lines: ['3-5'], + pageStart: 1, pageEnd: 1, cite: `attachment://${documentId(doc)}/page/1`, + content: '## Introduction\n\nIntro text alpha beta.', + }); + const byTitle = await read({ document: 'fixture paper', startLine: 3, endLine: 5 }, [doc], 2); + expect(byTitle.content).toBe(byId.content); + }); + + it('reads a page as the whole sections that touch it', async () => { + const { doc, read } = setup(); + const r = await read({ document: documentId(doc), page: 2 }); + expect(r.lines).toEqual(['7-12']); + expect(r.content!.startsWith('## Results')).toBe(true); + expect(r.content).toContain('| baseline | 803 |'); + expect([r.pageStart, r.pageEnd]).toEqual([2, 2]); + expect(r.cite).toBe(`attachment://${documentId(doc)}/page/2`); + }); + + it('the same agent re-reading gets a note; another agent reads', async () => { + const { doc, read } = setup(); + await read({ document: documentId(doc), page: 2 }); + const again = await read({ document: documentId(doc), page: 2 }); + expect(again.note).toMatch(/already read/); + expect(again.content).toBeUndefined(); + const other = await read({ document: documentId(doc), page: 2 }, [doc], 7); + expect(other.content).toContain('| baseline | 803 |'); + }); + + it('names what is attached for an unknown document, and bounds the page', async () => { + const { doc, read } = setup(); + expect((await read({ document: 'Nope' })).error).toMatch(/Unknown document: Nope\. Attached: Fixture Paper \([0-9a-f]{12}\)\./); + expect((await read({ document: documentId(doc), page: 9 })).error).toMatch(/out of range.*4 pages/); + expect((await read({ document: documentId(doc) }, [])).error).toBe(NO_DOCUMENTS); + }); +}); diff --git a/packages/abilities/documents/test/search-documents.test.ts b/packages/abilities/documents/test/search-documents.test.ts new file mode 100644 index 00000000..1c5b63a5 --- /dev/null +++ b/packages/abilities/documents/test/search-documents.test.ts @@ -0,0 +1,83 @@ +import { describe, it, expect } from 'vitest'; +import { run } from 'effection'; +import { Trace, NullTraceWriter } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId, NO_DOCUMENTS } from '../src/documents-index'; +import { SearchDocumentsTool } from '../src/tools/search-documents'; +import type { DocumentHit } from '../src/tools/search-documents'; +import { makeFixture, makeSecond, scoringReranker } from './helpers/fixture'; + +type Envelope = { hits: DocumentHit[]; totalScored: number; error?: string }; + +function setup() { + const fx = makeFixture(); + const reranker = scoringReranker(); + const indexFor = documentIndexer(fx.store, (t) => reranker.tokenize(t)); + const tool = new SearchDocumentsTool(indexFor, reranker); + const search = (args: { query: string; document?: string }, attachments: readonly Attachment[], agentId = 1) => + run(function* () { + yield* Trace.set(new NullTraceWriter()); + return (yield* tool.execute(args, { agentId, attachments } as ToolContext)) as Envelope; + }); + return { ...fx, tool, search }; +} + +describe('search_documents', () => { + it('hits carry the document, its id, its pages and a cite URL', async () => { + const { doc, image, search } = setup(); + const r = await search({ query: 'baseline cells' }, [image, doc]); + expect(r.error).toBeUndefined(); + expect(r.totalScored).toBeGreaterThan(0); + const results = r.hits.find((h) => h.heading === 'Results'); + expect(results).toMatchObject({ + document: 'Fixture Paper', id: documentId(doc), file: documentId(doc), + pageStart: 2, pageEnd: 2, cite: `attachment://${documentId(doc)}/page/2`, + }); + expect(results!.startLine).toBe(7); + }); + + it('scopes to one document by title or id, and names the attached ones for an unknown scope', async () => { + const { store, doc, search } = setup(); + const second = makeSecond(store); + const byTitle = await search({ query: 'delta', document: 'Second Report' }, [doc, second]); + expect(byTitle.hits.length).toBeGreaterThan(0); + expect(byTitle.hits.every((h) => h.id === documentId(second))).toBe(true); + const byId = await search({ query: 'delta', document: documentId(doc) }, [doc, second]); + expect(byId.hits.every((h) => h.id === documentId(doc))).toBe(true); + const unknown = await search({ query: 'delta', document: 'Third' }, [doc, second]); + expect(unknown.error).toMatch(/Unknown document: Third\. Attached: Fixture Paper \([0-9a-f]{12}\), Second Report/); + }); + + it('answers honestly when nothing is attached, and when only an image is', async () => { + const { image, search } = setup(); + expect((await search({ query: 'anything' }, [])).error).toBe(NO_DOCUMENTS); + expect((await search({ query: 'anything' }, [image])).error).toBe(NO_DOCUMENTS); + expect((await search({ query: ' ' }, [])).error).toMatch(/must not be empty/); + }); + + it('returns the best passages even when the judge says no to all of them — the ranker is relative, there is no floor', async () => { + const { doc, search } = setup(); + const r = await search({ query: 'zxqv plorth wibble' }, [doc]); + expect(r.error).toBeUndefined(); + expect(r.hits.length).toBeGreaterThan(0); + expect(r.hits.length).toBeLessThanOrEqual(5); + expect(r.hits.every((h) => h.score < 0)).toBe(true); + // The hits are the ranking's prefix: addresses intact, in score order. + expect(r.hits.every((h) => h.startLine >= 1 && h.cite.startsWith('attachment://'))).toBe(true); + }); + + it("scores in explore mode whatever the run's stance — the attached document is the on-topic universe, so the original question never vetoes a passage", async () => { + const { doc, tool } = setup(); + const scorer = { + scoreEntailmentBatch: async () => { throw new Error('the entailment scorer must not be consulted for a document search'); }, + }; + const r = await run(function* () { + yield* Trace.set(new NullTraceWriter()); + return (yield* tool.execute({ query: 'baseline cells' }, { agentId: 1, attachments: [doc], explore: false, scorer } as unknown as ToolContext)) as Envelope; + }); + expect(r.error).toBeUndefined(); + expect(r.hits.length).toBeGreaterThan(0); + }); +}); + diff --git a/packages/abilities/documents/test/source.test.ts b/packages/abilities/documents/test/source.test.ts new file mode 100644 index 00000000..c6f868d3 --- /dev/null +++ b/packages/abilities/documents/test/source.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect } from 'vitest'; +import type { Reranker, Document } from '@lloyal-labs/rig'; +import { DocumentsSource, buildToc } from '../src/source'; +import { makeFixture, makeSecond, wordTokenize } from './helpers/fixture'; + +const mockReranker = { tokenize: wordTokenize } as unknown as Reranker; + +describe('DocumentsSource.promptData', () => { + it('lists each document with its page count and top-level topics, skipping an image root', () => { + const { store, doc, image } = makeFixture(); + const second = makeSecond(store); + const source = new DocumentsSource(store, [], mockReranker); + expect(source.promptData([image, doc, second]).toc).toBe( + 'Fixture Paper — 4 pages (topics: Introduction, Results, Discussion, Appendix)\n' + + 'Second Report — 1 page (topics: Findings)', + ); + }); + + it('is empty for no attachments and for images only', () => { + const { store, image } = makeFixture(); + const source = new DocumentsSource(store, [], mockReranker); + expect(source.promptData().toc).toBe(''); + expect(source.promptData([image]).toc).toBe(''); + }); +}); + +describe('buildToc', () => { + const docWith = (title: string, headings: string[]): Document => ({ + meta: { + title, pageCount: 2, + sections: headings.map((h, i) => ({ heading: h, path: h, origin: 'heuristic' as const, startLine: i + 1, endLine: i + 1, pageStart: 1, pageEnd: 1 })), + pages: [], figures: [], tables: [], + derive: { profile: 'pdf.v1', pdfium: 't', dpi: 150, maxSide: 2048, maxPixels: 1, format: 'image/png', renderedPages: 0, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false }, + }, + } as unknown as Document); + + it('strips control characters, skips nested paths and the title, and caps topics at eight', () => { + const headings = Array.from({ length: 12 }, (_, i) => `Topic ${i + 1}`); + const toc = buildToc([docWith('BadTitle\nHere', ['Bad Title Here', 'Outer > Inner', ...headings])]); + expect(toc).not.toMatch(/\p{Cc}/u); + expect(toc.startsWith('BadTitle Here — 2 pages (topics: Bad Title Here, Topic 1,')).toBe(true); + expect(toc).not.toContain('Inner'); + expect(toc.match(/Topic \d+/g)).toHaveLength(7); + }); + + it('caps a long title', () => { + const toc = buildToc([docWith('T'.repeat(300), [])]); + expect(toc.length).toBeLessThan(140); + expect(toc).toContain('…'); + }); +}); diff --git a/packages/abilities/documents/test/view-page.test.ts b/packages/abilities/documents/test/view-page.test.ts new file mode 100644 index 00000000..2dff68dc --- /dev/null +++ b/packages/abilities/documents/test/view-page.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect, vi } from 'vitest'; +import { run } from 'effection'; +import { TOOL_ATTACHMENTS_KEY } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { documentIndexer, documentId } from '../src/documents-index'; +import { ViewPageTool, projectable } from '../src/tools/view-page'; +import { makeFixture, wordTokenize } from './helpers/fixture'; + +type View = Record<string, unknown> & { note?: string; error?: string; cite?: string; caption?: string }; + +function setup() { + const fx = makeFixture(); + const tool = new ViewPageTool(documentIndexer(fx.store, wordTokenize)); + const view = (args: { document: string; page: number; figure?: number }, attachments: readonly Attachment[] = [fx.doc], agentId = 1) => + run(function* () { return (yield* tool.execute(args, { agentId, attachments } as ToolContext)) as View; }); + return { ...fx, view }; +} + +describe('view_page', () => { + it('returns the page root as a descriptor under the attachment key, reading no blob for it', async () => { + const { doc, renders, store, view } = setup(); + // The first call builds the index (which reads the sidecar and markdown). + const text = await view({ document: documentId(doc), page: 3 }); + expect(text.note).toMatch(/text only/); + const reads = vi.spyOn(store, 'get'); + const r = await view({ document: documentId(doc), page: 2 }); + expect(r[TOOL_ATTACHMENTS_KEY]).toEqual([renders[2]]); + expect(r.cite).toBe(`attachment://${documentId(doc)}/page/2`); + expect(r).toMatchObject({ document: 'Fixture Paper', id: documentId(doc), page: 2 }); + expect(reads).not.toHaveBeenCalled(); + }); + + it('a figure returns its own root and caption; a figure that is not there is named', async () => { + const { doc, figure, view } = setup(); + const r = await view({ document: documentId(doc), page: 2, figure: 1 }); + expect(r[TOOL_ATTACHMENTS_KEY]).toEqual([figure]); + expect(r.caption).toBe('Figure 1. Cells per configuration.'); + expect(r.cite).toBe(`attachment://${documentId(doc)}/page/2`); + expect((await view({ document: documentId(doc), page: 2, figure: 2 })).error).toMatch(/1 figure\(s\); figure must be between 1 and 1/); + expect((await view({ document: documentId(doc), page: 3, figure: 1 })).error).toMatch(/no figures/); + }); + + it('a text-only page says so and carries no media key', async () => { + const { doc, view } = setup(); + const r = await view({ document: documentId(doc), page: 3 }); + expect(r.note).toMatch(/text only/); + expect(TOOL_ATTACHMENTS_KEY in r).toBe(false); + }); + + it('a page past the render bound is "not archived" — no range is ever named', async () => { + const { doc, view } = setup(); + const r = await view({ document: documentId(doc), page: 4 }); + expect(r.note).toMatch(/not archived/); + expect(r.note).not.toMatch(/\d+\s*[–-]\s*\d+/); + expect(TOOL_ATTACHMENTS_KEY in r).toBe(false); + }); + + it('page one is always projectable; a repeat by the same agent carries the page again with a note — admission is the only gate', async () => { + const { doc, renders, view } = setup(); + expect((await view({ document: documentId(doc), page: 1 }))[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); + // A settle rejection can replace the first result with a nudge; the tool + // cannot see that, so a repeat must still put the page in front of the model. + const again = await view({ document: documentId(doc), page: 1 }); + expect(again[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); + expect(again.note).toMatch(/viewed page 1 before/); + expect((await view({ document: documentId(doc), page: 1 }, [doc], 2))[TOOL_ATTACHMENTS_KEY]).toEqual([renders[1]]); + expect((await view({ document: documentId(doc), page: 5 })).error).toMatch(/out of range/); + }); + + it('the projection rule is a rule over the sidecar facts', () => { + const base = { page: 2, startLine: 1, endLine: 1, chars: 10, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }; + expect(projectable(base)).toBe(false); + expect(projectable({ ...base, page: 1 })).toBe(true); + expect(projectable({ ...base, chars: 0 })).toBe(true); + expect(projectable({ ...base, imageObjects: 1 })).toBe(true); + expect(projectable({ ...base, pathObjects: 20 })).toBe(true); + expect(projectable({ ...base, pathObjects: 19 })).toBe(false); + expect(projectable({ ...base, taggedTables: 1 })).toBe(true); + expect(projectable({ ...base, taggedFigures: 1 })).toBe(true); + }); +}); diff --git a/packages/abilities/documents/tsconfig.json b/packages/abilities/documents/tsconfig.json new file mode 100644 index 00000000..4f93912f --- /dev/null +++ b/packages/abilities/documents/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"], + "references": [ + { "path": "../../sdk" }, + { "path": "../../agents" }, + { "path": "../../media" }, + { "path": "../../rig" } + ] +} diff --git a/packages/abilities/web/src/tools/fetch-page.ts b/packages/abilities/web/src/tools/fetch-page.ts index bb98dd15..eda41c0d 100644 --- a/packages/abilities/web/src/tools/fetch-page.ts +++ b/packages/abilities/web/src/tools/fetch-page.ts @@ -96,7 +96,7 @@ export class FetchPageTool extends Tool<{ url: string; query?: string }> { ) { return { error: - "PDF documents cannot be extracted. Try searching for an HTML version of this content.", + "This is a PDF, which fetch_page cannot read. Ask the user to attach the file to the conversation.", url, }; } @@ -135,7 +135,7 @@ export class FetchPageTool extends Tool<{ url: string; query?: string }> { if (contentType.includes("application/pdf")) { return { error: - "PDF documents cannot be extracted. Try searching for an HTML version of this content.", + "This is a PDF, which fetch_page cannot read. Ask the user to attach the file to the conversation.", url, } as const; } diff --git a/packages/abilities/web/test/ability.test.ts b/packages/abilities/web/test/ability.test.ts index c7e798e9..d29161aa 100644 --- a/packages/abilities/web/test/ability.test.ts +++ b/packages/abilities/web/test/ability.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import { run } from 'effection'; -import { AbilityConfigStoreCtx } from '@lloyal-labs/lloyal-agents'; +import { AbilityConfigStoreCtx, Trace, NullTraceWriter } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; import { createInMemoryConfigStore } from '@lloyal-labs/rig'; import { createWebAbility } from '../src/index'; @@ -24,3 +25,19 @@ describe('createWebAbility', () => { expect(agentSrc).not.toContain('Apply the **'); }); }); + +describe('fetch_page on a PDF', () => { + it('names the way in — attach the file — instead of a dead end', async () => { + const ability = await run(function* () { + yield* AbilityConfigStoreCtx.set(createInMemoryConfigStore()); + yield* Trace.set(new NullTraceWriter()); + return yield* createWebAbility(); + }); + const fetchPage = ability.tools.find((t) => t.name === 'fetch_page')!; + const r = (await run(function* () { + yield* Trace.set(new NullTraceWriter()); + return yield* fetchPage.execute({ url: 'https://example.com/paper.pdf' }, { agentId: 1 } as ToolContext); + })) as { error?: string }; + expect(r.error).toBe('This is a PDF, which fetch_page cannot read. Ask the user to attach the file to the conversation.'); + }); +}); diff --git a/packages/agents/src/AgentPolicy.ts b/packages/agents/src/AgentPolicy.ts index 7c2f6b28..03c7c048 100644 --- a/packages/agents/src/AgentPolicy.ts +++ b/packages/agents/src/AgentPolicy.ts @@ -250,7 +250,7 @@ export interface AgentPolicy { * DISPATCH phase: should this tool call explore or exploit? * * When true (explore), content-boundary tools use agent-local scoring only. - * When false (exploit), tools apply dual scoring via scoreRelevanceBatch. + * When false (exploit), admission also scores against the original question and ranks by min(). * Non-monotonic — flips with live pressure. Separate from lifecycle. * Optional — defaults to true (explore) when absent. */ diff --git a/packages/agents/src/Tool.ts b/packages/agents/src/Tool.ts index 4e5100c2..ce823d50 100644 --- a/packages/agents/src/Tool.ts +++ b/packages/agents/src/Tool.ts @@ -1,5 +1,7 @@ import type { Operation } from 'effection'; import type { JsonSchema, ToolSchema, ToolContext } from './types'; +import { asAttachment } from '@lloyal-labs/media'; +import type { Attachment } from '@lloyal-labs/media'; /** * Abstract base class for tools usable by agents in the runtime @@ -188,7 +190,7 @@ export class ToolRetryError extends Error { * * | key | direction | does the model read it? | * |---|---|---| - * | `_images` | OUT of the result, before serializing | **no** — that is the point | + * | `_attachments` | OUT of the result, before serializing | **no** — that is the point | * | `_contextAvailablePercent` | INTO the result | yes | * | `_imageError` | INTO the result | yes — it exists to be read | * @@ -203,8 +205,15 @@ export class ToolRetryError extends Error { * Taken OUT before serializing, because these bytes must reach the cache down * the embedding rail and must never reach it as JSON. A 180 KB image * stringifies to ~700k characters of digits, which is not a degraded prefill - * but a destroyed one. */ -export const TOOL_MEDIA_KEY = '_images'; + * but a destroyed one. + * + * An entry is raw bytes — admitted through the ingress door — or a root + * descriptor already in the content store (a page render a tool looked up, a + * document a tool fetched): no bytes, no door, the ingest-time digest. The + * rail follows what the root materializes to: bitmaps ride the embedding + * rail; a root with none rides the token rail beside the tool's text and is + * booked as an asset available to the run. */ +export const TOOL_ATTACHMENTS_KEY = '_attachments'; /** Split a tool result into the images it carried and the result WITHOUT them. * @@ -215,30 +224,34 @@ export const TOOL_MEDIA_KEY = '_images'; * halves and hands each to exactly one consumer. * * `result` is returned unchanged when there is no media, so a text-only tool - * copies nothing. Entries that are not `Uint8Array` are ignored — one marker - * is emitted per SURVIVING entry, so the prompt and the bitmap list stay in - * step whatever a tool hands over. + * copies nothing. An entry is bytes or a root descriptor the store would + * recognise; anything else is ignored — markers are emitted per SURVIVING + * representation, so the prompt and the bitmap list stay in step whatever a + * tool hands over. * * @category Agents */ export function takeToolMedia( result: unknown, -): { media: Uint8Array[]; result: unknown } { +): { media: (Uint8Array | Attachment)[]; result: unknown } { if (!result || typeof result !== 'object' || Array.isArray(result)) { return { media: [], result }; } - if (!(TOOL_MEDIA_KEY in result)) return { media: [], result }; - const { [TOOL_MEDIA_KEY]: raw, ...rest } = result as Record<string, unknown>; + if (!(TOOL_ATTACHMENTS_KEY in result)) return { media: [], result }; + const { [TOOL_ATTACHMENTS_KEY]: raw, ...rest } = result as Record<string, unknown>; // The reserved key never survives into the serialized result, even when its // value is malformed — returning the original would JSON-encode byte // indices onto the token rail, the exact failure this helper exists to // prevent. An invalid value is simply zero media entries. - return { - media: Array.isArray(raw) - ? raw.filter((b): b is Uint8Array => b instanceof Uint8Array) - : [], - result: rest, - }; + const media: (Uint8Array | Attachment)[] = []; + if (Array.isArray(raw)) { + for (const entry of raw) { + if (entry instanceof Uint8Array) { media.push(entry); continue; } + const root = asAttachment(entry); + if (root) media.push(root); + } + } + return { media, result: rest }; } /** diff --git a/packages/agents/src/admission.ts b/packages/agents/src/admission.ts index 74a95f57..aa096f98 100644 --- a/packages/agents/src/admission.ts +++ b/packages/agents/src/admission.ts @@ -135,9 +135,11 @@ function selectTopChunks( * greps; discovery signals (`alsoOnPage`) compensate. * * **Exploit mode** (`context.explore === false`, set by - * `policy.shouldExplore`): every chunk is re-scored by the entailment scorer - * against the query and re-sorted — `min(toolQueryScore, originalQueryScore)` - * tightens focus when KV or time is short, at the cost of serendipity. + * `policy.shouldExplore`): every scored chunk is also scored by the + * entailment scorer against the ORIGINAL question — one extra pass; the + * tool-query scores are already in hand — and re-sorted by + * `min(toolQueryScore, originalQueryScore)`, which tightens focus when KV or + * time is short, at the cost of serendipity. * * Callers own everything around it: fetching/chunking (and * `tokenizeChunks` for budget mode), any first-stage narrowing (BM25), @@ -189,11 +191,13 @@ export function* admitChunks( ); return chunk?.text ?? ''; }); - const combinedScores: number[] = yield* call(() => - context.scorer!.scoreRelevanceBatch(chunkTexts, query), + // One more pass, against the original question; the tool-query scores + // are the ones already in hand. Both must be high to rank: min(). + const originalScores: number[] = yield* call(() => + context.scorer!.scoreEntailmentBatch(chunkTexts), ); const reordered: ScoredWithOriginal[] = scored - .map((sc, i) => ({ ...sc, score: combinedScores[i], _toolQueryScore: sc.score })) + .map((sc, i) => ({ ...sc, score: Math.min(sc.score, originalScores[i]), _toolQueryScore: sc.score })) .sort((a, b) => b.score - a.score); scored = reordered; diff --git a/packages/agents/src/agent-pool.ts b/packages/agents/src/agent-pool.ts index 86bdf665..8060b2a9 100644 --- a/packages/agents/src/agent-pool.ts +++ b/packages/agents/src/agent-pool.ts @@ -1,6 +1,7 @@ import { resource, ensure, createSignal, createChannel, createQueue, spawn, each, sleep, action, race } from 'effection'; import type { Operation, Subscription, Task, Signal } from 'effection'; import type { SessionContext, BranchStore } from '@lloyal-labs/sdk'; +import type { Attachment } from '@lloyal-labs/media'; import { buildTurnDelta } from '@lloyal-labs/sdk'; import { Ctx, Store, Trace, TraceParent, GrantStoreCtx, WindDown, CancelAgent, Pause, Attachments, Ingress } from './context'; import { useTraceScope } from './trace-scope'; @@ -75,6 +76,10 @@ export function useAgentPool(opts: AgentPoolOptions): Operation<Subscription<Age const tw = yield* Trace.expect(); const attachments = yield* Attachments.expect(); const ingress = yield* Ingress.expect(); + // Assets available to the run: staged by the host now, grown by every + // tool result that admits a root. One list, owned here — it outlives any + // agent, so a root agent A admitted stays available after A is pruned. + const available: Attachment[] = [...(opts.attachments ?? [])]; const { spine, orchestrate, toolsJson, tools, maxTurns = 100, terminalToolName, trace = false, pruneOnReturn = false, enableThinking = true, eagerGrammar } = opts; const toolIndexMap = new Map([...tools.keys()].map((name, i) => [name, i])); @@ -192,7 +197,7 @@ export function useAgentPool(opts: AgentPoolOptions): Operation<Subscription<Age permits: makePermits(opts.maxConcurrentTools ?? DEFAULT_MAX_CONCURRENT_TOOLS), completed, wake, progress, scorer: opts.scorer, toolIndexMap, toolkitSize: tools.size, terminalGrammar, eagerGrammar, enableThinking, spine, runNow, counters, totals, policy, - pressureOpts, ingress, attachments, ladder, trace, + pressureOpts, ingress, attachments, available, ladder, trace, }); // ── PoolContext — the orchestrator's API ───────────────────── diff --git a/packages/agents/src/create-agent-pool.ts b/packages/agents/src/create-agent-pool.ts index 15cfd899..323f5168 100644 --- a/packages/agents/src/create-agent-pool.ts +++ b/packages/agents/src/create-agent-pool.ts @@ -10,6 +10,7 @@ import { Events } from './context'; import { createToolkit } from './toolkit'; import { withSpine } from './spine'; import { useAgentPool } from './agent-pool'; +import type { Attachment } from '@lloyal-labs/media'; // ── CreateAgentPool opts ──────────────────────────────────── @@ -56,6 +57,8 @@ export interface CreateAgentPoolOpts { session?: Session; /** Entailment scorer for semantic coherence across recursive depths. */ scorer?: EntailmentScorer; + /** Assets available to the run — see {@link AgentPoolOptions.attachments}. */ + attachments?: readonly Attachment[]; /** Echo detection threshold. @default 0.8 */ echoThreshold?: number; /** Check ancestor tasks for echo. @default false */ @@ -160,6 +163,7 @@ export function* agentPool(opts: CreateAgentPoolOpts): Operation<AgentPoolResult trace: opts.trace, policy: opts.policy, scorer: opts.scorer, + attachments: opts.attachments, enableThinking: opts.enableThinking, }); diff --git a/packages/agents/src/emit.ts b/packages/agents/src/emit.ts index 5529b6fa..599c1494 100644 --- a/packages/agents/src/emit.ts +++ b/packages/agents/src/emit.ts @@ -172,11 +172,17 @@ export function project(t: Transition): Emission[] { ...(t.rc !== undefined ? { rc: t.rc } : {}), attempt: t.attempt, pressure: pressureRecord(t.pressure), } }]; case 'prefilled': - return [{ trace: { - type: 'branch:prefill', branchHandle: t.agent.id, cells: t.cells, role: t.role, - ...(t.attachments ? { attachments: t.attachments } : {}), - ...(t.probeText !== undefined ? { probeText: t.probeText } : {}), - } }]; + return [ + { bus: { + type: 'agent:prefilled', agentId: t.agent.id, cells: t.cells, role: t.role, + ...(t.attachments ? { attachments: t.attachments } : {}), + } }, + { trace: { + type: 'branch:prefill', branchHandle: t.agent.id, cells: t.cells, role: t.role, + ...(t.attachments ? { attachments: t.attachments } : {}), + ...(t.probeText !== undefined ? { probeText: t.probeText } : {}), + } }, + ]; case 'settleOrder': return [{ trace: { type: 'tool:settle_order', batch: t.batch } }]; case 'pruned': diff --git a/packages/agents/src/execute.ts b/packages/agents/src/execute.ts index b0385cd6..8055a9cb 100644 --- a/packages/agents/src/execute.ts +++ b/packages/agents/src/execute.ts @@ -5,13 +5,13 @@ import { CHAT_FORMAT_CONTENT_ONLY, CHAT_FORMAT_GENERIC, GrammarTriggerType, buildToolResultDelta, buildToolResultDeltaMultimodal, decodeErrorOf, deltaCells, } from '@lloyal-labs/sdk'; -import type { Attachment, AttachmentStore, ContentIngress } from '@lloyal-labs/media'; +import type { Attachment, AttachmentStore, ContentIngress, PreparedContent } from '@lloyal-labs/media'; import { waitUntilSettled } from './combinators'; import { Trace, TraceParent, CallingAgent, SpineFmt } from './context'; import { Agent, type FormatConfig } from './Agent'; import type { AgentPolicy, ToolRetryAction } from './AgentPolicy'; import { Tool, ToolRetryError, takeToolMedia, TOOL_CONTEXT_KEY, TOOL_IMAGE_ERROR_KEY } from './Tool'; -import type { Emitter } from './emit'; +import type { Emitter, Transition } from './emit'; import { ContextPressure } from './pressure'; import { prepareBatch } from './prepare-content'; import { runReplay } from './replay'; @@ -177,6 +177,10 @@ export interface ExecDeps { pressureOpts: PressureThresholds; ingress: ContentIngress; attachments: AttachmentStore; + /** Assets available to the run: the roots the host staged, then every root + * a tool result admitted, in admission order. Grows in `book()`; read by + * every tool call. Roots only — the pool never materializes an entry. */ + available: Attachment[]; ladder: Ladder; trace: boolean; } @@ -255,6 +259,10 @@ export class Executor { const landed: Agent[] = []; const order: { agentId: number; callId: string; cells: number }[] = []; const probes = new Map<number, string>(); + // Admissions to announce once the rails are done: `prefilled` rides the + // bus as well as the trace, and a bus send may suspend, so it leaves the + // synchronous bookkeeping below and is emitted from this generator. + const admissions: Transition[] = []; const tokenItems = items.filter((it): it is PrefillItem & { rail: 'token' } => it.rail === 'token'); const mediaItems = items.filter((it): it is PrefillItem & { rail: 'media' } => it.rail === 'media'); @@ -265,6 +273,11 @@ export class Executor { a.records.push({ kind: 'toolResult', resultStr: it.resultStr, callId: it.callId, ...(refs && refs.length > 0 ? { attachments: refs } : {}) }); } + // Admitted roots become assets available to the whole run — pool state, + // so they outlive the agent that admitted them. + for (const r of refs ?? []) { + if (!d.available.some((x) => x.digest === r.digest)) d.available.push(r); + } landed.push(a); order.push({ agentId: a.id, callId: it.callId, cells }); if (it.probe) probes.set(a.id, it.probe); @@ -272,7 +285,7 @@ export class Executor { const after = new ContextPressure(d.ctx, d.pressureOpts); a.recordToolResult({ name: it.toolName, args: it.args, resultCells: cells, contextAfterPercent: after.percentAvailable, timestamp: performance.now() }); - d.emit.trace({ kind: 'prefilled', agent: a, cells, + admissions.push({ kind: 'prefilled', agent: a, cells, role: it.kind === 'recovery' ? 'recovery' : 'toolResult', attachments: refs }); }; @@ -282,7 +295,7 @@ export class Executor { d.counters.warmPrefillCalls++; d.counters.warmPrefillBranches += tokenItems.length; d.ladder.consecutiveFatalRc = 0; - for (const t of tokenItems) book(t, t.tokens.length); + for (const t of tokenItems) book(t, t.tokens.length, t.attachments); out.tokenRail = { items: tokenItems, outcome: { ok: true } }; } catch (err) { const de = decodeErrorOf(err); @@ -312,6 +325,8 @@ export class Executor { } } + for (const t of admissions) yield* d.emit.emit(t); + if (landed.length > 0) { d.emit.trace({ kind: 'settleOrder', batch: order }); const probePairs: [Branch, number[]][] = []; @@ -326,7 +341,7 @@ export class Executor { if (probePairs.length > 0) { yield* prefill(d.store, probePairs); for (const m of probeMeta) { - d.emit.trace({ kind: 'prefilled', agent: m.agent, cells: m.cells, role: 'probe', probeText: m.text }); + yield* d.emit.emit({ kind: 'prefilled', agent: m.agent, cells: m.cells, role: 'probe', probeText: m.text }); m.agent.records.push({ kind: 'probe', text: m.text }); } } @@ -509,6 +524,7 @@ export class Executor { scorer: d.scorer, explore, pressurePercentAvailable: reading.percentAvailable, peerHistory, + attachments: [...d.available], }; if (tool?.fanout) { @@ -629,27 +645,39 @@ export class Executor { if (Array.isArray(obj.results)) agent.addNestedResults((obj.results as unknown[]).filter((f): f is string => typeof f === 'string')); if (Array.isArray(obj.nestedResults)) agent.addNestedResults((obj.nestedResults as unknown[]).filter((f): f is string => typeof f === 'string')); } - // Images come OUT before serializing; a model with no projector is TOLD. + // Media comes OUT before serializing. Bytes go through the ingress door; + // roots resolve through the store. The rail follows what the batch + // MATERIALIZES to, never the fact that media was there. const { media, result: told } = takeToolMedia(result); - if (media.length > 0 && !d.ctx.supportsVision()) { + const vision = d.ctx.supportsVision(); + const hasBytes = media.some((m) => m instanceof Uint8Array); + // THE BARRIER: normalized and committed before a marker exists, before + // admission, before any KV moves. A failure here is not a tool retry. + // A model with no projector never ingests bytes; it still resolves roots, + // because a root that expands to no bitmaps — a document — asks nothing + // of sight and is admitted on the token rail. + let prepared: PreparedContent | null = null; + if (media.length > 0 && (vision || !hasBytes)) { + prepared = yield* prepareBatch(d.ingress, d.attachments, media); + } + if (media.length > 0 && !vision && (hasBytes || (prepared?.bitmaps.length ?? 0) > 0)) { (told as Record<string, unknown>)[TOOL_IMAGE_ERROR_KEY] = `${tc.name} returned ${media.length} image(s), but this model cannot see images. ` + `Work from the text, or use a different source.`; + prepared = null; } const resultStr = JSON.stringify(told); yield* d.emit.emit({ kind: 'toolTold', agent, tool: tc.name, resultStr, contextAvailablePercent }); const common = { agent, toolName: tc.name, callId, args: tc.arguments, resultStr, probe: tool?.probe(told) ?? undefined }; let item: PrefillItem; - if (media.length > 0 && d.ctx.supportsVision()) { - // THE BARRIER: normalized and committed before a marker exists, before - // admission, before any KV moves. A failure here is not a tool retry. - const prepared = yield* prepareBatch(d.ingress, d.attachments, media); + if (prepared && prepared.bitmaps.length > 0) { const delta = buildToolResultDeltaMultimodal(d.ctx, resultStr, callId, prepared.bitmaps as Uint8Array[], { enableThinking: agent.fmt.enableThinking }); const cells = yield* measureCells(d.ctx, delta); item = { kind: 'toolResult', rail: 'media', ...common, media: { delta, cells, attachments: prepared.attachments } }; } else { const tokens = buildToolResultDelta(d.ctx, resultStr, callId, { enableThinking: agent.fmt.enableThinking }); - item = { kind: 'toolResult', rail: 'token', ...common, tokens }; + item = { kind: 'toolResult', rail: 'token', ...common, tokens, + ...(prepared && prepared.attachments.length > 0 ? { attachments: prepared.attachments } : {}) }; } d.emit.trace({ kind: 'toolResult', agent, tool: tc.name, result: told, cells: item.rail === 'media' ? item.media.cells : item.tokens.length, diff --git a/packages/agents/src/index.ts b/packages/agents/src/index.ts index d6da3009..69b95ace 100644 --- a/packages/agents/src/index.ts +++ b/packages/agents/src/index.ts @@ -40,7 +40,7 @@ export { prepareBatch } from './prepare-content'; // The one member of the framework-channel namespace a TOOL writes. The other // two are framework→model and no tool author ever sets them, so they stay // internal rather than growing the surface to describe a convention. -export { TOOL_MEDIA_KEY } from './Tool'; +export { TOOL_ATTACHMENTS_KEY } from './Tool'; export { useTraceScope } from './trace-scope'; export { admitChunks } from './admission'; export type { AdmitOpts, AdmitResult, AdmitSelect, AdmittedPassage } from './admission'; diff --git a/packages/agents/src/prepare-content.ts b/packages/agents/src/prepare-content.ts index 6d9f45d6..06bb322c 100644 --- a/packages/agents/src/prepare-content.ts +++ b/packages/agents/src/prepare-content.ts @@ -18,7 +18,9 @@ import { materialize } from '@lloyal-labs/media'; * * The barrier this enforces, in order: * - * 1. **Prepare** every item, committing each root manifest. + * 1. **Prepare** every item, committing each root manifest. An item that is + * already a root — a descriptor a tool read from the store — takes its + * place in order without touching the door; only bytes are ingested. * 2. **Materialize** every representation. * 3. **Flatten** preserving attachment order, then representation order within * each — markers correspond to REPRESENTATIONS, so a video contributes its @@ -54,7 +56,7 @@ import { materialize } from '@lloyal-labs/media'; export function* prepareBatch( ingress: ContentIngress, store: AttachmentStore, - items: readonly Uint8Array[], + items: readonly (Uint8Array | Attachment)[], ): Operation<PreparedContent> { // The scope's own signal, hoisted once. `call()` makes a halt OBSERVABLE at // this boundary but cannot stop the promise behind it — that is the leaked @@ -65,7 +67,8 @@ export function* prepareBatch( // Concurrent, in input order: normalization is the expensive step and the // normalizer already bounds itself process-wide, so a batch of N must not // cost the sum of N decodes while permits sit idle. `all` keeps the order. - const roots: Attachment[] = yield* all(items.map((bytes) => call(() => ingress.ingest(bytes, signal)))); + const roots: Attachment[] = yield* all(items.map((item) => + (item instanceof Uint8Array ? call(() => ingress.ingest(item, signal)) : call(() => item)))); // Resolve from the store rather than trusting what ingest returned, through // the SAME call replay uses — so a batch that materializes here is one that // can be rebuilt later, by construction rather than by assertion. diff --git a/packages/agents/src/replay.ts b/packages/agents/src/replay.ts index d808d928..de4d8c11 100644 --- a/packages/agents/src/replay.ts +++ b/packages/agents/src/replay.ts @@ -273,8 +273,9 @@ export type AgentTurnRecord = | { kind: 'assistant'; text: string } | { kind: 'toolResult'; resultStr: string; callId: string; - /** Roots for a media-bearing result — resolved through the run's - * attachment store at replay, exactly as the seed's are. */ + /** Roots admitted with this result — resolved through the run's + * attachment store at replay, exactly as the seed's are. The rail + * follows what they materialize to, as it did live. */ attachments?: readonly Attachment[]; } | { kind: 'probe'; text: string }; @@ -313,7 +314,13 @@ export function* prepareReplay( } else if (r.kind === 'probe') { tokenStep(ctx.tokenizeSync(r.text, false)); } else if (r.attachments && r.attachments.length > 0) { + // Rail by what the roots MATERIALIZE to: a document root expands to no + // bitmaps and replays as the tool text it was. const { bitmaps } = materialize(attachments, r.attachments); + if (bitmaps.length === 0) { + tokenStep(buildToolResultDelta(ctx, r.resultStr, r.callId, opts)); + continue; + } const delta = buildToolResultDeltaMultimodal(ctx, r.resultStr, r.callId, [...bitmaps], opts); const priced = yield* measureCells(ctx, delta); steps.push({ kind: 'media', delta, cells: priced }); diff --git a/packages/agents/src/source.ts b/packages/agents/src/source.ts index 83ca61d7..3b76190d 100644 --- a/packages/agents/src/source.ts +++ b/packages/agents/src/source.ts @@ -1,4 +1,5 @@ import type { Tool } from './Tool'; +import type { Attachment } from '@lloyal-labs/media'; /** * Entailment scorer — scores texts against an original query to @@ -11,12 +12,12 @@ import type { Tool } from './Tool'; * * | Concept | Scope | Field name in code | * |--------------------|--------------------------|-------------------------------------------| - * | **Tool query** | Per tool call | `localQuery` param of scoreRelevanceBatch | + * | **Tool query** | Per tool call | scored by the tool's own reranker pass | * | **Agent task** | Per agent lifetime | `reference` param of scoreSimilarityBatch | * | **Original query** | Per research invocation | Captured in closure by createScorer | * * - `scoreEntailmentBatch` scores against the **original query** (steering boundaries) - * - `scoreRelevanceBatch` combines **tool query** + **original query** via min() (exploit mode) + * - exploit mode (`admitChunks`) takes `min(tool-query score, scoreEntailmentBatch)` — one extra pass, never two * - `scoreSimilarityBatch` scores against an arbitrary **reference** (echo detection uses agent task) * * Conflating these produces wrong scores. When adding new scoring @@ -33,7 +34,6 @@ export interface EntailmentScorer { * @param texts - Content chunks to score * @param localQuery - The tool call's query argument (NOT the agent task) */ - scoreRelevanceBatch(texts: string[], localQuery: string): Promise<number[]>; /** Score texts against an arbitrary reference string. Returns 0–1 per text. */ scoreSimilarityBatch(reference: string, texts: string[]): Promise<number[]>; /** Threshold gate — returns true if the score is high enough to proceed. */ @@ -43,7 +43,6 @@ export interface EntailmentScorer { /** No-op scorer — all scores 1.0, all proceed. Used when no reranker is available. */ export const NULL_SCORER: EntailmentScorer = { scoreEntailmentBatch: async (texts) => texts.map(() => 1), - scoreRelevanceBatch: async (texts) => texts.map(() => 1), scoreSimilarityBatch: async (_ref, texts) => texts.map(() => 0), shouldProceed: () => true, }; @@ -109,18 +108,11 @@ export abstract class Source<TChunk = unknown> { if (!reranker || !originalQuery) return NULL_SCORER; const floor = this._entailmentFloor; - const combine = (local: number, orig: number) => Math.min(local, orig); return { async scoreEntailmentBatch(texts: string[]): Promise<number[]> { return reranker.scoreBatch(originalQuery, texts); }, - async scoreRelevanceBatch(texts: string[], localQuery: string): Promise<number[]> { - // SEQUENTIAL — single llama_context, no concurrent scoreBatch calls - const origScores = await reranker.scoreBatch(originalQuery, texts); - const localScores = await reranker.scoreBatch(localQuery, texts); - return texts.map((_, i) => combine(localScores[i], origScores[i])); - }, async scoreSimilarityBatch(reference: string, texts: string[]): Promise<number[]> { return reranker.scoreBatch(reference, texts); }, @@ -147,6 +139,10 @@ export abstract class Source<TChunk = unknown> { * (first-party corpora); it must not blanket-append data from untrusted * third-party abilities — `renderSpine` itself stays prose-free for exactly * this reason. + * + * `attachments` are the assets available to the run being staged — roots, + * as the host holds them. A source keyed on attachments builds its data + * from them; a source that is not ignores the argument. */ - promptData(): Record<string, unknown> { return {}; } + promptData(_attachments: readonly Attachment[] = []): Record<string, unknown> { return {}; } } diff --git a/packages/agents/src/state.ts b/packages/agents/src/state.ts index 00167906..9ecaf297 100644 --- a/packages/agents/src/state.ts +++ b/packages/agents/src/state.ts @@ -41,7 +41,10 @@ export type PrefillItem = { resultStr?: string; } & ( /** The token rail: the delta tokenized here and prefilled as tokens. */ - | { rail: 'token'; tokens: number[]; media?: never } + | { rail: 'token'; tokens: number[]; media?: never; + /** Roots admitted with a token-rail result — ones that materialize to + * no bitmaps — booked as assets available to the run. */ + attachments?: readonly Attachment[] } /** The embedding rail. `llama_batch` is token-XOR-embd, so this cannot * join a token batch — a separate call, not a separate strategy. */ | { rail: 'media'; tokens?: never; media: { delta: MultimodalDelta; cells: number; attachments: readonly Attachment[] } } diff --git a/packages/agents/src/trace-types.ts b/packages/agents/src/trace-types.ts index 388fba4e..1909d886 100644 --- a/packages/agents/src/trace-types.ts +++ b/packages/agents/src/trace-types.ts @@ -447,7 +447,7 @@ export type TraceEvent = | TraceEventBase & { /** Exploit-mode dual scoring at a content boundary (search/fetch_page). * Emitted when policy.shouldExplore() returns false and the tool - * applies scoreRelevanceBatch to tighten focus. */ + * ranks by min(tool score, original-question score) to tighten focus. */ type: 'entailment:content:exploit'; tool: string; /** Pressure snapshot that triggered exploit mode. Only the field the diff --git a/packages/agents/src/types.ts b/packages/agents/src/types.ts index 47173092..4dc0f53d 100644 --- a/packages/agents/src/types.ts +++ b/packages/agents/src/types.ts @@ -5,6 +5,7 @@ import type { AgentPolicy } from './AgentPolicy'; import type { EntailmentScorer } from './source'; import type { ToolHistoryEntry } from './Agent'; import type { TraceEvent } from './trace-types'; +import type { Attachment } from '@lloyal-labs/media'; // ── Tool base class types ────────────────────────────────────── @@ -78,7 +79,7 @@ export interface ToolContext { scorer?: EntailmentScorer; /** * When false, content-boundary tools apply dual scoring - * (scoreRelevanceBatch) for tighter focus. Computed per-DISPATCH + * (min with the original-question score) for tighter focus. Computed per-DISPATCH * by policy.shouldExplore(). @default true */ explore?: boolean; @@ -95,6 +96,14 @@ export interface ToolContext { * a "resource unavailable" error to force diversification. */ peerHistory?: ToolHistoryEntry[]; + /** + * Assets available to the run at this call: the roots the host staged the + * pool with, then every root any agent's tool result has admitted so far, + * in admission order. Roots only — reading one costs no KV; the content + * store resolves it. Availability is not projection: nothing here is in + * the model's context unless a branch admitted it. + */ + attachments?: readonly Attachment[]; } // ── Trace types ─────────────────────────────────────────────── @@ -301,6 +310,12 @@ export interface AgentPoolOptions { /** Entailment scorer for semantic coherence across recursive depths. * Passed to every tool via {@link ToolContext.scorer}. */ scorer?: EntailmentScorer; + /** + * Assets available to the run from the start — roots the host stages the + * pool with. Zero KV: the pool never materializes them. Every tool call + * reads them, plus whatever the run admits, as {@link ToolContext.attachments}. + */ + attachments?: readonly Attachment[]; /** * Eager GBNF grammar applied to every spawned agent's generating branch — * constrains generation from the first sampled token (no trigger). Used for @@ -405,6 +420,12 @@ export type AgentEvent = | { type: 'agent:tool_result'; agentId: number; tool: string; result: string; contextAvailablePercent?: number } | { type: 'agent:tool_progress'; agentId: number; tool: string; filled: number; total: number } | { type: 'agent:tool_retry'; agentId: number; tool: string; retryAfterMs: number; attempt: number } + /** A prefill LANDED on the agent's branch — a tool result, a recovery + * prompt or a probe — with the roots it admitted (`attachments`, present + * when the result carried any). Admission is a run-record fact the host + * books and shows, so it rides the bus like every other one; the trace's + * `branch:prefill` is the same moment written for replay. */ + | { type: 'agent:prefilled'; agentId: number; cells: number; role: 'toolResult' | 'recovery' | 'probe'; attachments?: readonly Attachment[] } | { type: 'agent:return'; agentId: number; result: string } | { type: 'agent:recovered'; agentId: number; result: string } | { type: 'agent:failed'; agentId: number; reason: string } diff --git a/packages/agents/test/admission.test.ts b/packages/agents/test/admission.test.ts index d260cb5f..ac7f51da 100644 --- a/packages/agents/test/admission.test.ts +++ b/packages/agents/test/admission.test.ts @@ -111,8 +111,8 @@ describe('admitChunks — exploit re-rank', () => { agentId: 1, explore: false, pressurePercentAvailable: 33, - // Reverses the tool-query order: chunk 2's text entails the query best. - scorer: { scoreRelevanceBatch: async (texts: string[]) => texts.map((_, i) => i) } as any, + // Original-question scores reverse the tool-query order: chunk 2's text entails the question best. + scorer: { scoreEntailmentBatch: async (texts: string[]) => texts.map((_, i) => i) } as any, }, ); expect(result.scored.map(s => s.heading)).toEqual(['H2', 'H1', 'H0']); @@ -123,11 +123,29 @@ describe('admitChunks — exploit re-rank', () => { expect(exploit.chunks[0]).toMatchObject({ heading: 'H2', toolQueryScore: 8, combinedScore: 2 }); }); + it('exploit asks the entailment scorer ONCE for the original-question scores and takes min() with the tool scores it already has', async () => { + const calls: string[][] = []; + const scorer = { + // Original-question scores only: chunk i entails the question at i. + scoreEntailmentBatch: async (texts: string[]) => { calls.push(texts); return texts.map((_, i) => i); }, + }; + const { result, tw } = await admit( + { tool: 'fetch_page', select: { mode: 'budget', topK: 3, tokenBudget: 10_000 } }, + mkChunks(3), + { agentId: 1, explore: false, scorer } as any, + ); + expect(calls).toHaveLength(1); + expect(calls[0]).toHaveLength(3); + // min(tool, original): H2 = min(8, 2) = 2, H1 = min(9, 1) = 1, H0 = min(10, 0) = 0. + expect(result.scored.map(s => [s.heading, s.score])).toEqual([['H2', 2], ['H1', 1], ['H0', 0]]); + expect(tw.ofType('entailment:content:exploit')[0].chunks[0]).toMatchObject({ heading: 'H2', toolQueryScore: 8, combinedScore: 2 }); + }); + it('an OMITTED explore flag defaults to explore — never exploit by accident', async () => { const { result, tw } = await admit( { tool: 'fetch_page', select: { mode: 'budget', topK: 3, tokenBudget: 10_000 } }, mkChunks(3), - { agentId: 1, scorer: { scoreRelevanceBatch: async () => [9, 9, 9] } as any }, + { agentId: 1, scorer: { scoreEntailmentBatch: async () => [9, 9, 9] } as any }, ); expect(result.scored.map(s => s.heading)).toEqual(['H0', 'H1', 'H2']); expect(tw.ofType('entailment:content:exploit')).toHaveLength(0); @@ -137,7 +155,7 @@ describe('admitChunks — exploit re-rank', () => { const { result, tw } = await admit( { tool: 'fetch_page', select: { mode: 'budget', topK: 3, tokenBudget: 10_000 } }, mkChunks(3), - { agentId: 1, explore: true, scorer: { scoreRelevanceBatch: async () => [9, 9, 9] } as any }, + { agentId: 1, explore: true, scorer: { scoreEntailmentBatch: async () => [9, 9, 9] } as any }, ); expect(result.scored.map(s => s.heading)).toEqual(['H0', 'H1', 'H2']); expect(tw.ofType('entailment:content:exploit')).toHaveLength(0); diff --git a/packages/agents/test/agent-pool.test.ts b/packages/agents/test/agent-pool.test.ts index 8881fded..4d118af8 100644 --- a/packages/agents/test/agent-pool.test.ts +++ b/packages/agents/test/agent-pool.test.ts @@ -9,7 +9,8 @@ * transitions, trace events, event emissions, ToolContext fields, recovery. */ import { describe, it, expect } from 'vitest'; -import { MediaTool, PNG_BYTES, MEDIA_TEST_NCTX, mediaFailures } from './helpers/media'; +import { MediaTool, RootTool, imageRoot, documentRoot, PNG_BYTES, MEDIA_TEST_NCTX, mediaFailures } from './helpers/media'; +import type { Attachment } from '@lloyal-labs/media'; import { run, createChannel, createSignal, spawn, each, scoped, call } from 'effection'; import type { Operation, Channel } from 'effection'; import { MockSessionContext, createMockSdk } from '../../sdk/src/testing.js'; @@ -58,11 +59,17 @@ async function runPool(opts: { mutateCtx?: (ctx: MockSessionContext) => void; /** Install an ingress that refuses everything, to exercise the barrier. */ refusingIngress?: boolean; + /** Assets available to the run from the start — what a host stages the pool with. */ + attachments?: readonly Attachment[]; + /** A content store the test committed roots into beforehand. */ + contentStore?: MemoryAttachmentStore; }): Promise<{ result: AgentPoolResult; events: AgentEvent[]; trace: CapturingTraceWriter; ctx: MockSessionContext; + /** How many times a tool result went through the ingress door. */ + ingressCalls: number; }> { const { ctx, store, root } = createMockSdk({ nCtx: opts.nCtx ?? 16384, @@ -103,6 +110,7 @@ async function runPool(opts: { const traceWriter = new CapturingTraceWriter(); const collectedEvents: AgentEvent[] = []; + let ingressCalls = 0; // Prefill root to simulate withSpine system prompt const rootTokens = ctx.tokenizeSync('system prompt'); @@ -117,15 +125,16 @@ async function runPool(opts: { // A real store: media paths now REFUSE to run without one, because // unaddressed media makes a run unreplayable. Tests that exercise them // must be configured the way a real harness is. - const contentStore = new MemoryAttachmentStore(); + const contentStore = opts.contentStore ?? new MemoryAttachmentStore(); yield* Attachments.set(contentStore); // Media now refuses to run without an ingress, because unnormalized, // unaddressed bytes make a run unreplayable. Tests use a raw one — they // exercise the rail, not the normalizer. + const raw = rawIngress(contentStore); yield* Ingress.set( opts.refusingIngress ? { ingest: () => Promise.reject(new Error('ingress refused')) } - : rawIngress(contentStore), + : { ingest: (bytes, signal) => { ingressCalls++; return raw.ingest(bytes, signal); } }, ); const windDownSignal = createSignal<void, void>(); @@ -152,6 +161,7 @@ async function runPool(opts: { terminalToolName: opts.terminalTool, trace: opts.trace ?? false, pruneOnReturn: opts.pruneOnReturn ?? false, + attachments: opts.attachments, }); // Drain Subscription — collect events, return close value let windDownFired = false; @@ -168,7 +178,7 @@ async function runPool(opts: { }); }); - return { result, events: collectedEvents, trace: traceWriter, ctx }; + return { result, events: collectedEvents, trace: traceWriter, ctx, ingressCalls }; } /** Minimal policy stub — every method overridable */ @@ -2359,7 +2369,7 @@ describe('tool results carrying images', () => { const shown = events.filter(e => e.type === 'agent:tool_result') .map(e => (e as { result: string }).result).join(''); expect(shown).not.toContain('137,80,78,71'); - expect(shown).not.toContain('_images'); + expect(shown).not.toContain('_attachments'); expect(shown).toContain('p1'); }); @@ -2506,3 +2516,139 @@ describe('tool results carrying images', () => { expect(shown).not.toContain('137,80,78,71'); }); }); + +// ── Assets available to the run ───────────────────────────────── +// Availability is not projection. A root the host stages, or one a tool +// admits through the key, is readable by every tool call in the run through +// ToolContext.attachments and costs no KV. Only admission onto a branch +// projects — and the rail follows what a root MATERIALIZES to, never the +// fact that a root was there. +describe('assets available to the run', () => { + const policy = () => stubPolicy({ + shouldExit: () => false, + onProduced: (_a, parsed) => { + if (parsed.toolCalls.length > 0) return { type: 'tool_call', tc: parsed.toolCalls[0] }; + if (parsed.content) return { type: 'free_text_return', content: parsed.content }; + return { type: 'idle', reason: 'free_text_stop' }; + }, + }); + /** A turn whose raw carries 't1' calls `first`; one carrying 't2' calls + * `second`; any other finishes. Keyed off raw, as the ladder tests are. */ + const calls = (first: string, second: string) => (raw: string) => + raw.includes('t2') + ? { content: '', reasoningContent: '', toolCalls: [{ name: second, arguments: '{}', id: 'c2' }] } + : raw.includes('t1') + ? { content: '', reasoningContent: '', toolCalls: [{ name: first, arguments: '{}', id: 'c1' }] } + : { content: 'done', reasoningContent: '', toolCalls: [] }; + const booked = (trace: CapturingTraceWriter) => + trace.events.filter(e => e.type === 'branch:prefill' && (e as { role: string }).role === 'toolResult') as { attachments?: readonly Attachment[] }[]; + const shownTo = (events: AgentEvent[]) => + events.filter(e => e.type === 'agent:tool_result').map(e => (e as { result: string }).result).join(''); + /** The roots each admission announced on the bus — the host's view, which + * the run record books and the UI shows without reading the trace. */ + const announced = (events: AgentEvent[]) => + events.filter(e => e.type === 'agent:prefilled' && ((e as { attachments?: readonly Attachment[] }).attachments?.length ?? 0) > 0) + .map(e => (e as { attachments?: readonly Attachment[] }).attachments); + + it('the first call reads the staged roots, and none of them is projected', async () => { + const store = new MemoryAttachmentStore(); + const img = imageRoot(store); + const doc = documentRoot(store); + const spy = new SpyTool('look'); + const { ctx, trace } = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, STOP]], + parseChatOutputFn: calls('look', 'look'), policy: policy(), + tools: new Map<string, Tool>([['look', spy]]), + contentStore: store, attachments: [img, doc], trace: true, + }); + expect(spy.capturedContexts).toHaveLength(1); + expect(spy.capturedContexts[0].attachments).toEqual([img, doc]); + // Zero projection from availability — even for the image. + expect(ctx.multimodalPrefills).toHaveLength(0); + expect(booked(trace).flatMap(p => p.attachments ?? [])).toEqual([]); + }); + + it('a result carrying a document root rides the token rail, is booked on branch:prefill, and the next call sees it', async () => { + const store = new MemoryAttachmentStore(); + const doc = documentRoot(store); + const spy = new SpyTool('look'); + const { ctx, trace, ingressCalls, events } = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, 2, STOP, STOP]], + parseChatOutputFn: calls('open', 'look'), policy: policy(), + tools: new Map<string, Tool>([['open', new RootTool([doc], 'open')], ['look', spy]]), + contentStore: store, trace: true, + }); + expect(ingressCalls).toBe(0); + expect(ctx.multimodalPrefills).toHaveLength(0); + expect(booked(trace)[0]?.attachments).toEqual([doc]); + // The same admission rides the bus: one event, the same roots, so a host + // books and shows it from the stream it already consumes. + expect(announced(events)).toEqual([[doc]]); + expect(spy.capturedContexts).toHaveLength(1); + expect(spy.capturedContexts[0].attachments).toEqual([doc]); + const cost = trace.events.find(e => e.type === 'tool:result') as { cells: number } | undefined; + expect(cost?.cells).toBeGreaterThan(0); + }); + + it('a result carrying a page root rides the media rail through the store — the ingress is never called', async () => { + const store = new MemoryAttachmentStore(); + const page = imageRoot(store); + const { ctx, trace, ingressCalls, events } = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, STOP]], + parseChatOutputFn: calls('view', 'view'), policy: policy(), + tools: new Map<string, Tool>([['view', new RootTool([page], 'view')]]), + contentStore: store, trace: true, + }); + expect(ingressCalls).toBe(0); + expect(ctx.multimodalPrefills).toHaveLength(1); + expect(ctx.multimodalPrefills[0].bitmapCounts).toEqual([1]); + expect(booked(trace)[0]?.attachments).toEqual([page]); + expect(announced(events)).toEqual([[page]]); + }); + + it('a root one agent admits is available to its siblings, and stays so after the admitting agent returns and is pruned', async () => { + const store = new MemoryAttachmentStore(); + const doc = documentRoot(store); + const spy = new SpyTool('look'); + const { result } = await runPool({ + nCtx: MEDIA_TEST_NCTX, taskCount: 2, + // A admits on its first turn and finishes. B generates for a while, + // then calls — well after A's result landed and A was pruned. + forkTokenQueues: [[1, STOP, STOP], [7, 7, 7, 7, 7, 7, 2, STOP, STOP]], + parseChatOutputFn: calls('open', 'look'), policy: policy(), + tools: new Map<string, Tool>([['open', new RootTool([doc], 'open')], ['look', spy]]), + contentStore: store, pruneOnReturn: true, trace: true, + }); + expect(result.agents[0].result).toBe('done'); + expect(spy.capturedContexts).toHaveLength(1); + expect(spy.capturedContexts[0].attachments).toEqual([doc]); + }); + + it('without a projector a document root is still admitted, while a page root is refused with the note', async () => { + const store = new MemoryAttachmentStore(); + const doc = documentRoot(store); + const page = imageRoot(store); + const spy = new SpyTool('look'); + const text = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, 2, STOP, STOP]], + parseChatOutputFn: calls('open', 'look'), policy: policy(), + tools: new Map<string, Tool>([['open', new RootTool([doc], 'open')], ['look', spy]]), + contentStore: store, trace: true, + mutateCtx: (c) => { c.mockSupportsVision = false; }, + }); + expect(spy.capturedContexts[0]?.attachments).toEqual([doc]); + expect(shownTo(text.events)).not.toContain('cannot see images'); + expect(booked(text.trace)[0]?.attachments).toEqual([doc]); + + const picture = await runPool({ + nCtx: MEDIA_TEST_NCTX, forkTokenQueues: [[1, STOP, STOP]], + parseChatOutputFn: calls('view', 'view'), policy: policy(), + tools: new Map<string, Tool>([['view', new RootTool([page], 'view')]]), + contentStore: store, trace: true, + mutateCtx: (c) => { c.mockSupportsVision = false; }, + }); + expect(shownTo(picture.events)).toContain('cannot see images'); + expect(picture.ctx.multimodalPrefills).toHaveLength(0); + expect(booked(picture.trace)[0]?.attachments).toBeUndefined(); + }); +}); diff --git a/packages/agents/test/attachments.test.ts b/packages/agents/test/attachments.test.ts index a028ae1d..b319a4c9 100644 --- a/packages/agents/test/attachments.test.ts +++ b/packages/agents/test/attachments.test.ts @@ -20,7 +20,7 @@ import { BranchStore } from '../../sdk/src/BranchStore'; import { NullAttachmentStore } from '@lloyal-labs/media'; import type { AttachmentStore } from '@lloyal-labs/media'; import { MemoryAttachmentStore } from './helpers/memory-store'; -import { representationsOf, sourceOf, ATTACHMENT_ARTIFACT_TYPE, EMPTY_DESCRIPTOR, MANIFEST_TYPE } from '@lloyal-labs/media'; +import { representationsOf, sourceOf, ATTACHMENT_ARTIFACT_TYPE, EMPTY_DESCRIPTOR, MANIFEST_TYPE, DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; import type { Attachment } from '@lloyal-labs/media'; /** A manifest descriptor, shaped the way a store would have returned one. @@ -187,6 +187,30 @@ describe('reconstructBranch', () => { }, store); }); + it('replayAgentTurns rails a record by what its roots MATERIALIZE to — a document root goes down the token rail', async () => { + // A root is not a picture. A document root (one text/markdown + // representation) expands to no bitmaps, so its record replays as the + // tool text it was — the same rule the live path applies at intake. + const store = new MemoryAttachmentStore(); + const doc = store.putAttachment({ + representations: [store.putBlob(new TextEncoder().encode('# Paper\n\nBody.\n'), 'text/markdown')], + config: { bytes: new TextEncoder().encode('{}'), mediaType: DOCUMENT_CONFIG_TYPE }, + }); + await withCtx(function*(ctx) { + const spine = yield* reconstructBranch(cp()); + const fork = spine.forkSync(); + let tokenPrefills = 0; + const orig = ctx._storePrefill.bind(ctx); + ctx._storePrefill = async (h, t) => { tokenPrefills++; return orig(h, t); }; + yield* replayAgentTurns(fork, [ + { kind: 'toolResult', resultStr: '{"page":"p1"}', callId: 'c1', attachments: [doc] }, + ], { enableThinking: false }); + expect(tokenPrefills).toBe(1); + expect(ctx.multimodalPrefills).toHaveLength(0); + return null; + }, store); + }); + it('refuses a marker with no attachment references', async () => { // The pre-attachments behaviour, preserved: a trace that recorded only // the marker still cannot be replayed. diff --git a/packages/agents/test/helpers/media.ts b/packages/agents/test/helpers/media.ts index a9402710..43c4c023 100644 --- a/packages/agents/test/helpers/media.ts +++ b/packages/agents/test/helpers/media.ts @@ -1,6 +1,8 @@ import type { Operation } from 'effection'; +import { DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, AttachmentStore } from '@lloyal-labs/media'; import { MockTool } from './mock-tool'; -import { TOOL_MEDIA_KEY } from '../../src/Tool'; +import { TOOL_ATTACHMENTS_KEY } from '../../src/Tool'; import type { AgentEvent } from '../../src/types'; /** @@ -18,7 +20,7 @@ export class MediaTool extends MockTool { *execute(): Operation<unknown> { // Through the constant, like a real tool author would: a fixture spelling // the literal is a fixture that keeps passing after the key changes. - return { page: 'p1', [TOOL_MEDIA_KEY]: this._bytes }; + return { page: 'p1', [TOOL_ATTACHMENTS_KEY]: this._bytes }; } } @@ -37,3 +39,37 @@ export const MEDIA_TEST_NCTX = 32768; export const mediaFailures = (events: AgentEvent[]): AgentEvent[] => events.filter(e => e.type === 'agent:failed' && (e as { reason?: string }).reason === 'media_prefill_failed'); + +/** A tool returning attachment ROOTS under the same key — what a tool that + * reads the content store hands back: descriptors, no bytes, no ingress. */ +export class RootTool extends MockTool { + constructor(private _roots: readonly Attachment[], name = 'view') { super(name); } + *execute(): Operation<unknown> { + return { page: 'p1', [TOOL_ATTACHMENTS_KEY]: this._roots }; + } +} + +/** An image root: one PNG representation — the shape a page render has. + * Materializes to one bitmap. */ +export function imageRoot(store: AttachmentStore, bytes: Uint8Array = PNG_BYTES): Attachment { + return store.putAttachment({ representations: [store.putBlob(bytes, 'image/png')] }); +} + +/** A document root: one `text/markdown` representation and the document + * sidecar as config. Materializes to NO bitmaps — that is the property the + * rail tests lean on; the sidecar's content is the media package's affair. */ +export function documentRoot(store: AttachmentStore, title = 'A Paper'): Attachment { + const markdown = new TextEncoder().encode(`# ${title}\n\nBody.\n`); + const sidecar = { + title, pageCount: 1, + sections: [{ heading: title, path: title, origin: 'heuristic', startLine: 1, endLine: 3, pageStart: 1, pageEnd: 1 }], + pages: [{ page: 1, startLine: 1, endLine: 3, chars: 5, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }], + figures: [], tables: [], + derive: { profile: 'pdf.v1', pdfium: 'test', dpi: 150, maxSide: 2048, maxPixels: 1, format: 'image/png', + renderedPages: 0, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false }, + }; + return store.putAttachment({ + representations: [store.putBlob(markdown, 'text/markdown')], + config: { bytes: new TextEncoder().encode(JSON.stringify(sidecar)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); +} diff --git a/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts b/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts index 4698275d..cd4beaf5 100644 --- a/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts +++ b/packages/agents/test/invariants/scenarios/heal-replay-admitted-by-fit.scenario.test.ts @@ -23,7 +23,7 @@ import { describe, it, expect } from 'vitest'; import type { Operation } from 'effection'; import type { AgentPolicy } from '../../../src/AgentPolicy'; -import { Tool, TOOL_MEDIA_KEY } from '../../../src/Tool'; +import { Tool, TOOL_ATTACHMENTS_KEY } from '../../../src/Tool'; import type { JsonSchema } from '../../../src/types'; import { runPool, STOP } from '../harness'; import { PNG_BYTES, mediaFailures } from '../../helpers/media'; @@ -38,7 +38,7 @@ class WideThenImage extends Tool<{ query: string }> { *execute(): Operation<unknown> { this.calls++; if (this.calls <= 4) return { results: ['x'.repeat(600)] }; - return { page: 'p1', [TOOL_MEDIA_KEY]: [PNG_BYTES] }; + return { page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES] }; } } diff --git a/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts b/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts index dcc37d56..5167238b 100644 --- a/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts +++ b/packages/agents/test/invariants/scenarios/no-projector-says-so.scenario.test.ts @@ -54,7 +54,7 @@ describe('scenario: a model with no projector is TOLD, not silently shorted', () const body = (run.channelEvents.find(e => e.type === 'agent:tool_result') as { result: string }).result; - expect(body).not.toContain('_images'); + expect(body).not.toContain('_attachments'); expect(body.length).toBeLessThan(2_000); }); diff --git a/packages/agents/test/prepare-content.test.ts b/packages/agents/test/prepare-content.test.ts index b2c357d2..05fd56d6 100644 --- a/packages/agents/test/prepare-content.test.ts +++ b/packages/agents/test/prepare-content.test.ts @@ -12,7 +12,7 @@ import { MemoryAttachmentStore } from './helpers/memory-store'; import { rawIngress } from './helpers/raw-ingress'; const img = (n: number): Uint8Array[] => - Array.from({ length: n }, (_, i) => new Uint8Array([i, i + 1, i + 2, 0x89])); + Array.from({ length: n }, (_, i) => new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, i])); describe('prepareBatch', () => { it('overlaps ingests instead of running them one after another', async () => { diff --git a/packages/agents/test/source.test.ts b/packages/agents/test/source.test.ts index c25f6ed0..213ff796 100644 --- a/packages/agents/test/source.test.ts +++ b/packages/agents/test/source.test.ts @@ -129,10 +129,6 @@ describe('NULL_SCORER', () => { expect(results).toEqual([0, 0]); }); - it('scoreRelevanceBatch returns 1.0 for all texts', async () => { - const results = await NULL_SCORER.scoreRelevanceBatch(['a', 'b'], 'any query'); - expect(results).toEqual([1, 1]); - }); }); describe('scoreSimilarityBatch', () => { @@ -170,78 +166,6 @@ describe('scoreSimilarityBatch', () => { }); }); -describe('scoreRelevanceBatch', () => { - it('returns min(localScore, originalScore) per text', async () => { - // scoreBatch returns different scores depending on the query - const scoreBatch = vi.fn(async (query: string, texts: string[]) => - texts.map((t) => { - if (query === 'original query') return t === 'bridging content' ? 0.2 : 0.8; - // local query scores everything high - return 0.9; - }), - ); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - - const scorer = source.createScorer('original query'); - const results = await scorer.scoreRelevanceBatch( - ['relevant content', 'bridging content'], - 'local tool query', - ); - - // min(0.9, 0.8) = 0.8, min(0.9, 0.2) = 0.2 - expect(results[0]).toBe(0.8); - expect(results[1]).toBe(0.2); - }); - - it('calls scoreBatch sequentially with originalQuery then localQuery', async () => { - const callOrder: string[] = []; - const scoreBatch = vi.fn(async (query: string, _texts: string[]) => { - callOrder.push(query); - return [0.5]; - }); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - - const scorer = source.createScorer('original'); - await scorer.scoreRelevanceBatch(['t'], 'local'); - - expect(callOrder).toEqual(['original', 'local']); - expect(scoreBatch).toHaveBeenCalledTimes(2); - }); - - it('when local > orig, result equals orig (min)', async () => { - const scoreBatch = vi.fn(async (q: string, _t: string[]) => - [q === 'orig' ? 0.3 : 0.9], - ); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - const scorer = source.createScorer('orig'); - const result = await scorer.scoreRelevanceBatch(['t'], 'local'); - expect(result[0]).toBe(0.3); - }); - - it('when local < orig, result equals local (min)', async () => { - const scoreBatch = vi.fn(async (q: string, _t: string[]) => - [q === 'orig' ? 0.9 : 0.3], - ); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - const scorer = source.createScorer('orig'); - const result = await scorer.scoreRelevanceBatch(['t'], 'local'); - expect(result[0]).toBe(0.3); - }); - - it('when local == orig, result equals both (min)', async () => { - const scoreBatch = vi.fn(async (_q: string, _t: string[]) => [0.6]); - const source = new TestSource(); - (source as any)._reranker = { scoreBatch }; - const scorer = source.createScorer('orig'); - const result = await scorer.scoreRelevanceBatch(['t'], 'local'); - expect(result[0]).toBe(0.6); - }); -}); - describe('ContextPressure fields', () => { it('percentAvailable: normal case', () => { // 8192 / 16384 = 50% diff --git a/packages/agents/test/spawn-agents.test.ts b/packages/agents/test/spawn-agents.test.ts index 560575cd..659f1b36 100644 --- a/packages/agents/test/spawn-agents.test.ts +++ b/packages/agents/test/spawn-agents.test.ts @@ -139,12 +139,10 @@ describe('Entailment boundary discipline', () => { // Verify EntailmentScorer interface has the right shape const scorer: EntailmentScorer = { scoreEntailmentBatch: async (texts) => texts.map(() => 0.5), - scoreRelevanceBatch: async (texts) => texts.map(() => 0.5), scoreSimilarityBatch: async (_ref, texts) => texts.map(() => 0), shouldProceed: (score) => score >= 0.25, }; expect(scorer.scoreEntailmentBatch).toBeDefined(); - expect(scorer.scoreRelevanceBatch).toBeDefined(); expect(scorer.shouldProceed).toBeDefined(); }); @@ -463,13 +461,13 @@ describe('Explore/exploit decoupled from lifecycle', () => { // ── EntailmentScorer interface shape ────────────────────── describe('EntailmentScorer interface', () => { - it('scoreRelevanceBatch exists on interface shape', () => { + it('scores against the original question, against a reference, and gates — exploit combines in admission, not here', () => { const scorer: EntailmentScorer = { scoreEntailmentBatch: async (texts) => texts.map(() => 0.5), - scoreRelevanceBatch: async (texts) => texts.map(() => 0.5), scoreSimilarityBatch: async (_ref, texts) => texts.map(() => 0), shouldProceed: (score) => score >= 0.25, }; - expect(scorer.scoreRelevanceBatch).toBeDefined(); + expect(scorer.scoreEntailmentBatch).toBeDefined(); + expect('scoreRelevanceBatch' in scorer).toBe(false); }); }); diff --git a/packages/agents/test/spine-multimodal.test.ts b/packages/agents/test/spine-multimodal.test.ts index e5bea146..04168fbc 100644 --- a/packages/agents/test/spine-multimodal.test.ts +++ b/packages/agents/test/spine-multimodal.test.ts @@ -25,7 +25,7 @@ import { CapturingTraceWriter } from './helpers/capturing-trace'; const SYSTEM = 'You are a research assistant.'; const img = (n: number): Uint8Array[] => - Array.from({ length: n }, (_, i) => new Uint8Array([i, i + 1, i + 2])); + Array.from({ length: n }, (_, i) => new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, i])); const markerCount = (s: string): number => (s.match(/<__media__>/g) ?? []).length; diff --git a/packages/agents/test/tool-media.test.ts b/packages/agents/test/tool-media.test.ts index 1b70be11..7c8e4482 100644 --- a/packages/agents/test/tool-media.test.ts +++ b/packages/agents/test/tool-media.test.ts @@ -7,12 +7,13 @@ * and the tool's own object left alone. */ import { describe, it, expect } from 'vitest'; -import { takeToolMedia, TOOL_MEDIA_KEY } from '../src/Tool'; +import { takeToolMedia, TOOL_ATTACHMENTS_KEY } from '../src/Tool'; +import { MANIFEST_TYPE } from '@lloyal-labs/media'; import { PNG_BYTES } from './helpers/media'; describe('takeToolMedia', () => { it('splits the images out from what the model is told', () => { - const { media, result } = takeToolMedia({ page: 'p1', [TOOL_MEDIA_KEY]: [PNG_BYTES] }); + const { media, result } = takeToolMedia({ page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES] }); expect(media).toEqual([PNG_BYTES]); expect(result).toEqual({ page: 'p1' }); @@ -22,20 +23,32 @@ describe('takeToolMedia', () => { // The bytes must reach neither the model's JSON nor the trace. Deleting // them in place made that a property of call order; this makes it a // property of the function. - const returned = { page: 'p1', [TOOL_MEDIA_KEY]: [PNG_BYTES] }; + const returned = { page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES] }; takeToolMedia(returned); - expect(returned[TOOL_MEDIA_KEY]).toEqual([PNG_BYTES]); + expect(returned[TOOL_ATTACHMENTS_KEY]).toEqual([PNG_BYTES]); }); it('drops entries that are not bytes, so markers and bitmaps stay in step', () => { const { media } = takeToolMedia({ - [TOOL_MEDIA_KEY]: [PNG_BYTES, 'not-an-image', null, PNG_BYTES], + [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES, 'not-an-image', null, PNG_BYTES], }); expect(media).toHaveLength(2); }); + it('keeps a descriptor the store would recognise, in order with bytes, and drops one it would not', () => { + // A tool that reads the content store returns ROOTS, not bytes: no + // ingress, no normalizer permit, the ingest-time digest. Anything that is + // neither bytes nor a root descriptor is not media and never was. + const root = { digest: 'sha256:' + 'a'.repeat(64), mediaType: MANIFEST_TYPE, size: 9 }; + const { media, result } = takeToolMedia({ + page: 'p1', [TOOL_ATTACHMENTS_KEY]: [PNG_BYTES, root, { digest: 'nope' }, 'x'], + }); + expect(media).toEqual([PNG_BYTES, root]); + expect(result).toEqual({ page: 'p1' }); + }); + it('returns a text-only result as-is, copying nothing', () => { const returned = { page: 'p1' }; const { media, result } = takeToolMedia(returned); @@ -51,15 +64,15 @@ describe('takeToolMedia', () => { }); it('strips a MALFORMED channel rather than serializing it', () => { - // `_images: Uint8Array` (not an array of them) used to return the + // `_attachments: Uint8Array` (not an array of them) used to return the // original object — JSON-encoding every byte index onto the token rail, // the exact failure this helper exists to prevent. The reserved key // never survives; an invalid value is zero media entries. for (const bad of [PNG_BYTES, 'nope', 42, { 0: 1 }]) { - const { media, result } = takeToolMedia({ page: 'p1', [TOOL_MEDIA_KEY]: bad }); + const { media, result } = takeToolMedia({ page: 'p1', [TOOL_ATTACHMENTS_KEY]: bad }); expect(media).toEqual([]); expect(result).toEqual({ page: 'p1' }); - expect(Object.keys(result as object)).not.toContain(TOOL_MEDIA_KEY); + expect(Object.keys(result as object)).not.toContain(TOOL_ATTACHMENTS_KEY); } }); }); diff --git a/packages/media/README.md b/packages/media/README.md index 71b1063d..3d70ad03 100644 --- a/packages/media/README.md +++ b/packages/media/README.md @@ -17,23 +17,26 @@ writes nothing new. ```bash npm i @lloyal-labs/media -npm i sharp # only in a process that accepts image uploads — see below +npm i sharp # only in a process that admits image uploads — see below +npm i @embedpdf/pdfium # only in a process that admits PDF uploads ``` ```ts import { materialize } from '@lloyal-labs/media'; -import { FileAttachmentStore, createImageIngress } from '@lloyal-labs/media/node'; +import { FileAttachmentStore, createContentIngress } from '@lloyal-labs/media/node'; const store = new FileAttachmentStore('media'); // a valid OCI Image Layout -const ingress = createImageIngress(store); // normalize → address → commit +const ingress = createContentIngress(store); // sniff → admit → address → commit const attachment = await ingress.ingest(bytes); // one root descriptor -const { bitmaps } = materialize(store, [attachment]); // the exact admitted pixels +const { bitmaps } = materialize(store, [attachment]); // the exact admitted pixels — none for a document ``` -`ingest` admits bytes — converting, downscaling and stripping as the admission -policy below requires — commits them, and returns a root descriptor small -enough to ride any wire. `materialize` is the inverse: given roots, it returns +`ingest` admits bytes and returns a root descriptor small enough to ride any +wire. The bytes decide the door: an image goes through the normalizer +(`createImageIngress` — converting, downscaling and stripping as the admission +policy below requires), a PDF through the document ingress +(`createDocumentIngress`, below); each is also exported on its own. `materialize` is the inverse: given roots, it returns the exact bytes to hand the projector (the model's vision encoder). Replay is `materialize` called later. @@ -46,13 +49,13 @@ trace and replay. This package is the format and the gate. | entry | holds | needs | |---|---|---| | `@lloyal-labs/media` | the OCI shapes, the store and ingress contracts, `materialize` | nothing — browser-safe, zero dependencies | -| `@lloyal-labs/media/node` | `FileAttachmentStore` (the layout on disk), `createImageIngress` (sharp) | `node:fs`; `sharp` as an optional peer | +| `@lloyal-labs/media/node` | `FileAttachmentStore` (the layout on disk), `createContentIngress` — `createImageIngress` (sharp) and `createDocumentIngress` (PDFium) | `node:fs`; `sharp` and `@embedpdf/pdfium` as optional peers | -The root entry never reaches `node:` or `sharp` — enforced by +The root entry never reaches `node:`, `sharp` or the codec — enforced by `npm run verify:packed`, which walks the packed artifact's require graph on -every push. `sharp` is required at call time, not module load, so a process -that never accepts an image pays nothing, and one that does gets an error -naming the install. +every push. Both are required at call time, not module load, so a process +that never accepts an image or a document pays nothing, and one that does +gets an error naming the install. Where things live: the **format** — how bytes are addressed and laid out — is this package. The **policy** — where a project keeps its store — is @@ -133,12 +136,72 @@ and nothing above it. ``` The `config` slot is empty for an image, which has nothing to say beyond its -layers. Timed media will use it: a timeline — timestamps, track descriptors, -sampling policy — is typed structured data, and +layers. A document fills it (next section), and timed media will: a timeline — +timestamps, track descriptors, sampling policy — is typed structured data, and [annotations are `map<string,string>`](https://github.com/opencontainers/image-spec/blob/main/annotations.md), -so the config blob is where it belongs. `putAttachment({ config })` already -accepts one, and a reader branches on `config.mediaType`, so introducing a -typed config later leaves every existing manifest valid. +so the config blob is where such things belong. A reader branches on +`config.mediaType`, so a manifest with the empty config stays valid beside one +with a sidecar. + +## A document is the same graph + +A PDF admitted through `createDocumentIngress` becomes one manifest whose +representation is what the model reads, whose source is the file as supplied, +and whose config is a **sidecar** of facts about the document: + +```jsonc +{ + "artifactType": "application/vnd.lloyal.attachment.v1", + "config": { "mediaType": "application/vnd.lloyal.document.v1+json", "digest": "sha256:…", "size": 2210 }, + "layers": [ + { "mediaType": "text/markdown", "digest": "sha256:…", "size": 18342, + "annotations": { "ai.lloyal.role": "representation" } }, // what the model reads + { "mediaType": "application/pdf", "digest": "sha256:…", "size": 812044, + "annotations": { "ai.lloyal.role": "source" } } // what the user supplied + ] +} +``` + +The sidecar (`DocumentMeta`, guarded by `asDocumentMeta` on the root entry) +records the title; the sections with their heading path, line span and pages; +a page map with per-page counts — characters, image objects, path objects, +tagged tables and figures; the figures with their captions; the tagged tables; +and the parameters the derivation ran under. Facts, not verdicts: whether a +page is worth spending a model's attention on is a rule the reader applies +over the counts. Lines are 1-based over the markdown's `\n` split, and +`pagesOf(meta, startLine, endLine)` is the one function that turns a line +span into pages. + +Every page within the render bound, and every figure crop, is its own +single-image manifest — a **page root** — named from the sidecar by +descriptor. A page root is an ordinary image attachment carrying `pdf.v1` +derive annotations, so the normalizer never runs on it and `materialize`, +replay and any OCI tool read it exactly as they read a photo: + +| key | value | +|---|---| +| `ai.lloyal.derive.profile` | `pdf.v1` | +| `ai.lloyal.derive.page` | the 1-based page it renders | +| `ai.lloyal.derive.dpi`, `.width`, `.height`, `.format` | how it was rendered | +| `ai.lloyal.derive.source` | the digest of the PDF blob it was rendered from | +| `ai.lloyal.derive.bbox` | a figure crop's box on the page, in PDF user space, clipped to the page | + +`materialize` projects only representations in a format the projector +decodes. A document root therefore yields no bitmaps — its text is for +retrieval, not the vision encoder — while a page root yields one. That single +rule is what lets a document ride the same wire, trace and replay as an image +without anything upstream learning a new kind. + +Bounds are declared constants on the node entry: `MAX_DOCUMENT_BYTES` (32 MiB, +refused before the codec sees a byte), `MAX_RENDERED_PAGES` (200 — page one +first, then graphics-bearing pages, then the rest, so the archive under the +bound is a deterministic function of the input; a page past it simply has no +render descriptor), `MAX_TEXT_PAGES` (400), `MAX_FIGURES` (16), and +`DOCUMENT_TIMEOUT_MS` (120 s — a failure bound that publishes nothing, never +a coverage knob). The same bytes under the same constants yield the same root +on any machine. A document holds one permit of the shared admission gate for +its whole duration, and one codec instance lives exactly as long as its +document. ### Annotations we define @@ -159,7 +222,9 @@ omitting it is a legitimate choice for a large original. ## Admission -`ingest` (and `normalizeImage` underneath it) applies one policy. +`createContentIngress` sniffs the bytes: an image goes to the policy below, a +PDF to the document ingress above, anything else is refused at the door. +For images, `ingest` (and `normalizeImage` underneath it) applies one policy. Byte-identical pass-through happens only when ALL of these hold: | | | @@ -219,6 +284,7 @@ pointing at content that is not there. | **Video derivation** | The manifest already has the slot: source + N frame representations. What is missing is a decoder, and that decision carries real licensing and codec-patent weight. | | **Live capture** | The *locator* is not addressable; every bounded frame that reaches the model still is. Attachments are per-prefill rather than per-run, so a live run is many prefills — no growing manifest. | | **Reachability GC** | Nothing here deletes. Deletion needs refcounting across runs that may share a digest. | +| **Rendering past the bound** | A page beyond `MAX_RENDERED_PAGES` has no render descriptor. Rendering on demand needs the source and a codec at read time; today a document is archived at admission only. | ## Known limitations diff --git a/packages/media/package.json b/packages/media/package.json index 9185d105..cd20c64c 100644 --- a/packages/media/package.json +++ b/packages/media/package.json @@ -1,7 +1,7 @@ { "name": "@lloyal-labs/media", "version": "0.2.0-alpha.3", - "description": "Content addressing for a harness — an OCI layout, and the image normalizer that feeds it", + "description": "Content addressing for a harness \u2014 an OCI layout, and the image and document ingresses that feed it", "main": "dist/index.js", "types": "dist/index.d.ts", "publishConfig": { @@ -22,7 +22,10 @@ "multimodal", "vision", "image", - "sharp" + "sharp", + "pdf", + "pdfium", + "document" ], "license": "SEE LICENSE IN LICENSE", "type": "commonjs", @@ -46,14 +49,20 @@ "LICENSE-FAQ.md" ], "devDependencies": { + "@embedpdf/pdfium": "^2.15.0", + "@types/emscripten": "^1.41.6", "sharp": "^0.35.4" }, "peerDependencies": { - "sharp": "^0.35.4" + "sharp": "^0.35.4", + "@embedpdf/pdfium": "^2.15.0" }, "peerDependenciesMeta": { "sharp": { "optional": true + }, + "@embedpdf/pdfium": { + "optional": true } } } diff --git a/packages/media/src/content-ingress.ts b/packages/media/src/content-ingress.ts new file mode 100644 index 00000000..027dfbf0 --- /dev/null +++ b/packages/media/src/content-ingress.ts @@ -0,0 +1,45 @@ +/** + * @file The one loading door: bytes in, an attachment out, the bytes deciding + * what they are. + * + * `ContentIngress` stays one port. A host installs this on its content routes + * and on the agents `Ingress` context, and an upload or a tool result that + * carries bytes goes through the same dispatch: an image to the normalizer, a + * PDF to the document ingress. A caller never declares a type; the signature + * does. + */ +import type { AttachmentStore } from './store'; +import type { ContentIngress } from './ingress'; +import { sniffMediaType } from './media-type'; +import { createImageIngress } from './image'; +import type { NormalizeOpts } from './image'; +import { createDocumentIngress } from './pdf'; +import type { DocumentOpts } from './pdf'; + +/** + * How long a host should allow an upload that may be a document, end to end: + * transfer plus the document's own time bound. The route's default is sized + * for images; a host that installs this ingress passes this beside + * `MAX_DOCUMENT_BYTES`, so every host that mounts the plane admits the same. + * + * @category Media + */ +export const DOCUMENT_UPLOAD_TIMEOUT_MS = 180_000; + +/** + * @category Media + */ +export function createContentIngress( + store: AttachmentStore, + opts: { image?: NormalizeOpts; document?: DocumentOpts } = {}, +): ContentIngress { + const image = createImageIngress(store, opts.image); + const document = createDocumentIngress(store, opts.document); + return { + ingest(bytes: Uint8Array, signal?: AbortSignal) { + return sniffMediaType(bytes) === 'application/pdf' + ? document.ingest(bytes, signal) + : image.ingest(bytes, signal); + }, + }; +} diff --git a/packages/media/src/document.ts b/packages/media/src/document.ts new file mode 100644 index 00000000..c27868c5 --- /dev/null +++ b/packages/media/src/document.ts @@ -0,0 +1,148 @@ +/** + * @file The document sidecar — facts about a document, kept in its manifest's + * config slot. + * + * Pure data and a guard, browser-safe like `attachment.ts`. A document + * attachment has ONE representation, `text/markdown`; its page renders and + * figure crops are ordinary single-image attachments named here by root + * descriptor. This file records facts only — counts, spans, roots, the + * derivation parameters — and no policy: which pages are worth projecting is + * a rule whoever consumes the sidecar applies over the counts. + */ +import type { Descriptor } from './attachment'; + +/** The config media type a document manifest carries; a reader branches on it. */ +export const DOCUMENT_CONFIG_TYPE = 'application/vnd.lloyal.document.v1+json' as const; + +/** + * Where a section came from: the PDF's structure tree, its bookmarks, or the + * font-size heuristics applied when neither covered the page. + */ +export type SectionOrigin = 'struct' | 'bookmark' | 'heuristic'; + +/** + * The sidecar. Lines are 1-based over the markdown representation's `\n` + * split — the `Chunk.startLine` convention — and the page map is the + * authority for line → page; the markdown carries no page anchors. Descriptors + * are complete so a consumer can hand them straight to `asAttachment`. + * + * @category Media + */ +export interface DocumentMeta { + title: string; + pageCount: number; + sections: { + heading: string; + /** Hierarchical path, `A > B > C`. */ + path: string; + origin: SectionOrigin; + startLine: number; + endLine: number; + pageStart: number; + pageEnd: number; + }[]; + pages: { + /** 1-based. */ + page: number; + startLine: number; + endLine: number; + /** Extracted characters; 0 for a scanned page. */ + chars: number; + imageObjects: number; + pathObjects: number; + taggedTables: number; + taggedFigures: number; + /** Root of the page's render, present for every page within the render bound. */ + render?: Descriptor; + }[]; + figures: { + page: number; + index: number; + /** PDF user-space points, `[x0, y0, x1, y1]`. */ + bbox: [number, number, number, number]; + caption?: string; + root: Descriptor; + }[]; + /** Tagged tables emitted into the markdown as pipe tables. */ + tables: { page: number; startLine: number; endLine: number }[]; + /** What derived this document — so a replay under other settings sees the difference. */ + derive: { + profile: 'pdf.v1'; + pdfium: string; + dpi: number; + maxSide: number; + maxPixels: number; + format: 'image/png'; + renderedPages: number; + maxFigures: number; + maxTextPages: number; + tagged: boolean; + structCoverage: number; + /** A data cap bit (pages, figures, text pages). Never a timeout: a timeout publishes nothing. */ + truncated: boolean; + }; +} + +const isInt = (v: unknown): v is number => typeof v === 'number' && Number.isSafeInteger(v); +const isNum = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v); +const isStr = (v: unknown): v is string => typeof v === 'string'; +const isObj = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null; +const isDescriptor = (v: unknown): v is Descriptor => + isObj(v) && isStr(v.mediaType) && isStr(v.digest) && isInt(v.size); + +/** + * Narrow parsed JSON to a {@link DocumentMeta}, or refuse it. Never throws: + * a sidecar arrives as bytes from a store, and a malformed one is a normal + * answer, not a fault. + * + * @category Media + */ +export function asDocumentMeta(json: unknown): DocumentMeta | null { + if (!isObj(json)) return null; + const { title, pageCount, sections, pages, figures, tables, derive } = json; + if (!isStr(title) || !isInt(pageCount)) return null; + if (!Array.isArray(sections) || !Array.isArray(pages) || !Array.isArray(figures) || !Array.isArray(tables)) return null; + const sectionOk = (s: unknown): boolean => + isObj(s) && isStr(s.heading) && isStr(s.path) && + (s.origin === 'struct' || s.origin === 'bookmark' || s.origin === 'heuristic') && + isInt(s.startLine) && isInt(s.endLine) && isInt(s.pageStart) && isInt(s.pageEnd); + const pageOk = (p: unknown): boolean => + isObj(p) && isInt(p.page) && isInt(p.startLine) && isInt(p.endLine) && isInt(p.chars) && + isInt(p.imageObjects) && isInt(p.pathObjects) && isInt(p.taggedTables) && isInt(p.taggedFigures) && + (p.render === undefined || isDescriptor(p.render)); + const figureOk = (f: unknown): boolean => + isObj(f) && isInt(f.page) && isInt(f.index) && Array.isArray(f.bbox) && f.bbox.length === 4 && + f.bbox.every(isNum) && (f.caption === undefined || isStr(f.caption)) && isDescriptor(f.root); + const tableOk = (t: unknown): boolean => isObj(t) && isInt(t.page) && isInt(t.startLine) && isInt(t.endLine); + const deriveOk = (d: unknown): boolean => + isObj(d) && d.profile === 'pdf.v1' && isStr(d.pdfium) && isNum(d.dpi) && isInt(d.maxSide) && isInt(d.maxPixels) && + d.format === 'image/png' && isInt(d.renderedPages) && isInt(d.maxFigures) && isInt(d.maxTextPages) && + typeof d.tagged === 'boolean' && isNum(d.structCoverage) && typeof d.truncated === 'boolean'; + if (!sections.every(sectionOk) || !pages.every(pageOk) || !figures.every(figureOk) || !tables.every(tableOk) || !deriveOk(derive)) return null; + return json as unknown as DocumentMeta; +} + +/** + * The pages a line span touches, by the sidecar's page map. A span past the + * map — lines the extractor emitted after the last page's text — belongs to + * the last page. + * + * @category Media + */ +export function pagesOf(meta: DocumentMeta, startLine: number, endLine: number): { pageStart: number; pageEnd: number } { + return pagesOfSpan(meta.pages, startLine, endLine); +} + +/** The same rule over a bare page map — what the layout uses before a sidecar exists. */ +export function pagesOfSpan( + pages: readonly { page: number; startLine: number; endLine: number }[], + startLine: number, + endLine: number, +): { pageStart: number; pageEnd: number } { + const touched = pages.filter((p) => p.endLine >= startLine && p.startLine <= endLine); + if (touched.length > 0) { + return { pageStart: touched[0].page, pageEnd: touched[touched.length - 1].page }; + } + const last = pages[pages.length - 1]?.page ?? 1; + return { pageStart: last, pageEnd: last }; +} diff --git a/packages/media/src/gate.ts b/packages/media/src/gate.ts new file mode 100644 index 00000000..4231b374 --- /dev/null +++ b/packages/media/src/gate.ts @@ -0,0 +1,151 @@ +/** + * @file The permit gate — the process-wide bound on decoded content in memory. + * + * Each concurrent normalization holds a fully decoded bitmap, and a document + * ingest holds a page bitmap and a codec heap, so N simultaneous admissions + * cost N such allocations however small the encoded bytes were. A served host + * takes uploads from anyone who can reach it. The bound therefore belongs to + * the PROCESS, not the request, and there is ONE gate: images and documents + * take permits from the same pool, so together they never exceed it. + * + * Pure Node, no `sharp`: safe for any module behind `./node` to import. + */ + +/** + * How many admissions may hold decoded content at once, process-wide. + * + * @category Media + */ +export const MAX_CONCURRENT_NORMALIZATIONS = 4; + +/** + * How long a caller may WAIT for a permit before the host refuses it. + * + * A queue with no deadline turns a burst into a pile of held request bodies + * that never drains; a caller that has waited this long is told so and can + * retry, and the bytes it held are released. + * + * @category Media + */ +export const PERMIT_WAIT_TIMEOUT_MS = 60_000; + +/** + * How many callers may wait for a permit. Past this depth the answer is an + * immediate busy, so memory is bounded by (permits + depth) admissions and + * nothing more. + * + * @category Media + */ +export const MAX_QUEUED_NORMALIZATIONS = MAX_CONCURRENT_NORMALIZATIONS * 4; + +/** The shape a caller can recognise without knowing this module. Matches what + * `fetch` throws on abort, because that is what callers already handle. */ +export function abortError(label: string): Error { + const e = new Error(`${label}: aborted`); + e.name = 'AbortError'; + return e; +} + +/** What {@link createGate} returns: one method, one permit per successful call. */ +export interface Gate { + /** + * Take one permit; resolve to its release. + * + * `signal` is how a caller that has GIVEN UP stops occupying the queue — + * an abandoned request holding a slot is a slot a live request cannot have. + * Callers inside an Effection scope get this for free: the scope's own + * signal aborts on halt. The signal covers the WAIT and the moment before + * work starts; work already running is bounded by its own timeout. + * + * Release is idempotent: a double release would MANUFACTURE a permit, the + * same bug as leaking one with the sign flipped. + * + * @throws `AbortError` when the signal is or becomes aborted before the + * permit is used; an error with `code: 'EBUSY'` when the queue is + * full; an error naming the wait when it outlasts `waitMs`. + */ + acquire(signal?: AbortSignal): Promise<() => void>; +} + +/** + * Build a gate. The shared, process-wide instance is {@link gate}; a separate + * one exists only for tests, which need small numbers. + * + * @category Media + */ +export function createGate(opts: { label: string; permits: number; maxQueued: number; waitMs: number }): Gate { + const { label, maxQueued, waitMs } = opts; + let permits = opts.permits; + const waiting: { grant: () => void; refuse: (e: Error) => void }[] = []; + + const busy = (): Error => { + const e = new Error(`${label}: ${opts.permits} in flight and ${maxQueued} queued — busy, retry later.`); + (e as Error & { code: string }).code = 'EBUSY'; + return e; + }; + const hand = (): void => { + const next = waiting.shift(); + if (next) next.grant(); else permits++; + }; + + return { + async acquire(signal?: AbortSignal): Promise<() => void> { + if (signal?.aborted) throw abortError(label); + + if (permits > 0) { + permits--; + } else { + if (waiting.length >= maxQueued) throw busy(); + await new Promise<void>((resolve, reject) => { + const leave = (): void => { + const at = waiting.indexOf(entry); + if (at >= 0) waiting.splice(at, 1); + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = (): void => { leave(); reject(abortError(label)); }; + const entry = { + grant: () => { leave(); resolve(); }, + refuse: (e: Error) => { leave(); reject(e); }, + }; + const timer = setTimeout(() => entry.refuse(new Error( + `${label}: waited ${waitMs}ms for one of ${opts.permits} slots and none came free.`, + )), waitMs); + // Do not hold the process open on account of a queued caller. + timer.unref?.(); + signal?.addEventListener('abort', onAbort, { once: true }); + waiting.push(entry); + }); + } + + // Granted, but the caller may have given up while it waited. Handing the + // permit straight to the next in line beats spending it on work nobody + // is waiting for. + if (signal?.aborted) { + hand(); + throw abortError(label); + } + + let released = false; + return () => { + if (released) return; + released = true; + hand(); + }; + }, + }; +} + +/** + * The one process-wide gate. Module-level on purpose: the resource it protects + * is host memory, shared by every session and every request, so a per-call or + * per-store limiter would not bound anything. + * + * @category Media + */ +export const gate: Gate = createGate({ + label: 'normalizeImage', + permits: MAX_CONCURRENT_NORMALIZATIONS, + maxQueued: MAX_QUEUED_NORMALIZATIONS, + waitMs: PERMIT_WAIT_TIMEOUT_MS, +}); diff --git a/packages/media/src/image.ts b/packages/media/src/image.ts index b340b7d3..0ceb3ad8 100644 --- a/packages/media/src/image.ts +++ b/packages/media/src/image.ts @@ -17,6 +17,7 @@ import { DERIVE_PREFIX } from './attachment'; import type { AttachmentStore } from './store'; import type { ContentIngress } from './ingress'; import { PROJECTOR_FORMATS, sniffMediaType, UNKNOWN_MEDIA_TYPE } from './media-type'; +import { gate, abortError } from './gate'; /** * The default pixel ceiling — mtmd's own (`image_max_pixels`, 2048²). @@ -54,46 +55,6 @@ export const MAX_INPUT_PIXELS = 100_000_000; */ export const NORMALIZE_TIMEOUT_SECONDS = 20; -/** - * How many images may be normalized at once, process-wide. - * - * The cap belongs to the PROCESS, not the request: each concurrent - * normalization holds a fully decoded bitmap in memory, so N simultaneous - * uploads cost N bitmaps regardless of how small the encoded bytes were. A - * served host takes uploads from anyone who can reach it. - * - * @category Media - */ -export const MAX_CONCURRENT_NORMALIZATIONS = 4; - -/** - * How long an image may WAIT for a permit before the host refuses it. - * - * An unbounded queue is not a bound: it is a memory leak with a politer name, - * and it turns any permit accounting bug into a process that hangs forever - * instead of failing. A busy host should say it is busy. - * - * Still earns its keep now that {@link NormalizeOpts.signal} exists: a caller - * inside a scope gives up the moment that scope halts, but the HTTP ingress - * route runs in a plain request handler and passes no signal, so this is the - * only bound it has. - * - * Sized well above the worst legitimate wait — a full queue of - * {@link NORMALIZE_TIMEOUT_SECONDS} decodes — so reaching it means genuine - * overload or a bug, never ordinary contention. - * - * @category Media - */ -export const PERMIT_WAIT_TIMEOUT_MS = 60_000; - -/** - * How many callers may WAIT for a permit. Permits bound the decoded bitmaps; - * this bounds the encoded ones parked behind them. Past it the answer is an - * immediate busy, so a burst costs at most (permits + queue) images of memory - * instead of one image per arrival for {@link PERMIT_WAIT_TIMEOUT_MS}. - */ -export const MAX_QUEUED_NORMALIZATIONS = MAX_CONCURRENT_NORMALIZATIONS * 4; - /** * @category Media */ @@ -180,100 +141,6 @@ export type NormalizeImage = ( opts?: NormalizeOpts, ) => Promise<NormalizedImage>; -/** - * A process-wide gate of {@link MAX_CONCURRENT_NORMALIZATIONS} permits. - * - * Module-level on purpose: the resource being protected is host memory, which - * is shared by every session and every request, so a per-call or per-store - * limiter would not bound anything. - */ -const gate = { - permits: MAX_CONCURRENT_NORMALIZATIONS, - waiting: [] as { grant: () => void; refuse: (e: Error) => void }[], -}; - -/** The shape a caller can recognise without knowing this module. Matches what - * `fetch` throws on abort, because that is what callers already handle. */ -const aborted = (): Error => { - const e = new Error('normalizeImage: aborted'); - e.name = 'AbortError'; - return e; -}; - -/** Thrown when the queue is full. `code` is Node's own errno name for the - * condition, so an HTTP layer can answer 503 without importing this module. */ -function busy(): Error { - const e = new Error( - `normalizeImage: ${MAX_CONCURRENT_NORMALIZATIONS} normalizations in flight and ` + - `${MAX_QUEUED_NORMALIZATIONS} queued — busy, retry later.`, - ); - (e as Error & { code: string }).code = 'EBUSY'; - return e; -} - -/** - * Take one of {@link MAX_CONCURRENT_NORMALIZATIONS} permits. - * - * `signal` is how a caller that has GIVEN UP stops occupying the queue. That - * matters more than it sounds: an abandoned request holding a slot is a slot a - * live request cannot have, so under load the queue fills with work whose - * results nobody will read. Callers inside an Effection scope get this for - * free — the scope's own signal aborts on halt. - * - * An in-flight sharp decode cannot be interrupted (sharp exposes no abort; - * {@link NORMALIZE_TIMEOUT_SECONDS} is the only bound on it), so the signal - * covers the WAIT and the moment before work starts, which is where a - * cancelled caller's cost actually accumulates. - */ -async function acquire(signal?: AbortSignal): Promise<() => void> { - if (signal?.aborted) throw aborted(); - - if (gate.permits > 0) { - gate.permits--; - } else { - if (gate.waiting.length >= MAX_QUEUED_NORMALIZATIONS) throw busy(); - await new Promise<void>((resolve, reject) => { - const leave = () => { - const at = gate.waiting.indexOf(entry); - if (at >= 0) gate.waiting.splice(at, 1); - clearTimeout(timer); - signal?.removeEventListener('abort', onAbort); - }; - const onAbort = () => { leave(); reject(aborted()); }; - const entry = { - grant: () => { leave(); resolve(); }, - refuse: (e: Error) => { leave(); reject(e); }, - }; - const timer = setTimeout(() => entry.refuse(new Error( - `normalizeImage: waited ${PERMIT_WAIT_TIMEOUT_MS}ms for one of ` + - `${MAX_CONCURRENT_NORMALIZATIONS} normalization slots and none came free.`, - )), PERMIT_WAIT_TIMEOUT_MS); - // Do not hold the process open on account of a queued upload. - timer.unref?.(); - signal?.addEventListener('abort', onAbort, { once: true }); - gate.waiting.push(entry); - }); - } - - // Granted, but the caller may have given up while it waited. Releasing here - // hands the permit straight to the next in line instead of spending it on - // work nobody is waiting for. - if (signal?.aborted) { - const next = gate.waiting.shift(); - if (next) next.grant(); else gate.permits++; - throw aborted(); - } - - let released = false; - return () => { - // Idempotent: a double release would MANUFACTURE a permit, which is the - // same bug as leaking one with the sign flipped. - if (released) return; - released = true; - const next = gate.waiting.shift(); - if (next) next.grant(); else gate.permits++; - }; -} /** * Does this ICC profile describe sRGB? @@ -396,7 +263,7 @@ export const normalizeImage: NormalizeImage = async (bytes, opts = {}) => { // upload is the failure mode that matters — a handful of bad files would // exhaust the gate permanently and wedge the host, and bad files are exactly // what arrives in volume. - const release = await acquire(opts.signal); + const release = await gate.acquire(opts.signal); try { const input = sharp(bytes, { animated: false, limitInputPixels: MAX_INPUT_PIXELS }) .timeout({ seconds: NORMALIZE_TIMEOUT_SECONDS }); @@ -579,7 +446,7 @@ export function createImageIngress( // have fired while it ran. The caller gave up; the route has answered // 408. Nothing may be committed on its behalf now — the decode was // discarded work, and it stays that way. - if (signal?.aborted) throw aborted(); + if (signal?.aborted) throw abortError('normalizeImage'); // A derivation record describes a derivation that HAPPENED. Writing it // on a pass-through would annotate bytes nobody re-encoded with a diff --git a/packages/media/src/index.ts b/packages/media/src/index.ts index ff1af849..0f407360 100644 --- a/packages/media/src/index.ts +++ b/packages/media/src/index.ts @@ -28,3 +28,8 @@ export { materialize, NoContentIngress } from './ingress'; export type { ContentIngress, PreparedContent } from './ingress'; export { PROJECTOR_FORMATS, sniffMediaType, UNKNOWN_MEDIA_TYPE } from './media-type'; + +// The document sidecar — facts about a document, kept in its manifest's config +// slot. Pure data and a guard; a reader branches on DOCUMENT_CONFIG_TYPE. +export { DOCUMENT_CONFIG_TYPE, asDocumentMeta, pagesOf } from './document'; +export type { DocumentMeta, SectionOrigin } from './document'; diff --git a/packages/media/src/ingress.ts b/packages/media/src/ingress.ts index 48171511..b2871239 100644 --- a/packages/media/src/ingress.ts +++ b/packages/media/src/ingress.ts @@ -6,6 +6,7 @@ * batch is an orchestration concern and lives with the orchestrator. */ import { representationsOf } from './attachment'; +import { PROJECTOR_FORMATS } from './media-type'; import type { Attachment } from './attachment'; import type { AttachmentStore } from './store'; @@ -23,9 +24,11 @@ import type { AttachmentStore } from './store'; export interface PreparedContent { /** Roots, in ingest order — what the trace and the fold carry. */ attachments: readonly Attachment[]; - /** Every root's representations, flattened in order — the EXACT bytes to - * hand a builder and then the projector. One image contributes one; a video - * contributes its sampled frames. */ + /** Every root's PROJECTOR-DECODABLE representations, flattened in order — + * the EXACT bytes to hand a builder and then the projector. One image + * contributes one; a video contributes its sampled frames; a document, + * whose one representation is text, contributes none: its text reaches the + * model through retrieval and the spine outline, never the decoder. */ bitmaps: readonly Uint8Array[]; } @@ -59,6 +62,10 @@ export function materialize( ); } for (const rep of representationsOf(manifest)) { + // Only what the projector decodes is a bitmap. A text representation is + // real content — retrieval reads it — but handing it to the image decoder + // would poison the branch it was prefilled on. + if (!PROJECTOR_FORMATS.includes(rep.mediaType)) continue; // Replay rebuilds KV from these bytes under this digest. The store // refuses bytes that drifted from their name, so null here means absent // or drifted — either must refuse, not decode as something else. diff --git a/packages/media/src/media-type.ts b/packages/media/src/media-type.ts index 3a0bfc09..695897b7 100644 --- a/packages/media/src/media-type.ts +++ b/packages/media/src/media-type.ts @@ -1,13 +1,14 @@ /** - * @file Identify an image format from its leading bytes. + * @file Identify a content format from its leading bytes. * - * Its own file because it depends on nothing else in the content surface, and - * because `spine.ts` and `agent-pool.ts` already import it on its own — the - * callers treated it as a separate module before it was one. + * Its own file because it depends on nothing else in the content surface. The + * image normalizer consults it on a decode error, and the content ingress + * dispatches on it: the bytes say what they are, a caller never does. */ /** - * The image formats a vision projector decodes, by their leading bytes. + * Formats this package identifies by their leading bytes: the four image + * formats the projector decodes, and PDF. * * A table rather than a chain of ifs. Anything unmatched is still stored — * validating pixels belongs to the normalizer that runs before ingress, and to @@ -18,6 +19,10 @@ const SIGNATURES: ReadonlyArray<{ mediaType: string; magic: readonly number[] }> { mediaType: 'image/png', magic: [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] }, { mediaType: 'image/gif', magic: [0x47, 0x49, 0x46, 0x38] }, { mediaType: 'image/bmp', magic: [0x42, 0x4d] }, + // `%PDF-`. Not a projector format (see below): the content ingress routes it + // to the document ingress; the image normalizer refuses it as it refuses any + // format it cannot decode. + { mediaType: 'application/pdf', magic: [0x25, 0x50, 0x44, 0x46, 0x2d] }, ]; /** @@ -41,7 +46,7 @@ export function sniffMediaType(bytes: Uint8Array): string { * **Its own list, deliberately NOT derived from {@link SIGNATURES}.** These are * two different questions and they have different answers: * - * - `SIGNATURES` — "what can I identify from leading bytes?" Four formats. + * - `SIGNATURES` — "what can I identify from leading bytes?" Four image formats and PDF. * - `PROJECTOR_FORMATS` — "what will mtmd decode?" Nine. * * Ground truth for this list is the kernel, not an assumption: diff --git a/packages/media/src/node.ts b/packages/media/src/node.ts index 66d57c51..de7b7834 100644 --- a/packages/media/src/node.ts +++ b/packages/media/src/node.ts @@ -13,3 +13,11 @@ export { createImageIngress, DEFAULT_MAX_PIXELS, normalizeImage } from './image' export type { NormalizedImage, NormalizeImage, NormalizeOpts } from './image'; export { FileAttachmentStore } from './file-store'; + +// The document codec and the dispatching ingress. `@embedpdf/pdfium` is an +// OPTIONAL peer like sharp: required at call time, never at import. +export { createDocumentIngress, createCodec, openDocument, PdfError, MAX_DOCUMENT_BYTES, DOCUMENT_TIMEOUT_MS, MAX_RENDERED_PAGES, MAX_TEXT_PAGES, MAX_FIGURES, RENDER_PROFILE } from './pdf'; +export type { DocumentOpts } from './pdf'; +export { createContentIngress, DOCUMENT_UPLOAD_TIMEOUT_MS } from './content-ingress'; +export { gate, createGate, MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS, PERMIT_WAIT_TIMEOUT_MS } from './gate'; +export type { Gate } from './gate'; diff --git a/packages/media/src/pdf-layout.ts b/packages/media/src/pdf-layout.ts new file mode 100644 index 00000000..fba177d0 Binary files /dev/null and b/packages/media/src/pdf-layout.ts differ diff --git a/packages/media/src/pdf.ts b/packages/media/src/pdf.ts new file mode 100644 index 00000000..0a00c6de --- /dev/null +++ b/packages/media/src/pdf.ts @@ -0,0 +1,779 @@ +/** + * @file The PDF codec at its seam: PDFium as WebAssembly, one instance per + * document, everything bounded. + * + * Why WASM and not a native build: a hostile PDF that traps a wasm instance + * takes that instance down, not the process the resident model lives in; and + * one artifact serves every platform the runtime ships for. The codec is an + * OPTIONAL peer like `sharp`, required at call time, so a harness that never + * accepts a document never loads it. + * + * Ownership is the whole design here. The module is compiled once per process + * (that is the expensive step); each document gets its OWN instance from that + * compiled module, created when its ingest starts and discarded when it ends. + * Nothing is shared between concurrent ingests, so there is no replacement + * rule, no heap threshold and no owner to hand over: a trap kills only its own + * document, and the shared permit gate bounds how many instances exist at once. + * + * Isolation is cooperative on the main thread: every FPDF call is synchronous. + * A Worker over a built entry, with `terminate()` as the hard bound, is the + * named upgrade path — not runnable from `src/*.ts` on Node 24 today. + */ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import type { WrappedPdfiumModule } from '@embedpdf/pdfium'; +import type { Attachment, Descriptor } from './attachment'; +import { DERIVE_PREFIX } from './attachment'; +import type { AttachmentStore } from './store'; +import type { ContentIngress } from './ingress'; +import { DOCUMENT_CONFIG_TYPE } from './document'; +import type { DocumentMeta } from './document'; +import { gate, abortError } from './gate'; +import { DEFAULT_MAX_PIXELS, MAX_INPUT_PIXELS } from './image'; +import { layoutDocument } from './pdf-layout'; +import type { Bookmark, LayoutResult, PageChar, PageFacts, PageImage, PageStruct } from './pdf-layout'; + +/** The failure a caller can name: which document, and why PDFium refused it. */ +export class PdfError extends Error { + readonly name = 'PdfError'; +} + +/** PDFium's `FPDF_GetLastError` codes, in its words. */ +const LOAD_ERRORS: Record<number, string> = { + 1: 'unknown error', + 2: 'file cannot be opened', + 3: 'malformed: not a PDF or a corrupt one', + 4: 'password protected', + 5: 'unsupported security scheme', + 6: 'page not found or content error', +}; + +/** One compiled module per process — the expensive step, paid once. */ +let compiled: Promise<WebAssembly.Module> | null = null; + +function compileOnce(): Promise<WebAssembly.Module> { + compiled ??= (async () => { + // Required at call time, not imported at module load: the codec is an + // optional peer, and a harness that never accepts a document should not + // pay 4.6 MB of wasm. The message names the package because the failure is + // a missing install, not a bad document. + let wasmPath: string; + try { + wasmPath = require.resolve('@embedpdf/pdfium/pdfium.wasm'); + } catch { + throw new Error( + 'createCodec: `@embedpdf/pdfium` is not installed. Add it to the harness ' + + 'that accepts document uploads (npm i @embedpdf/pdfium).', + ); + } + return WebAssembly.compile(readFileSync(wasmPath)); + })(); + return compiled; +} + +/** + * A live codec: one wasm instance, owned by one document's ingest. + * + * @category Media + */ +export interface Codec { + /** The wrapped module: FPDF calls as methods, plus `pdfium` for memory. */ + readonly pdfium: WrappedPdfiumModule; + /** Reserve `size` bytes on the wasm heap; `free` them with {@link Codec.free}. */ + malloc(size: number): number; + free(ptr: number): void; + /** Drop the instance. Idempotent. Every handle from it is dead afterwards. */ + dispose(): void; +} + +/** + * Instantiate the compiled module for one document. + * + * @category Media + */ +export async function createCodec(): Promise<Codec> { + const module = await compileOnce(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { init } = require('@embedpdf/pdfium') as typeof import('@embedpdf/pdfium'); + const wrapped = await init({ + // The glue honours this Emscripten hook: instantiate our compiled module + // instead of fetching and compiling the wasm again. Returning `{}` tells the + // glue the instantiation is asynchronous and will arrive via the callback. + instantiateWasm(imports, onInstantiated) { + WebAssembly.instantiate(module, imports).then((instance) => onInstantiated(instance)); + return {}; + }, + }); + wrapped.PDFiumExt_Init(); + + let disposed = false; + return { + pdfium: wrapped, + malloc(size: number): number { + const ptr = wrapped.pdfium.wasmExports.malloc(size); + if (ptr === 0) throw new PdfError(`codec: could not reserve ${size} bytes on the wasm heap`); + return ptr; + }, + free(ptr: number): void { + wrapped.pdfium.wasmExports.free(ptr); + }, + dispose(): void { + if (disposed) return; + disposed = true; + // Nothing to call: an instance with no references is collected, heap and + // all. Per-document lifetime IS the memory bound. + }, + }; +} + +/** + * An open PDF: its handle in one codec instance, and the heap it occupies. + * + * @category Media + */ +export interface OpenDocument { + readonly handle: number; + readonly pageCount: number; + /** Release the document and the bytes copied onto the heap. Idempotent. */ + close(): void; +} + +/** + * Open a PDF from bytes in `codec`. The bytes are copied onto the wasm heap + * and stay there until {@link OpenDocument.close}: PDFium reads lazily. + * + * @throws {PdfError} naming PDFium's reason — password, malformed, unsupported. + * + * @category Media + */ +export function openDocument(codec: Codec, bytes: Uint8Array): OpenDocument { + const { pdfium } = codec; + const ptr = codec.malloc(bytes.byteLength); + pdfium.pdfium.HEAPU8.set(bytes, ptr); + const handle = pdfium.FPDF_LoadMemDocument(ptr, bytes.byteLength, ''); + if (handle === 0) { + codec.free(ptr); + const code = pdfium.FPDF_GetLastError(); + throw new PdfError(`openDocument: ${LOAD_ERRORS[code] ?? `PDFium error ${code}`}`); + } + let closed = false; + return { + handle, + pageCount: pdfium.FPDF_GetPageCount(handle), + close(): void { + if (closed) return; + closed = true; + pdfium.FPDF_CloseDocument(handle); + codec.free(ptr); + }, + }; +} + +// ── bounds ─────────────────────────────────────────────────────── + +/** Largest PDF admitted, refused before a byte reaches the wasm heap. */ +export const MAX_DOCUMENT_BYTES = 32 * 1024 * 1024; +/** Time one document may take, end to end. A failure bound, never a coverage knob: past it nothing is published. */ +export const DOCUMENT_TIMEOUT_MS = 120_000; +/** Pages rendered and archived; the rest are text only. Graphics-bearing pages are chosen first. */ +export const MAX_RENDERED_PAGES = 200; +/** Pages whose text is extracted; past it the markdown says so. */ +export const MAX_TEXT_PAGES = 400; +/** Figure crops committed per document. */ +export const MAX_FIGURES = 16; +/** The render profile: what the derive annotations name. */ +export const RENDER_PROFILE = 'pdf.v1'; +export const RENDER_DPI = 150; +export const RENDER_MAX_SIDE = 2048; +/** A page is "graphics-bearing" at this many path objects — a chart, a rule-drawn table. */ +const GRAPHICS_PATH_FLOOR = 20; +const MIN_FIGURE_SIDE_PX = 64; +const MIN_FIGURE_AREA_RATIO = 0.02; + +// PDFium constants (fpdfview.h, fpdf_edit.h, fpdf_progressive.h). +const PAGEOBJ_PATH = 2; +const PAGEOBJ_IMAGE = 3; +const PAGEOBJ_FORM = 5; +/** How deep a walk follows Form XObjects nested in Form XObjects. */ +const MAX_FORM_DEPTH = 8; +/** How many page objects one page's walk will visit, forms included. */ +const MAX_WALKED_OBJECTS = 20_000; + +/** A PDF matrix `[a b c d e f]`: (x, y) ↦ (a·x + c·y + e, b·x + d·y + f). */ +type Matrix = [number, number, number, number, number, number]; +const IDENTITY: Matrix = [1, 0, 0, 1, 0, 0]; +/** `outer ∘ inner`: apply `inner` first, then `outer`. */ +function compose(outer: Matrix, inner: Matrix): Matrix { + const [a, b, c, d, e, f] = outer; const [a2, b2, c2, d2, e2, f2] = inner; + return [a * a2 + c * b2, b * a2 + d * b2, a * c2 + c * d2, b * c2 + d * d2, a * e2 + c * f2 + e, b * e2 + d * f2 + f]; +} +/** The axis-aligned bounds of a rectangle after a matrix. */ +function transformRect(m: Matrix, [x0, y0, x1, y1]: PageImage['bbox']): PageImage['bbox'] { + const xs: number[] = []; const ys: number[] = []; + for (const [x, y] of [[x0, y0], [x1, y0], [x0, y1], [x1, y1]] as const) { + xs.push(m[0] * x + m[2] * y + m[4]); ys.push(m[1] * x + m[3] * y + m[5]); + } + return [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)]; +} +/** The part of `r` inside the page, or null when nothing of it is. */ +function clipToPage(r: PageImage['bbox'], width: number, height: number): PageImage['bbox'] | null { + const c: PageImage['bbox'] = [Math.max(0, r[0]), Math.max(0, r[1]), Math.min(width, r[2]), Math.min(height, r[3])]; + return c[2] > c[0] && c[3] > c[1] ? c : null; +} +const BITMAP_BGRA = 4; +const FLAG_ANNOT = 0x01; +const FLAG_REVERSE_BYTE_ORDER = 0x10; +// fpdf_progressive.h: READY 0, TOBECONTINUED 1, DONE 2, FAILED 3. +const RENDER_TOBECONTINUED = 1; +const RENDER_DONE = 2; + +/** The failure the route can map: ETIMEDOUT names the class, the message the document. */ +function timeoutError(ms: number): Error { + const e = new PdfError(`createDocumentIngress: document exceeded ${ms}ms`); + (e as Error & { code: string }).code = 'ETIMEDOUT'; + return e; +} + +// ── reading strings and structs off the heap ───────────────────── + +/** The two-call pattern PDFium uses for UTF-16 strings: length, then fill. */ +function utf16(codec: Codec, call: (buf: number, len: number) => number): string { + const { pdfium } = codec; + const len = call(0, 0); + if (len <= 2) return ''; + const buf = codec.malloc(len); + try { + call(buf, len); + return pdfium.pdfium.UTF16ToString(buf); + } finally { + codec.free(buf); + } +} + +// ── page facts ─────────────────────────────────────────────────── + +function readStruct(codec: Codec, page: number): { struct: PageStruct; altByMcid: Map<number, string> } | undefined { + const { pdfium } = codec; + const tree = pdfium.FPDF_StructTree_GetForPage(page); + if (!tree) return undefined; + const roleByMcid = new Map<number, string>(); + const altByMcid = new Map<number, string>(); + const tables: PageStruct['tables'] = []; + let figures = 0; + try { + const typeOf = (el: number): string => utf16(codec, (b, l) => pdfium.FPDF_StructElement_GetType(el, b, l)); + const ownMcids = (el: number): number[] => { + const n = pdfium.FPDF_StructElement_GetMarkedContentIdCount(el); + const ids: number[] = []; + for (let i = 0; i < n; i++) { + const id = pdfium.FPDF_StructElement_GetMarkedContentIdAtIndex(el, i); + if (id >= 0) ids.push(id); + } + if (ids.length === 0) { + const one = pdfium.FPDF_StructElement_GetMarkedContentID(el); + if (one >= 0) ids.push(one); + } + return ids; + }; + const children = (el: number): number[] => { + const n = pdfium.FPDF_StructElement_CountChildren(el); + const out: number[] = []; + for (let i = 0; i < n; i++) { + const c = pdfium.FPDF_StructElement_GetChildAtIndex(el, i); + if (c) out.push(c); + } + return out; + }; + const descendantMcids = (el: number): number[] => [...ownMcids(el), ...children(el).flatMap(descendantMcids)]; + // Producers wrap marked content in structural wrappers (`NonStruct`, `Span`, + // `Div`, …) that carry no meaning of their own: a wrapper's content takes the + // role of the nearest semantic ancestor, or a heading under `H1 > NonStruct` + // would read as body text. + const TRANSPARENT = new Set(['NonStruct', 'Span', 'Div', 'Sect', 'Part', 'Art', 'Document', 'Link', 'Private']); + const readTable = (table: number): PageStruct['tables'][number] => { + const rows: { cells: { mcids: number[] }[] }[] = []; + const visitRows = (el: number): void => { + for (const c of children(el)) { + const t = typeOf(c); + if (t === 'TR') { + const cells = children(c).filter((k) => { const kt = typeOf(k); return kt === 'TD' || kt === 'TH'; }) + .map((k) => ({ mcids: descendantMcids(k) })); + rows.push({ cells }); + } else if (t !== 'Caption') visitRows(c); + } + }; + visitRows(table); + // The table's content is its cells; a caption or a border path is not. + return { mcids: rows.flatMap((r) => r.cells.flatMap((c) => c.mcids)), rows }; + }; + const visit = (el: number, cell: string | null, inherited: string | null): void => { + const type = typeOf(el); + const semantic = TRANSPARENT.has(type) ? inherited : type; + const effective = cell ?? semantic ?? type; + for (const m of ownMcids(el)) roleByMcid.set(m, effective); + if (type === 'Figure') { + figures++; + const alt = utf16(codec, (b, l) => pdfium.FPDF_StructElement_GetAltText(el, b, l)); + if (alt) for (const m of descendantMcids(el)) altByMcid.set(m, alt); + } + if (type === 'Table') tables.push(readTable(el)); + const nextCell = type === 'TD' || type === 'TH' ? type : cell; + for (const c of children(el)) visit(c, nextCell, semantic); + }; + const n = pdfium.FPDF_StructTree_CountChildren(tree); + for (let i = 0; i < n; i++) { + const c = pdfium.FPDF_StructTree_GetChildAtIndex(tree, i); + if (c) visit(c, null, null); + } + } finally { + pdfium.FPDF_StructTree_Close(tree); + } + return { struct: { roleByMcid, tables, figures }, altByMcid }; +} + +/** Everything the layout needs about one page, read in one pass over the codec. */ +function readPage(codec: Codec, doc: OpenDocument, index: number, opts: { text: boolean }): PageFacts { + const { pdfium } = codec; + const page = pdfium.FPDF_LoadPage(doc.handle, index); + if (!page) throw new PdfError(`readPage: page ${index + 1} cannot be loaded`); + const scratch = codec.malloc(96); + try { + const width = pdfium.FPDF_GetPageWidthF(page); + const height = pdfium.FPDF_GetPageHeightF(page); + const read = readStruct(codec, page); + + // Objects: image bounds (and a decompression-bomb check on their pixel size + // before anything could decode them), and the path count. The walk follows + // Form XObjects — rendering will decode what they hold, so the check and + // the counts must see it too — with each child's bounds carried into page + // space through the form matrices, and clipped to the page: an object may + // extend far past the page and the renderer clips it, so the visible part + // is the figure. + const images: PageImage[] = []; + let pathObjects = 0; + let walked = 0; + const matrixBuf = codec.malloc(24); + const walk = (obj: number, depth: number, m: Matrix): void => { + if (++walked > MAX_WALKED_OBJECTS) return; + const type = pdfium.FPDFPageObj_GetType(obj); + if (type === PAGEOBJ_PATH) { pathObjects++; return; } + if (type === PAGEOBJ_FORM) { + if (depth >= MAX_FORM_DEPTH) return; + let fm: Matrix = IDENTITY; + if (pdfium.FPDFPageObj_GetMatrix(obj, matrixBuf)) { + fm = [0, 1, 2, 3, 4, 5].map((k) => pdfium.pdfium.getValue(matrixBuf + k * 4, 'float')) as Matrix; + } + const inner = compose(m, fm); + const n = pdfium.FPDFFormObj_CountObjects(obj); + for (let k = 0; k < n; k++) walk(pdfium.FPDFFormObj_GetObject(obj, k), depth + 1, inner); + return; + } + if (type !== PAGEOBJ_IMAGE) return; + if (pdfium.FPDFImageObj_GetImagePixelSize(obj, scratch, scratch + 4)) { + const w = pdfium.pdfium.getValue(scratch, 'i32'); + const h = pdfium.pdfium.getValue(scratch + 4, 'i32'); + if (w * h > MAX_INPUT_PIXELS) { + throw new PdfError(`readPage: page ${index + 1} embeds a ${w}×${h} image, over the ${MAX_INPUT_PIXELS}-pixel ceiling`); + } + } + if (!pdfium.FPDFPageObj_GetBounds(obj, scratch, scratch + 4, scratch + 8, scratch + 12)) return; + const own: PageImage['bbox'] = [ + pdfium.pdfium.getValue(scratch, 'float'), pdfium.pdfium.getValue(scratch + 4, 'float'), + pdfium.pdfium.getValue(scratch + 8, 'float'), pdfium.pdfium.getValue(scratch + 12, 'float'), + ]; + const bbox = clipToPage(depth === 0 ? own : transformRect(m, own), width, height); + if (!bbox) return; + const mcid = pdfium.FPDFPageObj_GetMarkedContentID(obj); + const altText = mcid >= 0 ? read?.altByMcid.get(mcid) : undefined; + images.push({ index: images.length, bbox, ...(altText ? { altText } : {}) }); + }; + try { + const count = pdfium.FPDFPage_CountObjects(page); + for (let i = 0; i < count; i++) walk(pdfium.FPDFPage_GetObject(page, i), 0, IDENTITY); + } finally { + codec.free(matrixBuf); + } + + // Characters, with box, size, weight and marked-content id. + const chars: PageChar[] = []; + if (opts.text) { + const tp = pdfium.FPDFText_LoadPage(page); + if (tp) { + try { + const n = pdfium.FPDFText_CountChars(tp); + const mcidOfObject = new Map<number, number>(); + for (let i = 0; i < n; i++) { + const code = pdfium.FPDFText_GetUnicode(tp, i); + const text = code > 0 ? String.fromCodePoint(code) : ' '; + if (!pdfium.FPDFText_GetCharBox(tp, i, scratch, scratch + 8, scratch + 16, scratch + 24)) { + chars.push({ text: '\n', x0: 0, y0: 0, x1: 0, y1: 0, size: 0, weight: 400, mcid: -1 }); + continue; + } + // Rotated text — a journal's margin stamp, a landscape label — is + // not part of the page's reading flow; it would otherwise land as + // one "line" per glyph. Horizontal means the glyph's up is the + // page's up (a 180° flip counts as rotated too). + const angle = pdfium.FPDFText_GetCharAngle(tp, i); + if (Math.abs(Math.sin(angle)) > 0.1 || Math.cos(angle) < 0) continue; + // The LOOSE box is the glyph's advance box in page units — its width + // is the advance, so adjacent glyphs of one word abut and a word gap + // is a real gap. The tight box is the fallback when PDFium has none. + let left: number; let right: number; let bottom: number; let top: number; + const loose = pdfium.FPDFText_GetLooseCharBox(tp, i, scratch + 32); + if (loose) { + // FS_RECTF: left, top, right, bottom. + left = pdfium.pdfium.getValue(scratch + 32, 'float'); + top = pdfium.pdfium.getValue(scratch + 36, 'float'); + right = pdfium.pdfium.getValue(scratch + 40, 'float'); + bottom = pdfium.pdfium.getValue(scratch + 44, 'float'); + } else { + left = pdfium.pdfium.getValue(scratch, 'double'); + right = pdfium.pdfium.getValue(scratch + 8, 'double'); + bottom = pdfium.pdfium.getValue(scratch + 16, 'double'); + top = pdfium.pdfium.getValue(scratch + 24, 'double'); + } + // Size: the nominal font size × the text matrix's scale — the size + // the page was set in, whatever font drew the glyph (an em box + // varies by font: a math font's is taller than a text font's at + // one size) and however the producer arrived at it (a font set at + // size 1 under a ×9 matrix). The loose height is the fallback. + let size = pdfium.FPDFText_GetFontSize(tp, i); + if (pdfium.FPDFText_GetMatrix(tp, i, scratch + 64)) { + // FS_MATRIX: a, b, c, d, e, f. + const a = pdfium.pdfium.getValue(scratch + 64, 'float'); + const b = pdfium.pdfium.getValue(scratch + 68, 'float'); + const scale = Math.hypot(a, b); + if (scale > 0) size *= scale; + } + if (!(size > 0) && loose) size = top - bottom; + // The glyph origin's y is the baseline — the one thing a ligature + // drawn from a fallback font (a smaller em box) still shares with + // its neighbours, so lines group by it rather than by box edges. + let baseline = bottom; + if (pdfium.FPDFText_GetCharOrigin(tp, i, scratch + 48, scratch + 56)) { + baseline = pdfium.pdfium.getValue(scratch + 56, 'double'); + } + // A space PDFium synthesises between text runs has no box at all. + // It is a separator, not a glyph: give it the previous glyph's + // geometry so nothing downstream reads a zero-size box as a new + // baseline or a size-one font. + const last = chars[chars.length - 1]; + if (/\s/.test(text) && !(top - bottom > 0) && last && last.text !== '\n') { + left = last.x1; right = last.x1; bottom = last.y0; top = last.y1; size = last.size; + baseline = last.baseline ?? last.y0; + } + const obj = pdfium.FPDFText_GetTextObject(tp, i); + let mcid = -1; + if (obj) { + const cached = mcidOfObject.get(obj); + mcid = cached ?? pdfium.FPDFPageObj_GetMarkedContentID(obj); + mcidOfObject.set(obj, mcid); + } + const weight = pdfium.FPDFText_GetFontWeight(tp, i); + chars.push({ + text, x0: left, y0: bottom, x1: right, y1: top, + size, baseline, + weight: weight > 0 ? weight : 400, + mcid, + }); + } + } finally { + pdfium.FPDFText_ClosePage(tp); + } + } + } + + return { + page: index + 1, width, height, chars, images, pathObjects, textExtracted: opts.text, + ...(read ? { struct: read.struct } : {}), + }; + } finally { + codec.free(scratch); + pdfium.FPDF_ClosePage(page); + } +} + +function readBookmarks(codec: Codec, doc: OpenDocument): Bookmark[] { + const { pdfium } = codec; + const out: Bookmark[] = []; + const visit = (first: number, level: number): void => { + let bm = first; + while (bm) { + const title = utf16(codec, (b, l) => pdfium.FPDFBookmark_GetTitle(bm, b, l)); + const dest = pdfium.FPDFBookmark_GetDest(doc.handle, bm); + const page = dest ? pdfium.FPDFDest_GetDestPageIndex(doc.handle, dest) + 1 : 0; + out.push({ title, page, level }); + const child = pdfium.FPDFBookmark_GetFirstChild(doc.handle, bm); + if (child) visit(child, level + 1); + bm = pdfium.FPDFBookmark_GetNextSibling(doc.handle, bm); + } + }; + visit(pdfium.FPDFBookmark_GetFirstChild(doc.handle, 0), 0); + return out; +} + +// ── rendering ──────────────────────────────────────────────────── + +/** Pixel size for a page at `dpi`, held under the side and area ceilings. */ +function renderSize(widthPt: number, heightPt: number, dpi: number): { width: number; height: number; scale: number } { + const scale = fitScale(widthPt, heightPt, dpi / 72); + return { width: Math.max(1, Math.floor(widthPt * scale)), height: Math.max(1, Math.floor(heightPt * scale)), scale }; +} + +/** `scale`, lowered until a `widthPt` × `heightPt` region fits the side and area ceilings. */ +function fitScale(widthPt: number, heightPt: number, scale: number): number { + const longest = Math.max(widthPt, heightPt) * scale; + if (longest > RENDER_MAX_SIDE) scale *= RENDER_MAX_SIDE / longest; + const area = widthPt * heightPt * scale * scale; + if (area > DEFAULT_MAX_PIXELS) scale *= Math.sqrt(DEFAULT_MAX_PIXELS / area); + return scale; +} + +/** Copy a bitmap's RGBA rows off the heap (the stride may exceed the row). */ +function bitmapPixels(codec: Codec, bmp: number, width: number, height: number): Uint8Array { + const { pdfium } = codec; + const buf = pdfium.FPDFBitmap_GetBuffer(bmp); + const stride = pdfium.FPDFBitmap_GetStride(bmp); + const row = width * 4; + const out = new Uint8Array(row * height); + const heap = pdfium.pdfium.HEAPU8; + for (let y = 0; y < height; y++) out.set(heap.subarray(buf + y * stride, buf + y * stride + row), y * row); + return out; +} + +/** Render a whole page progressively, stopping at the deadline or an abort. */ +function renderPageRgba(codec: Codec, page: number, width: number, height: number, stop: () => boolean): Uint8Array { + const { pdfium } = codec; + const bmp = pdfium.FPDFBitmap_CreateEx(width, height, BITMAP_BGRA, 0, 0); + if (!bmp) throw new PdfError(`render: could not allocate a ${width}×${height} bitmap`); + const pause = codec.malloc(12); + const fn = pdfium.pdfium.addFunction(() => (stop() ? 1 : 0), 'ip'); + try { + pdfium.FPDFBitmap_FillRect(bmp, 0, 0, width, height, 0xffffffff); + pdfium.pdfium.setValue(pause, 1, 'i32'); + pdfium.pdfium.setValue(pause + 4, fn, 'i32'); + pdfium.pdfium.setValue(pause + 8, 0, 'i32'); + let status = pdfium.FPDF_RenderPageBitmap_Start(bmp, page, 0, 0, width, height, 0, FLAG_ANNOT | FLAG_REVERSE_BYTE_ORDER, pause); + while (status === RENDER_TOBECONTINUED && !stop()) status = pdfium.FPDF_RenderPage_Continue(page, pause); + pdfium.FPDF_RenderPage_Close(page); + if (status !== RENDER_DONE) throw new PdfError(status === RENDER_TOBECONTINUED ? 'render: stopped' : 'render: PDFium failed'); + return bitmapPixels(codec, bmp, width, height); + } finally { + pdfium.pdfium.removeFunction(fn); + codec.free(pause); + pdfium.FPDFBitmap_Destroy(bmp); + } +} + +/** Render a region of a page — a figure's bounds — at `scale` pixels per point. */ +function renderRegionRgba(codec: Codec, page: number, pageHeightPt: number, bbox: PageImage['bbox'], pageScale: number): { rgba: Uint8Array; width: number; height: number } { + const { pdfium } = codec; + const [x0, y0, x1, y1] = bbox; + // The bounds were clipped to the page when read, so at the page's scale the + // crop is never larger than the page render; the ceilings hold regardless. + const scale = fitScale(x1 - x0, y1 - y0, pageScale); + const width = Math.max(1, Math.round((x1 - x0) * scale)); + const height = Math.max(1, Math.round((y1 - y0) * scale)); + const bmp = pdfium.FPDFBitmap_CreateEx(width, height, BITMAP_BGRA, 0, 0); + if (!bmp) throw new PdfError(`render: could not allocate a ${width}×${height} bitmap`); + const matrix = codec.malloc(24); + const clip = codec.malloc(16); + try { + pdfium.FPDFBitmap_FillRect(bmp, 0, 0, width, height, 0xffffffff); + // Device space is top-left, y down; user space is bottom-left, y up. The + // display transform already flips y, so the region's top edge lands at the + // device origin when translated by the distance from the page top. + const m = [scale, 0, 0, scale, -x0 * scale, -(pageHeightPt - y1) * scale]; + m.forEach((v, i) => pdfium.pdfium.setValue(matrix + i * 4, v, 'float')); + [0, 0, width, height].forEach((v, i) => pdfium.pdfium.setValue(clip + i * 4, v, 'float')); + pdfium.FPDF_RenderPageBitmapWithMatrix(bmp, page, matrix, clip, FLAG_ANNOT | FLAG_REVERSE_BYTE_ORDER); + return { rgba: bitmapPixels(codec, bmp, width, height), width, height }; + } finally { + codec.free(matrix); + codec.free(clip); + pdfium.FPDFBitmap_Destroy(bmp); + } +} + +/** RGBA pixels → PNG, through sharp with its defaults, which are deterministic for a fixed libvips. */ +async function toPng(rgba: Uint8Array, width: number, height: number): Promise<Uint8Array> { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const sharp = require('sharp') as typeof import('sharp').default; + const png = await sharp(Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), { raw: { width, height, channels: 4 } }) + .removeAlpha() + .png() + .toBuffer(); + return new Uint8Array(png); +} + +/** Let the event loop breathe between pages: the socket, the model loop, an abort. */ +const breathe = (): Promise<void> => new Promise((r) => setImmediate(r)); + +function codecVersion(): string { + try { + const entry = require.resolve('@embedpdf/pdfium'); + const pkg = JSON.parse(readFileSync(join(dirname(entry), '..', 'package.json'), 'utf8')) as { version?: string }; + return pkg.version ?? 'unknown'; + } catch { + return 'unknown'; + } +} + +// ── the ingress ────────────────────────────────────────────────── + +/** + * What a host may size differently. Every default is the shipped bound above; + * tests use small numbers to reach the caps with small fixtures. + * + * @category Media + */ +export interface DocumentOpts { + maxRenderedPages?: number; + maxTextPages?: number; + maxFigures?: number; + timeoutMs?: number; + dpi?: number; +} + +/** + * The document ingress: a PDF in, a document attachment out. + * + * One permit of the shared gate for the whole document, one codec instance + * for the whole document. Extraction and rendering complete in memory first; + * the store is written only after the last check, so an abort or a timeout + * commits nothing. Coverage — which pages are rendered, how many figures — + * is a function of the input and the declared caps, never of time, so the + * same bytes yield the same root on any machine. + * + * @category Media + */ +export function createDocumentIngress(store: AttachmentStore, opts: DocumentOpts = {}): ContentIngress { + const maxRenderedPages = opts.maxRenderedPages ?? MAX_RENDERED_PAGES; + const maxTextPages = opts.maxTextPages ?? MAX_TEXT_PAGES; + const maxFigures = opts.maxFigures ?? MAX_FIGURES; + const timeoutMs = opts.timeoutMs ?? DOCUMENT_TIMEOUT_MS; + const dpi = opts.dpi ?? RENDER_DPI; + const label = 'createDocumentIngress'; + + return { + async ingest(bytes: Uint8Array, signal?: AbortSignal): Promise<Attachment> { + if (signal?.aborted) throw abortError(label); + if (bytes.byteLength > MAX_DOCUMENT_BYTES) { + throw new PdfError(`${label}: document is ${bytes.byteLength} bytes, over the ${MAX_DOCUMENT_BYTES}-byte ceiling`); + } + const release = await gate.acquire(signal); + try { + const deadline = Date.now() + timeoutMs; + const stop = (): boolean => Date.now() > deadline || !!signal?.aborted; + const check = (): void => { + if (signal?.aborted) throw abortError(label); + if (Date.now() > deadline) throw timeoutError(timeoutMs); + }; + + const codec = await createCodec(); + try { + const doc = openDocument(codec, bytes); + try { + const title = utf16(codec, (b, l) => codec.pdfium.FPDF_GetMetaText(doc.handle, 'Title', b, l)) || null; + const bookmarks = readBookmarks(codec, doc); + + const pages: PageFacts[] = []; + for (let i = 0; i < doc.pageCount; i++) { + check(); + pages.push(readPage(codec, doc, i, { text: i < maxTextPages })); + await breathe(); + } + const layout = layoutDocument({ + title, pages, bookmarks, maxTextPages, dpi, + minFigureSidePx: MIN_FIGURE_SIDE_PX, minFigureAreaRatio: MIN_FIGURE_AREA_RATIO, + }); + + // Which pages to archive under the cap: page one, then every + // graphics-bearing page, then the rest — a deterministic order. + const graphic = (p: LayoutResult['pages'][number]): boolean => + p.page === 1 || p.chars === 0 || p.imageObjects > 0 || p.pathObjects >= GRAPHICS_PATH_FLOOR || p.taggedTables > 0 || p.taggedFigures > 0; + const order = [...layout.pages] + .sort((a, b) => Number(graphic(b)) - Number(graphic(a)) || a.page - b.page) + .slice(0, maxRenderedPages) + .sort((a, b) => a.page - b.page); + + const renders = new Map<number, { png: Uint8Array; width: number; height: number }>(); + const crops: { figure: LayoutResult['figures'][number]; png: Uint8Array; width: number; height: number }[] = []; + const figures = layout.figures.slice(0, maxFigures); + for (const p of order) { + check(); + const facts = pages[p.page - 1]; + const size = renderSize(facts.width, facts.height, dpi); + const page = codec.pdfium.FPDF_LoadPage(doc.handle, p.page - 1); + if (!page) throw new PdfError(`render: page ${p.page} cannot be loaded`); + try { + const rgba = renderPageRgba(codec, page, size.width, size.height, stop); + check(); + renders.set(p.page, { png: await toPng(rgba, size.width, size.height), width: size.width, height: size.height }); + for (const f of figures.filter((x) => x.page === p.page)) { + check(); + const region = renderRegionRgba(codec, page, facts.height, f.bbox, size.scale); + crops.push({ figure: f, png: await toPng(region.rgba, region.width, region.height), width: region.width, height: region.height }); + } + } finally { + codec.pdfium.FPDF_ClosePage(page); + } + await breathe(); + } + // A figure on a page that fell outside the render cap is not cropped + // either: crops come from rendered pages, so the archive is consistent. + const truncated = layout.truncated || renders.size < doc.pageCount || layout.figures.length > figures.length; + + // The last check before the first write: from here nothing waits and + // nothing aborts, so a document is committed whole or not at all. + check(); + const source = store.putBlob(bytes, 'application/pdf'); + const derive = (page: number, width: number, height: number, extra: Record<string, string> = {}): Record<string, string> => ({ + [`${DERIVE_PREFIX}profile`]: RENDER_PROFILE, + [`${DERIVE_PREFIX}page`]: String(page), + [`${DERIVE_PREFIX}dpi`]: String(dpi), + [`${DERIVE_PREFIX}width`]: String(width), + [`${DERIVE_PREFIX}height`]: String(height), + [`${DERIVE_PREFIX}format`]: 'image/png', + [`${DERIVE_PREFIX}source`]: source.digest, + ...extra, + }); + const pageRoots = new Map<number, Descriptor>(); + for (const [page, r] of renders) { + const rep = store.putBlob(r.png, 'image/png', derive(page, r.width, r.height)); + pageRoots.set(page, store.putAttachment({ representations: [rep] })); + } + const figureRoots: DocumentMeta['figures'] = crops.map((c) => { + const rep = store.putBlob(c.png, 'image/png', derive(c.figure.page, c.width, c.height, { [`${DERIVE_PREFIX}bbox`]: c.figure.bbox.join(',') })); + return { ...c.figure, root: store.putAttachment({ representations: [rep] }) }; + }); + const markdown = store.putBlob(new TextEncoder().encode(layout.markdown), 'text/markdown'); + const meta: DocumentMeta = { + title: layout.title, + pageCount: doc.pageCount, + sections: layout.sections, + pages: layout.pages.map((p) => { const render = pageRoots.get(p.page); return render ? { ...p, render } : p; }), + figures: figureRoots, + tables: layout.tables, + derive: { + profile: RENDER_PROFILE, pdfium: codecVersion(), dpi, maxSide: RENDER_MAX_SIDE, maxPixels: DEFAULT_MAX_PIXELS, format: 'image/png', + renderedPages: renders.size, maxFigures, maxTextPages, tagged: layout.tagged, structCoverage: layout.structCoverage, truncated, + }, + }; + return store.putAttachment({ + representations: [markdown], + source, + config: { bytes: new TextEncoder().encode(JSON.stringify(meta)), mediaType: DOCUMENT_CONFIG_TYPE }, + }); + } finally { + doc.close(); + } + } finally { + codec.dispose(); + } + } finally { + release(); + } + }, + }; +} diff --git a/packages/media/test/content-ingress.test.ts b/packages/media/test/content-ingress.test.ts new file mode 100644 index 00000000..0a04e94a --- /dev/null +++ b/packages/media/test/content-ingress.test.ts @@ -0,0 +1,43 @@ +/** + * One door, the bytes deciding: an image to the normalizer, a PDF to the + * document ingress, junk refused. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { readFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import sharp from 'sharp'; +import { FileAttachmentStore } from '../src/node'; +import { createContentIngress, DOCUMENT_UPLOAD_TIMEOUT_MS } from '../src/content-ingress'; +import { DOCUMENT_CONFIG_TYPE } from '../src/document'; +import { EMPTY_DESCRIPTOR } from '../src/attachment'; + +const fixture = (name: string): Uint8Array => new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); + +describe('createContentIngress', () => { + it('admits an image as an image manifest', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'content-ingress-'))); + const png = new Uint8Array(await sharp({ create: { width: 32, height: 32, channels: 3, background: '#0a7' } }).png().toBuffer()); + const root = await createContentIngress(store).ingest(png); + const manifest = store.getManifest(root.digest)!; + expect(manifest.config.digest).toBe(EMPTY_DESCRIPTOR.digest); + expect(manifest.layers[0].mediaType).toBe('image/png'); + }); + + it('admits a PDF as a document manifest', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'content-ingress-'))); + const root = await createContentIngress(store).ingest(fixture('untagged.pdf')); + expect(store.getManifest(root.digest)!.config.mediaType).toBe(DOCUMENT_CONFIG_TYPE); + }, 60_000); + + it('refuses bytes that are neither', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'content-ingress-'))); + await expect(createContentIngress(store).ingest(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).rejects.toThrow(); + }); + + it('states the upload allowance a host passes beside the byte ceiling', () => { + expect(DOCUMENT_UPLOAD_TIMEOUT_MS).toBe(180_000); + }); +}); diff --git a/packages/media/test/document.test.ts b/packages/media/test/document.test.ts new file mode 100644 index 00000000..dece9b95 --- /dev/null +++ b/packages/media/test/document.test.ts @@ -0,0 +1,62 @@ +/** + * The document sidecar — facts about a document, as the content plane stores + * them in an attachment's config slot. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { DOCUMENT_CONFIG_TYPE, asDocumentMeta, pagesOf } from '../src/document'; +import type { DocumentMeta } from '../src/document'; + +const page = (n: number, startLine: number, endLine: number, extra: Partial<DocumentMeta['pages'][number]> = {}) => ({ + page: n, startLine, endLine, chars: 100, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0, ...extra, +}); + +const meta: DocumentMeta = { + title: 'A paper', + pageCount: 3, + sections: [{ heading: 'Intro', path: 'Intro', origin: 'heuristic', startLine: 1, endLine: 12, pageStart: 1, pageEnd: 2 }], + pages: [page(1, 1, 8), page(2, 9, 15, { imageObjects: 1 }), page(3, 16, 20, { chars: 0 })], + figures: [], + tables: [], + derive: { + profile: 'pdf.v1', pdfium: '2.15.0', dpi: 150, maxSide: 2048, maxPixels: 4_194_304, format: 'image/png', + renderedPages: 3, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false, + }, +}; + +describe('DOCUMENT_CONFIG_TYPE', () => { + it('is a versioned +json type, so a reader can branch on it', () => { + expect(DOCUMENT_CONFIG_TYPE).toBe('application/vnd.lloyal.document.v1+json'); + }); +}); + +describe('asDocumentMeta', () => { + it('accepts the shape and returns it typed', () => { + expect(asDocumentMeta(JSON.parse(JSON.stringify(meta)))).toEqual(meta); + }); + + it('refuses junk without throwing', () => { + expect(asDocumentMeta(null)).toBeNull(); + expect(asDocumentMeta('x')).toBeNull(); + expect(asDocumentMeta({})).toBeNull(); + expect(asDocumentMeta({ ...meta, title: 7 })).toBeNull(); + expect(asDocumentMeta({ ...meta, pages: 'no' })).toBeNull(); + expect(asDocumentMeta({ ...meta, pages: [{ page: 1 }] })).toBeNull(); + expect(asDocumentMeta({ ...meta, sections: [{ heading: 'x' }] })).toBeNull(); + }); +}); + +describe('pagesOf', () => { + it('maps a line span inside one page to that page', () => { + expect(pagesOf(meta, 2, 5)).toEqual({ pageStart: 1, pageEnd: 1 }); + }); + + it('maps a span across a page break to its first and last page', () => { + expect(pagesOf(meta, 7, 17)).toEqual({ pageStart: 1, pageEnd: 3 }); + }); + + it('maps a span past the page map to the last page', () => { + expect(pagesOf(meta, 40, 45)).toEqual({ pageStart: 3, pageEnd: 3 }); + }); +}); diff --git a/packages/media/test/fixtures/pdf/bigform.pdf b/packages/media/test/fixtures/pdf/bigform.pdf new file mode 100644 index 00000000..227597ea Binary files /dev/null and b/packages/media/test/fixtures/pdf/bigform.pdf differ diff --git a/packages/media/test/fixtures/pdf/bigimage.pdf b/packages/media/test/fixtures/pdf/bigimage.pdf new file mode 100644 index 00000000..9807e555 Binary files /dev/null and b/packages/media/test/fixtures/pdf/bigimage.pdf differ diff --git a/packages/media/test/fixtures/pdf/clipped.pdf b/packages/media/test/fixtures/pdf/clipped.pdf new file mode 100644 index 00000000..0100b3d5 Binary files /dev/null and b/packages/media/test/fixtures/pdf/clipped.pdf differ diff --git a/packages/media/test/fixtures/pdf/encrypted.pdf b/packages/media/test/fixtures/pdf/encrypted.pdf new file mode 100644 index 00000000..fe2d6729 Binary files /dev/null and b/packages/media/test/fixtures/pdf/encrypted.pdf differ diff --git a/packages/media/test/fixtures/pdf/form.pdf b/packages/media/test/fixtures/pdf/form.pdf new file mode 100644 index 00000000..e6364821 Binary files /dev/null and b/packages/media/test/fixtures/pdf/form.pdf differ diff --git a/packages/media/test/fixtures/pdf/make.sh b/packages/media/test/fixtures/pdf/make.sh new file mode 100755 index 00000000..3962c635 --- /dev/null +++ b/packages/media/test/fixtures/pdf/make.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Produces the PDF fixtures the ingress tests read. Run once; commit the outputs. +# Every producer and flag is recorded here so a regeneration is reproducible. +# +# tagged.pdf Chrome's print-to-pdf with a document outline: a structure tree +# (H1/H2/P/Table/Figure), bookmarks, three pages — text, table, image. +# scanned.pdf assembled by hand below: one page whose only content is a JPEG raster, +# no characters — what a scanner produces. +# untagged.pdf Ghostscript from PostScript: two pages, headings only by font size. +# encrypted.pdf untagged.pdf re-written by Ghostscript with a user password. +set -euo pipefail +cd "$(dirname "$0")" +CHROME="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" +PNG="$(node -e "require('sharp')({create:{width:96,height:64,channels:3,background:'#0a7'}}).png().toBuffer().then(b=>process.stdout.write(b.toString('base64')))")" + +cat > tagged.html <<HTML +<!doctype html><html lang="en"><head><meta charset="utf-8"><title>A Tagged Paper + + +

A Tagged Paper

+

Introduction

+

This introduction paragraph carries enough prose to survive any minimum-length floor a chunker may apply, and it mentions the retrieval budget explicitly so a search for the word budget lands here.

+

A second paragraph follows the first, so the section holds more than one block of text and the paragraph splitter has something to do.

+

Method

+

The method section describes the procedure in ordinary prose without any table or figure, so its page should count no image objects.

+
+

Results

+

Table 1 reports the measured values for each configuration.

+ + + + +
Table 1. Measured values.
ConfigurationCellsSeconds
baseline8034.2
four images15917.9
+
+

Discussion

+
A solid green rectangle
Figure 1. A solid green rectangle used as the chart stand-in.
+

The discussion refers to Figure 1 above and closes the document.

+ +HTML + + +# One fresh profile per call (a shared one blocks the second launch on its lock) +# and a hard alarm, so a wedged renderer fails the script instead of hanging it. +for name in tagged; do + PROFILE="$(mktemp -d)" + # Chrome sometimes lingers after the file is written; the alarm ends it and + # the file on disk is the verdict, not the exit code. + perl -e 'alarm shift; exec @ARGV' 45 "$CHROME" --headless=new --disable-gpu --no-first-run --no-default-browser-check \ + --user-data-dir="$PROFILE" --no-pdf-header-footer --generate-pdf-document-outline --virtual-time-budget=2000 \ + --print-to-pdf="$PWD/$name.pdf" "file://$PWD/$name.html" >/dev/null 2>&1 || true + rm -rf "$PROFILE" + [ -s "$PWD/$name.pdf" ] || { echo "chrome produced no $name.pdf" >&2; exit 1; } +done + + +# scanned.pdf — a minimal PDF assembled by hand: catalog, pages, one page, one +# DCTDecode image XObject painted full-bleed. No producer, no dates: byte-stable. +node - <<'JS' +const sharp = require('sharp'); const fs = require('fs'); +(async () => { + const W = 850, H = 1100; + const svg = `` + + `` + + ``; + const jpg = await sharp(Buffer.from(svg)).jpeg({ quality: 75 }).toBuffer(); + const w = 612, h = 792; + const parts = [Buffer.from('%PDF-1.4\n%\xE2\xE3\xCF\xD3\n', 'latin1')]; + let pos = parts[0].length; const offsets = []; + const add = (n, head, stream) => { + offsets[n] = pos; + let b = Buffer.from(`${n} 0 obj\n${head}\n`, 'latin1'); + if (stream) b = Buffer.concat([b, Buffer.from('stream\n', 'latin1'), stream, Buffer.from('\nendstream\n', 'latin1')]); + b = Buffer.concat([b, Buffer.from('endobj\n', 'latin1')]); + parts.push(b); pos += b.length; + }; + const content = Buffer.from(`q ${w} 0 0 ${h} 0 0 cm /Im1 Do Q`, 'latin1'); + add(1, '<< /Type /Catalog /Pages 2 0 R >>'); + add(2, '<< /Type /Pages /Kids [3 0 R] /Count 1 >>'); + add(3, `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 ${w} ${h}] /Resources << /XObject << /Im1 4 0 R >> >> /Contents 5 0 R >>`); + add(4, `<< /Type /XObject /Subtype /Image /Width ${W} /Height ${H} /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /DCTDecode /Length ${jpg.length} >>`, jpg); + add(5, `<< /Length ${content.length} >>`, content); + let x = `xref\n0 6\n0000000000 65535 f \n`; + for (let i = 1; i <= 5; i++) x += String(offsets[i]).padStart(10, '0') + ' 00000 n \n'; + x += `trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n${pos}\n%%EOF\n`; + parts.push(Buffer.from(x, 'latin1')); + fs.writeFileSync('scanned.pdf', Buffer.concat(parts)); +})(); +JS + +cat > untagged.ps <<'PS' +%!PS-Adobe-3.0 +/Helvetica-Bold findfont 18 scalefont setfont 72 720 moveto (An Untagged Report) show +/Helvetica-Bold findfont 14 scalefont setfont 72 688 moveto (First Section) show +/Helvetica findfont 10 scalefont setfont +72 670 moveto (Body text line one of the first section, set in the body size.) show +72 656 moveto (Body text line two of the first section continues the paragraph.) show +/Helvetica-Bold findfont 14 scalefont setfont 72 620 moveto (Second Section) show +/Helvetica findfont 10 scalefont setfont +72 602 moveto (Body text of the second section, also in the body size.) show +showpage +/Helvetica findfont 10 scalefont setfont +72 720 moveto (Page two carries a single body line and no heading.) show +showpage +PS +gs -q -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOutputFile=untagged.pdf untagged.ps +gs -q -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOwnerPassword=owner -sUserPassword=user -dEncryptionR=3 -dKeyLength=128 \ + -sOutputFile=encrypted.pdf untagged.pdf + +rm -f tagged.html untagged.ps +ls -la ./*.pdf + +# matrix.pdf — text set at font size 1 under scaled text matrices (22× title, +# 11× body), the way some publisher pipelines emit it. PDFium then reports a +# nominal font size of 1 for every glyph; the reader must measure size and +# spacing from the loose (advance) boxes instead. Written by hand: Ghostscript +# would fold the scale back into the font size and hide the case. +python3 - matrix.pdf <<'PY' +import io, sys +content = b"""BT +/F1 1 Tf +22 0 0 22 72 700 Tm +(Scaled Title Line) Tj +11 0 0 11 72 660 Tm +(Body text set at size one under a scaled matrix.) Tj +11 0 0 11 72 644 Tm +(Second line of body text with several words.) Tj +ET +""" +objs = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length " + str(len(content)).encode() + b" >>\nstream\n" + content + b"endstream", +] +out = io.BytesIO(); out.write(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n"); offsets = [] +for i, o in enumerate(objs, 1): + offsets.append(out.tell()); out.write(f"{i} 0 obj\n".encode() + o + b"\nendobj\n") +xref = out.tell() +out.write(f"xref\n0 {len(objs)+1}\n".encode() + b"0000000000 65535 f \n") +for off in offsets: out.write(f"{off:010d} 00000 n \n".encode()) +out.write(f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) +open(sys.argv[1], 'wb').write(out.getvalue()) +PY + +# Bounds fixtures, hand-written (Ghostscript would normalise them away): +# clipped.pdf — a 1×1 red image painted into a 2000-pt square on a 100-pt page: +# the visible figure is the page; the crop must not be the square. +# form.pdf — a 2×2 green image inside a Form XObject (/Matrix translate 10,10 +# under an outer cm translate 40,40): the image is nested, at +# page-space [50,50]–[150,150]. +# bigimage.pdf — a direct image whose dictionary says 10001×10000 pixels. +# bigform.pdf — the same image inside a Form XObject. +python3 - <<'PY' +import io +def pdf(path, objs): + out = io.BytesIO(); out.write(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n"); offsets = [] + for i, o in enumerate(objs, 1): + offsets.append(out.tell()); out.write(f"{i} 0 obj\n".encode() + o + b"\nendobj\n") + xref = out.tell() + out.write(f"xref\n0 {len(objs)+1}\n".encode() + b"0000000000 65535 f \n") + for off in offsets: out.write(f"{off:010d} 00000 n \n".encode()) + out.write(f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) + open(path, 'wb').write(out.getvalue()) +def stream(dict_head, data): return dict_head + b" /Length " + str(len(data)).encode() + b" >>\nstream\n" + data + b"\nendstream" +def image(w, h, rgb, data=None): + data = data if data is not None else bytes(rgb) * (w * h) + return stream(b"<< /Type /XObject /Subtype /Image /Width %d /Height %d /ColorSpace /DeviceRGB /BitsPerComponent 8" % (w, h), data) +def page(box, content, xobjects): + return b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %d %d] /Resources << /XObject << %s >> >> /Contents 4 0 R >>" % (box, box, xobjects) +CAT = b"<< /Type /Catalog /Pages 2 0 R >>"; PAGES = b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>" +pdf('clipped.pdf', [CAT, PAGES, page(100, None, b"/Im1 5 0 R"), + stream(b"<<", b"q 2000 0 0 2000 -950 -950 cm /Im1 Do Q"), image(1, 1, (255, 0, 0))]) +form = stream(b"<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] /Matrix [1 0 0 1 10 10] /Resources << /XObject << /Im1 6 0 R >> >>", b"q 100 0 0 100 0 0 cm /Im1 Do Q") +pdf('form.pdf', [CAT, PAGES, page(200, None, b"/Fx1 5 0 R"), + stream(b"<<", b"q 1 0 0 1 40 40 cm /Fx1 Do Q"), form, image(2, 2, (0, 170, 119))]) +big = image(10001, 10000, (0, 0, 0), data=b"\x00\x00\x00") +pdf('bigimage.pdf', [CAT, PAGES, page(100, None, b"/Im1 5 0 R"), stream(b"<<", b"q 100 0 0 100 0 0 cm /Im1 Do Q"), big]) +bigform = stream(b"<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] /Resources << /XObject << /Im1 6 0 R >> >>", b"q 100 0 0 100 0 0 cm /Im1 Do Q") +pdf('bigform.pdf', [CAT, PAGES, page(100, None, b"/Fx1 5 0 R"), stream(b"<<", b"q /Fx1 Do Q"), bigform, big]) +PY +ls -la clipped.pdf form.pdf bigimage.pdf bigform.pdf diff --git a/packages/media/test/fixtures/pdf/matrix.pdf b/packages/media/test/fixtures/pdf/matrix.pdf new file mode 100644 index 00000000..63f85eee Binary files /dev/null and b/packages/media/test/fixtures/pdf/matrix.pdf differ diff --git a/packages/media/test/fixtures/pdf/scanned.pdf b/packages/media/test/fixtures/pdf/scanned.pdf new file mode 100644 index 00000000..4345c277 Binary files /dev/null and b/packages/media/test/fixtures/pdf/scanned.pdf differ diff --git a/packages/media/test/fixtures/pdf/tagged.pdf b/packages/media/test/fixtures/pdf/tagged.pdf new file mode 100644 index 00000000..44a78771 Binary files /dev/null and b/packages/media/test/fixtures/pdf/tagged.pdf differ diff --git a/packages/media/test/fixtures/pdf/untagged.pdf b/packages/media/test/fixtures/pdf/untagged.pdf new file mode 100644 index 00000000..16dee38f Binary files /dev/null and b/packages/media/test/fixtures/pdf/untagged.pdf differ diff --git a/packages/media/test/gate.test.ts b/packages/media/test/gate.test.ts new file mode 100644 index 00000000..c6a42f6b --- /dev/null +++ b/packages/media/test/gate.test.ts @@ -0,0 +1,81 @@ +/** + * The permit gate — the process-wide bound on decoded content in memory. + * + * One construct, extracted from the image normalizer so a document ingest can + * hold a permit of the SAME gate for its whole duration: images and documents + * together never exceed the permit count. These cases pin the gate's own + * contract; the normalizer's suite pins that the normalizer still stands + * behind it. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { createGate, gate, MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS } from '../src/gate'; + +const tick = () => new Promise((r) => setTimeout(r, 0)); + +describe('createGate', () => { + it('bounds concurrency to its permits and hands a freed permit to the next waiter', async () => { + const g = createGate({ label: 'test', permits: 2, maxQueued: 4, waitMs: 1_000 }); + const r1 = await g.acquire(); + const r2 = await g.acquire(); + let third = false; + const p3 = g.acquire().then((r) => { third = true; return r; }); + await tick(); + expect(third).toBe(false); + r1(); + const r3 = await p3; + expect(third).toBe(true); + r2(); r3(); + }); + + it('refuses immediately with EBUSY once the queue is full', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 1, waitMs: 1_000 }); + const r1 = await g.acquire(); + const queued = g.acquire(); + await expect(g.acquire()).rejects.toMatchObject({ code: 'EBUSY' }); + r1(); + (await queued)(); + }); + + it('lets a queued caller give up before any slot frees, and refuses an already-aborted one', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 4, waitMs: 1_000 }); + const r1 = await g.acquire(); + const ctrl = new AbortController(); + const queued = g.acquire(ctrl.signal); + ctrl.abort(); + await expect(queued).rejects.toMatchObject({ name: 'AbortError' }); + await expect(g.acquire(AbortSignal.abort())).rejects.toMatchObject({ name: 'AbortError' }); + r1(); + // The gate is intact afterwards. + (await g.acquire())(); + }); + + it('times out a waiter and names the wait', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 4, waitMs: 20 }); + const r1 = await g.acquire(); + await expect(g.acquire()).rejects.toThrow(/waited 20ms/); + r1(); + }); + + it('makes release idempotent so a double release cannot manufacture a permit', async () => { + const g = createGate({ label: 'test', permits: 1, maxQueued: 4, waitMs: 1_000 }); + const r1 = await g.acquire(); + r1(); r1(); + const r2 = await g.acquire(); + let third = false; + const p3 = g.acquire().then((r) => { third = true; return r; }); + await tick(); + expect(third).toBe(false); + r2(); + (await p3)(); + }); +}); + +describe('the shared gate', () => { + it('is one instance sized by the normalizer constants', () => { + expect(typeof gate.acquire).toBe('function'); + expect(MAX_CONCURRENT_NORMALIZATIONS).toBe(4); + expect(MAX_QUEUED_NORMALIZATIONS).toBe(16); + }); +}); diff --git a/packages/media/test/ingress.test.ts b/packages/media/test/ingress.test.ts index 2f5d4e52..f4ef30e6 100644 --- a/packages/media/test/ingress.test.ts +++ b/packages/media/test/ingress.test.ts @@ -11,6 +11,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { FileAttachmentStore } from '../src/node'; import { createImageIngress } from '../src/image'; +import { materialize } from '../src/ingress'; const png = () => sharp({ create: { width: 64, height: 64, channels: 3, background: '#0a7' } }).png().toBuffer() @@ -47,3 +48,34 @@ describe('createImageIngress', () => { expect(store.getManifest(root.digest)).toBeTruthy(); }); }); + +describe('materialize — bitmaps are the projector-decodable representations', () => { + // A document's one representation is text. It reaches the model through + // retrieval and the spine outline, never through the projector, so a + // document root contributes NO bitmaps: feeding its bytes to the image + // decoder would poison the branch it was prefilled on. + it('yields no bitmaps for a root whose only representation is text/markdown', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'materialize-'))); + const md = store.putBlob(new TextEncoder().encode('# Title\n\nBody.\n'), 'text/markdown'); + const root = store.putAttachment({ representations: [md] }); + + const prepared = materialize(store, [root]); + + expect(prepared.attachments).toEqual([root]); + expect(prepared.bitmaps).toEqual([]); + }); + + it('yields exactly the image bitmaps for a mixed batch, in root order', async () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'materialize-'))); + const bytes = await png(); + const image = await createImageIngress(store).ingest(bytes); + const md = store.putBlob(new TextEncoder().encode('text'), 'text/markdown'); + const doc = store.putAttachment({ representations: [md] }); + + const prepared = materialize(store, [doc, image]); + + expect(prepared.attachments).toEqual([doc, image]); + expect(prepared.bitmaps).toHaveLength(1); + expect(prepared.bitmaps[0]).toEqual(bytes); + }); +}); diff --git a/packages/media/test/normalize.test.ts b/packages/media/test/normalize.test.ts index 29c5d5c6..6f133325 100644 --- a/packages/media/test/normalize.test.ts +++ b/packages/media/test/normalize.test.ts @@ -9,12 +9,13 @@ */ import { describe, it, expect } from 'vitest'; import sharp from 'sharp'; -import { normalizeImage, DEFAULT_MAX_PIXELS, MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS, MAX_INPUT_PIXELS } from '../src/image'; +import { normalizeImage, DEFAULT_MAX_PIXELS, MAX_INPUT_PIXELS } from '../src/image'; +import { MAX_CONCURRENT_NORMALIZATIONS, MAX_QUEUED_NORMALIZATIONS } from '../src/gate'; import type { NormalizedImage } from '../src/image'; // Its own list of nine, sourced from stb_image — NOT derived from the sniff // table, which knows four. Deriving it was a defect: the pass-through gate read // as covering six formats and covered one. -import { PROJECTOR_FORMATS } from '../src/index'; +import { PROJECTOR_FORMATS, sniffMediaType } from '../src/index'; const solid = (width: number, height: number, format: 'jpeg' | 'png' | 'webp' | 'tiff' = 'jpeg') => sharp({ create: { width, height, channels: 3, background: '#0a7' } })[format]().toBuffer() @@ -181,6 +182,22 @@ describe('the two format lists are different questions', () => { }); }); +describe('a PDF is identified, and is not an image', () => { + const pdf = new TextEncoder().encode('%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n%%EOF\n'); + + it('sniffs the PDF signature — the content ingress dispatches on it', () => { + expect(sniffMediaType(pdf)).toBe('application/pdf'); + }); + + it('is not something the projector decodes', () => { + expect(PROJECTOR_FORMATS).not.toContain('application/pdf'); + }); + + it('is refused by the image normalizer, exactly as an unknown format is', async () => { + await expect(normalizeImage(pdf, {})).rejects.toThrow(); + }); +}); + describe('KNOWN DEFECT — pinned, not endorsed', () => { it('DEFECT: metadata retention depends on whether the image hit the ceiling', async () => { // Re-encoding strips EXIF/ICC (sharp's default); the pass-through path diff --git a/packages/media/test/pdf-ingress.test.ts b/packages/media/test/pdf-ingress.test.ts new file mode 100644 index 00000000..6e0f3a21 --- /dev/null +++ b/packages/media/test/pdf-ingress.test.ts @@ -0,0 +1,126 @@ +/** + * The document ingress against real fixtures: the shape it commits, identity, + * the bounds, and that nothing is written on abort or timeout. + * + * @category Testing + */ +import { describe, it, expect, vi } from 'vitest'; +import { readFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { FileAttachmentStore } from '../src/node'; +import { createDocumentIngress, MAX_DOCUMENT_BYTES, PdfError } from '../src/pdf'; +import { DOCUMENT_CONFIG_TYPE, asDocumentMeta } from '../src/document'; +import { representationsOf, sourceOf } from '../src/attachment'; +import { materialize } from '../src/ingress'; +import type { DocumentMeta } from '../src/document'; + +const fixture = (name: string): Uint8Array => new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); +const freshStore = (): FileAttachmentStore => new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'pdf-ingress-'))); + +/** The committed sidecar of a document root. */ +function metaOf(store: FileAttachmentStore, digest: string): DocumentMeta { + const manifest = store.getManifest(digest)!; + expect(manifest.config.mediaType).toBe(DOCUMENT_CONFIG_TYPE); + const meta = asDocumentMeta(JSON.parse(new TextDecoder().decode(store.get(manifest.config.digest)!))); + if (!meta) throw new Error('sidecar did not validate'); + return meta; +} + +describe('createDocumentIngress — the shape', () => { + it('commits one text/markdown representation, the PDF as source, and the sidecar as config', async () => { + const store = freshStore(); + const root = await createDocumentIngress(store).ingest(fixture('untagged.pdf')); + const manifest = store.getManifest(root.digest)!; + const reps = representationsOf(manifest); + expect(reps).toHaveLength(1); + expect(reps[0].mediaType).toBe('text/markdown'); + expect(sourceOf(manifest)?.mediaType).toBe('application/pdf'); + const meta = metaOf(store, root.digest); + expect(meta.pageCount).toBe(2); + expect(meta.title).toBe('An Untagged Report'); + // A document materializes to no bitmaps: its text is for retrieval, never the projector. + expect(materialize(store, [root]).bitmaps).toEqual([]); + }, 60_000); + + it('reads headings by size on an untagged page and maps sections to pages', async () => { + const store = freshStore(); + const root = await createDocumentIngress(store).ingest(fixture('untagged.pdf')); + const meta = metaOf(store, root.digest); + const md = new TextDecoder().decode(store.get(representationsOf(store.getManifest(root.digest)!)[0].digest)!); + expect(md).toContain('# An Untagged Report'); + expect(md).toContain('\n## First Section\n'); + expect(md).toContain('\n## Second Section\n'); + expect(md).toContain('Body text line one of the first section, set in the body size. Body text line two of the first section continues the paragraph.'); + const first = meta.sections.find((s) => s.heading === 'First Section')!; + expect(first.origin).toBe('heuristic'); + expect([first.pageStart, first.pageEnd]).toEqual([1, 1]); + expect(meta.pages.map((p) => p.page)).toEqual([1, 2]); + expect(meta.pages[1].chars).toBeGreaterThan(0); + }, 60_000); + + it('yields the same root for the same bytes, twice, in two stores', async () => { + const a = await createDocumentIngress(freshStore()).ingest(fixture('untagged.pdf')); + const b = await createDocumentIngress(freshStore()).ingest(fixture('untagged.pdf')); + expect(a.digest).toBe(b.digest); + }, 60_000); +}); + +describe('createDocumentIngress — text at size one under a scaled matrix', () => { + it('measures size and spacing from the advance boxes, so words stay whole and the title is the big line', async () => { + // PDFium reports font size 1 for every glyph of matrix.pdf; a gap rule + // scaled by that size calls every glyph gap a space. The loose box carries + // the real advance and em height. + const store = freshStore(); + const root = await createDocumentIngress(store).ingest(fixture('matrix.pdf')); + const meta = metaOf(store, root.digest); + expect(meta.title).toBe('Scaled Title Line'); + const md = new TextDecoder().decode(store.get(representationsOf(store.getManifest(root.digest)!)[0].digest)!); + expect(md).toContain('# Scaled Title Line'); + expect(md).toContain('Body text set at size one under a scaled matrix. Second line of body text with several words.'); + expect(md).not.toMatch(/B o d y/); + }, 60_000); +}); + +describe('createDocumentIngress — the bounds', () => { + it('refuses a document over the byte ceiling before touching the codec', async () => { + const store = freshStore(); + const big = new Uint8Array(MAX_DOCUMENT_BYTES + 1); + big.set(new TextEncoder().encode('%PDF-1.4')); + await expect(createDocumentIngress(store).ingest(big)).rejects.toThrow(/ceiling/); + }); + + it('names a password-protected document and a malformed one', async () => { + const store = freshStore(); + await expect(createDocumentIngress(store).ingest(fixture('encrypted.pdf'))).rejects.toThrow(/password/i); + await expect(createDocumentIngress(store).ingest(new TextEncoder().encode('%PDF-1.4 not really'))).rejects.toThrow(PdfError); + }); + + it('commits nothing when the caller gives up while the document is being read', async () => { + const store = freshStore(); + const putBlob = vi.spyOn(store, 'putBlob'); + const putAttachment = vi.spyOn(store, 'putAttachment'); + const ctrl = new AbortController(); + const pending = createDocumentIngress(store).ingest(fixture('tagged.pdf'), ctrl.signal); + // The ingress yields to the loop between pages; the abort lands there. + setImmediate(() => ctrl.abort()); + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }); + expect(putBlob).not.toHaveBeenCalled(); + expect(putAttachment).not.toHaveBeenCalled(); + }, 60_000); + + it('publishes nothing when the time bound bites, and says it timed out', async () => { + const store = freshStore(); + const putAttachment = vi.spyOn(store, 'putAttachment'); + await expect(createDocumentIngress(store, { timeoutMs: 1 }).ingest(fixture('tagged.pdf'))) + .rejects.toMatchObject({ code: 'ETIMEDOUT' }); + expect(putAttachment).not.toHaveBeenCalled(); + }, 60_000); + + it('serves five concurrent ingests of one document through the shared gate to one root', async () => { + const store = freshStore(); + const ingress = createDocumentIngress(store); + const roots = await Promise.all(Array.from({ length: 5 }, () => ingress.ingest(fixture('untagged.pdf')))); + expect(new Set(roots.map((r) => r.digest)).size).toBe(1); + }, 120_000); +}); diff --git a/packages/media/test/pdf-layout.test.ts b/packages/media/test/pdf-layout.test.ts new file mode 100644 index 00000000..ee0e7678 --- /dev/null +++ b/packages/media/test/pdf-layout.test.ts @@ -0,0 +1,340 @@ +/** + * The layout heuristics, specified on synthetic character records. + * + * Nothing here touches the codec: a page is a list of characters with boxes, + * sizes and weights in PDF user space (origin bottom-left, y up), plus what + * the page's objects and structure tree said. That is the whole input, so the + * rules are stated where they can be read. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { layoutDocument } from '../src/pdf-layout'; +import type { PageChar, PageFacts } from '../src/pdf-layout'; + +/** Lay a string of characters on one baseline. Each glyph is `size * 0.5` wide. */ +function line(text: string, y: number, opts: { x?: number; size?: number; weight?: number; mcid?: number } = {}): PageChar[] { + const size = opts.size ?? 10; + const w = size * 0.5; + let x = opts.x ?? 72; + const out: PageChar[] = []; + for (const ch of text) { + out.push({ text: ch, x0: x, y0: y, x1: x + w, y1: y + size, size, weight: opts.weight ?? 400, mcid: opts.mcid ?? -1 }); + x += w; + } + return out; +} + +function page(n: number, chars: PageChar[], extra: Partial = {}): PageFacts { + return { page: n, width: 612, height: 792, chars, images: [], pathObjects: 0, textExtracted: true, ...extra }; +} + +const OPTS = { title: null, bookmarks: [], maxTextPages: 400, dpi: 150, minFigureSidePx: 64, minFigureAreaRatio: 0.02 }; + +describe('lines and paragraphs', () => { + it('groups characters by baseline into lines and joins lines of one block into a paragraph', () => { + const chars = [ + ...line('First line of the paragraph.', 700), + ...line('Second line of the same paragraph.', 686), + ...line('A new paragraph after a gap.', 640), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('First line of the paragraph. Second line of the same paragraph.'); + expect(r.markdown).toContain('\n\nA new paragraph after a gap.'); + }); + + it('joins a hyphenated line break without the hyphen when the next line starts lowercase', () => { + const chars = [...line('This is a hyphen-', 700), ...line('ated word here.', 686)]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('hyphenated word here.'); + }); + + it('inserts a space where glyphs leave a gap on one baseline', () => { + const chars = [...line('left', 700, { x: 72 }), ...line('right', 700, { x: 200 })]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('left right'); + }); +}); + +describe('headings and the title, by size and weight', () => { + const doc = () => [ + ...line('A Report Title', 720, { size: 18 }), + ...line('First Section', 688, { size: 14 }), + ...line('Body text in the first section, long enough to be body.', 670), + ...line('Body text continues for a second line here.', 656), + ...line('Subheading', 630, { size: 12 }), + ...line('More body text under the subheading.', 612), + ...line('Bold lead', 590, { weight: 700 }), + ...line('Body after a bold short line.', 576), + ]; + + it('takes the largest line on page one as the title when metadata has none', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, doc())] }); + expect(r.title).toBe('A Report Title'); + expect(r.markdown.startsWith('# A Report Title\n')).toBe(true); + }); + + it('trusts a metadata title only when a heading-size line on page one carries it', () => { + // A producer's Title field is often junk — here an e-mail that also runs + // in the footer at body size. The page's largest line is the title. + const chars = [...doc(), ...line('someone@example.com', 40, { size: 8 })]; + const r = layoutDocument({ ...OPTS, title: 'someone@example.com', pages: [page(1, chars)] }); + expect(r.title).toBe('A Report Title'); + expect(r.markdown.startsWith('# A Report Title\n')).toBe(true); + }); + + it('takes a metadata title the page carries, and does not repeat it as a heading', () => { + const r = layoutDocument({ ...OPTS, title: 'A Report Title', pages: [page(1, doc())] }); + expect(r.title).toBe('A Report Title'); + expect(r.markdown.match(/^# /gm)).toHaveLength(1); + expect(r.markdown).not.toMatch(/^## A Report Title$/m); + }); + + it('keeps a metadata title only for a page with no text to corroborate it', () => { + const r = layoutDocument({ ...OPTS, title: 'From Metadata', pages: [page(1, [], { images: [{ index: 0, bbox: [0, 0, 612, 792] }] })] }); + expect(r.title).toBe('From Metadata'); + }); + + it('folds a metadata title that wraps over two lines into the title, not into headings', () => { + // Chrome prints a long as two 22pt lines; each is a piece of the + // metadata title, neither equals it. They are the title, so the real + // sections stay top-level and the table of contents keeps its topics. + const chars = [ + ...line('Continuous Tree Batching: A', 720, { size: 22 }), + ...line('Measurement Note', 692, { size: 22 }), + ...line('Abstract', 660, { size: 14 }), + ...line('We measure how the cost of a run scales.', 642), + ...line('Method', 610, { size: 14 }), + ...line('Each configuration runs the same question.', 592), + ]; + const r = layoutDocument({ ...OPTS, title: 'Continuous Tree Batching: A Measurement Note', pages: [page(1, chars)] }); + expect(r.markdown.match(/^# /gm)).toHaveLength(1); + expect(r.markdown).not.toMatch(/^#+ Continuous Tree Batching: A$/m); + expect(r.markdown).not.toMatch(/^#+ Measurement Note$/m); + expect(r.sections.map((s) => [s.heading, s.path])).toEqual([ + ['Continuous Tree Batching: A Measurement Note', 'Continuous Tree Batching: A Measurement Note'], + ['Abstract', 'Abstract'], + ['Method', 'Method'], + ]); + }); + + it('ranks heading levels by size and treats a bold short body-size line as the lowest level', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, doc())] }); + expect(r.markdown).toContain('\n## First Section\n'); + expect(r.markdown).toContain('\n### Subheading\n'); + expect(r.markdown).toContain('\n#### Bold lead\n'); + expect(r.sections.map((s) => [s.heading, s.path, s.origin])).toEqual([ + ['A Report Title', 'A Report Title', 'heuristic'], + ['First Section', 'First Section', 'heuristic'], + ['Subheading', 'First Section > Subheading', 'heuristic'], + ['Bold lead', 'First Section > Subheading > Bold lead', 'heuristic'], + ]); + }); + + it('gives every section a contiguous line span that tiles the markdown', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, doc())] }); + const lines = r.markdown.split('\n'); + let next = 1; + for (const s of r.sections) { + expect(s.startLine).toBe(next); + expect(lines[s.startLine - 1]).toMatch(new RegExp(`^#+ ${s.heading}$`)); + expect(s.endLine).toBeGreaterThanOrEqual(s.startLine); + next = s.endLine + 1; + } + expect(next - 1).toBe(lines.length); + }); +}); + +describe('separators and wrapped titles', () => { + it('a synthesised space with no box is a separator, never a line break', () => { + // PDFium inserts a space between two text runs on one baseline and gives it + // an empty box. The reader hands it the previous glyph's geometry; the + // layout treats any whitespace as a separator — either way one line. + const chars = [ + ...line('2.1', 700, { size: 13 }), + { text: ' ', x0: 61, y0: 702.7, x1: 61, y1: 702.7, size: 1, weight: 400, mcid: -1 }, + ...line('Structure-Activity Relationship', 700, { x: 96, size: 13 }), + ...line('Body text under the heading, long enough to be body text.', 680), + ...line('And a second body line to settle the body size.', 666), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.markdown).toContain('2.1 Structure-Activity Relationship'); + expect(r.markdown).not.toMatch(/^#+ 2\.1$/m); + }); + + it('a ligature drawn from a fallback font with a small em box stays in its word and its line', () => { + // Chrome sets "fi" from another font: an 8pt-tall advance box on a 20pt + // line, same baseline. Grouping by baseline keeps "Configuration" whole. + const big = line('Con', 700, { size: 20 }); + const fi = { text: 'fi', x0: big[big.length - 1].x1 + 0.2, y0: 702, x1: big[big.length - 1].x1 + 3.4, y1: 710.4, size: 8.4, baseline: 700, weight: 400, mcid: -1 }; + const rest = line('guration matters here.', 700, { x: fi.x1 + 0.2, size: 20 }); + const body = [...line('Body text sets the body size for the page here.', 670), ...line('A second body line, so the heading rank has a body.', 656)]; + const r = layoutDocument({ ...OPTS, pages: [page(1, [...big.map((c) => ({ ...c, baseline: 700 })), fi, ...rest.map((c) => ({ ...c, baseline: 700 })), ...body])] }); + expect(r.markdown).toContain('Configuration matters here.'); + expect(r.markdown).not.toContain('Con fi'); + }); + + it('the title is the run of largest lettered lines, not a bare chapter number set larger', () => { + const chars = [ + ...line('2', 730, { size: 30 }), + ...line('Topical Corticosteroids:', 700, { size: 20 }), + ...line('Pharmacology', 676, { size: 20 }), + ...line('Gagandeep Kwatra and Sandip Mukhopadhyay', 650, { size: 12 }), + ...line('Abstract Topical corticosteroids are widely used for inflammatory disorders.', 630), + ...line('They are available in a number of formulations for the skin.', 616), + ]; + const r = layoutDocument({ ...OPTS, title: 'kwatragagandeep@gmail.com', pages: [page(1, chars)] }); + expect(r.title).toBe('Topical Corticosteroids: Pharmacology'); + expect(r.markdown.match(/^# /gm)).toHaveLength(1); + expect(r.markdown).not.toMatch(/^#+ Pharmacology$/m); + // The bare chapter number is not a heading either. + expect(r.markdown).not.toMatch(/^#+ 2$/m); + expect(r.sections.map((s) => s.heading)).toEqual(['Topical Corticosteroids: Pharmacology', 'Gagandeep Kwatra and Sandip Mukhopadhyay']); + }); + + it('orders a line by position when the producer emits its words out of sequence', () => { + // "DELTA" is drawn after "RULE" though it sits between "WITH" and "RULE"; + // and the title contains "DELTA", which must not swallow the subtitle's. + // 20 glyphs of 7pt from x=72 end at 212; DELTA sits at 220..255, RULE after it. + const a = line('IMPROVING MAMBA WITH', 700, { size: 14 }); + const rule = line('RULE', 700, { x: 262, size: 14 }); + const delta = line('DELTA', 700, { x: 220, size: 14 }); + const chars = [ + ...line('GATED DELTA NETWORKS', 730, { size: 17 }), + ...a, ...rule, ...delta, + ...line('Body text of the abstract, long enough to be body text here.', 670), + ...line('And a second body line to settle the body size of the page.', 656), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.title).toBe('GATED DELTA NETWORKS'); + expect(r.markdown).toContain('\n## IMPROVING MAMBA WITH DELTA RULE\n'); + }); + + it('a small-caps heading is a heading: its capitals are set larger than the rest', () => { + const chars = [ + ...line('A Report Title', 730, { size: 18 }), + ...line('1 I', 700, { size: 12 }), + ...line('NTRODUCTION', 700, { x: 72 + 3 * 6, size: 9.6 }), + ...line('Body text under the small-caps heading, long enough to be body.', 680), + ...line('And a second body line to settle the body size of the page.', 666), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.sections.map((s) => s.heading)).toEqual(['A Report Title', '1 INTRODUCTION']); + }); + + it('a heading wrapped over two adjacent lines is one heading', () => { + const chars = [ + ...line('A Report Title', 730, { size: 18 }), + ...line('2.1 Structure-Activity', 700, { size: 13 }), + ...line('Relationship', 685, { size: 13 }), + ...line('Body text under the wrapped heading, long enough to be body text.', 665), + ...line('And a second body line to settle the body size of the page.', 651), + ...line('2.2 Potency', 620, { size: 13 }), + ...line('More body text under the second heading of the page here.', 602), + ]; + const r = layoutDocument({ ...OPTS, pages: [page(1, chars)] }); + expect(r.sections.map((s) => s.heading)).toEqual(['A Report Title', '2.1 Structure-Activity Relationship', '2.2 Potency']); + expect(r.markdown).toContain('\n## 2.1 Structure-Activity Relationship\n'); + expect(r.markdown).not.toMatch(/^## Relationship$/m); + }); +}); + +describe('pages, counts and the page map', () => { + it('records one span per page, contiguous across pages, and the per-page counts', () => { + const p1 = page(1, [...line('Page one body text.', 700)], { images: [{ index: 0, bbox: [72, 300, 372, 500] }], pathObjects: 3 }); + const p2 = page(2, [...line('Page two body text.', 700)]); + const r = layoutDocument({ ...OPTS, pages: [p1, p2] }); + expect(r.pages.map((p) => [p.page, p.chars, p.imageObjects, p.pathObjects])).toEqual([[1, 19, 1, 3], [2, 19, 0, 0]]); + expect(r.pages[1].startLine).toBe(r.pages[0].endLine + 1); + const total = r.markdown.split('\n').length; + expect(r.pages[1].endLine).toBe(total); + }); + + it('emits a note for a page with no characters and counts zero chars', () => { + const r = layoutDocument({ ...OPTS, pages: [page(1, [], { images: [{ index: 0, bbox: [0, 0, 612, 792] }] })] }); + expect(r.markdown).toContain('*[page 1: no extractable text — scanned image]*'); + expect(r.pages[0].chars).toBe(0); + expect(r.pages[0].imageObjects).toBe(1); + }); + + it('stops extracting text past the page cap, says so, and marks the result truncated', () => { + const pages = [1, 2, 3].map((n) => page(n, line(`Text of page ${n}.`, 700), { textExtracted: n <= 2 })); + const r = layoutDocument({ ...OPTS, maxTextPages: 2, pages }); + expect(r.truncated).toBe(true); + expect(r.markdown).toContain('*[pages 3–3 not extracted: page limit]*'); + expect(r.markdown).not.toContain('Text of page 3.'); + expect(r.pages).toHaveLength(3); + }); +}); + +describe('figures and captions', () => { + it('keeps an image object as a figure when it is large enough, anchored to a nearby caption line', () => { + const chars = [ + ...line('Some text above the figure.', 700), + ...line('Figure 1. The green rectangle.', 280), + ]; + const p = page(1, chars, { images: [{ index: 3, bbox: [72, 300, 372, 500] }, { index: 4, bbox: [500, 700, 520, 720] }] }); + const r = layoutDocument({ ...OPTS, pages: [p] }); + expect(r.figures).toEqual([{ page: 1, index: 3, bbox: [72, 300, 372, 500], caption: 'Figure 1. The green rectangle.' }]); + expect(r.pages[0].imageObjects).toBe(2); + }); + + it('falls back to the structure tree alt text when no caption line is near', () => { + const p = page(1, line('Unrelated text far away.', 750), { + images: [{ index: 0, bbox: [72, 100, 372, 300], altText: 'A chart of cells per image' }], + }); + const r = layoutDocument({ ...OPTS, pages: [p] }); + expect(r.figures[0].caption).toBe('A chart of cells per image'); + }); +}); + +describe('a tagged page', () => { + it('takes heading levels from structure roles, emits a tagged table as a pipe table, and reports coverage', () => { + const roles = new Map<number, string>([[0, 'H1'], [1, 'H2'], [2, 'P'], [3, 'TH'], [4, 'TH'], [5, 'TD'], [6, 'TD'], [7, 'P']]); + const chars = [ + ...line('A Tagged Paper', 720, { size: 18, mcid: 0 }), + ...line('Results', 690, { size: 12, mcid: 1 }), + ...line('Table 1 reports the values.', 672, { mcid: 2 }), + ...line('Config', 640, { x: 72, mcid: 3 }), ...line('Cells', 640, { x: 200, mcid: 4 }), + ...line('baseline', 624, { x: 72, mcid: 5 }), ...line('803', 624, { x: 200, mcid: 6 }), + ...line('Closing paragraph.', 590, { mcid: 7 }), + ]; + const p = page(1, chars, { + struct: { + roleByMcid: roles, + tables: [{ mcids: [3, 4, 5, 6], rows: [{ cells: [{ mcids: [3] }, { mcids: [4] }] }, { cells: [{ mcids: [5] }, { mcids: [6] }] }] }], + figures: 0, + }, + }); + const r = layoutDocument({ ...OPTS, title: 'A Tagged Paper', pages: [p] }); + expect(r.tagged).toBe(true); + expect(r.structCoverage).toBeGreaterThanOrEqual(0.9); + // H1 equals the title: emitted once as the title, not again as a heading. + expect(r.markdown.match(/A Tagged Paper/g)).toHaveLength(1); + expect(r.markdown).toContain('\n### Results\n'); + expect(r.markdown).toContain('| Config | Cells |\n| --- | --- |\n| baseline | 803 |'); + expect(r.tables).toHaveLength(1); + expect(r.pages[0].taggedTables).toBe(1); + expect(r.sections.find((s) => s.heading === 'Results')?.origin).toBe('struct'); + }); +}); + +describe('bookmarks', () => { + it('takes a heading\'s path from the bookmark chain when a bookmark matches it', () => { + const chars = [ + ...line('Doc', 720, { size: 18 }), + ...line('Methods', 690, { size: 14 }), + ...line('Body of methods.', 672), + ...line('Sampling', 640, { size: 12 }), + ...line('Body of sampling.', 622), + ]; + const bookmarks = [ + { title: 'Methods', page: 1, level: 0 }, + { title: 'Sampling', page: 1, level: 1 }, + ]; + const r = layoutDocument({ ...OPTS, bookmarks, pages: [page(1, chars)] }); + const s = r.sections.find((x) => x.heading === 'Sampling')!; + expect(s.path).toBe('Methods > Sampling'); + expect(s.origin).toBe('bookmark'); + }); +}); diff --git a/packages/media/test/pdf-render.test.ts b/packages/media/test/pdf-render.test.ts new file mode 100644 index 00000000..4834071e --- /dev/null +++ b/packages/media/test/pdf-render.test.ts @@ -0,0 +1,167 @@ +/** + * Page renders and figure crops: the archive, the projection facts, the tagged + * structure, and the scanned page — against real fixtures. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { readFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import sharp from 'sharp'; +import { FileAttachmentStore } from '../src/node'; +import { createDocumentIngress } from '../src/pdf'; +import { normalizeImage } from '../src/image'; +import { asDocumentMeta } from '../src/document'; +import { RENDER_MAX_SIDE } from '../src/pdf'; +import { DEFAULT_MAX_PIXELS } from '../src/image'; +import { representationsOf } from '../src/attachment'; +import { materialize } from '../src/ingress'; +import type { DocumentMeta } from '../src/document'; + +const fixture = (name: string): Uint8Array => new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); +const freshStore = (): FileAttachmentStore => new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'pdf-render-'))); + +async function ingest(name: string, opts: Parameters<typeof createDocumentIngress>[1] = {}) { + const store = freshStore(); + const root = await createDocumentIngress(store, opts).ingest(fixture(name)); + const manifest = store.getManifest(root.digest)!; + const meta = asDocumentMeta(JSON.parse(new TextDecoder().decode(store.get(manifest.config.digest)!)))!; + const markdown = new TextDecoder().decode(store.get(representationsOf(manifest)[0].digest)!); + return { store, root, meta, markdown }; +} + +describe('the archive', () => { + it('renders every page within the bound as its own image root the projector can read, byte-stable under the normalizer', async () => { + const { store, meta } = await ingest('tagged.pdf'); + expect(meta.pages.map((p) => !!p.render)).toEqual([true, true, true]); + for (const p of meta.pages) { + const prepared = materialize(store, [p.render!] as never); + expect(prepared.bitmaps).toHaveLength(1); + const png = prepared.bitmaps[0]; + const norm = await normalizeImage(png, {}); + expect(norm.derived).toBe(false); + expect(norm.bytes).toBe(png); + expect(norm.mime).toBe('image/png'); + } + expect(meta.derive.renderedPages).toBe(3); + expect(meta.derive.truncated).toBe(false); + }, 90_000); + + it('records per-page facts: the image page and the table page carry objects, the text page does not', async () => { + const { meta } = await ingest('tagged.pdf'); + const [p1, p2, p3] = meta.pages; + expect(p1.chars).toBeGreaterThan(100); + expect(p1.imageObjects).toBe(0); + expect(p2.taggedTables + p2.pathObjects).toBeGreaterThan(0); + expect(p3.imageObjects).toBeGreaterThan(0); + }, 90_000); + + it('chooses graphics-bearing pages first under a page cap, marks the result truncated, and stays deterministic', async () => { + const a = await ingest('tagged.pdf', { maxRenderedPages: 2 }); + const rendered = a.meta.pages.filter((p) => p.render).map((p) => p.page); + // Page one always. Pages two (a tagged, rule-drawn table) and three (an + // image) are both graphics-bearing, so the tie falls to page order and + // the cap keeps page two; the text-only page would lose to either. + expect(rendered).toEqual([1, 2]); + expect(a.meta.derive.truncated).toBe(true); + const b = await ingest('tagged.pdf', { maxRenderedPages: 2 }); + expect(b.root.digest).toBe(a.root.digest); + const one = await ingest('tagged.pdf', { maxRenderedPages: 1 }); + expect(one.meta.pages.filter((p) => p.render).map((p) => p.page)).toEqual([1]); + }, 120_000); +}); + +describe('the tagged path', () => { + it('takes sections from the structure tree, emits the table as pipe rows, paths from bookmarks, and reports coverage', async () => { + const { meta, markdown } = await ingest('tagged.pdf'); + expect(meta.derive.tagged).toBe(true); + expect(meta.derive.structCoverage).toBeGreaterThanOrEqual(0.9); + expect(markdown).toContain('# A Tagged Paper'); + expect(markdown).toMatch(/\n### Introduction\n|\n## Introduction\n/); + expect(markdown).toContain('| Configuration | Cells | Seconds |'); + expect(markdown).toContain('| baseline | 803 | 4.2 |'); + expect(meta.tables).toHaveLength(1); + const results = meta.sections.find((s) => s.heading === 'Results')!; + expect(['struct', 'bookmark']).toContain(results.origin); + expect(results.pageStart).toBe(2); + }, 90_000); + + it('crops the figure with its caption and the crop is the green rectangle, not the page', async () => { + const { store, meta } = await ingest('tagged.pdf'); + expect(meta.figures).toHaveLength(1); + const fig = meta.figures[0]; + expect(fig.page).toBe(3); + expect(fig.caption).toMatch(/^Figure 1\./); + const png = materialize(store, [fig.root] as never).bitmaps[0]; + const stats = await sharp(Buffer.from(png)).stats(); + const [r, g, b] = stats.channels.map((c) => c.mean); + // #0a7 is rgb(0, 170, 119): green dominant, red near zero. + expect(g).toBeGreaterThan(120); + expect(r).toBeLessThan(60); + expect(b).toBeGreaterThan(60); + }, 90_000); +}); + +describe('the scanned page', () => { + it('says the page has no extractable text, counts zero chars, and still archives the render', async () => { + const { meta, markdown } = await ingest('scanned.pdf'); + expect(markdown).toContain('*[page 1: no extractable text — scanned image]*'); + expect(meta.pages[0].chars).toBe(0); + expect(meta.pages[0].imageObjects).toBe(1); + expect(meta.pages[0].render).toBeTruthy(); + }, 60_000); +}); + +describe('the bounds on figures', () => { + /** The crop's own record of its size: the derive annotations on its representation. */ + function cropSize(store: FileAttachmentStore, meta: DocumentMeta, i = 0): { width: number; height: number } { + const manifest = store.getManifest(meta.figures[i].root.digest)!; + const a = manifest.layers[0].annotations ?? {}; + return { width: Number(a['ai.lloyal.derive.width']), height: Number(a['ai.lloyal.derive.height']) }; + } + + it('an image painted far past the page is cropped to its visible part, under the same ceilings as a page', async () => { + // A 1×1 red image into a 2000-pt square on a 100-pt page: the object's + // bounds are 20× the page; only the page is visible. + const { store, meta } = await ingest('clipped.pdf'); + expect(meta.pages[0].imageObjects).toBe(1); + expect(meta.figures).toHaveLength(1); + const [x0, y0, x1, y1] = meta.figures[0].bbox; + expect([x0, y0]).toEqual([0, 0]); + expect(x1).toBeCloseTo(100, 0); + expect(y1).toBeCloseTo(100, 0); + const { width, height } = cropSize(store, meta); + expect(Math.max(width, height)).toBeLessThanOrEqual(RENDER_MAX_SIDE); + expect(width * height).toBeLessThanOrEqual(DEFAULT_MAX_PIXELS); + // The crop is the visible page, so it is no larger than the page render. + const page = store.getManifest(meta.pages[0].render!.digest)!.layers[0].annotations ?? {}; + expect(width).toBeLessThanOrEqual(Number(page['ai.lloyal.derive.width'])); + const png = materialize(store, [meta.figures[0].root] as never).bitmaps[0]; + const [r, g] = (await sharp(Buffer.from(png)).stats()).channels.map((c) => c.mean); + expect(r).toBeGreaterThan(200); + expect(g).toBeLessThan(40); + }, 90_000); + + it('an image inside a Form XObject is counted, placed on the page through the form matrices, and cropped', async () => { + // A 2×2 green image in a form with /Matrix translate (10,10), drawn under + // an outer translate (40,40): page space [50,50]–[150,150] on a 200-pt page. + const { store, meta } = await ingest('form.pdf'); + expect(meta.pages[0].imageObjects).toBe(1); + expect(meta.figures).toHaveLength(1); + const [x0, y0, x1, y1] = meta.figures[0].bbox; + expect(x0).toBeCloseTo(50, 0); expect(y0).toBeCloseTo(50, 0); + expect(x1).toBeCloseTo(150, 0); expect(y1).toBeCloseTo(150, 0); + const png = materialize(store, [meta.figures[0].root] as never).bitmaps[0]; + const [r, g, b] = (await sharp(Buffer.from(png)).stats()).channels.map((c) => c.mean); + expect(g).toBeGreaterThan(120); + expect(r).toBeLessThan(60); + expect(b).toBeGreaterThan(60); + }, 90_000); + + it('an image over the pixel ceiling is refused before anything renders — direct, and nested in a form alike', async () => { + await expect(ingest('bigimage.pdf', { maxRenderedPages: 0 })).rejects.toThrow(/pixel ceiling/); + await expect(ingest('bigform.pdf', { maxRenderedPages: 0 })).rejects.toThrow(/pixel ceiling/); + }, 90_000); +}); + diff --git a/packages/media/test/pdfium-load.test.ts b/packages/media/test/pdfium-load.test.ts new file mode 100644 index 00000000..3c4b9517 --- /dev/null +++ b/packages/media/test/pdfium-load.test.ts @@ -0,0 +1,63 @@ +/** + * The codec loads offline, once compiled, one instance per document. + * + * This is the step that proves the dependency: the wasm ships in the package + * and is read from disk (no CDN), the module compiles once and instantiates + * per document through the glue's `instantiateWasm` hook, and a document + * opened in one instance is invisible to another. Everything above this in + * the ingress rests on these three facts. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { createCodec, openDocument, PdfError } from '../src/pdf'; + +const fixture = (name: string): Uint8Array => + new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'pdf', name))); + +describe('createCodec', () => { + it('loads the wasm from the package on disk and answers a page count', async () => { + const codec = await createCodec(); + try { + const doc = openDocument(codec, fixture('untagged.pdf')); + try { + expect(doc.pageCount).toBe(2); + } finally { + doc.close(); + } + } finally { + codec.dispose(); + } + }); + + it('gives each document its own instance: a handle from one is meaningless in another', async () => { + const a = await createCodec(); + const b = await createCodec(); + try { + expect(a.pdfium).not.toBe(b.pdfium); + const doc = openDocument(a, fixture('untagged.pdf')); + try { + // The same handle number in the other instance is not a document. + expect(b.pdfium.FPDF_GetPageCount(doc.handle)).not.toBe(2); + } finally { + doc.close(); + } + } finally { + a.dispose(); + b.dispose(); + } + }, 30_000); + + it('names the reason a document cannot be opened', async () => { + const codec = await createCodec(); + try { + expect(() => openDocument(codec, fixture('encrypted.pdf'))).toThrow(PdfError); + expect(() => openDocument(codec, fixture('encrypted.pdf'))).toThrow(/password/i); + expect(() => openDocument(codec, new TextEncoder().encode('%PDF-1.4 not really'))).toThrow(/format|malformed/i); + } finally { + codec.dispose(); + } + }); +}); diff --git a/packages/media/tsconfig.json b/packages/media/tsconfig.json index 152ea8e0..dc561f10 100644 --- a/packages/media/tsconfig.json +++ b/packages/media/tsconfig.json @@ -2,7 +2,11 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "outDir": "dist", - "rootDir": "src" + "rootDir": "src", + "lib": [ + "ES2022", + "DOM" + ] }, "include": [ "src/**/*.ts" diff --git a/packages/abilities/corpus/src/bm25.ts b/packages/rig/src/bm25.ts similarity index 100% rename from packages/abilities/corpus/src/bm25.ts rename to packages/rig/src/bm25.ts diff --git a/packages/rig/src/content-routes.ts b/packages/rig/src/content-routes.ts index 681aafc3..0ba2af3a 100644 --- a/packages/rig/src/content-routes.ts +++ b/packages/rig/src/content-routes.ts @@ -1,5 +1,5 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; -import { representationsOf, DIGEST_PATTERN, type Attachment } from '@lloyal-labs/media'; +import { representationsOf, sourceOf, DIGEST_PATTERN, MANIFEST_TYPE, type Attachment } from '@lloyal-labs/media'; import type { AttachmentStore, Descriptor } from '@lloyal-labs/media'; /** Thrown when a body exceeds the cap, so the caller can answer 413 rather @@ -48,7 +48,10 @@ export interface ContentRoutesOpts { * that question, and the ingress is where they are decoded. */ ingest?: (bytes: Uint8Array, signal?: AbortSignal) => Promise<Attachment>; - /** Ceiling on a single upload body. @default 8 MiB */ + /** Ceiling on a single upload body. @default 8 MiB — sized for an image. A + * host that installs the content ingress passes `MAX_DOCUMENT_BYTES` from + * `@lloyal-labs/media/node` beside `DOCUMENT_UPLOAD_TIMEOUT_MS`, so every + * host that mounts the plane admits the same. */ maxUploadBytes?: number; /** * Ceiling on how long one upload may take, end to end. @@ -86,7 +89,10 @@ export interface ContentRoutesOpts { * else it serves. * * ``` - * POST /v1/media/ingress upload → normalize → root descriptor + * POST /v1/media/ingress upload → admit → root descriptor + * GET /v1/media/<manifest> the manifest itself, by digest + * GET /v1/media/<manifest>/config its typed config blob (a document's sidecar; OCI's empty blob for an image) + * GET /v1/media/<manifest>/source the original as supplied, when the ingest retained it * GET /v1/media/<manifest>/representations/<i> the bytes the model actually saw * HEAD /v1/content/<digest> existence, for pre-flight dedupe * ``` @@ -344,7 +350,12 @@ export function createContentRoutes( // A full normalization queue is overload: retryable, not a client // fault and not ours. `EBUSY` is the errno the ingress sets for it. const busy = typeof e === 'object' && e !== null && (e as { code?: unknown }).code === 'EBUSY'; - const code = tooLarge ? 413 : tooSlow ? 408 : busy ? 503 : 400; + // The ingress's own time bound is the same class as a slow upload. + const timedOut = typeof e === 'object' && e !== null && (e as { code?: unknown }).code === 'ETIMEDOUT'; + // An error carrying a syscall came from the filesystem under the + // store — disk full, read-only volume. That is ours, not the client's. + const ours = typeof e === 'object' && e !== null && typeof (e as { syscall?: unknown }).syscall === 'string'; + const code = tooLarge ? 413 : (tooSlow || timedOut) ? 408 : busy ? 503 : ours ? 500 : 400; fail(res, code, e instanceof Error ? e.message : 'ingress failed'); // Now that the status is on the wire, stop the upload. A stalled // client will not close on its own — that is the whole problem — @@ -355,6 +366,48 @@ export function createContentRoutes( return true; } + // GET /v1/media/<manifest> — the manifest itself, by digest. A manifest is + // a blob, and serving it by digest is what any OCI reader expects; it is + // how a UI learns what an attachment IS (its config media type) and what + // roots it names, without a "kind" field anywhere on the wire. + const man = /^\/v1\/media\/([^/]+)$/.exec(path); + if (man && (method === 'GET' || method === 'HEAD')) { + const digest = decodeSegment(man[1]); + if (digest === null || !DIGEST_PATTERN.test(digest)) { fail(res, 400, 'malformed digest'); return true; } + if (!opts.store.getManifest(digest)) { fail(res, 404, 'no such attachment manifest'); return true; } + serveBlob(req, res, { mediaType: MANIFEST_TYPE, digest, size: 0 }, method === 'HEAD'); + return true; + } + + // GET /v1/media/<manifest>/config — the typed config blob, resolved + // THROUGH the manifest the way representations are. A document's + // sidecar lives here; an image's config is OCI's canonical empty blob. + const cfg = /^\/v1\/media\/([^/]+)\/config$/.exec(path); + if (cfg && (method === 'GET' || method === 'HEAD')) { + const digest = decodeSegment(cfg[1]); + if (digest === null || !DIGEST_PATTERN.test(digest)) { fail(res, 400, 'malformed digest'); return true; } + const manifest = opts.store.getManifest(digest); + if (!manifest) { fail(res, 404, 'no such attachment manifest'); return true; } + serveBlob(req, res, manifest.config, method === 'HEAD'); + return true; + } + + // GET /v1/media/<manifest>/source — the original as supplied, when the + // ingest retained it: a document's PDF, an image before normalization. + // Resolved THROUGH the manifest by ROLE, never by a blob digest a + // client typed — raw blobs stay HEAD-only. + const src = /^\/v1\/media\/([^/]+)\/source$/.exec(path); + if (src && (method === 'GET' || method === 'HEAD')) { + const digest = decodeSegment(src[1]); + if (digest === null || !DIGEST_PATTERN.test(digest)) { fail(res, 400, 'malformed digest'); return true; } + const manifest = opts.store.getManifest(digest); + if (!manifest) { fail(res, 404, 'no such attachment manifest'); return true; } + const source = sourceOf(manifest); + if (!source) { fail(res, 404, 'no source retained for this attachment'); return true; } + serveBlob(req, res, source, method === 'HEAD'); + return true; + } + fail(res, 405, 'unsupported method or path'); return true; } catch (e) { diff --git a/packages/rig/src/index.ts b/packages/rig/src/index.ts index 4f520ee6..7ec14ff9 100644 --- a/packages/rig/src/index.ts +++ b/packages/rig/src/index.ts @@ -38,6 +38,15 @@ export type { SourceContext } from './sources/types'; export { chunkFetchedPages, chunkHtml } from './sources/chunking'; export type { FetchedPage } from './sources/chunking'; +// Retrieval primitives shared by every source ability (pure TS — RN-safe). +export { BM25Index } from './bm25'; +export type { Bm25Opts, Bm25Hit } from './bm25'; +export { fitChunks, splitParagraphs, DEFAULT_CHUNK_TOKENS } from './resources/fit'; +export type { FitOpts } from './resources/fit'; +export { mergeRanges, subtractRanges } from './ranges'; +export { loadDocuments } from './resources/documents'; +export type { Document } from './resources/documents'; + // Resource types (pure TS — RN-safe) export type { Resource, Chunk } from './resources/types'; diff --git a/packages/rig/src/ranges.ts b/packages/rig/src/ranges.ts new file mode 100644 index 00000000..9db21d87 --- /dev/null +++ b/packages/rig/src/ranges.ts @@ -0,0 +1,46 @@ +/** + * @file Half-open line-range arithmetic. + * + * Shared by the read tools that remember which lines each agent has already + * been shown, so a second read returns only what is new. Pure; ranges are + * `[start, end)`. + */ + +/** + * The parts of `target` not covered by any range in `covered`, in order. + * + * @category Rig + */ +export function subtractRanges( + [s, e]: [number, number], + covered: readonly [number, number][], +): [number, number][] { + let ranges: [number, number][] = [[s, e]]; + for (const [cs, ce] of covered) { + ranges = ranges.flatMap(([a, b]): [number, number][] => { + if (ce <= a || cs >= b) return [[a, b]]; + const result: [number, number][] = []; + if (a < cs) result.push([a, cs]); + if (ce < b) result.push([ce, b]); + return result; + }); + } + return ranges; +} + +/** + * Overlapping or touching ranges collapsed into the minimal set, sorted by start. + * + * @category Rig + */ +export function mergeRanges(ranges: readonly [number, number][]): [number, number][] { + if (ranges.length === 0) return []; + const sorted = ranges.map(([a, b]): [number, number] => [a, b]).sort((x, y) => x[0] - y[0]); + const merged: [number, number][] = [sorted[0]]; + for (let i = 1; i < sorted.length; i++) { + const last = merged[merged.length - 1]; + if (sorted[i][0] <= last[1]) last[1] = Math.max(last[1], sorted[i][1]); + else merged.push(sorted[i]); + } + return merged; +} diff --git a/packages/rig/src/registry.ts b/packages/rig/src/registry.ts index bbaaffe3..603469a2 100644 --- a/packages/rig/src/registry.ts +++ b/packages/rig/src/registry.ts @@ -37,7 +37,7 @@ import { AbilityRegistryCtx, AbilityConfigStoreCtx, GrantStoreCtx, - RerankerCtx, + RerankerCtx, Attachments, } from '@lloyal-labs/lloyal-agents'; import type { Ability, @@ -125,6 +125,10 @@ export function* createAbilityRegistry( reranker = undefined; } + // The content store the harness installed (the null store when none was): + // an ability that reads documents resolves them through it. + const attachments = yield* Attachments.expect(); + const [scope, destroy] = createScope(); let added = false; return yield* scoped(function* () { @@ -154,6 +158,7 @@ export function* createAbilityRegistry( yield* AbilityConfigStoreCtx.set(configStore); yield* AbilityRegistryCtx.set(registry); if (reranker !== undefined) yield* RerankerCtx.set(reranker); + yield* Attachments.set(attachments); const constructed = yield* factory(); resolve(constructed); yield* suspend(); diff --git a/packages/rig/src/reranker.ts b/packages/rig/src/reranker.ts index a16d4e61..f184ac67 100644 --- a/packages/rig/src/reranker.ts +++ b/packages/rig/src/reranker.ts @@ -19,12 +19,14 @@ export interface RerankerLoadOpts { /** Decode batch size (default floor(nCtx / nSeqMax)). */ nBatch?: number; /** - * KV cache types for the reranker context. Both default to `q4_0` in this - * version. + * KV cache types for the reranker context. Both default to `q8_0`. * - * The score is a logit difference, so KV precision bounds the smallest - * score difference that is meaningful. Set these explicitly if you need a - * known resolution. + * The score is a logit difference read back through these cells, so KV + * precision bounds the smallest score difference that is meaningful. + * Measured on real document windows (2026-09-07): at `q4_0` ten identical + * passages spread 4–6 logits across the leaves and the verdict changed + * sign; at `q8_0` they spread 0.05–0.12 at the same pass time. + * `test/reranker-resolution.test.ts` holds the default to that floor. */ typeK?: KvCacheType; typeV?: KvCacheType; @@ -85,8 +87,8 @@ export function createReranker( nCtx, nSeqMax, nBatch, - typeK: opts?.typeK ?? 'q4_0', - typeV: opts?.typeV ?? 'q4_0', + typeK: opts?.typeK ?? 'q8_0', + typeV: opts?.typeV ?? 'q8_0', })); const rerank = yield* call(() => Rerank.create(ctx as unknown as SessionContext, { diff --git a/packages/rig/src/resources/documents.ts b/packages/rig/src/resources/documents.ts new file mode 100644 index 00000000..6771dbd7 --- /dev/null +++ b/packages/rig/src/resources/documents.ts @@ -0,0 +1,80 @@ +/** + * @file From document attachments in the content store to the resources and + * chunks the retrieval stack already understands. + * + * A document attachment is a manifest whose one representation is + * `text/markdown` and whose config is the sidecar (`DocumentMeta`). Nothing + * here parses markdown: the sidecar's sections already say where each one + * starts and ends, so a chunk is a line slice. No codec, no native addon — + * this module is pure and sits on the platform-agnostic barrel. + */ +import { representationsOf, asDocumentMeta, DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, AttachmentManifest, AttachmentStore, DocumentMeta } from '@lloyal-labs/media'; +import type { Resource, Chunk } from '@lloyal-labs/lloyal-agents'; + +/** + * A document as retrieval sees it: the root it came from, the sidecar, and + * the resource and section chunks cut from its markdown. + * + * @category Rig + */ +export interface Document { + attachment: Attachment; + manifest: AttachmentManifest; + meta: DocumentMeta; + resource: Resource; + /** One chunk per sidecar section, `tokens` empty — size them with `fitChunks`. */ + chunks: Chunk[]; +} + +/** + * Load the documents among `roots`. A root whose config is not a document + * sidecar (an image, say) is skipped — it is legitimately something else. + * A root whose manifest or blobs are missing or drifted THROWS, naming the + * digest: silent degradation here would index an empty document behind a + * digest that promises content. + * + * Resource names are the documents' titles, deduplicated in order + * (`Title`, `Title (2)`), so two uploads with one title stay addressable. + * + * @category Rig + */ +export function loadDocuments(store: AttachmentStore, roots: readonly Attachment[]): Document[] { + const out: Document[] = []; + const seen = new Map<string, number>(); + const short = (digest: string): string => `${digest.slice(0, 19)}…`; + for (const attachment of roots) { + const manifest = store.getManifest(attachment.digest); + if (!manifest) { + throw new Error(`loadDocuments: attachment manifest ${short(attachment.digest)} is not in the content store, or its bytes no longer hash to its digest.`); + } + if (manifest.config.mediaType !== DOCUMENT_CONFIG_TYPE) continue; + const configBytes = store.get(manifest.config.digest); + if (!configBytes) throw new Error(`loadDocuments: sidecar ${short(manifest.config.digest)} is missing from the content store or has drifted.`); + const meta = asDocumentMeta(JSON.parse(new TextDecoder().decode(configBytes))); + if (!meta) throw new Error(`loadDocuments: sidecar ${short(manifest.config.digest)} is not a ${DOCUMENT_CONFIG_TYPE} record.`); + const markdown = representationsOf(manifest).find((r) => r.mediaType === 'text/markdown'); + if (!markdown) throw new Error(`loadDocuments: document ${short(attachment.digest)} carries no text/markdown representation.`); + const bytes = store.get(markdown.digest); + if (!bytes) throw new Error(`loadDocuments: markdown ${short(markdown.digest)} is missing from the content store or has drifted.`); + const content = new TextDecoder().decode(bytes); + + const n = (seen.get(meta.title) ?? 0) + 1; + seen.set(meta.title, n); + const name = n === 1 ? meta.title : `${meta.title} (${n})`; + const lines = content.split('\n'); + const chunks: Chunk[] = meta.sections + .map((s) => ({ + resource: name, + heading: s.heading, + section: s.path, + text: lines.slice(s.startLine - 1, s.endLine).join('\n'), + tokens: [], + startLine: s.startLine, + endLine: s.endLine, + })) + .filter((c) => c.text.trim().length > 0); + out.push({ attachment, manifest, meta, resource: { name, content }, chunks }); + } + return out; +} diff --git a/packages/rig/src/resources/files.ts b/packages/rig/src/resources/files.ts index 71613b6b..49a390d6 100644 --- a/packages/rig/src/resources/files.ts +++ b/packages/rig/src/resources/files.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import ignoreFactory = require("ignore"); import { loadBinary } from "@lloyal-labs/lloyal.node"; import type { Resource, Chunk } from "./types"; +import { splitParagraphs } from "./fit"; interface Section { heading: string; @@ -126,26 +127,20 @@ export function loadResources(input: string): Resource[] { function chunkByParagraph(res: Resource): Chunk[] { const lines = res.content.split("\n"); const chunks: Chunk[] = []; - let start = 0; - for (let i = 0; i <= lines.length; i++) { - const blank = i === lines.length || !lines[i].trim(); - if (blank && i > start) { - const text = lines.slice(start, i).join("\n").trim(); - if (text) { - chunks.push({ - resource: res.name, - heading: - text.slice(0, 60).replace(/\n/g, " ") + - (text.length > 60 ? "\u2026" : ""), - section: '', - text, - tokens: [], - startLine: start + 1, - endLine: i, - }); - } - } - if (blank) start = i + 1; + for (const [start, end] of splitParagraphs(lines)) { + const text = lines.slice(start, end).join("\n").trim(); + if (!text) continue; + chunks.push({ + resource: res.name, + heading: + text.slice(0, 60).replace(/\n/g, " ") + + (text.length > 60 ? "\u2026" : ""), + section: '', + text, + tokens: [], + startLine: start + 1, + endLine: end, + }); } return chunks; } diff --git a/packages/rig/src/resources/fit.ts b/packages/rig/src/resources/fit.ts new file mode 100644 index 00000000..2c013d0c --- /dev/null +++ b/packages/rig/src/resources/fit.ts @@ -0,0 +1,129 @@ +/** + * @file Passages sized for a reranker leaf, by lines. + * + * Pure: it knows nothing about rerankers beyond the `tokenize` it is handed, + * and nothing about where chunks came from. The size is a retrieval choice the + * caller makes; the reranker's capacity is the reranker's own affair. Kept + * free of `./files` on purpose — that module statically imports the native + * package, and nothing on the platform-agnostic barrel may depend on it — so + * the paragraph splitter lives here and `./files` imports it. + */ +import type { Chunk } from '@lloyal-labs/lloyal-agents'; + +/** + * Default passage size, in tokens of the scoring reranker's vocabulary. + * + * A granularity, not a capacity: passages of a few hundred tokens are what + * first-stage and cross-encoder retrieval work best over, whatever the + * hardware. It is sized so that a passage of this length plus a 64-token + * query fits the smallest reranker sizing rig ships (nSeqMax 10 · nCtx 4096), + * which `test/reranker-capacity.test.ts` pins against real weights. + * + * @category Rig + */ +export const DEFAULT_CHUNK_TOKENS = 256; + +/** + * Half-open line ranges `[start, end)` of the blank-separated paragraphs in + * `lines`. A whitespace-only line is blank; runs of blank lines separate + * exactly as one does. + * + * @category Rig + */ +export function splitParagraphs(lines: readonly string[]): [number, number][] { + const out: [number, number][] = []; + let start = -1; + for (let i = 0; i <= lines.length; i++) { + const blank = i === lines.length || lines[i].trim() === ''; + if (!blank && start < 0) start = i; + if (blank && start >= 0) { + out.push([start, i]); + start = -1; + } + } + return out; +} + +/** What {@link fitChunks} needs: a size, and the tokenizer whose vocabulary the size is in. */ +export interface FitOpts { + /** Longest window, in tokens. Positive integer. */ + maxTokens: number; + /** The scoring reranker's tokenizer — chunk tokens must be in ITS vocabulary. */ + tokenize: (text: string) => Promise<number[]>; +} + +/** + * Cut chunks into windows of at most `maxTokens`, returning them tokenized. + * + * A chunk within the size is returned whole. A longer one is split on its + * own line structure: whole paragraphs are packed greedily into windows, a + * paragraph over the size is packed by lines, and a single line over the size + * stays one window — the reranker head-truncates it. **Never inside a line**: + * admission joins scored chunks back to their source by `(resource, + * startLine)`, so two windows must never share a start line. Windows tile + * their parent's non-blank lines, carry real `startLine`/`endLine`, inherit + * `resource`, `heading` and `section`, and never overlap (BM25 would count + * the overlap twice). + * + * Every window is measured once more after packing, so the tokens it carries + * are exactly what the reranker will score — not a sum of its parts. + * + * @throws When `maxTokens` is not a positive integer: an uncapped chunker is + * a configuration error, never a silent fallback. + * + * @category Rig + */ +export async function fitChunks(chunks: readonly Chunk[], opts: FitOpts): Promise<Chunk[]> { + const { maxTokens, tokenize } = opts; + if (!Number.isInteger(maxTokens) || maxTokens < 1) { + throw new Error(`fitChunks: maxTokens must be a positive integer, got ${String(maxTokens)}`); + } + + const out: Chunk[] = []; + for (const chunk of chunks) { + const tokens = await tokenize(chunk.text); + if (tokens.length <= maxTokens) { + out.push({ ...chunk, tokens }); + continue; + } + + const lines = chunk.text.split('\n'); + const slice = (s: number, e: number): string => lines.slice(s, e).join('\n'); + + // Units to pack: paragraphs, or the lines of a paragraph too long to be one. + const units: [number, number][] = []; + for (const [s, e] of splitParagraphs(lines)) { + const n = (await tokenize(slice(s, e))).length; + if (n <= maxTokens) units.push([s, e]); + else for (let i = s; i < e; i++) units.push([i, i + 1]); + } + + // Greedy packing of consecutive units; a candidate window is measured + // exactly rather than summed, because tokenization is not additive across + // the joins. + let ws = -1; + let we = -1; + const flush = async (): Promise<void> => { + if (ws < 0) return; + const text = slice(ws, we); + out.push({ + ...chunk, + text, + tokens: await tokenize(text), + startLine: chunk.startLine + ws, + endLine: chunk.startLine + we - 1, + }); + ws = -1; + we = -1; + }; + for (const [s, e] of units) { + if (ws < 0) { ws = s; we = e; continue; } + if ((await tokenize(slice(ws, e))).length <= maxTokens) { we = e; continue; } + await flush(); + ws = s; + we = e; + } + await flush(); + } + return out; +} diff --git a/packages/rig/src/sources/chunking.ts b/packages/rig/src/sources/chunking.ts index ea64eaef..0ac9660d 100644 --- a/packages/rig/src/sources/chunking.ts +++ b/packages/rig/src/sources/chunking.ts @@ -12,6 +12,7 @@ */ import type { Chunk } from '../resources/types'; +import { splitParagraphs } from '../resources/fit'; /** * Raw page content buffered during web research for post-research reranking @@ -46,35 +47,32 @@ export interface FetchedPage { export function chunkFetchedPages(pages: FetchedPage[]): Chunk[] { const chunks: Chunk[] = []; for (const page of pages) { - const paragraphs = page.text - .split(/\n\s*\n/) - .map((p) => p.trim()) - .filter((p) => p.length > 40); - - if (paragraphs.length === 0) { - if (page.text.trim().length > 40) { - chunks.push({ - resource: page.url, - heading: page.title || page.url, - section: '', - text: page.text.trim(), - tokens: [], - startLine: 1, - endLine: 1, - }); - } - continue; + // Real line ranges over the page text: a consumer holding the same text can + // resolve any chunk, and windows over these chunks keep distinct start lines. + const lines = page.text.split('\n'); + const before = chunks.length; + for (const [start, end] of splitParagraphs(lines)) { + const text = lines.slice(start, end).join('\n').trim(); + if (text.length <= 40) continue; + chunks.push({ + resource: page.url, + heading: page.title || page.url, + section: '', + text, + tokens: [], + startLine: start + 1, + endLine: end, + }); } - - for (let i = 0; i < paragraphs.length; i++) { + if (chunks.length === before && page.text.trim().length > 40) { chunks.push({ resource: page.url, heading: page.title || page.url, section: '', - text: paragraphs[i], + text: page.text.trim(), tokens: [], - startLine: i + 1, - endLine: i + 1, + startLine: 1, + endLine: lines.length, }); } } @@ -107,21 +105,27 @@ export async function chunkHtml(html: string, url: string, title: string): Promi const chunks: Chunk[] = []; let currentHeading = title; let currentText = ''; - let chunkIndex = 0; + // Real line ranges over the text this function builds: the emitted sections + // joined by one blank line. Distinct start lines per chunk, contiguous + // ranges, and a line count that matches each chunk's text. + let cursor = 0; function flushSection() { const text = currentText.trim(); if (text.length > 40) { + if (chunks.length > 0) cursor += 1; + const startLine = cursor + 1; + const endLine = cursor + text.split('\n').length; + cursor = endLine; chunks.push({ resource: url, heading: currentHeading || title || url, section: '', text, tokens: [], - startLine: chunkIndex + 1, - endLine: chunkIndex + 1, + startLine, + endLine, }); - chunkIndex++; } currentText = ''; } diff --git a/packages/rig/src/tools/delegate.ts b/packages/rig/src/tools/delegate.ts index dc8a2bf6..2f3994c3 100644 --- a/packages/rig/src/tools/delegate.ts +++ b/packages/rig/src/tools/delegate.ts @@ -212,6 +212,10 @@ export class DelegateTool extends Tool<Record<string, unknown>> { parent: context?.branch, pruneOnReturn: opts.pruneOnReturn ?? true, scorer: context?.scorer, + // The child pool starts with the assets this call can see: the run's + // staged roots plus everything any agent admitted so far. The static + // pool options predate the run and cannot carry them. + attachments: context?.attachments, }); const result = { diff --git a/packages/abilities/corpus/test/bm25.test.ts b/packages/rig/test/bm25.test.ts similarity index 100% rename from packages/abilities/corpus/test/bm25.test.ts rename to packages/rig/test/bm25.test.ts diff --git a/packages/rig/test/chunking.test.ts b/packages/rig/test/chunking.test.ts new file mode 100644 index 00000000..8cdaac1d --- /dev/null +++ b/packages/rig/test/chunking.test.ts @@ -0,0 +1,45 @@ +/** + * The HTML chunkers address the text they build by real, contiguous line + * ranges — not ordinals — so a consumer holding the same built text can + * resolve any chunk, and windows over these chunks keep distinct start lines. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { chunkFetchedPages, chunkHtml } from '../src/sources/chunking'; + +const LONG = (tag: string) => `${tag} paragraph with enough words in it to clear the forty character floor.`; + +describe('chunkFetchedPages — real line ranges over the page text', () => { + it('numbers each paragraph by its lines in the page text and keeps the ranges contiguous', () => { + const text = `${LONG('First')}\nsecond line of the first\n\n${LONG('Second')}\n\n\n${LONG('Third')}`; + const chunks = chunkFetchedPages([{ url: 'https://x/y', title: 'T', text }]); + expect(chunks.map((c) => [c.startLine, c.endLine])).toEqual([[1, 2], [4, 4], [7, 7]]); + for (const c of chunks) { + expect(c.text.split('\n')).toHaveLength(c.endLine - c.startLine + 1); + } + }); + + it('gives a page with no paragraph over the floor one chunk spanning its lines', () => { + const text = `${LONG('Only')}\nand a tail line`; + const chunks = chunkFetchedPages([{ url: 'u', title: 't', text }]); + expect(chunks).toHaveLength(1); + expect([chunks[0].startLine, chunks[0].endLine]).toEqual([1, 2]); + }); +}); + +describe('chunkHtml — real line ranges over the built text', () => { + it('assigns contiguous, distinct line ranges across sections of one page', async () => { + const html = `<article><h1>Intro</h1><p>${LONG('Intro')}</p><p>${LONG('More intro')}</p>` + + `<h2>Body</h2><p>${LONG('Body')}</p></article>`; + const chunks = await chunkHtml(html, 'https://x/y', 'T'); + expect(chunks.length).toBeGreaterThanOrEqual(2); + let expectedStart = 1; + for (const c of chunks) { + expect(c.startLine).toBeGreaterThanOrEqual(expectedStart); + expect(c.text.split('\n')).toHaveLength(c.endLine - c.startLine + 1); + expectedStart = c.endLine + 1; + } + expect(new Set(chunks.map((c) => c.startLine)).size).toBe(chunks.length); + }); +}); diff --git a/packages/rig/test/content-routes.test.ts b/packages/rig/test/content-routes.test.ts index e52ddac5..8ab8cfbf 100644 --- a/packages/rig/test/content-routes.test.ts +++ b/packages/rig/test/content-routes.test.ts @@ -343,3 +343,101 @@ describe('content routes', () => { }); }); }); + +describe('content routes — the manifest and its config, two plain doors', () => { + // A document attachment's page renders are their own roots named in its + // sidecar, which lives in the manifest's config blob. The UI resolves a + // citation through those two records, so both need a door — OCI-shaped, by + // digest, through the manifest — while raw blobs stay HEAD-only. + it('serves a manifest by digest with the caching headers a blob gets, and 304 on a match', async () => { + const { store, root, rep } = fixture(); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/vnd.oci.image.manifest.v1+json'); + expect(res.headers.get('etag')).toBe(`"${root.digest}"`); + const manifest = await res.json() as { layers: { digest: string }[] }; + expect(manifest.layers.some((l) => l.digest === rep.digest)).toBe(true); + const again = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}`, { headers: { 'If-None-Match': `"${root.digest}"` } }); + expect(again.status).toBe(304); + const missing = await fetch(`${base}/v1/media/sha256:${'0'.repeat(64)}`); + expect(missing.status).toBe(404); + }); + }); + + it('serves the retained source through the manifest under its own media type, and 404 when none was kept', async () => { + // The original as supplied — a document's PDF, an image before + // normalization — resolved by ROLE through the manifest. Raw blobs stay + // HEAD-only; this door exists because a manifest names the source. + const { store, root, source } = fixture(); + const md = store.putBlob(new TextEncoder().encode('# T\n'), 'text/markdown'); + const sourceless = store.putAttachment({ representations: [md] }); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}/source`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('image/jpeg'); + expect(res.headers.get('etag')).toBe(`"${source.digest}"`); + expect(new Uint8Array(await res.arrayBuffer())).toEqual(JPEG); + const head = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}/source`, { method: 'HEAD' }); + expect(head.status).toBe(200); + const none = await fetch(`${base}/v1/media/${encodeURIComponent(sourceless.digest)}/source`); + expect(none.status).toBe(404); + const missing = await fetch(`${base}/v1/media/sha256:${'0'.repeat(64)}/source`); + expect(missing.status).toBe(404); + }); + }); + + it('serves the config blob through the manifest under its own media type', async () => { + const dir = mkdtempSync(join(tmpdir(), 'lloyal-routes-')); + const store = new FileAttachmentStore(dir); + const md = store.putBlob(new TextEncoder().encode('# T\n'), 'text/markdown'); + const sidecar = new TextEncoder().encode(JSON.stringify({ title: 'T', pages: [] })); + const root = store.putAttachment({ representations: [md], config: { bytes: sidecar, mediaType: 'application/vnd.lloyal.document.v1+json' } }); + const { root: imageRoot } = fixture(); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/media/${encodeURIComponent(root.digest)}/config`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/vnd.lloyal.document.v1+json'); + expect(await res.json()).toEqual({ title: 'T', pages: [] }); + // An image manifest's config is OCI's canonical empty blob, served as such. + const empty = await fetch(`${base}/v1/media/${encodeURIComponent(fixture().root.digest)}/config`).catch(() => null); + void empty; void imageRoot; + }); + }); + + it('still refuses GET on a raw blob: the manifest doors do not open one for sources', async () => { + const { store, source } = fixture(); + await withServer({ store }, async (base) => { + const res = await fetch(`${base}/v1/content/${encodeURIComponent(source.digest)}`); + expect(res.status).toBe(405); + }); + }); +}); + +describe('content routes — whose fault an ingest failure is', () => { + it('answers 5xx when the store cannot write, not 400: disk full is not the client\'s doing', async () => { + const { store } = fixture(); + const noSpace = (): never => { + const e = new Error('ENOSPC: no space left on device, write') as Error & { code: string; syscall: string }; + e.code = 'ENOSPC'; e.syscall = 'write'; + throw e; + }; + await withServer({ store, ingest: async () => noSpace() }, async (base) => { + const res = await fetch(`${base}/v1/media/ingress`, { method: 'POST', body: PNG }); + expect(res.status).toBe(500); + }); + }); + + it('answers 408 when the ingress reports its own time bound, as it does for a slow upload', async () => { + const { store } = fixture(); + const timedOut = (): never => { + const e = new Error('document exceeded 120000ms') as Error & { code: string }; + e.code = 'ETIMEDOUT'; + throw e; + }; + await withServer({ store, ingest: async () => timedOut() }, async (base) => { + const res = await fetch(`${base}/v1/media/ingress`, { method: 'POST', body: PNG }); + expect(res.status).toBe(408); + }); + }); +}); diff --git a/packages/rig/test/delegate-assets.test.ts b/packages/rig/test/delegate-assets.test.ts new file mode 100644 index 00000000..1d96deb0 --- /dev/null +++ b/packages/rig/test/delegate-assets.test.ts @@ -0,0 +1,43 @@ +/** + * A delegated child pool starts with the assets the delegating call can see — + * the run's staged roots plus everything admitted so far — not with the + * static pool options alone. + * + * @category Testing + */ +import { describe, it, expect, vi } from 'vitest'; +import { run } from 'effection'; + +const seen = vi.hoisted(() => ({ poolOpts: [] as Record<string, unknown>[] })); +vi.mock('@lloyal-labs/lloyal-agents', async (importOriginal) => { + const actual = await importOriginal<typeof import('@lloyal-labs/lloyal-agents')>(); + return { + ...actual, + agentPool: vi.fn(function* (opts: Record<string, unknown>) { + seen.poolOpts.push(opts); + return { agents: [], totalTokens: 0, totalToolCalls: 0 }; + }), + }; +}); + +import { Trace, NullTraceWriter } from '@lloyal-labs/lloyal-agents'; +import type { ToolContext } from '@lloyal-labs/lloyal-agents'; +import type { Attachment } from '@lloyal-labs/media'; +import { DelegateTool } from '../src/tools/delegate'; + +const root = (hex: string): Attachment => + ({ mediaType: 'application/vnd.oci.image.manifest.v1+json', digest: `sha256:${hex.repeat(64)}`, size: 700 }) as unknown as Attachment; + +describe('delegate — the child pool inherits the run\'s assets', () => { + it('forwards the delegating call\'s attachments into agentPool, admitted roots included', async () => { + const staged = root('a'); + const admittedMidRun = root('b'); + const tool = new DelegateTool({ poolOpts: {}, systemPrompt: 'sys', extractTasks: (a) => a.tasks as string[] }); + await run(function* () { + yield* Trace.set(new NullTraceWriter()); + return yield* tool.execute({ tasks: ['look into it'] }, { agentId: 7, attachments: [staged, admittedMidRun] } as unknown as ToolContext); + }); + expect(seen.poolOpts).toHaveLength(1); + expect(seen.poolOpts[0].attachments).toEqual([staged, admittedMidRun]); + }); +}); diff --git a/packages/rig/test/documents.test.ts b/packages/rig/test/documents.test.ts new file mode 100644 index 00000000..a5f359ae --- /dev/null +++ b/packages/rig/test/documents.test.ts @@ -0,0 +1,79 @@ +/** + * `loadDocuments` — from document attachments in the content store to the + * resources and chunks the retrieval stack already understands. No codec, no + * addon: a document root is a manifest whose config is the sidecar. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { FileAttachmentStore } from '@lloyal-labs/media/node'; +import { DOCUMENT_CONFIG_TYPE } from '@lloyal-labs/media'; +import type { Attachment, DocumentMeta } from '@lloyal-labs/media'; +import { loadDocuments } from '../src/resources/documents'; + +const MARKDOWN = ['# A paper', '', '## Intro', '', 'First line of the intro.', 'Second line of the intro.', '', '## Method', '', 'Method body.'].join('\n'); + +const meta = (title: string): DocumentMeta => ({ + title, pageCount: 2, + sections: [ + { heading: title, path: title, origin: 'heuristic', startLine: 1, endLine: 2, pageStart: 1, pageEnd: 1 }, + { heading: 'Intro', path: 'Intro', origin: 'heuristic', startLine: 3, endLine: 7, pageStart: 1, pageEnd: 1 }, + { heading: 'Method', path: 'Method', origin: 'heuristic', startLine: 8, endLine: 10, pageStart: 2, pageEnd: 2 }, + ], + pages: [ + { page: 1, startLine: 1, endLine: 7, chars: 60, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }, + { page: 2, startLine: 8, endLine: 10, chars: 12, imageObjects: 0, pathObjects: 0, taggedTables: 0, taggedFigures: 0 }, + ], + figures: [], tables: [], + derive: { profile: 'pdf.v1', pdfium: 'test', dpi: 150, maxSide: 2048, maxPixels: 4_194_304, format: 'image/png', renderedPages: 2, maxFigures: 16, maxTextPages: 400, tagged: false, structCoverage: 0, truncated: false }, +}); + +function commitDocument(store: FileAttachmentStore, title: string, markdown = MARKDOWN): Attachment { + const md = store.putBlob(new TextEncoder().encode(markdown), 'text/markdown'); + return store.putAttachment({ + representations: [md], + config: { bytes: new TextEncoder().encode(JSON.stringify(meta(title))), mediaType: DOCUMENT_CONFIG_TYPE }, + }); +} + +describe('loadDocuments', () => { + it('turns a document root into a resource and one chunk per section, with real lines and the section path', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const root = commitDocument(store, 'A paper'); + const [doc] = loadDocuments(store, [root]); + expect(doc.attachment).toEqual(root); + expect(doc.meta.title).toBe('A paper'); + expect(doc.resource).toEqual({ name: 'A paper', content: MARKDOWN }); + expect(doc.chunks.map((c) => [c.heading, c.section, c.startLine, c.endLine])).toEqual([ + ['A paper', 'A paper', 1, 2], ['Intro', 'Intro', 3, 7], ['Method', 'Method', 8, 10], + ]); + expect(doc.chunks[1].text).toBe('## Intro\n\nFirst line of the intro.\nSecond line of the intro.\n'); + expect(doc.chunks.every((c) => c.resource === 'A paper' && c.tokens.length === 0)).toBe(true); + }); + + it('skips roots that are not documents — an image root is legitimately something else', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const image = store.putAttachment({ representations: [store.putBlob(new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2]), 'image/png')] }); + const doc = commitDocument(store, 'Only one'); + expect(loadDocuments(store, [image, doc]).map((d) => d.meta.title)).toEqual(['Only one']); + }); + + it('dedupes resource names so two documents with one title stay addressable', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const a = commitDocument(store, 'Same'); + const b = commitDocument(store, 'Same', MARKDOWN + '\nextra'); + expect(loadDocuments(store, [a, b]).map((d) => d.resource.name)).toEqual(['Same', 'Same (2)']); + }); + + it('throws naming the digest when the markdown blob is missing or drifted', () => { + const store = new FileAttachmentStore(mkdtempSync(join(tmpdir(), 'documents-'))); + const root = commitDocument(store, 'Drifted'); + const md = store.getManifest(root.digest)!.layers[0]; + const path = join((store as unknown as { _dir: string })._dir, 'blobs', 'sha256', md.digest.slice(7)); + require('node:fs').writeFileSync(path, 'tampered'); + expect(() => loadDocuments(store, [root])).toThrow(new RegExp(md.digest.slice(0, 19))); + }); +}); diff --git a/packages/rig/test/fit-chunks.test.ts b/packages/rig/test/fit-chunks.test.ts new file mode 100644 index 00000000..be3d3e72 --- /dev/null +++ b/packages/rig/test/fit-chunks.test.ts @@ -0,0 +1,104 @@ +/** + * `fitChunks` — passages sized for a reranker leaf, by lines. + * + * A word-count tokenizer makes every expectation readable by hand. The size + * is a granularity choice the caller makes; the function knows nothing about + * rerankers beyond the tokenize it is handed. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import type { Chunk } from '@lloyal-labs/lloyal-agents'; +import { fitChunks, splitParagraphs, DEFAULT_CHUNK_TOKENS } from '../src/resources/fit'; + +const words = async (text: string): Promise<number[]> => + text.split(/\s+/).filter(Boolean).map((_, i) => i + 1); + +function chunk(text: string, startLine = 1, extra: Partial<Chunk> = {}): Chunk { + const lines = text.split('\n'); + return { + resource: 'doc.md', heading: 'H', section: 'A > H', text, tokens: [], + startLine, endLine: startLine + lines.length - 1, ...extra, + }; +} + +const PARA = (n: number, w = 'w') => Array.from({ length: n }, () => w).join(' '); + +describe('splitParagraphs', () => { + it('returns half-open line ranges of blank-separated paragraphs, skipping blank runs', () => { + const lines = ['a', 'b', '', '', 'c', '', 'd', 'e', 'f']; + expect(splitParagraphs(lines)).toEqual([[0, 2], [4, 5], [6, 9]]); + }); + + it('treats whitespace-only lines as blank and an empty input as no paragraphs', () => { + expect(splitParagraphs([' ', 'x', ' \t', 'y'])).toEqual([[1, 2], [3, 4]]); + expect(splitParagraphs([])).toEqual([]); + }); +}); + +describe('fitChunks', () => { + it('exports a default passage size', () => { + expect(DEFAULT_CHUNK_TOKENS).toBe(256); + }); + + it('keeps a chunk within the size as one tokenized window', async () => { + const c = chunk(`${PARA(4)}\n\n${PARA(3)}`, 10); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out).toHaveLength(1); + expect(out[0]).toMatchObject({ startLine: 10, endLine: 12, text: c.text }); + expect(out[0].tokens).toHaveLength(7); + }); + + it('packs whole paragraphs into windows that tile the parent lines, inheriting heading and section', async () => { + // 6 + 6 + 6 words: two fit in ten? no — 6+6 = 12 > 10, so one paragraph per window. + const c = chunk(`${PARA(6)}\n\n${PARA(6)}\n\n${PARA(6)}`, 1); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[1, 1], [3, 3], [5, 5]]); + for (const w of out) { + expect(w.tokens.length).toBeLessThanOrEqual(10); + expect(w.heading).toBe('H'); + expect(w.section).toBe('A > H'); + expect(w.resource).toBe('doc.md'); + expect(w.text.split('\n')).toHaveLength(w.endLine - w.startLine + 1); + } + }); + + it('packs consecutive short paragraphs together, keeping the blank lines between them', async () => { + const c = chunk(`${PARA(3)}\n\n${PARA(3)}\n\n${PARA(3)}\n\n${PARA(3)}`, 1); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + // 3+3+3 = 9 ≤ 10, then 3. + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[1, 5], [7, 7]]); + expect(out[0].tokens).toHaveLength(9); + }); + + it('splits a paragraph over the size by lines', async () => { + const c = chunk([PARA(4), PARA(4), PARA(4)].join('\n'), 20); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[20, 21], [22, 22]]); + }); + + it('keeps a single line longer than the size as one window and never splits inside a line', async () => { + const c = chunk(`${PARA(3)}\n${PARA(25)}\n${PARA(3)}`, 5); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => [w.startLine, w.endLine])).toEqual([[5, 5], [6, 6], [7, 7]]); + expect(out[1].tokens).toHaveLength(25); + }); + + it('never yields two windows with the same (resource, startLine)', async () => { + const c = chunk(`${PARA(30)}\n\n${PARA(30)}`, 1); + const out = await fitChunks([c], { maxTokens: 10, tokenize: words }); + const keys = out.map((w) => `${w.resource}:${w.startLine}`); + expect(new Set(keys).size).toBe(keys.length); + }); + + it('preserves input order across chunks and resources', async () => { + const a = chunk(PARA(3), 1, { resource: 'a.md' }); + const b = chunk(`${PARA(8)}\n\n${PARA(8)}`, 1, { resource: 'b.md' }); + const out = await fitChunks([a, b], { maxTokens: 10, tokenize: words }); + expect(out.map((w) => `${w.resource}:${w.startLine}`)).toEqual(['a.md:1', 'b.md:1', 'b.md:3']); + }); + + it('throws on a size below one instead of silently not capping', async () => { + await expect(fitChunks([chunk('x')], { maxTokens: 0, tokenize: words })).rejects.toThrow(/maxTokens/); + }); +}); diff --git a/packages/rig/test/helpers/rerank-model.ts b/packages/rig/test/helpers/rerank-model.ts new file mode 100644 index 00000000..f8705ae2 --- /dev/null +++ b/packages/rig/test/helpers/rerank-model.ts @@ -0,0 +1,17 @@ +/** + * Where the weights-gated reranker tests find a Qwen3-Reranker GGUF: the + * environment first, then the usual local homes. `null` skips the suite, + * as the SDK's integration tests do without weights. + * + * @category Testing + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +export const RERANK_MODEL_PATH: string | null = [ + process.env.LLAMA_RERANK_MODEL, + path.join(os.homedir(), '.cache/lloyal/models/qwen3-reranker-0.6b-q4_k_m.gguf'), + path.join(os.homedir(), 'dev/apps/lloyal-node/models/qwen3-reranker-0.6b-q4_k_m.gguf'), + path.join(os.homedir(), '.cache/lloyal/models/qwen3-reranker-0.6b-q8_0.gguf'), +].find((p): p is string => !!p && fs.existsSync(p)) ?? null; diff --git a/packages/rig/test/ranges.test.ts b/packages/rig/test/ranges.test.ts new file mode 100644 index 00000000..9d548bc4 --- /dev/null +++ b/packages/rig/test/ranges.test.ts @@ -0,0 +1,34 @@ +/** + * Half-open line-range arithmetic shared by the read tools' per-agent dedup. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { mergeRanges, subtractRanges } from '../src/ranges'; + +describe('subtractRanges', () => { + it('returns the target untouched when nothing is covered', () => { + expect(subtractRanges([0, 10], [])).toEqual([[0, 10]]); + expect(subtractRanges([0, 10], [[10, 20]])).toEqual([[0, 10]]); + }); + + it('cuts covered spans out, splitting around a hole in the middle', () => { + expect(subtractRanges([0, 10], [[3, 5]])).toEqual([[0, 3], [5, 10]]); + expect(subtractRanges([0, 10], [[0, 4], [6, 10]])).toEqual([[4, 6]]); + }); + + it('returns nothing when the target is fully covered', () => { + expect(subtractRanges([2, 8], [[0, 10]])).toEqual([]); + }); +}); + +describe('mergeRanges', () => { + it('collapses overlapping and touching ranges, sorted by start', () => { + expect(mergeRanges([[5, 8], [0, 3], [3, 6]])).toEqual([[0, 8]]); + }); + + it('keeps disjoint ranges apart and an empty input empty', () => { + expect(mergeRanges([[0, 2], [4, 6]])).toEqual([[0, 2], [4, 6]]); + expect(mergeRanges([])).toEqual([]); + }); +}); diff --git a/packages/rig/test/registry.test.ts b/packages/rig/test/registry.test.ts index 696d9596..7eddbef5 100644 --- a/packages/rig/test/registry.test.ts +++ b/packages/rig/test/registry.test.ts @@ -33,6 +33,8 @@ import { describe, it, expect, vi } from 'vitest'; import { run, ensure } from 'effection'; import { AbilityConfigStoreCtx } from '@lloyal-labs/lloyal-agents'; +import { Attachments } from '@lloyal-labs/lloyal-agents'; +import type { AttachmentStore } from '@lloyal-labs/media'; import type { Ability, AbilityManifest, AbilityFactory } from '@lloyal-labs/lloyal-agents'; import { createAbilityRegistry } from '../src/registry'; import { createInMemoryConfigStore } from '../src/config-store'; @@ -134,6 +136,25 @@ describe('createAbilityRegistry', () => { expect(seen).toEqual({ key: 'value' }); }); + it('seeds the content store into the factory scope, beside the config store and the reranker', async () => { + // An ability that reads documents resolves them through the content store + // at enable time and in its tools; the detached scope must carry the store + // the harness installed, not the null default. + const seen = await run(function* () { + const store = { marker: 'the harness store' } as unknown as AttachmentStore; + yield* Attachments.set(store); + let inFactory: unknown; + const factory: AbilityFactory = function* () { + inFactory = yield* Attachments.expect(); + return fakeApp({ name: 'docs' }); + }; + const registry = yield* createAbilityRegistry({ configStore: createInMemoryConfigStore() }); + yield* registry.enable(factory); + return inFactory === store; + }); + expect(seen).toBe(true); + }); + it('runs the factory body (setup) when enabled', async () => { const onSetup = vi.fn(); await run(function* () { diff --git a/packages/rig/test/reranker-capacity.test.ts b/packages/rig/test/reranker-capacity.test.ts new file mode 100644 index 00000000..72834e8b --- /dev/null +++ b/packages/rig/test/reranker-capacity.test.ts @@ -0,0 +1,90 @@ +/** + * The default passage size fits the smallest reranker sizing rig ships. + * + * `DEFAULT_CHUNK_TOKENS` is a retrieval granularity choice, not a capacity + * contract: the reranker's per-leaf budget depends on the instruction and the + * chat template, which only a loaded `Rerank` knows. This test therefore + * asks the loaded reranker one question through its existing truncation + * callback — does a passage of the default size, under a query of the + * reserve we assume, score whole at nSeqMax 10 / nCtx 4096? — and, separately, + * proves the callback path is live with a passage no leaf could hold. No + * assertion names a boundary. + * + * Runs only when a Qwen3 reranker GGUF is present (same discovery as the SDK's + * integration suite): `LLAMA_RERANK_MODEL`, then the cache and sibling paths. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import * as path from 'node:path'; +import { createContext } from '@lloyal-labs/lloyal.node'; +import { Rerank } from '@lloyal-labs/sdk'; +import type { RerankTruncation, SessionContext } from '@lloyal-labs/sdk'; +import { DEFAULT_CHUNK_TOKENS } from '../src/resources/fit'; +import { RERANK_MODEL_PATH } from './helpers/rerank-model'; + +const MODEL_PATH = RERANK_MODEL_PATH; + +/** The smallest sizing `createReranker` ships when a host says nothing. */ +const N_SEQ_MAX = 10; +const N_CTX = 4096; +/** The query allowance the default chunk size is stated against. */ +const QUERY_TOKENS = 64; + +/** A text that tokenizes to exactly `n` tokens: one repeated word is monotone + * in its count, so the search converges in a few steps. */ +async function textOfTokens(rerank: Rerank, n: number): Promise<string> { + let words = n; + for (let i = 0; i < 40; i++) { + const text = Array.from({ length: words }, () => 'alpha').join(' '); + const len = (await rerank.tokenize(text)).length; + if (len === n) return text; + words += n - len; + } + throw new Error(`textOfTokens: could not build a ${n}-token text`); +} + +async function drain<T>(iter: AsyncIterable<T>): Promise<void> { + for await (const _ of iter) { /* consume */ } +} + +const describeWithModel = MODEL_PATH ? describe : describe.skip; + +describeWithModel(`DEFAULT_CHUNK_TOKENS fits the shipped reranker sizing — ${MODEL_PATH ? path.basename(MODEL_PATH) : 'no model'}`, () => { + it( + 'a default-size passage under the query reserve scores whole; a passage no leaf could hold is truncated', + async () => { + const truncations: RerankTruncation[] = []; + const ctx = (await createContext({ + modelPath: MODEL_PATH!, + nCtx: N_CTX, + nSeqMax: N_SEQ_MAX, + typeK: 'q4_0', + typeV: 'q4_0', + })) as unknown as SessionContext; + const rerank = await Rerank.create(ctx, { + nSeqMax: N_SEQ_MAX, + nCtx: N_CTX, + onTruncate: (e) => truncations.push(e), + }); + try { + const query = await textOfTokens(rerank, QUERY_TOKENS); + const filler = await rerank.tokenize('word '.repeat(2 * Math.floor(N_CTX / N_SEQ_MAX) + 16)); + + // Granularity fits: the default passage is scored whole. + await drain(rerank.score(query, [filler.slice(0, DEFAULT_CHUNK_TOKENS)], 1)); + expect(truncations).toEqual([]); + + // The callback path is live: twice the per-sequence slice cannot fit any leaf. + const oversize = filler.slice(0, 2 * Math.floor(N_CTX / N_SEQ_MAX)); + await drain(rerank.score(query, [oversize], 1)); + expect(truncations).toHaveLength(1); + expect(truncations[0]).toMatchObject({ docIndex: 0, origLen: oversize.length }); + expect(truncations[0].maxLen).toBeGreaterThanOrEqual(DEFAULT_CHUNK_TOKENS); + } finally { + rerank.dispose(); + } + }, + 120_000, + ); +}); diff --git a/packages/rig/test/reranker-options.test.ts b/packages/rig/test/reranker-options.test.ts index 5637c4e7..6a8f5f10 100644 --- a/packages/rig/test/reranker-options.test.ts +++ b/packages/rig/test/reranker-options.test.ts @@ -67,12 +67,12 @@ describe('createReranker — KV precision', () => { yield* createReranker('/fake/reranker.gguf', opts); }); - it('requests q4_0 for both KV types when the caller specifies neither', async () => { + it('requests q8_0 for both KV types when the caller specifies neither — the resolution the verdicts need', async () => { await load(); expect(createContext).toHaveBeenCalledTimes(1); const args = createContext.mock.calls[0][0]; - expect(args.typeK).toBe('q4_0'); - expect(args.typeV).toBe('q4_0'); + expect(args.typeK).toBe('q8_0'); + expect(args.typeV).toBe('q8_0'); }); it('requests the caller\'s KV types when given', async () => { diff --git a/packages/rig/test/reranker-resolution.test.ts b/packages/rig/test/reranker-resolution.test.ts new file mode 100644 index 00000000..65caa0de --- /dev/null +++ b/packages/rig/test/reranker-resolution.test.ts @@ -0,0 +1,43 @@ +/** + * The reranker's resolution under the KV type `createReranker` ships by + * default: ten identical passages in one call fill the ten leaves, so any + * spread between their scores is KV noise, not content. Measured 2026-09-07 + * on real document windows: at q4_0 the spread was 4–6 logits and the verdict + * changed sign between leaves; at q8_0 it was 0.05–0.12 at the same speed. + * Every admission threshold stands on this number. + * + * Weights-gated like `reranker-capacity.test.ts`. + * + * @category Testing + */ +import { describe, it, expect } from 'vitest'; +import { run, call } from 'effection'; +import * as path from 'node:path'; +import { createReranker } from '../src/reranker'; +import { RERANK_MODEL_PATH } from './helpers/rerank-model'; + +const describeWithModel = RERANK_MODEL_PATH ? describe : describe.skip; + +const QUERY = 'What is the capital of France?'; +const RELEVANT = 'Paris is the capital and largest city of France, on the river Seine.'; +const IRRELEVANT = 'The recipe calls for two eggs, a cup of flour and a pinch of salt.'; +const COPIES = 10; +/** The widest spread the default KV type may show between identical leaves. */ +const MAX_SPREAD = 0.5; + +describeWithModel(`the default KV type resolves the reranker's verdicts — ${RERANK_MODEL_PATH ? path.basename(RERANK_MODEL_PATH) : 'no model'}`, () => { + it('ten identical passages score within half a logit of each other, and the relevant passage stays above the irrelevant one', async () => { + const scores: number[] = await run(function* () { + const reranker = yield* createReranker(RERANK_MODEL_PATH!, { nSeqMax: COPIES, nCtx: 4096 }); + const texts = [...Array<string>(COPIES).fill(RELEVANT), ...Array<string>(COPIES).fill(IRRELEVANT)]; + return yield* call(() => reranker.scoreBatch(QUERY, texts)); + }); + const relevant = scores.slice(0, COPIES); + const irrelevant = scores.slice(COPIES); + const spread = (xs: number[]): number => Math.max(...xs) - Math.min(...xs); + const show = (xs: number[]): string => xs.map((s) => s.toFixed(2)).join(' '); + expect(spread(relevant), `relevant copies: ${show(relevant)}`).toBeLessThan(MAX_SPREAD); + expect(spread(irrelevant), `irrelevant copies: ${show(irrelevant)}`).toBeLessThan(MAX_SPREAD); + expect(Math.min(...relevant), `relevant ${show(relevant)} vs irrelevant ${show(irrelevant)}`).toBeGreaterThan(Math.max(...irrelevant)); + }, 120_000); +}); diff --git a/scripts/cut-alpha.test.ts b/scripts/cut-alpha.test.ts index 4def725e..7358b274 100644 --- a/scripts/cut-alpha.test.ts +++ b/scripts/cut-alpha.test.ts @@ -166,12 +166,16 @@ describe('the abilities admit the set', () => { }; const view = (name: string) => { if (name in registry) return registry[name]; throw e404; }; const stamped = planAlphas({ cut: 99, packages: arcPackages(CUTS, EXTERNAL, manifestOf), view }); - for (const dir of ['packages/abilities/web', 'packages/abilities/corpus', 'packages/abilities/wikipedia']) { + for (const dir of ['packages/abilities/web', 'packages/abilities/corpus', 'packages/abilities/wikipedia', 'packages/abilities/documents']) { const peers = manifestOf(dir).peerDependencies ?? {}; for (const [name, range] of Object.entries(peers)) { if (!(name in stamped)) continue; expect(satisfies(stamped[name], range), `${dir}: ${name} ${range} admits ${stamped[name]}`).toBe(true); - expect(satisfies(registry[name], range), `${dir}: ${name} ${range} admits stable ${registry[name]}`).toBe(true); + // A member the registry has never seen stable (media, whose first + // publish was an alpha) has no stable to admit yet. + if (registry[name] !== undefined) { + expect(satisfies(registry[name], range), `${dir}: ${name} ${range} admits stable ${registry[name]}`).toBe(true); + } } } }); diff --git a/scripts/verify-packed-install.sh b/scripts/verify-packed-install.sh index a2fc50da..89d975ec 100755 --- a/scripts/verify-packed-install.sh +++ b/scripts/verify-packed-install.sh @@ -90,14 +90,25 @@ node -e " if (opt !== true) throw new Error('sharp peer is not marked optional'); if ((p.devDependencies||{}).sharp && !peer) throw new Error('devDependency only: never installed for consumers'); if (!(p.files||[]).includes('README.md')) throw new Error('README.md missing from files — the format spec never ships'); - console.log(' ok — peer '+peer+', optional, README packaged'); + // The PDF codec sits beside sharp on the same terms: a consumer that never + // admits a document installs nothing for it. + const pdf = (p.peerDependencies||{})['@embedpdf/pdfium']; + const pdfOpt = ((p.peerDependenciesMeta||{})['@embedpdf/pdfium']||{}).optional; + if (!pdf) throw new Error('@embedpdf/pdfium is not a peerDependency — a consumer cannot discover it'); + if (pdfOpt !== true) throw new Error('@embedpdf/pdfium peer is not marked optional'); + console.log(' ok — peers sharp '+peer+' and @embedpdf/pdfium '+pdf+', both optional, README packaged'); " echo "3. './node' normalizes from the packed build once sharp is installed" npm install --silent --no-audit --no-fund sharp@^0.35.4 node -e " - const { normalizeImage, FileAttachmentStore } = require('@lloyal-labs/media/node'); + const { normalizeImage, FileAttachmentStore, createContentIngress, createDocumentIngress } = require('@lloyal-labs/media/node'); if (typeof FileAttachmentStore !== 'function') throw new Error('the node entry is incomplete'); + // The document ingress is exported, but loading the entry must not have + // loaded the codec: it is required at call time, like sharp, and it is not + // installed in this consumer. + if (typeof createContentIngress !== 'function' || typeof createDocumentIngress !== 'function') throw new Error('the content ingress is missing from the node entry'); + if (Object.keys(require.cache).some(k => k.includes('@embedpdf'))) throw new Error('the node entry loaded @embedpdf/pdfium at module load'); const fs = require('fs'); const src = new Uint8Array(fs.readFileSync('$CAT')); normalizeImage(src, { maxPixels: 65536 }).then(o => { diff --git a/tsconfig.test.json b/tsconfig.test.json index 4c24952e..55ef82a5 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -54,7 +54,8 @@ "@lloyal-labs/relay": ["packages/relay/src/index.ts"], "@lloyal-labs/corpus-ability": ["packages/abilities/corpus/src/index.ts"], "@lloyal-labs/web-ability": ["packages/abilities/web/src/index.ts"], - "@lloyal-labs/wikipedia-ability": ["packages/abilities/wikipedia/src/index.ts"] + "@lloyal-labs/wikipedia-ability": ["packages/abilities/wikipedia/src/index.ts"], + "@lloyal-labs/documents-ability": ["packages/abilities/documents/src/index.ts"] } }, "include": [