feat(media,rig,agents,abilities)!: documents — PDF ingress, a documents ability, assets on the run

packages/abilities/corpus/src/index.ts

@@ -1,8 +1,8 @@
 /**
  * `@lloyal-labs/corpus-ability` — HDK reference ability: local-corpus research.
  *
- * Requires a reranker (its `search` tool scores chunks); loads + tokenizes the
- * corpus at construction, and returns a validated {@link Ability} whose
+ * Requires a reranker (its `search` tool scores chunks); loads the corpus and
+ * fits it into reranker-sized windows at construction, and returns a validated {@link Ability} whose
  * {@link CorpusSource} is already-bound.
  *
  * @packageDocumentation
@@ -13,15 +13,13 @@ import { join } from "node:path";
 import { call } from "effection";
 import { AbilityConfigStoreCtx, RerankerCtx } from "@lloyal-labs/lloyal-agents";
 import type { AbilityManifest, Tool } from "@lloyal-labs/lloyal-agents";
-import { defineAbility } from "@lloyal-labs/rig";
+import { defineAbility, fitChunks, DEFAULT_CHUNK_TOKENS } from "@lloyal-labs/rig";
 import type { Reranker } from "@lloyal-labs/rig";
 import { loadResources, chunkResources } from "@lloyal-labs/rig/node";
 import { CorpusSource } from "./source";
 
 export { CorpusSource } from "./source";
 export type { CorpusSourceOpts, CorpusPromptData } from "./source";
-export { BM25Index } from "./bm25";
-export type { Bm25Opts, Bm25Hit } from "./bm25";
 
 // The declarative manifest + skill template, read once at module load. The
 // manifest is handed to defineAbility, which advertises it on the factory — so the
@@ -63,8 +61,11 @@ export const createCorpusAbility = defineAbility(manifest, function* () {
   }
 
   const resources = loadResources(corpusPath);
-  const chunks = chunkResources(resources);
-  yield* call(() => reranker.tokenizeChunks(chunks));
+  // Sections become windows the reranker scores whole; the size is a retrieval
+  // choice (see DEFAULT_CHUNK_TOKENS), the tokens are the reranker's own.
+  const chunks = yield* call(() =>
+    fitChunks(chunkResources(resources), { maxTokens: DEFAULT_CHUNK_TOKENS, tokenize: (t) => reranker.tokenize(t) }),
+  );
 
   const source = new CorpusSource(resources, chunks, reranker);
   const tools: Record<string, Tool> = {};

packages/abilities/corpus/src/tools/read-file.ts

@@ -2,64 +2,8 @@ import type { Operation } from 'effection';
 import { Tool } from '@lloyal-labs/lloyal-agents';
 import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents';
 import type { Resource, Chunk } from '@lloyal-labs/rig';
+import { mergeRanges, subtractRanges } from '@lloyal-labs/rig';
 
-/**
- * Subtract previously-covered ranges from a target range
- *
- * Given a target half-open interval `[s, e)` and an array of
- * already-covered intervals, returns the sub-ranges of `[s, e)`
- * that have not yet been covered. Used by {@link ReadFileTool}
- * to avoid re-reading lines the agent has already seen.
- *
- * @param range - Target range `[start, end)` (0-indexed)
- * @param covered - Array of previously-covered `[start, end)` ranges
- * @returns Uncovered sub-ranges of the target
- *
- * @category Rig
- */
-export function subtractRanges(
-  [s, e]: [number, number],
-  covered: [number, number][],
-): [number, number][] {
-  let ranges: [number, number][] = [[s, e]];
-  for (const [cs, ce] of covered) {
-    ranges = ranges.flatMap(([a, b]): [number, number][] => {
-      if (ce <= a || cs >= b) return [[a, b]];
-      const result: [number, number][] = [];
-      if (a < cs) result.push([a, cs]);
-      if (ce < b) result.push([ce, b]);
-      return result;
-    });
-  }
-  return ranges;
-}
-
-/**
- * Merge overlapping or adjacent half-open ranges into a minimal set
- *
- * Sorts the input ranges by start position, then collapses any
- * overlapping or touching intervals. Used by {@link ReadFileTool}
- * to maintain a compact record of lines already read per agent.
- *
- * @param ranges - Array of `[start, end)` ranges to merge
- * @returns Merged non-overlapping ranges sorted by start
- *
- * @category Rig
- */
-export function mergeRanges(ranges: [number, number][]): [number, number][] {
-  if (ranges.length === 0) return [];
-  const sorted = [...ranges].sort((a, b) => a[0] - b[0]);
-  const merged: [number, number][] = [sorted[0]];
-  for (let i = 1; i < sorted.length; i++) {
-    const last = merged[merged.length - 1];
-    if (sorted[i][0] <= last[1]) {
-      last[1] = Math.max(last[1], sorted[i][1]);
-    } else {
-      merged.push(sorted[i]);
-    }
-  }
-  return merged;
-}
 
 /**
  * Read content from corpus files by line range

packages/abilities/corpus/src/tools/search.ts

@@ -4,7 +4,7 @@ import { Tool, Trace, admitChunks } from '@lloyal-labs/lloyal-agents';
 import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents';
 import type { Chunk } from '@lloyal-labs/rig';
 import type { Reranker } from '@lloyal-labs/rig';
-import { BM25Index } from '../bm25';
+import { BM25Index } from '@lloyal-labs/rig';
 
 /**
  * Default score floor for search hits — a useful **discrimination signal**,

packages/abilities/corpus/test/ability.test.ts

@@ -10,9 +10,12 @@ import { createInMemoryConfigStore } from '@lloyal-labs/rig';
 import { createCorpusAbility } from '../src/index';
 import { SearchTool } from '../src/tools/search';
 
-// The factory only calls reranker.tokenizeChunks at construction; search
-// scoring (which needs a real cross-encoder) isn't exercised here.
-const mockReranker = { tokenizeChunks() {} } as unknown as Reranker;
+// The factory fits the corpus into windows at construction, which needs only
+// the reranker's tokenizer; search scoring (a real cross-encoder) isn't
+// exercised here. Words stand in for tokens.
+const mockReranker = {
+  tokenize: async (text: string) => text.split(/\s+/).filter(Boolean).map((_, i) => i + 1),
+} as unknown as Reranker;
 
 let dir: string;
 beforeAll(() => {
@@ -47,6 +50,55 @@ describe('createCorpusAbility', () => {
     ).rejects.toThrow(/requires a reranker/);
   });
 
+  it('fits a long section into more than one window with real line ranges', async () => {
+    // One heading over 360 words on 60 lines: longer than DEFAULT_CHUNK_TOKENS
+    // under the word tokenizer, so the factory must cut it into windows the
+    // reranker can score whole. The source keeps its chunks private; the
+    // search envelope's `totalScored` is the chunk count, and each hit carries
+    // the window's real lines.
+    const longDir = mkdtempSync(join(tmpdir(), 'corpus-long-'));
+    const body = Array.from({ length: 60 }, (_, i) => `line ${i + 1} alpha beta gamma delta`).join('\n');
+    writeFileSync(join(longDir, 'long.md'), `# Long\n\n${body}\n`);
+    try {
+      const wordy: Reranker = {
+        ...mkScoringReranker(new Map()),
+        tokenize: async (text: string) => text.split(/\s+/).filter(Boolean).map((_, i) => i + 1),
+        score(_query: string, chunks: Chunk[]) {
+          return (async function* () {
+            yield {
+              filled: chunks.length, total: chunks.length,
+              results: chunks.map((c) => ({
+                file: c.resource, heading: c.heading, section: c.section, snippet: c.text,
+                score: 1, startLine: c.startLine, endLine: c.endLine,
+              })),
+            };
+          })();
+        },
+      };
+      const result = (await run(function* () {
+        yield* Trace.set(new NullTraceWriter());
+        const store = createInMemoryConfigStore();
+        yield* store.set('corpus', { corpusPath: longDir });
+        yield* AbilityConfigStoreCtx.set(store);
+        yield* RerankerCtx.set(wordy);
+        const ability = yield* createCorpusAbility();
+        const search = ability.tools.find((t) => t.name === 'search')!;
+        return yield* search.execute({ query: 'alpha' });
+      })) as { hits: ScoredChunk[]; totalScored: number };
+
+      expect(result.totalScored).toBeGreaterThan(1);
+      const starts = result.hits.map((h) => h.startLine);
+      expect(new Set(starts).size).toBe(starts.length);
+      for (const h of result.hits) {
+        expect(h.endLine).toBeGreaterThanOrEqual(h.startLine);
+        expect(h.startLine).toBeGreaterThanOrEqual(1);
+        expect(h.endLine).toBeLessThanOrEqual(62);
+      }
+    } finally {
+      rmSync(longDir, { recursive: true, force: true });
+    }
+  });
+
   it('throws when corpusPath config is missing', async () => {
     await expect(
       run(function* () {

packages/abilities/documents/LICENSE

@@ -0,0 +1,107 @@
+# Functional Source License, Version 1.1, Apache 2.0 Future License
+
+## Abbreviation
+
+FSL-1.1-Apache-2.0
+
+## Notice
+
+Copyright 2026 Lloyal Labs
+
+## Terms and Conditions
+
+### Licensor ("We")
+
+The party offering the Software under these Terms and Conditions.
+
+### The Software
+
+The "Software" is each version of the software that we make available under
+these Terms and Conditions, as indicated by our inclusion of these Terms and
+Conditions with the Software.
+
+### License Grant
+
+Subject to your compliance with this License Grant and the Patents,
+Redistribution and Trademark clauses below, we hereby grant you the right to
+use, copy, modify, create derivative works, publish, and distribute the
+Software for any Permitted Purpose identified below.
+
+### Permitted Purpose
+
+A "Permitted Purpose" is any purpose other than a Competing Use. A
+"Competing Use" means making the Software available to others in a
+commercial product or service that:
+
+1. substitutes for the Software;
+
+2. substitutes for any other product or service we offer using the Software
+   that exists as of the date we make the Software available; or
+
+3. offers the same or substantially similar functionality as the Software.
+
+Permitted Purposes specifically include using the Software:
+
+1. for your internal use and access;
+
+2. for non-commercial education;
+
+3. for non-commercial research; and
+
+4. in connection with professional services that you provide to a Licensee
+   using the Software in accordance with these Terms and Conditions.
+
+### Patents
+
+To the extent your use for a Permitted Purpose would necessarily infringe our
+patents, the license grant above includes a license under our patents. If you
+make a claim against any party that the Software infringes or contributes to
+the infringement of any patent, then your patent license to the Software ends
+immediately.
+
+### Redistribution
+
+The Terms and Conditions apply to all copies, modifications and derivatives
+of the Software.
+
+If you redistribute any copies, modifications or derivatives of the Software,
+you must include a copy of or a link to these Terms and Conditions and not
+remove any copyright notices provided in or with the Software.
+
+### Disclaimer
+
+THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND,
+INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
+
+IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO
+THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL
+DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
+
+### Trademarks
+
+Except for displaying the License Details and identifying us as the origin of
+the Software, you have no right under these Terms and Conditions to use our
+trademarks, trade names, service marks or product names.
+
+## Grant of Future License
+
+We hereby irrevocably grant you an additional license to use the Software
+under the Apache License, Version 2.0 that is effective on the second
+anniversary of the date we make the Software available. On or after that
+date, you may use the Software under the Apache License, Version 2.0, in
+which case the following will apply:
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+See the License for the specific language governing permissions and
+limitations under the License.