From 5783451148713d06a8f4c6676b9d4334a2e0bcbc Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Wed, 18 Nov 2026 10:39:09 +0800 Subject: [PATCH 01/15] Unify default knowledge discovery and versioned delivery Use Find to rank managed knowledge questions before collecting pinned, budgeted context. Resolve installed entries inside each runtime so base installations and DSH can follow a hit to the original material. Retain exact catalog and explicit legacy operations for existing callers. Agent-assisted-by: Codex (managed discovery, readback and DSH implementation) Signed-off-by: Zhou ziheng --- docs/dsh-plugin.md | 10 +- docs/knowledge-operations.md | 110 ++++----- docs/knowledge-retrieval-trace.md | 27 +-- docs/operations.md | 76 +++--- docs/optimization-workflow-unification.md | 2 +- docs/quickstart.md | 67 +++--- plugins/inferrsi-dsh/remote.cjs | 2 +- plugins/inferrsi/skills/inferrsi/SKILL.md | 28 +-- .../inferrsi/skills/inferrsi/SKILL.md | 49 ++-- src/inferrsi/apps/campaign.py | 22 +- src/inferrsi/apps/cli/__init__.py | 8 +- src/inferrsi/apps/cli/arguments.py | 63 +++-- src/inferrsi/apps/context_provider.py | 13 +- src/inferrsi/apps/knowledge.py | 12 +- src/inferrsi/apps/views/capture_delivery.py | 3 +- src/inferrsi/apps/views/search_archive.py | 2 +- src/inferrsi/knowledge/_skill_selection.py | 38 ++- src/inferrsi/knowledge/knowledge_find.py | 155 ++++++++----- .../knowledge/knowledge_operations.py | 216 +++++++++++++----- tests/check_dsh_remote.cjs | 4 + tests/fixtures/knowledge_retrieval.py | 6 +- tests/test_apps_context_provider.py | 21 +- tests/test_campaign_record_injection.py | 13 +- tests/test_campaign_task_selection.py | 60 ++++- tests/test_dsh_plugin.py | 35 +++ tests/test_knowledge_find.py | 12 +- tests/test_knowledge_install.py | 39 +++- tests/test_optimization_session.py | 44 ++-- tests/test_project_optimization.py | 17 ++ 29 files changed, 773 insertions(+), 381 deletions(-) diff --git a/docs/dsh-plugin.md b/docs/dsh-plugin.md index 6cf14409c..d8f90237d 100644 --- a/docs/dsh-plugin.md +++ b/docs/dsh-plugin.md @@ -232,8 +232,14 @@ the launcher is `scripts/inferrsi.py`. - Use `--plugin-info` to check the selected interpreter, bundled runtime version, and content SHA-256. An unbuilt source checkout can report `0+unknown` as the runtime version. -- Use `collect` to find relevant packaged knowledge and `read` to inspect a - selected revision and its resources. +- Use `find ""` to search Cases, Skills and Records. Follow the + emitted `read` command to inspect the selected material and its revision. + On a remote connection, use `inferrsi_remote` with `operation: "cli"` and + `args: ["find", "", "--json"]`. Send the hit's `read_args` as + `args` in the next CLI call. + The base install supports reading bundled Cases and Skills that have a + packaged projection. Reading Skills from a custom `--skill-root` requires + PyYAML in the interpreter running InferRSI. - Use `plan` to preview a comparison. Running a comparison requires the accepted scope, frozen gates, and prepared target environment. - Use `status` and `report` to inspect an existing run and retain its recorded diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md index 7003561a8..5b7950359 100644 --- a/docs/knowledge-operations.md +++ b/docs/knowledge-operations.md @@ -19,22 +19,23 @@ source = reader.resource(item["ref"], "methods/calibrate-serving-simulation/refe ## CLI access -The `collect`, `read` and `resource` commands expose these operations with the -same rules (see the [operation guide](operations.md) for flags, boundaries and -examples). Sources are selected explicitly: the installed package by default, a -an explicit skills root via `--skill-root`, an existing store read-only via `--store`; -the store records the skills root it reserves and rejects a mismatched or missing -`--skill-root` on open. `resource` requires the exact revision returned by -`collect`/`read` and writes bytes to `--output`. +For discovery, use `find` and follow the relevant hit's `open` command. Use +`collect` to take a batch of known materials, and `read` or `resource` for an +exact bundle or resource. The [operation guide](operations.md) lists the flags. +The package catalog is available by default. Select a custom library with +`--skill-root` or select the bundled library with `--installed-library` on +`collect`, `read` or `resource`. The read-only `--store` option requires the +store's recorded skills root if one was configured. `resource` takes the +revision returned by `collect` or `read` and writes bytes to `--output`. ```bash -inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/inferrsi/knowledge/data/skill_library -inferrsi read calibrate-serving-simulation --skill-root src/inferrsi/knowledge/data/skill_library +inferrsi find 'Ascend OOM after caching per-step masks' --kinds case ``` -Reading checkout skill bodies requires the `skills` extra (`python -m pip install -'.[skills]'`). Package-only operations and checkout case reads work with the base -install. +Direct reads of checkout Skill bodies require the `skills` extra +(`python -m pip install '.[skills]'`). On a base install, `find` opens a matching +indexed installed Skill through its package projection. Package-only operations +and direct Case reads also work with the base install. Every item has an owning `authority` (`package`, `checkout` or `store`) and a qualified `ref` of the form `:`. Public IDs are unchanged and may be used @@ -66,18 +67,23 @@ Deterministic identity order is not a learned relevance ranking. ## Optional packaged skill selection and context -Two independent opt-ins extend the same `collect` operation: +Standalone Collect supports legacy query-based selection and budgeted +assembly. The explicit-ID path can assemble context without a ranking query: ```python reader = KnowledgeReader() -# Existing query and material semantics are unchanged. legacy = reader.collect("cpu.stat.local", material=True, limit=3) -selected = reader.collect("cpu.stat.local", relevance="cgroupv2", - context_bytes=65536, limit=3) +selected = reader.collect( + ids=["package:inferrsi.ascend.skills.ascend_graph_mode", + "package:inferrsi.ascend.skills.ascend_device_health"], + context_bytes=65536, +) ``` ```bash -inferrsi collect 'cpu.stat.local' --relevance cgroupv2 --context-bytes 65536 --json +inferrsi collect --id package:inferrsi.ascend.skills.ascend_graph_mode \ + --id package:inferrsi.ascend.skills.ascend_device_health \ + --context-bytes 65536 --json ``` `relevance` is a separate lexical cue, not a replacement for `query`. Discovery @@ -102,16 +108,21 @@ from its JSON escape. Literal backslash-letter text remains searchable. Nonstring fields, including the combined tags/applicability value, retain their legacy JSON representation. Both methods keep the same substring matching, distinct cue words, weights, positive-score admission and identity tie order. -The nondefault method requires `relevance`; unsupported methods are rejected. +The raw-string method requires `relevance`. Unsupported methods are rejected. Explicit IDs still bypass ranking. This option changes only owner discovery ranking, not query matching or optional context-resource scoring. It is a lexical-presence contract option, not evidence of general retrieval quality. -Campaign discovery (`campaign capture-context --relevance ...`) defaults to -`task-lexical-v1`. This method searches registered skills and package knowledge -records, including GPU procedures under the engine's repository pack. -`collect --relevance-method task-lexical-v1 --relevance ...` runs the same -search. Whole words in titles, summaries, tags, repository/scenario labels, +For questions in `campaign knowledge`, `benchmarks session knowledge` and +`campaign capture-context`, the default ranking method is +`find-lexical-v1`. It uses the same ranker as `find` to select Cases, Skills +and Records, then assembles the selected material with its revisions and +budget omissions. Explicit IDs bypass ranking. Captures using an explicit +`--store` use the established store selection. To select a different ranking +method, pass `--relevance-method` explicitly. + +`task-lexical-v1` remains available for package Skill and Record searches. +Whole words in titles, summaries, tags, repository/scenario labels, applicability and body text contribute to rank. A portable owner or one matching the backend receives twice the lexical score of an unscoped specialization. Without an explicit backend, platform words in the cue @@ -121,26 +132,24 @@ Use `campaign init --backend ` to retain the runtime backend for later searches. An explicit capture backend takes precedence. Existing campaigns can pass `--backend` directly on the capture. The search returns the best matches up to the requested limit and records the matched and -returned counts. For engine records it captures the complete `record.json`, -including the source, applicable geometry and limitations. Linked skill -documents can be read at the returned revision. Registered skills keep the -resource rules below. In task discovery, `--scenario` admits records matching -the workload and registered skills tagged `skills.discovery`. That tag describes -the discovery operation. The task cue ranks these procedures, whose suitability -must be checked against their own conditions. Captured hits retain their -declared scenario. Explicit IDs and the older ranking methods still filter on -the owner's declared catalog scenario. - -`context_bytes` requests a separate context view for returned generated packaged -skills. It requires an explicit positive integer budget, with no token estimate -or default. It can be used without `relevance`; optional resources then match the -existing query, or use path order for an empty query. The existing `material=True` -still returns `record.json` for these owners, not `SKILL.md`. +returned counts. For engine Records it captures the complete `record.json`, +including the source, applicable geometry and limitations. For Cases it can +capture the complete entrypoint. Linked documents can be read at the returned +revision. Indexed Skills keep the resource rules below. In Find and task +discovery, `--scenario` admits package Records matching the workload and +packaged Skill records tagged `skills.discovery`. Check each procedure's own +conditions before using it. Captured hits retain the scenario declared by the +owner. Explicit IDs and legacy catalog selectors filter on the declared scenario. + +`context_bytes` requests a separate context view. It takes a positive integer +byte budget. It can be used without `relevance`. For indexed Skills, optional +resources then match the existing query, or use path order for an empty query. +`material=True` on a package Skill record returns its `record.json` entrypoint. | Additional result | Meaning | | --- | --- | | `retrieval` | Recorded entry, filters, ranking execution, returned-item scores and budget counts; outside budgeted item/resource payloads. See [retrieval observations](knowledge-retrieval-trace.md#execution-observations) | -| `selection` | Method, cue, scope and whether discovery ranking was applied; only with `relevance` | +| `selection` | Method, cue, scope and whether discovery ranking was applied | | `contexts` | Selected owner views, each with `ref`, exact Bundle `revision`, resources, `content_bytes`, resource omissions, sources, attribution and limitations | | `context_omissions` | Owner context unavailable: `unsupported_context`, `index_unavailable` or `context_limit` | | `context_bytes`, `context_budget` | Used body bytes, requested ceiling and accounting method | @@ -204,8 +213,10 @@ or installation rejects an old revision; it never substitutes new bytes. This reader does not itself retain history or claim standalone exports. `read --material --omit-resources` returns the full entrypoint, provenance, -applicability and revision. Remove `--omit-resources` to list resource paths and -digests. Both forms use the same full-bundle revision. +applicability and revision. Use `read --resource ` to display a specific +UTF-8 resource in `content` and its key in `content_resource`. Remove +`--omit-resources` to list resource paths and digests. These views use the same +full-bundle revision. - Local skills include their complete owner folder (except incidental `__pycache__`, `*.pyc` and `.DS_Store`) and declared cross-skill references, @@ -239,21 +250,18 @@ supplied code never runs it or grants permissions. ## Retrieve diagnostic options -Rejected directions use the same checkout Case bundles as other observations: +Use `find --kinds case` to locate rejected directions. Given the identity and +revision of a hit, reopen the exact Case through the same library: ```python reader = KnowledgeReader(skill_root=Path("src/inferrsi/knowledge/data/skill_library")) -options = reader.collect("Ascend OOM", kind="case", material=True, limit=3) -item = options["items"][0] # Inspect no_match/omissions first in a general client. -case = reader.read(item["ref"], revision=item["revision"]) +case = reader.read("checkout:case.qwen-ascend-rejected-directions", + revision="") ``` -Other example queries are `B300 RMSNorm` and `Drama3 prefix reuse`. Query terms -match metadata and entry content as substrings; combine a mechanism and context -to narrow results. An explicit ID such as -`checkout:case.qwen-ascend-rejected-directions` selects exactly one record. -Package repo/backend/scenario filters still exclude checkout cases; inspect the -returned applicability rather than assuming those filters qualify a case. +Other example queries are `B300 RMSNorm` and `Drama3 prefix reuse`. Include the +mechanism and symptom to narrow the results. Read the Case's conditions and +limitations before applying the observation. For explicit CLI resource-path resolution from the installed library (optional `skills` extra required), or a custom library selected with `--root`: diff --git a/docs/knowledge-retrieval-trace.md b/docs/knowledge-retrieval-trace.md index 8e0e783cd..81c36b7f1 100644 --- a/docs/knowledge-retrieval-trace.md +++ b/docs/knowledge-retrieval-trace.md @@ -13,22 +13,22 @@ interfaces do not depend on unfinished general statistics or workflow proposals. ## Execution observations -`knowledge search` and `collect` remain separate operations. Neither calls the -other. `campaign capture-context` calls `collect`; its default task relevance -policy is unchanged. There is no global listener or automatic capture. +`campaign capture-context` uses Find ranking for a single free-text question, +then assembles selected material through Collect. The capture records both +operations. Explicit IDs select material in caller order. Exact catalog search +and explicit legacy ranking methods remain available for existing workflows. +Capture is explicit. ```sh -inferrsi knowledge search 'decode launch' --explain -inferrsi collect --relevance 'decode launch' --json +inferrsi find 'decode launch' --explain --json inferrsi campaign capture-context --directory --id r1-context \ --relevance 'decode launch' --context-bytes 65536 ``` `knowledge search --explain` returns a JSON envelope with `items` and `retrieval`. -Without that flag, its list API, JSON and tabular output are unchanged. Python -callers use `search_knowledge(..., explain=True)`. `collect` adds a `retrieval` -member outside its budgeted item/resource payloads. Existing item revisions, -return order, `material_bytes`, context assembly and byte budgets are unchanged. +Python callers use `search_knowledge(..., explain=True)` for the same +catalog envelope. `collect` includes a `retrieval` member outside its +budgeted item and resource payloads. | `retrieval` field | Meaning | | --- | --- | @@ -47,11 +47,12 @@ remain repeated components. Ties use ascending ID. A catalog observation's `catalog-sha256:` revision hashes the returned catalog record using sorted, UTF-8 JSON. It identifies that record, not a collect bundle or an upstream commit. -The two weighted methods record title, summary, tags/applicability and body -contributions. Task relevance records title, summary, tags, repo/scenario, +Find records each matching term's contribution and uses recorded state, +identity and qualified reference to break score ties. The two legacy weighted +methods record title, summary, tags/applicability and body contributions. Task relevance records title, summary, tags, repo/scenario, applicability and body contributions plus its platform multiplier. Ties use -ascending qualified ref. Explicit IDs preserve caller order and skip ranking; -no relevance request records `not_requested`. Filters use first-exclusion +ascending qualified ref. Explicit IDs preserve caller order and skip ranking. +A plain inventory records `not_requested`. Filters use first-exclusion counts; these are not overlapping marginal totals. Per-item explanations cover returned items only. Existing omissions and resource hashes remain available. diff --git a/docs/operations.md b/docs/operations.md index 806b26023..1a48ce02a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -32,11 +32,11 @@ Effect classes used below: | Prepare and record a fresh native Agent | CLI `agents prepare/dispatch/bind/record/status` | Local records / explicit host handoff | The initiating Agent calls native host tools. Records prevent repeated dispatch; exported host observations do not establish task acceptance. See [native sessions](native-agent-sessions.md) | | Discover packaged tasks | CLI `benchmarks list` / `benchmarks show`; Python `benchmark.registry.builtin_tasks` / `describe_task` | Read-only | Reads the task library's registry and opaque manifests from installed resources | | Execute one packaged task | CLI `benchmarks run`; Python `apps.benchmark_operations.run_task` | Experiment-executing | Application launches the task entry, supplies the execution callback and preserves arguments and exit status. The task owns its device, measurement and assessment protocol | -| Search packaged records | CLI `knowledge search` / `knowledge show`; catalog API | Read-only | Installed package catalog with the existing tier/repo/backend/scenario filters and IDs; these package-only filters exclude skills and cases from every other authority | +| Find relevant cases, skills and records | CLI `find` | Read-only | Default discovery across the registered library and package records. Follow the emitted command to read a hit and its revision | +| Inspect the package catalog | CLI `knowledge search` / `knowledge show`; catalog API | Read-only | Filtered inventory and existing catalog scripts. Uses the installed package's tier/repo/backend/scenario filters and IDs | | Prepare model-aware engine research | CLI `knowledge research`; Python `research_knowledge` | Read-only | Independent model/framework/task/modality/mechanism queries over the same package catalog; source-grounded known profiles, deduplicated matches and explicit target annotations; no engine/hardware qualification | | Discover/read packaged or custom skills | CLI `skills list` / `show` / `resource` / `reference` | Read-only | Metadata-first listing; `show` loads one named instruction body; `resource`/`reference` resolve paths for your file reader without loading or executing them | -| Rank relevant cases, skills and records | CLI `find` | Read-only | Search the registered library and package records in one list. Follow the emitted command to read a hit and its revision | -| Collect normalized knowledge | CLI `collect`; Python `KnowledgeReader.collect` | Read-only | Explicit sources, authority-qualified identities, exact revisions, bounded output and explicit omissions | +| Collect selected knowledge | CLI `collect`; Python `KnowledgeReader.collect` | Read-only | Batch selection and budgeted assembly with exact IDs, revisions and omissions. Existing query and ranking options are also supported | | Read exact material/resources | CLI `read` / `resource`; Python `KnowledgeReader.read` / `KnowledgeReader.resource` | Read-only | Returns the requested revision or an explicit unavailable/stale error; never substitutes newer bytes | | Export legacy cited material | CLI `context` | Read-only | Cited package export for manual use with a coding agent; not a conversation store and not a full Collect equivalent | | Create/update a store candidate | Python `KnowledgeStore.create` / `KnowledgeStore.update` | Candidate-writing | One explicitly authorized local store; expected-revision updates with reason and evidence; retained history; no promotion and no edits to package/checkout content | @@ -50,7 +50,7 @@ Effect classes used below: | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | -| Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Runs `collect` with a context budget and keeps the full envelope and resource hashes in a new `knowledge/.json`; proves what was prepared, not what a model read | +| Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Uses Find for ordinary questions and supports explicit IDs and legacy ranking options | | Retain an executed retrieval | Python `apps.campaign.save_retrieval`; [capture contract](knowledge-retrieval-trace.md#retain-a-standalone-result) | Local evidence-writing | Saves an existing catalog/collect result under a fresh campaign capture ID; no new search or automatic publication | | Import one round of knowledge searches | CLI `campaign import-search`; [round import](dashboard.md#import-one-rounds-searches) | Local evidence-writing | Reads a Codex rollout window or context captures. With `--archive-review`, retains excerpts of attributable tool returns and reviewed decisions for readers of the evidence bundle. | | Record the optimization log | CLI `campaign log`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Validates and stores a new numbered campaign log version; the dashboard cross-checks it against captures and runs | @@ -112,12 +112,15 @@ are distinct from `knowledge search`, whose package catalog filters ### Search packaged records — `knowledge search` / `knowledge show` -- **Inputs:** query words plus optional `--tier`, `--repo-type`, `--backend`, `--scenario`, `--limit`, `--json`, `--explain`; `show` takes one record ID. +Use `find` for ordinary discovery. Use this catalog for filtered inventory and +existing scripts that depend on its exact queries or record JSON. + +- **Inputs:** optional query words and `--tier`, `--repo-type`, `--backend`, `--scenario`, `--limit`, `--json`, `--explain`; `show` takes one record ID. - **Scope:** the installed package catalog (`repo_packs`, `knowledge_packs`) only. - **Output:** tab-separated record lines (id, status, backend, title), or a JSON list with `--json`; `--explain` returns a JSON envelope with `items` and `retrieval` (entry, ranking, scores and counts). Python uses `search_knowledge(..., explain=True)` for that envelope. `show` prints one full record as JSON. - **Side effects:** none. - **Failure behavior:** no match prints nothing and exits 0 — an empty result is not an error (an empty JSON list with `--json`, or empty `items` in the `--explain` envelope); an unknown `show` ID and unsupported filter values exit 2. -- **Example:** `inferrsi knowledge search 'Qwen-Image-2.1' --repo-type vllm_omni --backend ascend` +- **Example:** `inferrsi knowledge search --repo-type vllm_omni --backend ascend --json` The tier/repo/backend/scenario filters are package-only: an empty filtered result does not mean that no checkout or store knowledge exists. See the @@ -178,14 +181,10 @@ checkout cases. Run a result's `open` command to read its complete entry and receive a revision. Use that revision with the same ID and source selection for a pinned `read` or `resource` call. -Three properties distinguish it from the Collect selection path below. Cases are -ranked beside skills instead of being reachable only after their owner is -chosen, because a recorded rejected direction is the material retrieval can -supply that re-reading current source cannot. One matching term is enough to -rank an entry, and `--limit` caps the result list. Scoring reads corpus -files directly rather than Bundles, whose metadata lists the whole packaged -library as one dependency closure. The reader validates the returned material -without pinning a revision during search. +Cases are ranked alongside skills, so prior rejected directions can surface +directly. One matching term is enough to rank an entry, and `--limit` caps the +result list. Scoring reads the corpus files directly. The reader opens bundles +to validate material for the returned hits. Scoring is `idf x field weight x sqrt(occurrences)` summed over query terms, with titles, summaries, tags and case findings weighted above bodies. Bodies @@ -237,7 +236,8 @@ question calls for prior evidence. `source_location` (`installed` or `checkout`), `kind`, recorded `state`, `score`, `title` and corpus-relative `path`. `source_location` identifies where the material was indexed. For installed skill and case bodies, the - `checkout:` read reference uses the installed library as `--skill-root`. + `checkout:` read reference uses `--installed-library` to resolve the source + from the reader's installation. A custom root uses `--skill-root`. Where recorded, items carry source, summary, applicability, limitations and local verification status. Case findings are ordered by query overlap, with `findings_total` and `findings_withheld` describing the available preview. @@ -248,19 +248,17 @@ question calls for prior evidence. - **Reading a hit.** Plain output prints a POSIX `open` command or a labelled PowerShell command on Windows. JSON exposes the rendered command as `ref`, full subprocess arguments as `open_argv` and InferRSI CLI arguments as - `read_args`. The arguments include the source-qualified ID and a skill root - where needed. They use `read --material --omit-resources` to return the - entrypoint, provenance and revision with a smaller response. Remove + `read_args`. The arguments include the source-qualified ID and any needed + source flag. Most hits use `read --material --omit-resources` to return the + full entrypoint, provenance and revision. Remove `--omit-resources` to list the bundle's resources. Hits that fail the reader's material check carry `unreadable` with the reason. -- **Base installation fallback.** When a Skill hit requires PyYAML, run a new - search of the packaged Skills with - `inferrsi collect --relevance "" --limit 1 --material --json`. - It may select a different Skill. Check that the returned Skill fits your - task, then fetch its `skill_documents.original` with - `inferrsi resource --revision --output ` using - the `ref` and `revision` from the new result. -- **Side effects:** none. No revision is pinned; use `read`/`resource` when a capture or evidence record needs exact bytes. +- **Base installation.** When PyYAML is absent, an installed Skill with a + matching valid package record opens its original text through that record, + using `read --resource` and its exact revision. The result names the selected + resource in `content_resource`. A Skill in a custom root may require the + `skills` extra. +- **Side effects:** none. Retain the revision returned by `read` for evidence and resource followups. - **Example:** `inferrsi find "NPU OOM after caching per-step masks" --limit 3` returns the casebook entry recording that attempt first. A vaguer phrasing ranks the procedural skills above it, which is the first limit below rather than a defect. #### External channels are not installed with this package @@ -302,30 +300,32 @@ from this integration mechanism. ### Collect — `collect` / `KnowledgeReader.collect` -Optional `--relevance ` ranks generated packaged skills after the existing -query and filters; `--context-bytes ` returns their complete core plus bounded -whole reference resources in a separate view. They preserve default calls, -explicit-ID membership/order, `material` and the serialized-item `max_bytes` -budget. Context counts UTF-8 body bytes separately and reports unavailable or -oversized context explicitly. See [selection and context](knowledge-operations.md#optional-packaged-skill-selection-and-context) -for scoring, provenance and accounting. For example: -`inferrsi collect 'cpu.stat.local' --relevance cgroupv2 --context-bytes 65536 --json`. +Use Collect to take known material in caller-specified order. With +`--context-bytes `, it assembles complete files up to a UTF-8 body budget and +reports unavailable or oversized context. Existing queries and ranking methods +remain available for compatibility. See [selection and context](knowledge-operations.md#optional-packaged-skill-selection-and-context) +for scoring, provenance and accounting. + +`campaign knowledge` and `benchmarks session knowledge` use Find to select +material by default, then assemble and retain the selected context with its +revisions and byte budget. Explicit `--relevance-method` selects a named +ranking method. - **Inputs:** optional query, `--kind`, repeatable explicit `--id`, repeatable `--revision ID=REV`, `--limit`, `--max-bytes`, `--material`; `--skill-root` selects an explicit skills root, `--store` opens an existing store read-only (the store records its reserved skills root and rejects a mismatched or missing `--skill-root`). - **Scope:** installed package records plus (with `--skill-root`) skills and cases from the explicitly selected root, plus (with `--store`) store candidates; bounded output. - **Output:** with `--json`, the complete **normalized** envelope — items carrying the bundle's normalized metadata, resources and dependencies, plus `no_match`, `omissions`, `collisions`, `material_bytes` and the recorded limitations; fields that exist only in a package record's original entrypoint document are not projected into the normalized metadata and are read via `--material` or `resource`. Without `--json`, the envelope's selection is projected line-wise: one tab-separated line per item (`ref`, `revision`, `kind`, `state`, recorded `applicability`; free-text fields are control-character-escaped so one item stays one line), `collision` lines for multi-authority IDs, `omitted` lines with reason codes, an explicit `no_match` line when nothing was selected (query, filters or limits; when items were skipped the `omitted` lines above it carry the per-item causes), and — with `--material` — the entrypoint content in delimited blocks. The text mode does not print `material_bytes` or the envelope limitations; use `--json` for the complete envelope. The recorded `state` and applicability are printed as recorded and no validation status is invented. Exit 0 covers a result envelope, including no-match and omissions. - **Side effects:** none; both adapters and the read-only store connection are read-only. - **Failure behavior:** discovery-wide invalid identities fail the call; per-item omissions are reported individually. Package repo/tier/backend/scenario filters exclude checkout items. -- **Example:** `inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/inferrsi/knowledge/data/skill_library` +- **Example:** `inferrsi collect --id package:inferrsi.ascend.skills.ascend_graph_mode --id package:inferrsi.ascend.skills.ascend_device_health --context-bytes 65536 --json` ### Read exact material/resources — `read` / `resource` / `KnowledgeReader.read` / `.resource` -- **Inputs:** `read` takes a qualified or unambiguous public ID and an optional `--revision` (the current revision when omitted); `resource` takes the ID, a relative resource key, and the **required** exact `--revision` returned by collect/read plus the required `--output` path. +- **Inputs:** `read` takes a qualified or unambiguous public ID and an optional `--revision` (the current revision when omitted). Use `--material` for the entrypoint or `--resource ` for a specific UTF-8 bundle resource. The `resource` command takes the ID, resource key, exact `--revision` and `--output` path. - **Scope:** one selected bundle from the package, checkout or store adapter. -- **Output:** the bundle view at the requested revision, or the resource bytes (`resource` writes them to the required `--output` path). +- **Output:** `read` returns the bundle view at the requested revision. With `--resource`, it identifies the returned text with `content_resource`. The `resource` command writes raw bytes to the requested output path. Use the same source flag when following up with either command. - **Side effects:** `resource` writes only the caller-chosen output file. - **Failure behavior:** ambiguous identity names the qualified choices; unavailable/stale revisions and missing/unsafe resources fail explicitly — no fallback to newer bytes. -- **Example:** `inferrsi read calibrate-serving-simulation --skill-root src/inferrsi/knowledge/data/skill_library --material` +- **Example:** `inferrsi read checkout:case.qwen-ascend-rejected-directions --installed-library --material` ## Store candidates (Python) diff --git a/docs/optimization-workflow-unification.md b/docs/optimization-workflow-unification.md index c93ed9219..576f98706 100644 --- a/docs/optimization-workflow-unification.md +++ b/docs/optimization-workflow-unification.md @@ -199,7 +199,7 @@ inferrsi campaign final --directory --reason '' inferrsi dashboard --input --output ``` -知识检索和 profiling 按当前问题需要进行,不要求每轮重查或每轮 profile。知识关闭时受管 `knowledge` 操作拒绝执行;这不是对宿主所有文件访问的隔离。 +两个受管 `knowledge` 入口都用 Find 为当前问题排序,再按预算装配选中的正文和需要的资料,留存引用、版本和省略原因。显式传入旧 `--relevance-method` 可以复用对应的排序。知识检索和 profiling 按当前问题需要进行。冻结条件为 `--knowledge off` 时,受管 `knowledge` 操作拒绝执行。 ## 实际保留什么、保护什么 diff --git a/docs/quickstart.md b/docs/quickstart.md index 01c5b488b..8630cde08 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -14,11 +14,9 @@ inferrsi --help inferrsi doctor --repo . --repo-type vllm_omni --backend cpu ``` -Package-only knowledge commands (`knowledge`, `context`, package `collect`/`read`) -work with this base install. Reading skill libraries — the `skills` command and -any `--skill-root` selection — additionally requires -`python -m pip install '.[skills]'` (PyYAML); without it those operations fail -with that install hint instead of a traceback. +This base install supports `find`, Case reads and package-only knowledge +operations. The `skills` command and direct reads of checkout Skill bodies +require `python -m pip install '.[skills]'` (PyYAML). For the maintained model examples, see [Qwen B300](qwen-image-2.1-turbo-b300.md) and [Qwen Ascend](qwen-image-2.1-turbo-ascend.md). Continue below for the generic @@ -58,48 +56,55 @@ inferrsi find '算子测过了但整图质量没过' --limit 3 Run the `open` command on a relevant hit to read the entry and its source, applicability, limitations and revision. Add `--kinds case` to focus the search -on recorded observations and failed directions. +on recorded observations and failed directions. On a base install, an indexed +Skill in the installed library opens its original text through the corresponding +package record. Unmatched query terms do not exclude results that match other terms. If no query terms match the index, `find` returns `no_match`. Read the reported state, -`scope` and bounds before acting on a hit. The remaining commands are -exact-lookup and export — the [operation guide](operations.md) maps every entry -point, effect class and boundary: +`scope` and bounds before acting on a hit. Use the package catalog to list +records with exact filters and `context` to export a cited package selection. +See the [operation guide](operations.md) for command boundaries. ```bash -inferrsi knowledge search 'Qwen-Image-2.1' --repo-type vllm_omni --backend ascend +inferrsi knowledge search --repo-type vllm_omni --backend ascend --json inferrsi context --repo-type vllm_omni --backend cuda --output context.md inferrsi skills list --json # requires python -m pip install '.[skills]' -inferrsi read inferrsi.ascend.skills.ascend_graph_mode --material ``` Normalized Collect, exact-revision reads and resource access are available as CLI commands and as the Python `KnowledgeReader` ([contract](knowledge-operations.md)); authorized local candidate Create/Update remains a Python `KnowledgeStore` operation ([contract](knowledge-store.md)). -The `--skill-root` examples below require the `skills` extra from above: +Use `collect` with explicit IDs when you already know which material you want. +For example, assemble the Ascend graph-mode and device-health Skills within +a 65,536-byte context budget. ```bash -inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/inferrsi/knowledge/data/skill_library -inferrsi read calibrate-serving-simulation --skill-root src/inferrsi/knowledge/data/skill_library --material -inferrsi resource weight-loading-optimization methods/weight-loading-optimization/SKILL.md \ - --revision --skill-root src/inferrsi/knowledge/data/skill_library --output SKILL.md +inferrsi collect --id package:inferrsi.ascend.skills.ascend_graph_mode \ + --id package:inferrsi.ascend.skills.ascend_device_health \ + --context-bytes 65536 --json +inferrsi read checkout:case.qwen-ascend-rejected-directions --installed-library --material +inferrsi resource checkout:case.qwen-ascend-rejected-directions qwen-ascend-rejected-directions.md \ + --revision '' --installed-library --output case.md ``` The revision for `resource` is the second field of the `collect` text-mode line, -or the `revision` field in `collect --json` / `read` output. The JSON mode returns -the complete normalized envelope; fields kept only in a package record's original -entrypoint document are read with `--material` or `resource`, not projected into -the normalized metadata. - -Sources are selected explicitly: the installed package by default, an explicit -skills root via `--skill-root`, an existing store opened read-only via `--store`. -A store records the skills root it reserves and rejects a mismatched or missing -`--skill-root` on open. - -The `skills` command defaults to the installed library's registered skills; -its full bodies and resources are packaged. `--skill-root` in the normalized -examples above explicitly adds that root's skill and case IDs. +or the `revision` field in `collect --json` / `read` output. For a known revision +in a batch, pass `--revision 'ID=REV'` to `collect`. Use `read --material` for the +entrypoint or `read --resource ` for a specific UTF-8 resource. The latter +names the selected resource in `content_resource`. + +The installed package catalog is available by default. For Case and Skill +bodies, `read` and `resource` accept `--installed-library` for the bundled +library or `--skill-root` for a custom library. Reuse the source flag when +following a result. An existing store opened read-only with `--store` requires +its recorded skills root if one was configured. + +During a managed optimization, `campaign knowledge` and +`benchmarks session knowledge` use Find when the frozen plan permits knowledge. +They assemble the selected Case, Skill or Record material within the requested +budget and retain its revisions with the campaign. ## Establish acceptance criteria @@ -130,7 +135,8 @@ plan/run: all doctor flags, plus --workload-id --model --model-revision --output-dir --dry-run status/report: --run-id --output-dir find: QUERY --limit --kinds --skill-root --explain --json -knowledge search: QUERY --tier --repo-type --backend --scenario --limit --json +read: ID --revision --material --resource --omit-resources --installed-library --skill-root --store +knowledge search: [QUERY] --tier --repo-type --backend --scenario --limit --json knowledge show: ID context: QUERY --tier --repo-type --backend --scenario --limit --output ``` @@ -195,7 +201,6 @@ A trial script can read `INFERRSI_RESULT_PATH` when it constructs the native `-- ```bash inferrsi status --run-id RUN_ID --output-dir /work/inferrsi-runs inferrsi report --run-id RUN_ID --output-dir /work/inferrsi-runs -inferrsi knowledge search 'Qwen-Image-2.1' --repo-type vllm_omni --backend ascend inferrsi context --repo-type vllm_omni --backend cuda \ --scenario diffusion.image_generation --output /work/image-context.md ``` diff --git a/plugins/inferrsi-dsh/remote.cjs b/plugins/inferrsi-dsh/remote.cjs index deb600a57..f00dbd44f 100644 --- a/plugins/inferrsi-dsh/remote.cjs +++ b/plugins/inferrsi-dsh/remote.cjs @@ -568,7 +568,7 @@ function createRemoteManager({ } if (request.operation === 'cli') { const args = checkedArgs(request.args); - if (!args.length || !['doctor', 'knowledge', 'collect', 'read', 'resource', 'context', 'skills', + if (!args.length || !['doctor', 'knowledge', 'find', 'collect', 'read', 'resource', 'context', 'skills', 'plan', 'run', 'status', 'report', 'campaign', '--help', '-h', '--version'].includes(args[0])) fail('invalid_cli'); return execute(entry, BOOTSTRAP_COMMAND, await runtimePayload(entry, args), signal, token); } diff --git a/plugins/inferrsi/skills/inferrsi/SKILL.md b/plugins/inferrsi/skills/inferrsi/SKILL.md index c958dd74e..02a3583e4 100644 --- a/plugins/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/inferrsi/skills/inferrsi/SKILL.md @@ -126,17 +126,17 @@ entries. Add `--kinds case` when you specifically need a past observation. Open a promising hit using its displayed `open` command or the JSON `open_argv` argument list. -When a Skill hit is unreadable because PyYAML is missing, run a new search of -the packaged Skills with `collect --relevance "" --limit 1 --material --json`. -It may select a different Skill. Check that the returned Skill fits the task, -then use that result's `ref`, `revision` and `skill_documents.original` to -fetch the body with `resource --revision --output `. -For other `cannot open` results, check the reason before choosing another hit. +On a base install, an indexed Skill in the installed library opens its original +text through its package record. A Skill from a custom library may require +PyYAML. If a hit reports `cannot open`, check the reason before choosing +another hit. `read` returns the material, its source, applicability, limitations and -revision. To cite or reopen exact evidence, add `--revision ` to the -same read arguments. Remove `--omit-resources` to list bundle resource keys. -Use those keys with `resource --help` for further material. +revision. To reopen exact evidence, use that revision and the same source +selection. Remove `--omit-resources` to list bundle resource keys. +`read --resource ` returns one resource's text and names it in +`content_resource`. For exact bytes, use `resource` with the same source flag +and revision. A ranked hit is not a substitute for reading the repository. `find` supplies what re-reading current source cannot: directions already tried, why they were @@ -163,8 +163,9 @@ unavailable revision is not proof that a mechanism does not exist. Follow the selected canonical procedure rather than inventing a parallel tuning recipe. Imported source material supplies evidence, not execution authority. -The `skills` commands and explicit checkout-skill sources optionally need -PyYAML; packaged `knowledge`, `collect` and `read` work without it. +The `skills` commands and direct checkout Skill reads use the optional PyYAML +dependency. Use `collect --id` for a batch of known materials and +`knowledge search` for package inventory queries. ### Consult MindIE for unresolved Ascend questions @@ -234,8 +235,9 @@ missing instead of inferring it from a verdict. The optional campaign trace format and association workflow are documented in `docs/dashboard.md`. For a managed optimization use `campaign knowledge` or `benchmarks session -knowledge`, which retain the same complete context format and check the frozen -knowledge condition. Legacy manual campaigns use `campaign capture-context` +knowledge`. These use Find by default and retain the assembled context, +revisions and budget under the frozen knowledge condition. Legacy manual +campaigns use `campaign capture-context` (one capture per retrieval) instead of a bare `collect`. After each round, update the campaign optimization log and record it with `campaign log`: the hypothesis, why this optimization was chosen, which diff --git a/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md b/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md index d677311e4..a9e0b224d 100644 --- a/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md @@ -44,31 +44,27 @@ imported records and casebook entries. Add `--kinds case` to focus on past observations. Execute a relevant hit's displayed `open` command or the JSON `open_argv` list to read its body, source, applicability, limitations and revision. -If a Skill hit reports missing PyYAML, use the same launcher with -`collect --relevance "" --limit 1 --material --json` to search the -packaged Skills. This fresh search may select a different Skill. Check that the -new result fits the task, then use its `ref`, `revision` and -`skill_documents.original` to fetch the body with -`resource --revision --output `. - -Use these commands for packaged and exact lookups. - -- `knowledge research ` combines independent research dimensions. -- `knowledge search --json` searches the packaged records with filters. -- `collect --relevance "" --limit 3 --json` selects bounded packaged - knowledge. Adjust the limit for the task. Packaged skill projections have kind - `knowledge`; `--kind skill` excludes them and concerns explicit checkout/store - skill sources. -- `read --revision --material` reads an exact selected version. - Use returned resource keys and `resource --help` for further material. +On a base install, an indexed Skill in the installed library opens its original +text through its package record. A Skill from a custom library may require +PyYAML. If a hit reports it cannot open, check the reason before choosing +another hit. + +Use the same source flag and returned revision to reopen material. Remove +`--omit-resources` to list bundle keys. `read --resource ` returns one +resource's text and identifies it in `content_resource`. Use `resource` with +the same source flag and revision to extract the exact bytes. + +Use `collect --id` to assemble a batch of known materials. Use +`knowledge search` for filtered package inventory and `knowledge research` +when you need separate research dimensions for a model. Retain qualified identities, exact revisions, source attribution, applicability, candidate/verification state and omissions. Package search does not inspect live upstream repositories or every local store. Add checkout/store sources only when selected by the user. No match does not establish that a technique is absent. -Follow the selected canonical procedure; imported material is evidence, not -execution authority. Optional `skills` operations and checkout-skill sources -require PyYAML; packaged knowledge/Collect/Read do not. +Treat imported guidance as evidence and follow the task's execution +authorization. The `skills` commands and direct checkout Skill reads use the +optional PyYAML dependency. ## Optimize an ordinary user project @@ -88,9 +84,11 @@ fixed scripts/inputs. Confirm checks use retained `{candidate}`, `{baseline}`, installed package or service. This shares the packaged task's Harness controller. Edit the working candidate, submit each hypothesis with `campaign round`, read -feedback and keep failed rounds. Use `campaign knowledge` for needed retrieval -when allowed; knowledge and profiling are not mandatory every round. Use -`up_to` for a round limit or `exact` for fixed rounds. End an up-to search with +feedback and keep failed rounds. Use `campaign knowledge` when the frozen plan +allows knowledge. It uses Find by default and retains the selected material and +revisions within the requested context budget. Search and profile when the +current question needs it. Use `up_to` for a round limit or `exact` for fixed +rounds. End an up-to search with `campaign final --reason `; final retests the retained best candidate, not unsubmitted working edits. Reserve final time and report its actual scope. Import only observed token exports; unknown is not zero. `dashboard` reads the @@ -128,8 +126,9 @@ In the execution conversation, use `benchmarks session --help` through the selected runtime. The first iterative task is `qwen21-denoise-npu`. Start a new session with the user's round/time budget, condition label, evaluation prefix and sampling arguments. Read its returned frozen TASK.md and -edit the returned working candidate. When knowledge is permitted, retrieve it -through `session knowledge` and read the delivered material. Submit each changed +edit the returned working candidate. When knowledge is permitted, use +`session knowledge` for the same Find selection and budgeted context, and +read the delivered material. Submit each changed candidate and hypothesis with `session round`, then inspect the independent feedback before the next change. Keep failed rounds. Complete the frozen budget and use `session final` for independent retesting. Report `summary.json` and its diff --git a/src/inferrsi/apps/campaign.py b/src/inferrsi/apps/campaign.py index b30c2aefb..efbdd46fb 100644 --- a/src/inferrsi/apps/campaign.py +++ b/src/inferrsi/apps/campaign.py @@ -12,7 +12,7 @@ from inferrsi import __version__ from inferrsi.harness.records import campaign_records, knowledge_capture -from inferrsi.knowledge._skill_selection import TASK_RELEVANCE +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE, TASK_RELEVANCE from inferrsi.knowledge.catalog import BACKENDS from inferrsi.knowledge.context_projection import ( PROVENANCE_FIELDS, @@ -47,20 +47,32 @@ def capture_context(root: Path, *, context_id: str, reader, query: str = "", """ campaign = knowledge_capture.capture_campaign(root, context_id) if relevance_method is None: - relevance_method = TASK_RELEVANCE if relevance and ids is None else "weighted-lexical-v1" - if relevance_method == TASK_RELEVANCE and ids is None and backend is None: + has_store = "store" in sources or getattr(reader, "store", None) is not None + if ids is None and not has_store and bool(relevance) != bool(query.strip()): + relevance_method = FIND_RELEVANCE + elif relevance and ids is None: + relevance_method = TASK_RELEVANCE + else: + relevance_method = "weighted-lexical-v1" + if relevance_method in (FIND_RELEVANCE, TASK_RELEVANCE) and ids is None and backend is None: backend = campaign.get("backend") result = reader.collect( query, ids=ids, kind=kind, repo=repo, tier=tier, backend=backend, scenario=scenario, limit=limit, relevance=relevance, relevance_method=relevance_method, context_bytes=context_bytes, ) + observation = result.get("retrieval") + source_types = list(sources) + if relevance_method == FIND_RELEVANCE and isinstance(observation, dict): + source_types = observation.get("request", {}).get("sources", source_types) request = { "query": query, "relevance": relevance, - "relevance_method": relevance_method if relevance else None, + "relevance_method": ( + relevance_method if relevance or relevance_method == FIND_RELEVANCE else None + ), "ids": ids, "kind": kind, "repo": repo, "tier": tier, "backend": backend, "scenario": scenario, "limit": limit, - "context_bytes": context_bytes, "sources": list(sources), + "context_bytes": context_bytes, "sources": source_types, } if _supplies_no_observation(result): # Such a reader gets the record this entry wrote before observations diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 974f3d9a3..1ecaa3d32 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -70,7 +70,12 @@ def _fixed_attempt_command(args) -> int: def _knowledge_reader(args) -> KnowledgeReader: - return knowledge_reader(skill_root=args.skill_root, store_root=args.store) + root = args.skill_root + if getattr(args, "installed_library", False): + from ...knowledge.knowledge_find import installed_skill_root + + root = installed_skill_root() + return knowledge_reader(skill_root=root, store_root=args.store) def _revisions(pairs: list[str]) -> dict[str, str]: @@ -531,6 +536,7 @@ def main(argv: list[str] | None = None) -> int: if args.command == "read": _, _, view = read_knowledge( _knowledge_reader(args), args.id, revision=args.revision, material=args.material, + resource=args.resource, ) if args.omit_resources: view.pop("resources") diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 0b0c5da18..90a1803d5 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -13,6 +13,22 @@ from ...knowledge.catalog import BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES +def _read_sources(parser: argparse.ArgumentParser) -> None: + library = parser.add_mutually_exclusive_group() + library.add_argument( + "--skill-root", type=Path, + help="Select a skills root; reading checkout skills requires the 'skills' extra", + ) + library.add_argument( + "--installed-library", action="store_true", + help="Select the library bundled with this InferRSI installation", + ) + parser.add_argument( + "--store", type=Path, + help="Open an existing store read-only; its database must already exist", + ) + + def _fixed_attempt_args( parser: argparse.ArgumentParser, *, legacy_flag: str | None = None, ) -> None: @@ -96,7 +112,7 @@ def _optimization_actions(stages, *, project=False): if name == "knowledge": stage.add_argument("--query", required=True) stage.add_argument("--relevance-method", choices=RELEVANCE_METHODS, - default="weighted-lexical-v1") + help="Explicit ranking method; defaults to Find") stage.add_argument("--context-bytes", type=int, required=True) stage.add_argument("--limit", type=int, default=5) @@ -342,14 +358,14 @@ def parser() -> argparse.ArgumentParser: help="Caller-reviewed campaign-relative optimization trace JSON") capture = campaign_actions.add_parser( "capture-context", - help="Run collect and keep the exact knowledge context it assembled", + help="Save the exact knowledge context selected for a question or explicit IDs", ) capture.add_argument("--directory", type=Path, required=True) capture.add_argument("--id", required=True, help="New context id, e.g. r1-diagnosis") capture.add_argument("query", nargs="?", default="") capture.add_argument("--relevance", help="Task cue for package knowledge and skill discovery") capture.add_argument("--relevance-method", choices=RELEVANCE_METHODS, - help="Campaign discovery defaults to task-lexical-v1") + help="Explicit ranking method; questions default to Find") capture.add_argument("--id-ref", action="append", dest="ids", metavar="ID", help="Exact knowledge identity to include (repeatable)") capture.add_argument("--kind", choices=("skill", "knowledge", "case")) @@ -418,8 +434,7 @@ def parser() -> argparse.ArgumentParser: actions = knowledge.add_subparsers(dest="knowledge_action", required=True) search = actions.add_parser( "search", - help="Search installed package records only; the repo/tier/backend/scenario " - "filters exclude checkout and store items", + help="Filter or search the installed record catalog. Use find for general discovery.", ) search.add_argument("query", nargs="?", default="") search.add_argument("--repo-type", choices=KNOWLEDGE_REPOS) @@ -514,7 +529,7 @@ def parser() -> argparse.ArgumentParser: collect.add_argument("--limit", type=int, default=10) collect.add_argument("--max-bytes", type=int) collect.add_argument("--material", action="store_true") - collect.add_argument("--relevance", help="Lexical cue for package knowledge discovery") + collect.add_argument("--relevance", help="Compatibility ranking for packaged knowledge") collect.add_argument( "--relevance-method", choices=RELEVANCE_METHODS, default="weighted-lexical-v1", help="Package ranking method; nondefault methods require --relevance", @@ -523,21 +538,12 @@ def parser() -> argparse.ArgumentParser: "--context-bytes", type=int, help="Opt-in context view; total UTF-8 resource-body byte budget", ) - collect.add_argument( - "--skill-root", - type=Path, - help="Select a skills root; reading checkout skills requires the 'skills' extra", - ) - collect.add_argument( - "--store", - type=Path, - help="Open an existing store read-only; its database must already exist", - ) + _read_sources(collect) collect.add_argument("--json", action="store_true") read = commands.add_parser( "read", - help="Read one bundle; select sources explicitly with --skill-root/--store", + help="Read one bundle from the package, a skill library, or a store", ) read.add_argument("id") read.add_argument("--revision", help="Exact revision returned by collect or read") @@ -546,16 +552,8 @@ def parser() -> argparse.ArgumentParser: "--omit-resources", action="store_true", help="Omit the resource inventory from output; the revision still covers the full bundle", ) - read.add_argument( - "--skill-root", - type=Path, - help="Select a skills root; reading checkout skills requires the 'skills' extra", - ) - read.add_argument( - "--store", - type=Path, - help="Open an existing store read-only; its database must already exist", - ) + read.add_argument("--resource", help="Include the UTF-8 text at this bundle resource key") + _read_sources(read) resource = commands.add_parser( "resource", @@ -566,16 +564,7 @@ def parser() -> argparse.ArgumentParser: resource.add_argument( "--revision", required=True, help="Exact revision returned by collect or read" ) - resource.add_argument( - "--skill-root", - type=Path, - help="Select a skills root; reading checkout skills requires the 'skills' extra", - ) - resource.add_argument( - "--store", - type=Path, - help="Open an existing store read-only; its database must already exist", - ) + _read_sources(resource) resource.add_argument("--output", type=Path, required=True, help="File receiving the raw bytes") context = commands.add_parser( diff --git a/src/inferrsi/apps/context_provider.py b/src/inferrsi/apps/context_provider.py index 58fdc6824..22efe36e4 100644 --- a/src/inferrsi/apps/context_provider.py +++ b/src/inferrsi/apps/context_provider.py @@ -5,6 +5,8 @@ from hashlib import sha256 from inferrsi.contracts.json_v1 import MAX_SAFE_INTEGER +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE +from inferrsi.knowledge.knowledge_find import installed_skill_root from inferrsi.knowledge.knowledge_operations import KnowledgeReader _REQUEST_FIELDS = {"query", "ids", "pins", "selection_limits", "max_bytes", "round_id"} @@ -57,7 +59,16 @@ def collect(self, request: dict) -> DeliveredContextDraft: ): raise ValueError("invalid Knowledge selection option value") - result = self.reader.collect( + has_query = bool(query and query.strip()) + has_cue = bool(limits.get("relevance")) + if not ids and self.reader.store is None and has_query != has_cue: + limits = {"relevance_method": FIND_RELEVANCE, **limits} + reader = self.reader + if reader.skill_root is None and reader.store is None and any( + identity.startswith("checkout:") for identity in ids + ): + reader = KnowledgeReader(skill_root=installed_skill_root()) + result = reader.collect( query or "", ids=ids or None, revisions=_revisions(pins), context_bytes=max_bytes or None, **limits, ) diff --git a/src/inferrsi/apps/knowledge.py b/src/inferrsi/apps/knowledge.py index 881a70cb3..f6adedd1b 100644 --- a/src/inferrsi/apps/knowledge.py +++ b/src/inferrsi/apps/knowledge.py @@ -2,7 +2,7 @@ from pathlib import Path -from inferrsi.knowledge.knowledge_operations import KnowledgeReader +from inferrsi.knowledge.knowledge_operations import KnowledgeReader, safe_path from inferrsi.knowledge.knowledge_store import KnowledgeStore @@ -28,9 +28,15 @@ def knowledge_reader( def read( - reader: KnowledgeReader, reference: str, *, revision: str | None = None, material: bool = False, + reader: KnowledgeReader, reference: str, *, revision: str | None = None, + material: bool = False, resource: str | None = None, ) -> tuple[str, str, dict]: """Return the qualified reference, actual revision, and public Bundle view.""" qualified, bundle = reader.read_resolved(reference, revision=revision) - view = bundle.view(material=material) + view = bundle.view(material=material and resource is None) + if resource is not None: + path = safe_path(resource) + if path not in bundle.content: + raise ValueError(f"Unknown resource: {path}") + view.update(content=bundle.content[path].decode("utf-8"), content_resource=path) return qualified, view["revision"], view diff --git a/src/inferrsi/apps/views/capture_delivery.py b/src/inferrsi/apps/views/capture_delivery.py index 5bae55f46..99851bb43 100644 --- a/src/inferrsi/apps/views/capture_delivery.py +++ b/src/inferrsi/apps/views/capture_delivery.py @@ -158,7 +158,8 @@ def capture_event(context_id, captured, *, archive=False, return_chunks=None, materials.append({"ref": ref, "revision": revision, "resources": resources.get(ref, [])}) observation = doc.get("retrieval") or {} calls = observation.get("calls", []) - tool = calls[-1] if calls and calls[-1] in ("collect", "knowledge search") else "collect" + recorded_tool = calls[-1] if calls else None + tool = recorded_tool if recorded_tool in ("collect", "find", "knowledge search") else "collect" event = {"action": "search", "tool": tool, "context_id": context_id, "source": source, "sha256": digest, "request": doc["request"], "selection": doc.get("selection"), "materials": materials} diff --git a/src/inferrsi/apps/views/search_archive.py b/src/inferrsi/apps/views/search_archive.py index c46751ae4..2e75b48fa 100644 --- a/src/inferrsi/apps/views/search_archive.py +++ b/src/inferrsi/apps/views/search_archive.py @@ -123,7 +123,7 @@ def reviewed_uses(review, events, decisions): return_line = delivery.get("return_line") if type(return_line) is not int or item["decision_line"] <= return_line: raise ValueError("archive review decision must follow the visible tool return") - indexer_event = event.get("tool") == "collect" + indexer_event = event.get("tool") in ("collect", "find") if indexer_event and not item.get("passage_path"): raise ValueError("Indexer review must name the exact captured resource path") if not indexer_event and "passage_path" in item: diff --git a/src/inferrsi/knowledge/_skill_selection.py b/src/inferrsi/knowledge/_skill_selection.py index f36ca8ec7..4b8c6d58f 100644 --- a/src/inferrsi/knowledge/_skill_selection.py +++ b/src/inferrsi/knowledge/_skill_selection.py @@ -7,6 +7,7 @@ from ._source_snapshot import source_entry TASK_RELEVANCE = "task-lexical-v1" +FIND_RELEVANCE = "find-lexical-v1" RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE) TASK_STOP_WORDS = frozenset({"a", "an", "and", "for", "in", "of", "on", "the", "to", "with"}) @@ -157,18 +158,35 @@ def terms(text: str) -> set[str]: def record_context(record: dict, view: dict, content: dict[str, bytes], available: int) -> tuple: - """Keep the canonical package record including its sources and limits.""" - path = view["entrypoint"] - body = content[path] - if len(body) > available: + """Deliver whole files, including the declared closure of a Case.""" + paths = {view["entrypoint"]: "record" if view["kind"] == "knowledge" else "core"} + omissions = [ + {"path": path, "reason": "linked_document"} + for path in dict.fromkeys(record.get("skill_documents", {}).values()) + ] + if view["kind"] == "case": + for path in view.get("required_resources", ()): + paths.setdefault(path, "reference") + # The checkout Bundle exposes the case card both as case.json and, when + # declared, under its authored filename. Deliver the named copy once. + if "case.json" in paths and any( + path != "case.json" and path.endswith(".json") + and content[path] == content["case.json"] for path in paths + ): + del paths["case.json"] + omissions.append({"path": "case.json", "reason": "duplicate_resource"}) + required_bytes = sum(len(content[path]) for path in paths) + if required_bytes > available: return None, {"ref": view["ref"], "reason": "context_limit", - "required_bytes": len(body), "available_bytes": available} + "required_bytes": required_bytes, "available_bytes": available} return { - "ref": view["ref"], "revision": view["revision"], "content_bytes": len(body), - "resources": [{"path": path, "sha256": view["resources"][path], "role": "record", - "locator": "whole-resource", "content": body.decode("utf-8")}], - "omissions": [{"path": p, "reason": "linked_document"} for p in - dict.fromkeys(record.get("skill_documents", {}).values())], + "ref": view["ref"], "revision": view["revision"], "content_bytes": required_bytes, + "resources": [ + {"path": path, "sha256": view["resources"][path], "role": role, + "locator": "whole-resource", "content": content[path].decode("utf-8")} + for path, role in paths.items() + ], + "omissions": omissions, "sources": view["sources"], "attribution": view["attribution"], "limitations": view["limitations"], }, None diff --git a/src/inferrsi/knowledge/knowledge_find.py b/src/inferrsi/knowledge/knowledge_find.py index f73fdfde1..9355ac6c3 100644 --- a/src/inferrsi/knowledge/knowledge_find.py +++ b/src/inferrsi/knowledge/knowledge_find.py @@ -38,7 +38,9 @@ if TYPE_CHECKING: from .channels import ChannelRegistry -from ._skill_selection import TASK_STOP_WORDS, named_backends +from ._knowledge_errors import MissingSkillsExtra +from ._skill_selection import TASK_STOP_WORDS, indexed_skill, named_backends +from ._source_snapshot import BUNDLE_LIBRARY from .catalog import BACKENDS # Author-written intent outranks bulk prose: a term in a title is evidence the @@ -258,7 +260,8 @@ def relevant_findings(self, query_tokens: list[str]) -> tuple[list[Finding], int return shown, max(0, len(self.findings) - len(shown)) def view( - self, *, score: float, query_tokens: list[str] = (), unreadable: str | None = None + self, *, score: float, query_tokens: list[str] = (), + read_args: tuple[str, ...] = (), unreadable: str | None = None, ) -> dict: record = { "id": self.identity, @@ -270,13 +273,12 @@ def view( } if self.qualified: record["authority"] = self.qualified.partition(":")[0] - if unreadable is None: - read_args = _read_args(self.qualified, self.root) + if read_args: argv = [*launcher(), *read_args] record["read_args"] = list(read_args) record["open_argv"] = argv record["ref"] = shell_command(argv) - else: + elif unreadable is not None: record["unreadable"] = unreadable if self.source_location: record["source_location"] = self.source_location @@ -306,20 +308,6 @@ def view( return record -def _read_args(identity: str, root: Path | None = None) -> tuple[str, ...]: - """CLI arguments that return the entry's text, not just its metadata. - - Without `--material`, `read` reports the entrypoint's filename in `content` - rather than the file; a caller following the reference lands on a name. - Kept as an argument list so a program can run it without re-parsing, and - so quoting is decided once, for the shell the command will be shown in. - """ - parts = ["read", identity, "--material", "--omit-resources"] - if root is not None: - parts += ["--skill-root", str(root)] - return tuple(parts) - - def launcher() -> list[str]: """How to start this CLI from where the caller is. @@ -443,6 +431,7 @@ def _case_documents(root: Path, *, source_location: str) -> list[Document]: applicability=record.get("applicability"), limitations=tuple(record.get("limitations", [])), findings=findings, + backend=record.get("backend"), )) return documents @@ -620,12 +609,42 @@ def score( return total, detail -def load_corpus( +def scoped_backend(document: Document) -> str | None: + """Backend declared by a source, including the recorded cohorts of older Cases.""" + if document.backend or document.kind != "case": + return document.backend + terms = set(tokens(_flatten(document.applicability))) + # Older observations record the hardware in their applicability list. + aliases = {**_BACKEND_NAME_ALIASES, "b300": "cuda"} + backends = named_backends({aliases.get(term, term) for term in terms}) + return next(iter(backends)) if len(backends) == 1 else None + + +def installed_skill_root() -> Path: + """Resolve the library from the current installation.""" + return Path(files("inferrsi").joinpath("knowledge", "data", "skill_library")).resolve() + + +def packaged_skill_projection(document: Document, reader) -> dict | None: + """Find the generated record for this exact installed Skill path.""" + if document.kind != "skill" or document.source_location != "installed" or not document.path: + return None + if document.root != installed_skill_root(): + return None + resource = f"{BUNDLE_LIBRARY}/{document.path}" + return next(( + record for record in reader.records().values() + if record.get("index_type") == "packaged-skill" + and record.get("skill_documents", {}).get("original") == resource + ), None) + + +def load_documents( skill_root: Path | None = None, *, kinds: tuple[str, ...] = ("case", "skill", "record") -) -> Corpus: +) -> list[Document]: documents: list[Document] = [] if "case" in kinds or "skill" in kinds: - installed = Path(files("inferrsi").joinpath("knowledge", "data", "skill_library")).resolve() + installed = installed_skill_root() root = installed if skill_root is None else Path(skill_root).resolve() if not root.is_dir(): raise ValueError(f"Skill root is not a directory: {root}") @@ -640,7 +659,13 @@ def load_corpus( if len(set(identities)) != len(identities): duplicate = next(i for i, c in Counter(identities).items() if c > 1) raise ValueError(f"Duplicate identity in corpus: {duplicate}") - return Corpus(documents) + return documents + + +def load_corpus( + skill_root: Path | None = None, *, kinds: tuple[str, ...] = ("case", "skill", "record") +) -> Corpus: + return Corpus(load_documents(skill_root, kinds=kinds)) @lru_cache(maxsize=1) @@ -703,33 +728,59 @@ def expand(query: str) -> tuple[list[str], dict[str, tuple[str, ...]]]: return [token for token in dict.fromkeys(found) if token not in _QUERY_STOP_WORDS], applied -def _why_unreadable(document: Document, readers: dict) -> str | None: - """Open the entry the way `read` would; return why not, or None if it opens. - - Ranking reads files directly, which is what lets it skip Bundle closures, - but it means a hit can be indexed and still refused by `read`: a source - record missing the version `read` requires, or a case whose body file was - never copied. Offering a command known to fail is worse than offering - none, so only the hits actually returned are checked -- a handful of - reads, not the corpus -- and a failing one reports why instead. - - A missing optional dependency is reported against the hit like any other - reason it cannot be opened here. Ranking itself needs no parser, so the - search must not fail for it; the caller learns which hits need PyYAML - to open, and can still open every case, which does not. - """ +def _open_args(document: Document, readers: dict) -> tuple[str, ...]: + """Validate material and build the read command for a returned hit.""" from .knowledge_operations import KnowledgeReader if not document.qualified: - return None + return () reader = readers.get(document.root) if reader is None: reader = readers[document.root] = KnowledgeReader(skill_root=document.root) try: reader.read(document.qualified).view(material=True) - except (ValueError, KeyError, OSError, TypeError) as exc: - return str(exc) or type(exc).__name__ - return None + except MissingSkillsExtra: + record = packaged_skill_projection(document, reader) + if record is None: + raise + package_reader = readers.get(None) + if package_reader is None: + package_reader = readers[None] = KnowledgeReader() + ref = f"package:{record['id']}" + bundle = package_reader.read(ref) + view = bundle.view() + core, _, _ = indexed_skill(record, {**view, "authority": "package"}, bundle.content) + bundle.content[core].decode("utf-8") + return ( + "read", ref, "--omit-resources", "--resource", core, "--revision", view["revision"], + ) + + parts = ("read", document.qualified, "--material", "--omit-resources") + if document.root is None: + return parts + if document.source_location == "installed": + return (*parts, "--installed-library") + return (*parts, "--skill-root", str(document.root)) + + +def rank(query: str, corpus: Corpus) -> tuple[ + list[str], dict[str, tuple[str, ...]], list[tuple[float, int, int, dict]] +]: + """Score matches without opening a Bundle or formatting a delivery.""" + # A topic alias can add backend terms for matching, as 图模式 adds cuda graph. + explicit_terms = {_BACKEND_NAME_ALIASES.get(term, term) for term in tokens(query)} + backends = named_backends(explicit_terms) + query_backend = next(iter(backends)) if len(backends) == 1 else None + query_tokens, applied = expand(query) + scored = [] + for index, document in enumerate(corpus.documents): + value, detail = corpus.score(index, query_tokens, query_backend=query_backend) + if value > 0: + scored.append((value, STATE_RANK.get(document.state, 0), index, detail)) + scored.sort(key=lambda row: ( + -row[0], -row[1], corpus.documents[row[2]].identity, corpus.documents[row[2]].qualified, + )) + return query_tokens, applied, scored def find( @@ -746,25 +797,17 @@ def find( ) -> dict: if type(limit) is not int or limit < 1: raise ValueError("limit must be a positive integer") - # A topic alias can add backend terms for matching, as 图模式 adds cuda graph. - explicit_terms = {_BACKEND_NAME_ALIASES.get(term, term) for term in tokens(query)} - backends = named_backends(explicit_terms) - query_backend = next(iter(backends)) if len(backends) == 1 else None - query_tokens, applied = expand(query) - scored = [] - for index, document in enumerate(corpus.documents): - value, detail = corpus.score(index, query_tokens, query_backend=query_backend) - if value > 0: - scored.append((value, STATE_RANK.get(document.state, 0), index, detail)) - scored.sort(key=lambda row: ( - -row[0], -row[1], corpus.documents[row[2]].identity, corpus.documents[row[2]].qualified, - )) + query_tokens, applied, scored = rank(query, corpus) items = [] readers: dict = {} for value, _, index, detail in scored[:limit]: document = corpus.documents[index] + try: + read_args, unreadable = _open_args(document, readers), None + except (ValueError, KeyError, OSError, TypeError) as exc: + read_args, unreadable = (), str(exc) or type(exc).__name__ view = document.view( - score=value, query_tokens=query_tokens, unreadable=_why_unreadable(document, readers) + score=value, query_tokens=query_tokens, read_args=read_args, unreadable=unreadable, ) if explain: view["matched"] = detail diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py index 9c5e1e7b3..69b8ac62e 100644 --- a/src/inferrsi/knowledge/knowledge_operations.py +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -11,6 +11,7 @@ from ._knowledge_errors import MissingSkillsExtra from ._skill_selection import ( + FIND_RELEVANCE, RELEVANCE_METHODS, TASK_RELEVANCE, context_view, @@ -513,6 +514,48 @@ def resource(self, identity: str, path: str, *, revision: str) -> bytes: raise ValueError(f"Unknown resource: {path}") return bundle.content[path] + def _search_order( + self, cue: str, matches, *, kind: str | None, package_only: bool, + ) -> tuple[dict, int, int]: + from .knowledge_find import ( + Corpus, + installed_skill_root, + load_documents, + packaged_skill_projection, + rank, + scoped_backend, + ) + + root = None if package_only else self.skill_root + documents = load_documents(root) + if root is not None and root != installed_skill_root() and kind not in ("case", "skill"): + # A custom library adds checkout sources to Collect. Installed + # projections remain available under their package authority. + documents.extend( + document for document in load_documents(kinds=("skill",)) + if packaged_skill_projection(document, self) is not None + ) + eligible = {} + for index, document in enumerate(documents): + authority, _, identity = document.qualified.partition(":") + # Keep built-in Skills under their package references for context + # delivery, including their indexed required and optional files. + projection = packaged_skill_projection(document, self) if kind != "skill" else None + if projection is not None: + authority, identity = "package", projection["id"] + if authority == "package": + metadata = {**self.records()[identity], "kind": "knowledge"} + else: + metadata = {"kind": document.kind, "backend": scoped_backend(document)} + if matches(metadata, authority): + eligible[index] = f"{authority}:{identity}" + _, _, scored = rank(cue, Corpus(documents)) + results = { + eligible[index]: {"score": round(score, 2), "components": components} + for score, _, index, components in scored if index in eligible + } + return results, len(documents), len(eligible) + def collect( self, query: str = "", @@ -543,9 +586,14 @@ def collect( raise ValueError("Unknown content kind") if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): raise ValueError("relevance requires at least one word") - if relevance_method not in RELEVANCE_METHODS: + use_find = relevance_method == FIND_RELEVANCE + find_ranking = use_find and ids is None + if not use_find and relevance_method not in RELEVANCE_METHODS: raise ValueError("Unknown relevance method") - if relevance is None and relevance_method != "weighted-lexical-v1": + if use_find: + if bool(query.strip()) == bool(relevance): + raise ValueError("Find selection requires one query or relevance cue") + elif relevance is None and relevance_method != "weighted-lexical-v1": raise ValueError("relevance_method requires relevance") if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): raise ValueError("context_bytes must be a positive integer") @@ -558,21 +606,31 @@ def collect( raise ValueError("revisions must map identity strings to revision strings") if ids is not None and len(set(ids)) != len(ids): raise ValueError("Duplicate requested identity") - owners = self.owners() - requested = ids if ids is not None else self.identities() + reader = self + if find_ranking: + if self.store is not None: + raise ValueError("Use explicit IDs or catalog collection for stores") + if self.skill_root is None: + from .knowledge_find import installed_skill_root + + reader = KnowledgeReader(skill_root=installed_skill_root()) + owners = reader.owners() + requested = ids if ids is not None else sorted( + f"{authority}:{identity}" for identity, values in owners.items() for authority in values + ) def qualified(reference: str) -> str | None: # Revision keys may use any spelling the API accepts for an identity. try: - return self.resolve(reference, owners) + return reader.resolve(reference, owners) except IdentityError: return None - selected = {identity for identity in map(qualified, requested) if identity is not None} + selected_refs = {qualified(reference) for reference in requested} if revisions else set() pinned: dict[str, str] = {} for reference, revision in revisions.items(): target = qualified(reference) - if target is None or target not in selected: + if target is None or target not in selected_refs: raise ValueError( f"Revision requested for an unselected identity: {reference}" ) @@ -583,8 +641,10 @@ def qualified(reference: str) -> str | None: if tier is not None and tier not in KNOWLEDGE_SCOPES: raise ValueError(f"Unknown knowledge tier: {tier}") scenario = SCENARIO_ALIASES.get(scenario, scenario) - package_filters = any(value is not None for value in (repo, tier, backend, scenario)) - task_ranking = relevance_method == TASK_RELEVANCE + package_filters = any( + value is not None for value in (repo, tier, scenario) + ) or (backend is not None and not find_ranking) + task_ranking = relevance_method in (TASK_RELEVANCE, FIND_RELEVANCE) def matches(metadata: dict, authority: str) -> bool: declared = metadata.get("scenario") @@ -595,7 +655,7 @@ def matches(metadata: dict, authority: str) -> bool: discovery_skill = ( task_ranking and ids is None and authority == "package" and declared == "skills.discovery" - and self.records()[metadata["id"]].get("index_type") == "packaged-skill" + and reader.records()[metadata["id"]].get("index_type") == "packaged-skill" ) return not ( (kind and metadata.get("kind") != kind) @@ -611,14 +671,25 @@ def matches(metadata: dict, authority: str) -> bool: ) ) - cases = self._cases() if kind else {} + if find_ranking: + found, considered, eligible = reader._search_order( + relevance or query, matches, kind=kind, + package_only=package_filters or kind == "knowledge", + ) + requested = list(found) + else: + found, considered, eligible = {}, len(requested), len(requested) + cases = reader._cases() if kind and not find_ranking else {} selected, omitted, seen = [], [], set() used = 0 ranked, contexts, context_omissions = [], [], [] context_used = 0 - explanations = {} - excluded = {"ranking_scope": 0, "filters": 0, "query": 0, "zero_score": 0, - "item_limit": 0, "material_bytes": 0} + explanations = found + excluded = { + "ranking_scope": 0, "filters": considered - eligible, "query": 0, + "zero_score": eligible - len(found) if find_ranking else 0, + "item_limit": 0, "material_bytes": 0, + } matched_count = 0 def select(reference: str, bundle: Bundle, view: dict) -> None: @@ -639,9 +710,15 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: used += size if context_bytes is None: return - record = self.records().get(view["id"], {}) + packaged = view["authority"] == "package" + record = reader.records().get(view["id"], {}) if packaged else {} is_skill = record.get("index_type") == "packaged-skill" - if view["authority"] != "package" or (not is_skill and not task_ranking): + pinned_item = ids is not None and view["ref"] in pinned + checkout_context = ( + view["authority"] == "checkout" and view["kind"] in ("case", "skill") + and (use_find or pinned_item) + ) + if not ((packaged and (is_skill or task_ranking or pinned_item)) or checkout_context): context_omissions.append({"ref": view["ref"], "reason": "unsupported_context"}) return try: @@ -664,34 +741,38 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: context_used += context["content_bytes"] for reference in requested: + if find_ranking and len(selected) >= limit: + excluded["item_limit"] += len(requested) - len(seen) + break try: - qualified = self.resolve(reference, owners) + qualified = reader.resolve(reference, owners) if qualified in seen: raise IdentityError("duplicate_identity", f"Already selected: {qualified}") seen.add(qualified) authority, _, identity = qualified.partition(":") - # Cheap metadata checks first, so filtered items never load resource bytes. - if authority == "package": - early = {**self.records()[identity], "kind": "knowledge"} - elif authority == "checkout": - early = {"kind": "case" if identity in cases else "skill"} - else: - early = None - if relevance is not None and ids is None and ( - authority != "package" - or (not task_ranking and early.get("index_type") != "packaged-skill") - ): - excluded["ranking_scope"] += 1 - continue - if early is not None and not matches(early, authority): - excluded["filters"] += 1 - if ids is not None: - omitted.append({"id": reference, "reason": "filters"}) - continue - bundle = self._read(qualified, pinned.get(qualified)) + if not find_ranking: + # Cheap metadata checks first, before loading bundle resources. + if authority == "package": + early = {**reader.records()[identity], "kind": "knowledge"} + elif authority == "checkout": + early = {"kind": "case" if identity in cases else "skill"} + else: + early = None + if relevance is not None and ids is None and ( + authority != "package" + or (not task_ranking and early.get("index_type") != "packaged-skill") + ): + excluded["ranking_scope"] += 1 + continue + if early is not None and not matches(early, authority): + excluded["filters"] += 1 + if ids is not None: + omitted.append({"id": reference, "reason": "filters"}) + continue + bundle = reader._read(qualified, pinned.get(qualified)) view = {**bundle.view(material=material), "authority": authority, "ref": qualified} - entry = bundle.content[bundle.metadata["entrypoint"]] - haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() + if find_ranking and view["kind"] == "case": + view.setdefault("title", identity.removeprefix("case.")) except IdentityError as exc: omitted.append({"id": reference, "reason": exc.code, "detail": str(exc)}) continue @@ -700,7 +781,10 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: {"id": reference, "reason": "unavailable_revision", "detail": str(exc)} ) continue - except MissingSkillsExtra: + except MissingSkillsExtra as exc: + if find_ranking: + omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) + continue # The install cannot parse any checkout skill, not just this one. # As an omission it would return `no_match`, which reads as # "nothing found" rather than "install the skills extra". @@ -708,18 +792,21 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: except (ValueError, KeyError, TypeError, AttributeError, OSError, sqlite3.Error) as exc: omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) continue - if not matches(view, authority): + if not find_ranking and not matches(view, authority): excluded["filters"] += 1 if ids is not None: omitted.append({"id": reference, "reason": "filters"}) continue - if any(token not in haystack for token in query.casefold().split()): - excluded["query"] += 1 - if ids is not None: - omitted.append({"id": reference, "reason": "query"}) - continue - if relevance is not None and ids is None: - record = self.records().get(identity, {}) + entry = bundle.content[bundle.metadata["entrypoint"]] + if not find_ranking and query: + haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() + if any(token not in haystack for token in query.casefold().split()): + excluded["query"] += 1 + if ids is not None: + omitted.append({"id": reference, "reason": "query"}) + continue + if not find_ranking and relevance is not None and ids is None: + record = reader.records().get(identity, {}) try: is_skill = record.get("index_type") == "packaged-skill" if is_skill: @@ -762,7 +849,16 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: "Read references must be resolved with their returned exact revision.", ], } - if relevance is not None: + if find_ranking: + result["selection"] = { + "method": FIND_RELEVANCE, "cue": relevance or query, + "scope": "package-records-and-library", "ranking_applied": True, + "matched_count": len(found), "returned_count": len(selected), + } + result["limitations"][0] = ( + "Find ranks textual matches; check each item's applicability and limitations." + ) + elif relevance is not None: result["selection"] = { "method": relevance_method, "cue": relevance, "scope": ("package-skills-and-records" if task_ranking @@ -785,28 +881,34 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: }) from .retrieval import observation - ranking = relevance is not None and ids is None + ranking = find_ranking or (relevance is not None and ids is None) + selection = result.get("selection", {}) result["retrieval"] = observation( entrypoint="collect", request={ "query": query, "relevance": relevance, "relevance_method": relevance_method, "ids": ids, "revisions": revisions, "kind": kind, "repo": repo, "tier": tier, "backend": backend, "scenario": scenario, "limit": limit, "max_bytes": max_bytes, "context_bytes": context_bytes, - "sources": ["package"] + (["checkout"] if self.skill_root else []) - + (["store"] if self.store is not None else []), + "sources": ["package"] + (["checkout"] if reader.skill_root else []) + + (["store"] if reader.store is not None else []), }, selection={ - "method": relevance_method if relevance is not None else None, - "scope": (("package-skills-and-records" if task_ranking else - "generated-packaged-skills") if relevance is not None else None), + "method": selection.get("method"), + "scope": selection.get("scope"), "ranking_applied": ranking, "state": "applied" if ranking else "explicit_ids" if ids is not None else "not_requested", - "tie_break": "ref_ascending" if ranking else None, + "tie_break": ( + "state_then_id_then_ref" if find_ranking else "ref_ascending" + ) if ranking else None, }, items=[{"ref": item["ref"], "revision": item["revision"], "rank": n, **explanations.get(item["ref"], {})} for n, item in enumerate(selected, 1)], - counts={"considered": len(requested), "matched": matched_count, - "returned": len(selected), "excluded": excluded}, + counts={ + "considered": considered, "matched": len(found) if find_ranking else matched_count, + "returned": len(selected), "excluded": excluded, + }, assembly="not_requested" if context_bytes is None else "performed", ) + if find_ranking: + result["retrieval"]["calls"].append("find") return result diff --git a/tests/check_dsh_remote.cjs b/tests/check_dsh_remote.cjs index a6b23ae24..a7e8eecb4 100644 --- a/tests/check_dsh_remote.cjs +++ b/tests/check_dsh_remote.cjs @@ -305,6 +305,10 @@ async function main() { assert.equal(compilation.status, 0, 'Remote Python bootstrap did not compile'); await manager.run(sessionId, id, { operation: 'cli', args: ['collect', 'synthetic', '--limit', '1'] }); assert.equal(Buffer.from(payloads.at(-1).archive, 'base64').toString(), 'synthetic-runtime-archive'); + const findArgs = ['find', 'synthetic question', '--limit', '1', '--json']; + const search = await manager.run(sessionId, id, { operation: 'cli', args: findArgs }); + assert.equal(search.exitCode, 0); + assert.deepEqual(payloads.at(-1).args, findArgs); assert.ok(commands.every(command => command === 'uname -s' || command === commands.find(item => item !== 'uname -s')), 'user data changed the SSH shell command'); assert.ok(!JSON.stringify(payloads).includes(fakePassword)); diff --git a/tests/fixtures/knowledge_retrieval.py b/tests/fixtures/knowledge_retrieval.py index 454fc30cb..0c341621e 100644 --- a/tests/fixtures/knowledge_retrieval.py +++ b/tests/fixtures/knowledge_retrieval.py @@ -69,8 +69,10 @@ def build(root, *, runs=DEFAULT_RUNS): source = reader() budget = 2 * len(b"Synthetic launch guidance.\n") captures = {} - path, captures["capture"] = capture_context(root, context_id="capture", reader=source, - relevance="launch", context_bytes=budget) + path, captures["capture"] = capture_context( + root, context_id="capture", reader=source, relevance="launch", + relevance_method="task-lexical-v1", context_bytes=budget, + ) captured = json.loads(path.read_text()) with patch("inferrsi.knowledge.catalog.knowledge_records", return_value=list(source.records().values())): diff --git a/tests/test_apps_context_provider.py b/tests/test_apps_context_provider.py index 353a47810..84b7f095a 100644 --- a/tests/test_apps_context_provider.py +++ b/tests/test_apps_context_provider.py @@ -85,7 +85,26 @@ def test_discovery_delegates_selection_to_knowledge(): assert [item["pin"]["reference"] for item in draft.items] == [_CGROUP] assert draft.selection["ranking_applied"] is True assert draft.selection["state"] == "applied" - assert draft.selection["tie_break"] == "ref_ascending" + assert draft.selection["method"] == "find-lexical-v1" + assert draft.selection["tie_break"] == "state_then_id_then_ref" + + +def test_discovered_installed_case_can_be_replayed_at_its_exact_revision(): + provider = ContextProvider(knowledge_reader()) + discovered = provider.collect(_request( + query="cache per-step masks ran out of device memory", ids=[], + selection_limits={"limit": 1}, max_bytes=8192, + )) + item, = discovered.items + pin = item["pin"] + assert pin["reference"] == "checkout:case.qwen-ascend-rejected-directions" + replay = provider.collect(_request( + ids=[pin["reference"]], pins=[{"name": pin["reference"], "revision": pin["revision"]}], + max_bytes=8192, + )) + assert replay.content and replay.content == discovered.content + assert replay.items == discovered.items + assert replay.selection["state"] == "explicit_ids" @pytest.mark.parametrize("overrides", [ diff --git a/tests/test_campaign_record_injection.py b/tests/test_campaign_record_injection.py index eca883bd9..ba61470c6 100644 --- a/tests/test_campaign_record_injection.py +++ b/tests/test_campaign_record_injection.py @@ -50,20 +50,20 @@ def collect(self, query="", **options): # Each row: capture options, the method and backend stored in the record, and -# the method handed to ``collect``. These are the values recorded before the -# capture moved out of the observability package. +# the method handed to ``collect``. Include the legacy methods explicitly to +# cover both current defaults and older callers. SELECTION = { - "cue": ({"relevance": "decode launch"}, "task-lexical-v1", "cuda", "task-lexical-v1"), + "cue": ({"relevance": "decode launch"}, "find-lexical-v1", "cuda", "find-lexical-v1"), "cue-and-ids": ({"relevance": "decode launch", "ids": ["package:graphs"]}, "weighted-lexical-v1", None, "weighted-lexical-v1"), - "query-only": ({"query": "decode"}, None, None, "weighted-lexical-v1"), + "query-only": ({"query": "decode"}, "find-lexical-v1", "cuda", "find-lexical-v1"), "explicit-raw": ({"relevance": "decode launch", "relevance_method": "weighted-raw-string-v1"}, "weighted-raw-string-v1", None, "weighted-raw-string-v1"), "task-without-cue": ({"relevance_method": "task-lexical-v1"}, None, "cuda", "task-lexical-v1"), "explicit-backend": ({"relevance": "decode launch", "backend": "ascend"}, - "task-lexical-v1", "ascend", "task-lexical-v1"), + "find-lexical-v1", "ascend", "find-lexical-v1"), "explicit-legacy": ({"relevance": "decode launch", "relevance_method": "weighted-lexical-v1"}, "weighted-lexical-v1", None, "weighted-lexical-v1"), @@ -210,7 +210,8 @@ def test_reader_without_observation_serializes_the_exact_earlier_bytes(tmp_path, monkeypatch.setattr(knowledge_capture, "datetime", FixedClock) campaign.create_campaign(tmp_path, "layout") path, digest = campaign.capture_context(tmp_path, context_id="r1", reader=Reader(), - relevance="decode launch") + relevance="decode launch", + relevance_method="task-lexical-v1") expected = legacy_record_bytes(__version__) assert path.read_bytes() == expected diff --git a/tests/test_campaign_task_selection.py b/tests/test_campaign_task_selection.py index 5d1a2da1a..fc9b20dc5 100644 --- a/tests/test_campaign_task_selection.py +++ b/tests/test_campaign_task_selection.py @@ -1,5 +1,7 @@ import json +import pytest + from inferrsi.apps.campaign import capture_context, create_campaign, verify_capture from inferrsi.knowledge.knowledge_operations import KnowledgeReader @@ -9,7 +11,8 @@ def test_campaign_search_returns_portable_methods_and_records_counts(tmp_path): create_campaign(tmp_path, "diffusion") path, _ = capture_context(tmp_path, context_id="first", reader=KnowledgeReader(), - relevance=BASELINE_CUE, limit=2, context_bytes=14000) + relevance=BASELINE_CUE, relevance_method="task-lexical-v1", + limit=2, context_bytes=14000) document, state = verify_capture(path.read_bytes()) assert state == "verified" assert [item["title"] for item in document["items"]] == [ @@ -60,3 +63,58 @@ def test_workload_scenario_discovers_skills_and_compatible_engine_records(): assert reader.collect(ids=[portable], **options)["omissions"] == [ {"id": portable, "reason": "filters"} ] + + +@pytest.mark.parametrize(("backend", "case"), [ + ("cuda", "qwen-ascend-rejected-directions"), + ("ascend", "qwen-b300-rejected-directions"), +]) +def test_find_filters_cases_to_the_campaign_backend(tmp_path, backend, case): + create_campaign(tmp_path, "target", backend=backend) + path, _ = capture_context( + tmp_path, context_id="r1", reader=KnowledgeReader(), kind="case", + relevance=case.replace("-", " "), limit=5, + ) + capture, state = verify_capture(path.read_bytes()) + assert state == "verified" + assert capture["items"] + assert all(item["ref"] != f"checkout:case.{case}" for item in capture["items"]) + counts = capture["retrieval"]["counts"] + assert counts["excluded"]["filters"] > 0 + assert counts["considered"] == ( + counts["matched"] + counts["excluded"]["filters"] + counts["excluded"]["zero_score"] + ) + + +def test_find_with_custom_library_keeps_packaged_projections(tmp_path): + create_campaign(tmp_path, "target") + library = tmp_path / "custom" + library.mkdir() + path, _ = capture_context( + tmp_path, context_id="r1", reader=KnowledgeReader(skill_root=library), + relevance="cgroupv2", limit=1, + ) + capture, state = verify_capture(path.read_bytes()) + assert state == "verified" + item, = capture["items"] + assert item["ref"] == "package:inferrsi.common.skills.cgroup_resource_controls" + + +def test_find_preserves_required_case_readset(tmp_path): + from inferrsi.knowledge.knowledge_find import installed_skill_root + + create_campaign(tmp_path, "target") + path, _ = capture_context( + tmp_path, context_id="r1", reader=KnowledgeReader(), kind="case", + relevance="vllm gdn source contract", limit=1, + ) + capture, state = verify_capture(path.read_bytes()) + assert state == "verified" + context, = capture["contexts"] + assert context["ref"] == "checkout:case.vllm-gdn-source-contract" + resource = next( + item for item in context["resources"] if item["path"] == "vllm-gdn-causal-conv-contract.md" + ) + assert resource["content"].encode("utf-8") == KnowledgeReader( + skill_root=installed_skill_root() + ).resource(context["ref"], resource["path"], revision=context["revision"]) diff --git a/tests/test_dsh_plugin.py b/tests/test_dsh_plugin.py index e9e52dd2e..c72aa2dfa 100644 --- a/tests/test_dsh_plugin.py +++ b/tests/test_dsh_plugin.py @@ -1,5 +1,6 @@ """Check the native bundle and its isolated launcher without requiring DSH in CI.""" +import base64 import hashlib import io import json @@ -91,6 +92,10 @@ def invoke(*args): record = invoke("read", item["ref"], "--revision", item["revision"]) assert record["revision"] == item["revision"] + # The remote bootstrap runs on Linux and installs POSIX signal handlers. + if os.name == "posix": + _check_remote_case_read(package, target) + campaign = target / "campaign" campaign.mkdir() (campaign / "campaign.json").write_text( @@ -105,6 +110,36 @@ def invoke(*args): build_bundle(archive) +def _check_remote_case_read(package, target): + # The remote bootstrap unpacks a fresh runtime for each command. A read + # that depends on the find invocation's temporary path fails here. + source = (package / "remote.cjs").read_text(encoding="utf-8") + bootstrap = source.split("const BOOTSTRAP = String.raw`", 1)[1].split("`;", 1)[0] + remote_payload = { + "mode": "cli", + "repository": str(target.resolve()), + "archive": base64.b64encode((package / "remote-runtime.zip").read_bytes()).decode("ascii"), + } + + def invoke_remote(*args): + response = subprocess.run( + [sys.executable, "-I", "-S", "-u", "-c", bootstrap], + input=json.dumps({**remote_payload, "args": args}), + capture_output=True, text=True, encoding="utf-8", timeout=45, + ) + assert response.returncode == 0, response.stderr + return json.loads(response.stdout) + + found = invoke_remote("find", "Qwen Ascend", "--kinds", "case", "--limit", "1", "--json") + hit = found["items"][0] + assert hit["id"] == "case.qwen-ascend-rejected-directions" + opened = invoke_remote(*hit["read_args"]) + assert opened["id"] == hit["id"] + assert opened["revision"].startswith("sha256:") + library = package / "runtime/inferrsi/knowledge/data/skill_library" + assert opened["content"] == (library / hit["path"]).read_bytes().decode("utf-8") + + @pytest.mark.skipif(shutil.which("node") is None, reason="Node.js is needed for native SSH checks") def test_remote_ssh_connection_and_scoped_operations(): subprocess.run(["node", str(PROJECT / "tests/check_dsh_remote.cjs")], diff --git a/tests/test_knowledge_find.py b/tests/test_knowledge_find.py index 309996d61..a0ac666c4 100644 --- a/tests/test_knowledge_find.py +++ b/tests/test_knowledge_find.py @@ -241,7 +241,7 @@ def test_result_carries_a_reference_the_caller_can_open(corpus): assert item["source"].startswith("https://") -def test_read_accepts_the_reference_find_reports(capsys): +def test_read_accepts_the_reference_find_reports(capsys, tmp_path): """Run the reported command as given and check it returns the text. Metadata is not what a reader following a hit is after, and `read` reports @@ -249,12 +249,20 @@ def test_read_accepts_the_reference_find_reports(capsys): """ result = find("mask caching ran out of device memory", load_corpus(ROOT), limit=10) item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + assert "--installed-library" in item["read_args"] + assert "--skill-root" not in item["read_args"] assert main(item["read_args"]) == 0 payload = json.loads(capsys.readouterr().out) assert payload["id"] == "case.qwen-ascend-rejected-directions" body = payload["content"] assert "rejected directions" in body.casefold() assert "910B1" in body + target = tmp_path / "case.md" + assert main([ + "resource", item["read_args"][1], payload["entrypoint"], "--installed-library", + "--revision", payload["revision"], "--output", str(target), + ]) == 0 + assert target.read_bytes() == body.encode("utf-8") def test_reference_survives_a_root_whose_path_has_spaces(tmp_path, capsys): @@ -265,6 +273,7 @@ def test_reference_survives_a_root_whose_path_has_spaces(tmp_path, capsys): result = find("mask caching ran out of device memory", corpus, limit=10) item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") assert item["source_location"] == "checkout" + assert item["read_args"][-2:] == ["--skill-root", str(root.resolve())] assert main(item["read_args"]) == 0 assert "910B1" in json.loads(capsys.readouterr().out)["content"] if os.name != "nt": @@ -730,6 +739,7 @@ def test_cli_json_mode_states_its_limits(capsys): assert item["id"] == "ascend-device-health" assert item["source_location"] == "installed" assert item["read_args"][:2] == ["read", "checkout:ascend-device-health"] + assert "--installed-library" in item["read_args"] assert payload["corpus_size"] == len(load_corpus(ROOT).documents) assert any("not a learned ranking" in line for line in payload["limitations"]) diff --git a/tests/test_knowledge_install.py b/tests/test_knowledge_install.py index fb672c112..d6880788e 100644 --- a/tests/test_knowledge_install.py +++ b/tests/test_knowledge_install.py @@ -61,18 +61,25 @@ def test_package_only_commands_work_without_pyyaml(without_yaml): def test_packaged_skill_search_and_resource_work_without_pyyaml(without_yaml, tmp_path): env = _cli_env(without_yaml) - query = "cuda graph warmup" + query = "ascend graph mode" found = _run(["find", query, "--kinds", "skill", "--limit", "1", "--json"], env) assert found.returncode == 0, found.stderr hit, = json.loads(found.stdout)["items"] - assert "PyYAML" in hit["unreadable"] - assert "open_argv" not in hit + assert hit["id"] == "ascend-graph-mode" + opened = _run(hit["read_args"], env) + assert opened.returncode == 0, opened.stderr + read = json.loads(opened.stdout) collected = _run(["collect", "--relevance", query, "--limit", "1", "--material", "--json"], env) assert collected.returncode == 0, collected.stderr packaged, = json.loads(collected.stdout)["items"] assert packaged["authority"] == "package" original = packaged["skill_documents"]["original"] + expected = (SKILLS / Path(original).relative_to("data/skill_library")).read_bytes() + assert read["id"] == packaged["id"] + assert read["revision"] == packaged["revision"] + assert read["content_resource"] == original + assert read["content"].encode("utf-8") == expected output = tmp_path / "packaged-skill.md" fetched = _run([ @@ -80,8 +87,30 @@ def test_packaged_skill_search_and_resource_work_without_pyyaml(without_yaml, tm "--output", str(output), ], env) assert fetched.returncode == 0, fetched.stderr - expected = SKILLS / Path(original).relative_to("data/skill_library") - assert output.read_bytes() == expected.read_bytes() + assert output.read_bytes() == expected + + +def test_custom_skill_does_not_open_the_installed_projection_without_pyyaml(without_yaml, tmp_path): + relative = "hardware/ascend/ascend-graph-mode" + directory = tmp_path / relative + directory.mkdir(parents=True) + (directory / "SKILL.md").write_text( + "---\nname: ascend-graph-mode\ndescription: Ascend graph mode experiment\n---\n" + "This is a private local observation.\n", + encoding="utf-8", + ) + (tmp_path / "skill-index.json").write_text(json.dumps({ + "schema_version": 1, + "skills": {"ascend-graph-mode": {"path": relative, "backend": "ascend"}}, + }), encoding="utf-8") + result = _run([ + "find", "ascend graph mode", "--skill-root", str(tmp_path), "--kinds", "skill", "--json", + ], _cli_env(without_yaml)) + assert result.returncode == 0, result.stderr + hit, = json.loads(result.stdout)["items"] + assert hit["source_location"] == "checkout" + assert "PyYAML" in hit["unreadable"] + assert "read_args" not in hit def test_skill_root_without_pyyaml_fails_with_install_hint(without_yaml): diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index 168d7cb7e..543de3292 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -350,29 +350,37 @@ def test_task_runner_does_not_expose_harness_as_top_level_modules(task, tmp_path assert json.loads(output.read_text())["isolated"] -def test_knowledge_delivery_is_retained_and_bound_to_round(session, monkeypatch, capsys): +def test_knowledge_delivery_is_retained_and_bound_to_round(session, capsys): + from inferrsi.apps.campaign import verify_capture + from inferrsi.knowledge.knowledge_find import installed_skill_root from inferrsi.knowledge.knowledge_operations import KnowledgeReader - def collect(self, query, **options): - import hashlib - - assert query == "" and options["relevance"] == "normalization" - assert options["context_bytes"] == 2048 - content = "source-grounded synthetic context" - return {"contexts": [{"ref": "package:test", "revision": "0" * 64, - "content_bytes": len(content), "omissions": [], - "resources": [{"path": "SKILL.md", "role": "body", - "content": content, - "sha256": hashlib.sha256(content.encode()).hexdigest()} - ]}]} - - monkeypatch.setattr(KnowledgeReader, "collect", collect) - assert main(["benchmarks", "session", "knowledge", "--session", str(session), - "--query", "normalization", "--context-bytes", "2048"]) == 0 + assert main([ + "benchmarks", "session", "knowledge", "--session", str(session), + "--query", "cache per-step masks ran out of device memory", + "--limit", "1", "--context-bytes", "8192", + ]) == 0 delivered = json.loads(capsys.readouterr().out) + path = session / delivered["reference"]["path"] + capture, state = verify_capture(path.read_bytes()) + assert state == "verified" + assert capture["selection"]["method"] == "find-lexical-v1" + assert capture["retrieval"]["calls"] == ["campaign capture-context", "collect", "find"] + item, = capture["items"] + assert item["ref"] == "checkout:case.qwen-ascend-rejected-directions" + context, = capture["contexts"] + assert context["revision"] == item["revision"] + assert context["content_bytes"] <= 8192 + assert {resource["path"] for resource in context["resources"]} == { + "qwen-ascend-rejected-directions.md", "qwen-ascend-rejected-directions.json", + } + reader = KnowledgeReader(skill_root=installed_skill_root()) + for resource in context["resources"]: + assert resource["content"].encode("utf-8") == reader.resource( + item["ref"], resource["path"], revision=item["revision"] + ) result = submit(session, 2, capsys) assert result["latest_observation"]["contexts"] == [delivered["reference"]] - assert (session / delivered["reference"]["path"]).is_file() def test_runtime_identity_covers_migrated_process_domain_and_not_removed_executor() -> None: diff --git a/tests/test_project_optimization.py b/tests/test_project_optimization.py index 9e08239c3..9854e96a9 100644 --- a/tests/test_project_optimization.py +++ b/tests/test_project_optimization.py @@ -210,6 +210,23 @@ def test_fixed_rounds_and_frozen_protocol_are_not_bypassed(project, capsys): submit(project, capsys, 30) +def test_project_knowledge_uses_find_and_records_budget_omission(project, capsys): + start(project, capsys) + root = project[3] + assert main([ + "campaign", "knowledge", "--directory", str(root), + "--query", "cache per-step masks ran out of device memory", + "--limit", "1", "--context-bytes", "1", + ]) == 0 + capture = json.loads(capsys.readouterr().out)["context"] + assert capture["selection"]["method"] == "find-lexical-v1" + item, = capture["items"] + assert item["ref"] == "checkout:case.qwen-ascend-rejected-directions" + omission, = capture["context_omissions"] + assert (omission["ref"], omission["reason"]) == (item["ref"], "context_limit") + assert capture["contexts"] == [] + + def test_project_knowledge_off_and_manual_record_writes_are_rejected(project, capsys): start(project, capsys, extra=("--knowledge", "off")) root = project[3] From 5f4c1cfaa56a344205f6519c2d6ee891de86c5c0 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Wed, 18 Nov 2026 15:30:22 +0800 Subject: [PATCH 02/15] Declare backend scope in five recorded cases Agent-assisted-by: Codex / gpt-6.2-sol [effort=max] (Case backend metadata) Signed-off-by: Zhou ziheng --- .../cases/admission-drain-occupancy.json | 1 + .../cases/drama3-rejected-directions.json | 1 + .../cases/qwen-ascend-rejected-directions.json | 1 + .../cases/qwen-b300-rejected-directions.json | 1 + .../kernel-writing-guide/cases/vllm-gdn-source-contract.json | 1 + 5 files changed, 5 insertions(+) diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json index 7f793f216..24c35b4c8 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json @@ -1,6 +1,7 @@ { "id": "case.admission-drain-occupancy", "kind": "case", + "backend": "cuda", "origin": ["internal"], "state": "historical_observation", "content": "admission-drain-occupancy.md", diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json index b1c1fa0c8..188f6534d 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json @@ -1,6 +1,7 @@ { "id": "case.drama3-rejected-directions", "kind": "case", + "backend": "cuda", "origin": ["internal"], "state": "historical_observation", "content": "drama3-rejected-directions.md", diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json index 99352d96d..191e319c8 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json @@ -1,6 +1,7 @@ { "id": "case.qwen-ascend-rejected-directions", "kind": "case", + "backend": "ascend", "origin": ["external"], "state": "author_reported", "content": "qwen-ascend-rejected-directions.md", diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json index 173b0afca..dc608b0bd 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json @@ -1,6 +1,7 @@ { "id": "case.qwen-b300-rejected-directions", "kind": "case", + "backend": "cuda", "origin": ["internal"], "state": "historical_observation", "content": "qwen-b300-rejected-directions.md", diff --git a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json index edcb5adf1..6b008fe6b 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json @@ -1,6 +1,7 @@ { "id": "case.vllm-gdn-source-contract", "kind": "case", + "backend": "cuda", "origin": [ "external" ], From 6f261c9dea5f5becd2927506b02d9a3155493a9c Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Wed, 18 Nov 2026 15:32:16 +0800 Subject: [PATCH 03/15] Align scoped Find ranking and repair installed context capture Rank the eligible kind corpus, open packaged skills without the optional YAML parser, and use authored Case backend scope. Retain legacy catalogue and dual-cue selectors. Allow exact installed Case capture and record the actual discovery sources. Agent-assisted-by: Codex (implementation and capture regression coverage) Signed-off-by: Zhou ziheng --- docs/knowledge-operations.md | 39 ++++-- docs/knowledge-retrieval-trace.md | 26 +++- docs/operations.md | 20 ++-- src/inferrsi/apps/campaign.py | 18 +-- src/inferrsi/apps/cli/__init__.py | 8 +- src/inferrsi/apps/cli/arguments.py | 11 +- src/inferrsi/apps/context_provider.py | 6 +- src/inferrsi/knowledge/_skill_selection.py | 11 +- .../inferrsi/knowledge/_provenance.json | 2 +- .../inference-optimization-casebook.json | 8 +- .../common/skills/kernel-writing-guide.json | 2 +- src/inferrsi/knowledge/knowledge_find.py | 21 +--- .../knowledge/knowledge_operations.py | 111 ++++++++++++------ tests/test_campaign_record_injection.py | 4 + tests/test_campaign_task_selection.py | 64 +++++++++- tests/test_knowledge_cli.py | 14 +++ tests/test_knowledge_install.py | 23 ++++ tests/test_optimization_story.py | 48 +++++++- 18 files changed, 322 insertions(+), 114 deletions(-) diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md index 5b7950359..59b672a99 100644 --- a/docs/knowledge-operations.md +++ b/docs/knowledge-operations.md @@ -24,8 +24,9 @@ For discovery, use `find` and follow the relevant hit's `open` command. Use exact bundle or resource. The [operation guide](operations.md) lists the flags. The package catalog is available by default. Select a custom library with `--skill-root` or select the bundled library with `--installed-library` on -`collect`, `read` or `resource`. The read-only `--store` option requires the -store's recorded skills root if one was configured. `resource` takes the +`collect`, `read`, `resource` or `campaign capture-context`. The read-only +`--store` option requires the store's recorded skills root if one was configured. +`resource` takes the revision returned by `collect` or `read` and writes bytes to `--output`. ```bash @@ -113,13 +114,22 @@ Explicit IDs still bypass ranking. This option changes only owner discovery ranking, not query matching or optional context-resource scoring. It is a lexical-presence contract option, not evidence of general retrieval quality. -For questions in `campaign knowledge`, `benchmarks session knowledge` and -`campaign capture-context`, the default ranking method is -`find-lexical-v1`. It uses the same ranker as `find` to select Cases, Skills -and Records, then assembles the selected material with its revisions and -budget omissions. Explicit IDs bypass ranking. Captures using an explicit -`--store` use the established store selection. To select a different ranking -method, pass `--relevance-method` explicitly. +For a single question in `campaign knowledge`, `benchmarks session knowledge` +or `campaign capture-context`, the default ranking method is `find-lexical-v1`. +It uses the local `find` ranker to select Cases, Skills and Records before +assembling the material with revisions and budget omissions. To use it in +standalone Collect, pass `--relevance-method find-lexical-v1` with either a +positional query or `--relevance`. The existing selectors support an exact +query filter plus a separate relevance cue. Explicit IDs bypass discovery +ranking. Store collection uses the existing store selection. + +The Find selector limits the scoring corpus to the requested `--kind`. Installed +Skills can be selected with `--kind skill` and delivered through their package +records, which have bundle kind `knowledge` and provide the original `SKILL.md` +through assembled context or `read --resource`. This works on a base install. +In a mixed search with a custom `--skill-root`, the custom library supplies its +registered Skills and installed projections supply names absent from that root. +Each version remains available by its qualified reference. `task-lexical-v1` remains available for package Skill and Record searches. Whole words in titles, summaries, tags, repository/scenario labels, @@ -130,10 +140,13 @@ can supply this preference. Qualification by identity breaks ties. Use `campaign init --backend ` to retain the runtime backend for later searches. An explicit capture backend takes precedence. Existing -campaigns can pass `--backend` directly on the capture. The search returns -the best matches up to the requested limit and records the matched and -returned counts. For engine Records it captures the complete `record.json`, -including the source, applicable geometry and limitations. For Cases it can +campaigns can pass `--backend` directly on the capture. For scoped Find selection, +a Case's optional top-level `backend` can name a supported backend or `common`. +Case applicability text stays searchable. Cases without a backend declaration +remain in unscoped discovery and count toward `ranking_scope` exclusions in +scoped selection. The search returns the best matches up to the requested limit +and records the matched and returned counts. For engine Records it captures the +complete `record.json`, including the source, applicable geometry and limitations. For Cases it can capture the complete entrypoint. Linked documents can be read at the returned revision. Indexed Skills keep the resource rules below. In Find and task discovery, `--scenario` admits package Records matching the workload and diff --git a/docs/knowledge-retrieval-trace.md b/docs/knowledge-retrieval-trace.md index 81c36b7f1..922c82f69 100644 --- a/docs/knowledge-retrieval-trace.md +++ b/docs/knowledge-retrieval-trace.md @@ -15,9 +15,10 @@ interfaces do not depend on unfinished general statistics or workflow proposals. `campaign capture-context` uses Find ranking for a single free-text question, then assembles selected material through Collect. The capture records both -operations. Explicit IDs select material in caller order. Exact catalog search -and explicit legacy ranking methods remain available for existing workflows. -Capture is explicit. +operations. Explicit IDs select material in caller order. The legacy form that +supplies both a positional query and `--relevance` keeps the exact query filter +and uses the separate cue for task ranking. Exact catalog search and explicit +legacy ranking methods remain available for existing workflows. ```sh inferrsi find 'decode launch' --explain --json @@ -25,6 +26,18 @@ inferrsi campaign capture-context --directory --id r1-conte --relevance 'decode launch' --context-bytes 65536 ``` +To capture an installed Case at its observed revision, select the bundled +library and use the ID and revision from the read: + +```sh +inferrsi campaign capture-context --directory '' --id r1-case \ + --installed-library --id-ref 'checkout:' \ + --revision 'checkout:=' +``` + +The capture records the requested pin. A stale revision is recorded as +`unavailable_revision`. + `knowledge search --explain` returns a JSON envelope with `items` and `retrieval`. Python callers use `search_knowledge(..., explain=True)` for the same catalog envelope. `collect` includes a `retrieval` member outside its @@ -100,8 +113,11 @@ assembly byte budget are retained independently. with `campaign log`; do not edit a previously retained log or input. 4. Import the round's captures and, when available, its explicitly bounded local trace with `campaign import-search`. Use `--archive-review` for the existing - portable v2 passage-to-decision contract. See [the import contract](dashboard.md#import-one-rounds-searches) - for its input fields. Mark complete coverage only when established. + portable v2 passage-to-decision contract. A decision citing a rendered + resource from a verified `find` or `collect` capture supplies `passage_path` + with the resource's bundle-relative key. Its `passage_line` counts from the + first line of that resource. See [the import contract](dashboard.md#import-one-rounds-searches) + for the input fields. Mark complete coverage only when established. 5. Append results and decisions using existing campaign operations. A closing import may extend the same trace window/add captures and runs, subject to the importer's preservation, overlap and repeated-import rules. diff --git a/docs/operations.md b/docs/operations.md index 1a48ce02a..8ddfaf368 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -112,8 +112,10 @@ are distinct from `knowledge search`, whose package catalog filters ### Search packaged records — `knowledge search` / `knowledge show` -Use `find` for ordinary discovery. Use this catalog for filtered inventory and -existing scripts that depend on its exact queries or record JSON. +Use `find` for ordinary discovery. The package catalog retains its own exact +query behavior for filtered inventory and existing scripts. Its +`catalog-occurrences-v1` ranking requires every query token to match and orders +hits by total occurrences. The `context` export shares this selector. - **Inputs:** optional query words and `--tier`, `--repo-type`, `--backend`, `--scenario`, `--limit`, `--json`, `--explain`; `show` takes one record ID. - **Scope:** the installed package catalog (`repo_packs`, `knowledge_packs`) only. @@ -136,8 +138,8 @@ result does not mean that no checkout or store knowledge exists. See the - **Example:** `inferrsi knowledge research MiniMax-H3 --framework mlx --target-backend metal --target-device "M5 Mac Studio" --target-memory-gib 64 --limit 12` - **Python:** `from inferrsi.knowledge.research import research_knowledge`; `research_knowledge("MiniMax-H3", frameworks=["mlx"], modalities=["video"], limit=12)`. -Exact `knowledge search` and `context` still AND query tokens. Independent -research queries broaden discovery without relaxing the active package filters. +Each research query uses the exact catalog selector described above. Querying +separate dimensions broadens discovery while preserving the active package filters. Keep those filters explicit when transferring mechanisms from other backends; target intent does not qualify their applicability. Read the canonical [preparation procedure](../src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/references/model-aware-preparation.md) @@ -306,10 +308,12 @@ reports unavailable or oversized context. Existing queries and ranking methods remain available for compatibility. See [selection and context](knowledge-operations.md#optional-packaged-skill-selection-and-context) for scoring, provenance and accounting. -`campaign knowledge` and `benchmarks session knowledge` use Find to select -material by default, then assemble and retain the selected context with its -revisions and byte budget. Explicit `--relevance-method` selects a named -ranking method. +`campaign knowledge`, `benchmarks session knowledge` and `campaign capture-context` +use Find for single questions by default. They then assemble and retain the +selected context with its revisions and byte budget. Standalone Collect can +opt in with `--relevance-method find-lexical-v1`. To capture an installed Case +exactly, follow the `find` result to `read`, then pass its ref and revision to +`campaign capture-context --installed-library --id-ref REF --revision REF=REV`. - **Inputs:** optional query, `--kind`, repeatable explicit `--id`, repeatable `--revision ID=REV`, `--limit`, `--max-bytes`, `--material`; `--skill-root` selects an explicit skills root, `--store` opens an existing store read-only (the store records its reserved skills root and rejects a mismatched or missing `--skill-root`). - **Scope:** installed package records plus (with `--skill-root`) skills and cases from the explicitly selected root, plus (with `--store`) store candidates; bounded output. diff --git a/src/inferrsi/apps/campaign.py b/src/inferrsi/apps/campaign.py index efbdd46fb..33795fe1d 100644 --- a/src/inferrsi/apps/campaign.py +++ b/src/inferrsi/apps/campaign.py @@ -12,7 +12,7 @@ from inferrsi import __version__ from inferrsi.harness.records import campaign_records, knowledge_capture -from inferrsi.knowledge._skill_selection import FIND_RELEVANCE, TASK_RELEVANCE +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE, TASK_RELEVANCE, single_query from inferrsi.knowledge.catalog import BACKENDS from inferrsi.knowledge.context_projection import ( PROVENANCE_FIELDS, @@ -34,8 +34,8 @@ def create_campaign(root: Path, campaign_id: str, *, target: float | None = None def capture_context(root: Path, *, context_id: str, reader, query: str = "", relevance: str | None = None, relevance_method: str | None = None, - ids: list[str] | None = None, kind: str | None = None, - repo: str | None = None, tier: str | None = None, + ids: list[str] | None = None, revisions: dict[str, str] | None = None, + kind: str | None = None, repo: str | None = None, tier: str | None = None, backend: str | None = None, scenario: str | None = None, limit: int = 5, context_bytes: int = 65536, sources: tuple[str, ...] = ("package",)) -> tuple[Path, str]: @@ -45,21 +45,25 @@ def capture_context(root: Path, *, context_id: str, reader, query: str = "", not by a caller narrative. Returns the new campaign-relative file and its SHA-256. """ + if revisions and not ids: + raise ValueError("Revision pins require explicit IDs (--id-ref)") campaign = knowledge_capture.capture_campaign(root, context_id) if relevance_method is None: has_store = "store" in sources or getattr(reader, "store", None) is not None - if ids is None and not has_store and bool(relevance) != bool(query.strip()): + if ids is None and not has_store and single_query(query, relevance): relevance_method = FIND_RELEVANCE - elif relevance and ids is None: + elif ids is None and relevance: + # Preserve the exact query filter and separate cue for dual-input callers. relevance_method = TASK_RELEVANCE else: relevance_method = "weighted-lexical-v1" if relevance_method in (FIND_RELEVANCE, TASK_RELEVANCE) and ids is None and backend is None: backend = campaign.get("backend") + pins = {"revisions": revisions} if revisions else {} result = reader.collect( query, ids=ids, kind=kind, repo=repo, tier=tier, backend=backend, scenario=scenario, limit=limit, relevance=relevance, - relevance_method=relevance_method, context_bytes=context_bytes, + relevance_method=relevance_method, context_bytes=context_bytes, **pins, ) observation = result.get("retrieval") source_types = list(sources) @@ -72,7 +76,7 @@ def capture_context(root: Path, *, context_id: str, reader, query: str = "", ), "ids": ids, "kind": kind, "repo": repo, "tier": tier, "backend": backend, "scenario": scenario, "limit": limit, - "context_bytes": context_bytes, "sources": source_types, + "context_bytes": context_bytes, "sources": source_types, **pins, } if _supplies_no_observation(result): # Such a reader gets the record this entry wrote before observations diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 1ecaa3d32..c376d15c3 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -639,12 +639,16 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.campaign_action == "capture-context": require_manual_campaign(Path(args.directory).resolve()) - sources = ("package",) + (("checkout",) if args.skill_root else ()) + ( - ("store",) if args.store else ()) + sources = ("package",) + if args.skill_root is not None or args.installed_library: + sources += ("checkout",) + if args.store is not None: + sources += ("store",) path, digest = capture_context( args.directory, context_id=args.id, reader=_knowledge_reader(args), query=args.query, relevance=args.relevance, relevance_method=args.relevance_method, ids=args.ids, kind=args.kind, + revisions=_revisions(args.revision or []), repo=args.knowledge_repo, tier=args.tier, backend=args.backend, scenario=args.scenario, limit=args.limit, context_bytes=args.context_bytes, sources=sources, diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 90a1803d5..58619a75a 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -112,7 +112,7 @@ def _optimization_actions(stages, *, project=False): if name == "knowledge": stage.add_argument("--query", required=True) stage.add_argument("--relevance-method", choices=RELEVANCE_METHODS, - help="Explicit ranking method; defaults to Find") + help="Ranking method, defaults to find-lexical-v1") stage.add_argument("--context-bytes", type=int, required=True) stage.add_argument("--limit", type=int, default=5) @@ -365,9 +365,11 @@ def parser() -> argparse.ArgumentParser: capture.add_argument("query", nargs="?", default="") capture.add_argument("--relevance", help="Task cue for package knowledge and skill discovery") capture.add_argument("--relevance-method", choices=RELEVANCE_METHODS, - help="Explicit ranking method; questions default to Find") + help="Defaults to Find for one query or cue without IDs or a store") capture.add_argument("--id-ref", action="append", dest="ids", metavar="ID", help="Exact knowledge identity to include (repeatable)") + capture.add_argument("--revision", action="append", metavar="ID=REV", + help="Require this revision of an explicit --id-ref (repeatable)") capture.add_argument("--kind", choices=("skill", "knowledge", "case")) capture.add_argument("--repo-type", dest="knowledge_repo", choices=KNOWLEDGE_REPOS) capture.add_argument("--tier", choices=KNOWLEDGE_SCOPES) @@ -375,8 +377,7 @@ def parser() -> argparse.ArgumentParser: capture.add_argument("--scenario") capture.add_argument("--limit", type=int, default=5) capture.add_argument("--context-bytes", type=int, default=65536) - capture.add_argument("--skill-root", type=Path) - capture.add_argument("--store", type=Path) + _read_sources(capture) search = campaign_actions.add_parser( "import-search", help="Import the observed knowledge searches for one campaign round" ) @@ -532,7 +533,7 @@ def parser() -> argparse.ArgumentParser: collect.add_argument("--relevance", help="Compatibility ranking for packaged knowledge") collect.add_argument( "--relevance-method", choices=RELEVANCE_METHODS, default="weighted-lexical-v1", - help="Package ranking method; nondefault methods require --relevance", + help="Keep the packaged ranking or opt into Find with one question or cue", ) collect.add_argument( "--context-bytes", type=int, diff --git a/src/inferrsi/apps/context_provider.py b/src/inferrsi/apps/context_provider.py index 22efe36e4..bcee5b520 100644 --- a/src/inferrsi/apps/context_provider.py +++ b/src/inferrsi/apps/context_provider.py @@ -5,7 +5,7 @@ from hashlib import sha256 from inferrsi.contracts.json_v1 import MAX_SAFE_INTEGER -from inferrsi.knowledge._skill_selection import FIND_RELEVANCE +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE, single_query from inferrsi.knowledge.knowledge_find import installed_skill_root from inferrsi.knowledge.knowledge_operations import KnowledgeReader @@ -59,9 +59,7 @@ def collect(self, request: dict) -> DeliveredContextDraft: ): raise ValueError("invalid Knowledge selection option value") - has_query = bool(query and query.strip()) - has_cue = bool(limits.get("relevance")) - if not ids and self.reader.store is None and has_query != has_cue: + if not ids and self.reader.store is None and single_query(query, limits.get("relevance")): limits = {"relevance_method": FIND_RELEVANCE, **limits} reader = self.reader if reader.skill_root is None and reader.store is None and any( diff --git a/src/inferrsi/knowledge/_skill_selection.py b/src/inferrsi/knowledge/_skill_selection.py index 4b8c6d58f..2e464628e 100644 --- a/src/inferrsi/knowledge/_skill_selection.py +++ b/src/inferrsi/knowledge/_skill_selection.py @@ -8,7 +8,9 @@ TASK_RELEVANCE = "task-lexical-v1" FIND_RELEVANCE = "find-lexical-v1" -RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE) +RELEVANCE_METHODS = ( + "weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE, FIND_RELEVANCE, +) TASK_STOP_WORDS = frozenset({"a", "an", "and", "for", "in", "of", "on", "the", "to", "with"}) _BACKEND_TERMS = { @@ -21,6 +23,13 @@ class SkillIndexError(ValueError): pass +def single_query(query: str | None, relevance: str | None) -> str | None: + """Find takes one question; the legacy selectors can also take a filter and cue.""" + if query and query.strip(): + return None if relevance else query + return relevance or None + + def words(cue: str) -> set[str]: return set(re.findall(r"\w+", cue.casefold())) diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json index 047a69a76..ed0c58dec 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json @@ -1,7 +1,7 @@ { "schema_version": 1, "repository": "https://github.com/InferMatrixLab/InferRSI", - "source_revision": "1cfecf2872788f3615e64452c79c2b29be667bda", + "source_revision": "36e69caf1f2d5ceb2493143f67cc6ea59aad3855", "registry": { "path": "src/inferrsi/knowledge/data/skill_library/skill-index.json", "sha256": "7904275c3383380db179287171e1430617aaa9ac4f439291eff8a71426987fbb" diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json index e2a3332c6..20c8cc2c1 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json @@ -25,13 +25,13 @@ "checkout_ref": "checkout:inference-optimization-casebook", "source_files": { "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/SKILL.md": "89a6a82c692f7f5192f4d7f4c3a5ff08a4df3b1c4bdaa0aafb7e04292d8aa5b4", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json": "8cd0042edde7b1d74edec463cb901f689c3327fac0c34f70692ca10679dc8a1d", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json": "4f8544292b2cc3391f4d4d75994cb81fdd295eff99e55b7e60374c6c122de09e", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.md": "e88bbac5d253a8e7e3a3d5e21047f23f2970c11ac23887d4303ab94fca52dd27", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json": "eba990c460265bab19a74093ea5a65196d7df1b17d99939e1b3fe28e5a066c28", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json": "a2dc5db8aba62081c5743b12f854e74ae500c53bd57e2486260d0b5ae478638a", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.md": "6f143bc363a3290fc008c08e57cb075f1776d1c8cca8a6a59297d5b1fbbf61bf", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json": "7f0ca00b2e5d56f363a2eb8611d7e68778755aa133fa6f951151dc6e9b40b0a9", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json": "b2c1808919ae525efb82148a24b465459799422fefc6d420891d7dbeb36cd31a", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.md": "be396983812262515899a3e2ba4f01b23ff92e1b836881882c7580372b8ae552", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json": "52b5dd233f869e8836b6d2e3cf9bda1b0877a7bf72c1ab8ca3132cf1adc06d23", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json": "90e3cdca085e2d122e648aae527bd6a61de6d4fddb66c393ca25ac4541cd9122", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.md": "cce2930f544a0f78f21797fba4d8ded298e4a81f76c34d7977b0e31d361bc738", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/metadata.json": "963488658f327d2379183d70357eabf832d70a6a70db2aa67f548493f4088d97" }, diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json index a92e4a3d3..4c2419600 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json @@ -48,7 +48,7 @@ "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json": "5237ac437eb76e910bb1d08bfcc364b87dea1b947de5e67eebce511ea1ef1d23", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.md": "8b093faf0ab775d5605ecdd9653fb3689c2b839cba2641c5d5cda17c721bb2ac", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-causal-conv-contract.md": "2f3dcee66e2dfe7a98bee21a36b892538998de5de3caf1bc63257137f9984236", - "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json": "56af6b07bbb32cd6aad44e831404ec041e0450db9623fbe6b048fcbbca32e893", + "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json": "31cfac1e780113c8233b451af7a7562af983aa2f4209938017cc67347ee5dd66", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.md": "7f9ebe8de7e55d3db2a119b6a56e15c37448fd7d4e022a47122a143206d31687", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/metadata.json": "766aa688d573c790789e2836f952315e91c4e06806fca3d04523a8a748bf9c32", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/references/akg-affine-tail-guards.md": "113deb2db10b171db3425bb3f1792bee763706b794c8cc2e222b4cc0d0114118", diff --git a/src/inferrsi/knowledge/knowledge_find.py b/src/inferrsi/knowledge/knowledge_find.py index 9355ac6c3..431033ee2 100644 --- a/src/inferrsi/knowledge/knowledge_find.py +++ b/src/inferrsi/knowledge/knowledge_find.py @@ -408,6 +408,7 @@ def _case_documents(root: Path, *, source_location: str) -> list[Document]: except (ValueError, OSError): body_path = None findings = _findings(record, identity) + backend = record.get("backend") documents.append(Document( identity=identity, kind="case", @@ -431,7 +432,7 @@ def _case_documents(root: Path, *, source_location: str) -> list[Document]: applicability=record.get("applicability"), limitations=tuple(record.get("limitations", [])), findings=findings, - backend=record.get("backend"), + backend=backend if backend in (*BACKENDS, "common") else None, )) return documents @@ -609,17 +610,6 @@ def score( return total, detail -def scoped_backend(document: Document) -> str | None: - """Backend declared by a source, including the recorded cohorts of older Cases.""" - if document.backend or document.kind != "case": - return document.backend - terms = set(tokens(_flatten(document.applicability))) - # Older observations record the hardware in their applicability list. - aliases = {**_BACKEND_NAME_ALIASES, "b300": "cuda"} - backends = named_backends({aliases.get(term, term) for term in terms}) - return next(iter(backends)) if len(backends) == 1 else None - - def installed_skill_root() -> Path: """Resolve the library from the current installation.""" return Path(files("inferrsi").joinpath("knowledge", "data", "skill_library")).resolve() @@ -631,12 +621,7 @@ def packaged_skill_projection(document: Document, reader) -> dict | None: return None if document.root != installed_skill_root(): return None - resource = f"{BUNDLE_LIBRARY}/{document.path}" - return next(( - record for record in reader.records().values() - if record.get("index_type") == "packaged-skill" - and record.get("skill_documents", {}).get("original") == resource - ), None) + return reader.packaged_skill_records.get(f"{BUNDLE_LIBRARY}/{document.path}") def load_documents( diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py index 69b8ac62e..1edc8410f 100644 --- a/src/inferrsi/knowledge/knowledge_operations.py +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -6,6 +6,7 @@ import sqlite3 from dataclasses import dataclass from datetime import date, datetime +from functools import cached_property from importlib.resources import files from pathlib import Path, PurePosixPath @@ -17,6 +18,7 @@ context_view, indexed_skill, record_context, + single_query, skill_score, task_score, words, @@ -279,6 +281,16 @@ def records(self) -> dict[str, dict]: self._records = {record["id"]: record for record in knowledge_records()} return self._records + @cached_property + def packaged_skill_records(self) -> dict[str, dict]: + """Index this catalog snapshot by the original library resource.""" + return { + record["skill_documents"]["original"]: record + for record in self.records().values() + if record.get("index_type") == "packaged-skill" + and (record.get("skill_documents") or {}).get("original") + } + def _skill_library(self) -> tuple[dict[str, bytes], str]: if self._library is None: # K02 moved the physical corpus under the Knowledge module. The bundle @@ -515,46 +527,66 @@ def resource(self, identity: str, path: str, *, revision: str) -> bytes: return bundle.content[path] def _search_order( - self, cue: str, matches, *, kind: str | None, package_only: bool, - ) -> tuple[dict, int, int]: + self, cue: str, matches, *, kind: str | None, package_only: bool, backend: str | None, + ) -> tuple[dict, int, dict[str, int]]: from .knowledge_find import ( Corpus, installed_skill_root, load_documents, packaged_skill_projection, rank, - scoped_backend, ) + kinds = { + None: ("case", "skill", "record"), + "case": ("case",), + "skill": ("skill",), + "knowledge": ("skill", "record"), + } root = None if package_only else self.skill_root - documents = load_documents(root) - if root is not None and root != installed_skill_root() and kind not in ("case", "skill"): - # A custom library adds checkout sources to Collect. Installed - # projections remain available under their package authority. + documents = load_documents(root, kinds=kinds[kind]) + if root is not None and root != installed_skill_root() and kind is None: + # A selected custom library owns its registered skills. The package + # supplies missing skills and remains available for exact reads. + custom_skills = { + document.identity for document in documents if document.kind == "skill" + } documents.extend( document for document in load_documents(kinds=("skill",)) - if packaged_skill_projection(document, self) is not None + if document.identity not in custom_skills + and packaged_skill_projection(document, self) is not None ) - eligible = {} - for index, document in enumerate(documents): + + eligible, references = [], [] + excluded = {"filters": 0, "ranking_scope": 0} + for document in documents: authority, _, identity = document.qualified.partition(":") - # Keep built-in Skills under their package references for context - # delivery, including their indexed required and optional files. - projection = packaged_skill_projection(document, self) if kind != "skill" else None + projection = packaged_skill_projection(document, self) if projection is not None: authority, identity = "package", projection["id"] if authority == "package": - metadata = {**self.records()[identity], "kind": "knowledge"} + material_kind = "skill" if kind == "skill" else "knowledge" + metadata = {**self.records()[identity], "kind": material_kind} + else: + metadata = {"kind": document.kind, "backend": document.backend} + if ( + backend and not package_only and document.kind == "case" + and document.backend is None + ): + excluded["ranking_scope"] += 1 + elif matches(metadata, authority): + eligible.append(document) + references.append(f"{authority}:{identity}") else: - metadata = {"kind": document.kind, "backend": scoped_backend(document)} - if matches(metadata, authority): - eligible[index] = f"{authority}:{identity}" - _, _, scored = rank(cue, Corpus(documents)) + excluded["filters"] += 1 + + _, _, scored = rank(cue, Corpus(eligible)) results = { - eligible[index]: {"score": round(score, 2), "components": components} - for score, _, index, components in scored if index in eligible + references[index]: {"score": round(score, 2), "components": components} + for score, _, index, components in scored } - return results, len(documents), len(eligible) + excluded["zero_score"] = len(eligible) - len(results) + return results, len(documents), excluded def collect( self, @@ -586,12 +618,15 @@ def collect( raise ValueError("Unknown content kind") if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): raise ValueError("relevance requires at least one word") - use_find = relevance_method == FIND_RELEVANCE - find_ranking = use_find and ids is None - if not use_find and relevance_method not in RELEVANCE_METHODS: + if relevance_method not in RELEVANCE_METHODS: raise ValueError("Unknown relevance method") - if use_find: - if bool(query.strip()) == bool(relevance): + find_ranking = relevance_method == FIND_RELEVANCE + cue = relevance or query + if find_ranking: + if ids is not None: + raise ValueError("Find ranking cannot be combined with explicit IDs") + cue = single_query(query, relevance) + if cue is None: raise ValueError("Find selection requires one query or relevance cue") elif relevance is None and relevance_method != "weighted-lexical-v1": raise ValueError("relevance_method requires relevance") @@ -671,25 +706,25 @@ def matches(metadata: dict, authority: str) -> bool: ) ) + excluded = { + "ranking_scope": 0, "filters": 0, "query": 0, "zero_score": 0, + "item_limit": 0, "material_bytes": 0, + } + considered = len(requested) + found = {} if find_ranking: - found, considered, eligible = reader._search_order( - relevance or query, matches, kind=kind, + found, considered, rank_excluded = reader._search_order( + cue, matches, kind=kind, backend=backend, package_only=package_filters or kind == "knowledge", ) requested = list(found) - else: - found, considered, eligible = {}, len(requested), len(requested) + excluded.update(rank_excluded) cases = reader._cases() if kind and not find_ranking else {} selected, omitted, seen = [], [], set() used = 0 ranked, contexts, context_omissions = [], [], [] context_used = 0 explanations = found - excluded = { - "ranking_scope": 0, "filters": considered - eligible, "query": 0, - "zero_score": eligible - len(found) if find_ranking else 0, - "item_limit": 0, "material_bytes": 0, - } matched_count = 0 def select(reference: str, bundle: Bundle, view: dict) -> None: @@ -716,7 +751,7 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: pinned_item = ids is not None and view["ref"] in pinned checkout_context = ( view["authority"] == "checkout" and view["kind"] in ("case", "skill") - and (use_find or pinned_item) + and (find_ranking or pinned_item) ) if not ((packaged and (is_skill or task_ranking or pinned_item)) or checkout_context): context_omissions.append({"ref": view["ref"], "reason": "unsupported_context"}) @@ -725,7 +760,7 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: available = context_bytes - context_used if is_skill: context, omission = context_view( - record, view, bundle.content, relevance or query, available + record, view, bundle.content, cue, available ) else: context, omission = record_context(record, view, bundle.content, available) @@ -851,7 +886,7 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: } if find_ranking: result["selection"] = { - "method": FIND_RELEVANCE, "cue": relevance or query, + "method": FIND_RELEVANCE, "cue": cue, "scope": "package-records-and-library", "ranking_applied": True, "matched_count": len(found), "returned_count": len(selected), } diff --git a/tests/test_campaign_record_injection.py b/tests/test_campaign_record_injection.py index ba61470c6..1f7761249 100644 --- a/tests/test_campaign_record_injection.py +++ b/tests/test_campaign_record_injection.py @@ -57,6 +57,8 @@ def collect(self, query="", **options): "cue-and-ids": ({"relevance": "decode launch", "ids": ["package:graphs"]}, "weighted-lexical-v1", None, "weighted-lexical-v1"), "query-only": ({"query": "decode"}, "find-lexical-v1", "cuda", "find-lexical-v1"), + "query-and-cue": ({"query": "decode", "relevance": "kernel launch"}, + "task-lexical-v1", "cuda", "task-lexical-v1"), "explicit-raw": ({"relevance": "decode launch", "relevance_method": "weighted-raw-string-v1"}, "weighted-raw-string-v1", None, "weighted-raw-string-v1"), @@ -85,6 +87,8 @@ def test_capture_record_keeps_its_relevance_method(tmp_path, name): assert document["request"]["relevance_method"] == stored assert document["request"]["backend"] == backend assert reader.calls[0]["relevance_method"] == delivered + assert reader.calls[0]["query"] == options.get("query", "") + assert reader.calls[0]["relevance"] == options.get("relevance") assert reader.calls[0]["backend"] == backend assert hashlib.sha256(raw).hexdigest() == digest assert campaign.verify_capture(raw)[1] == "verified" diff --git a/tests/test_campaign_task_selection.py b/tests/test_campaign_task_selection.py index fc9b20dc5..4dda1f73b 100644 --- a/tests/test_campaign_task_selection.py +++ b/tests/test_campaign_task_selection.py @@ -1,8 +1,10 @@ import json +import shutil import pytest from inferrsi.apps.campaign import capture_context, create_campaign, verify_capture +from inferrsi.knowledge.knowledge_find import find, installed_skill_root, load_corpus from inferrsi.knowledge.knowledge_operations import KnowledgeReader BASELINE_CUE = "reproducible diffusion baseline correctness fixed seed end to end benchmark" @@ -82,7 +84,9 @@ def test_find_filters_cases_to_the_campaign_backend(tmp_path, backend, case): counts = capture["retrieval"]["counts"] assert counts["excluded"]["filters"] > 0 assert counts["considered"] == ( - counts["matched"] + counts["excluded"]["filters"] + counts["excluded"]["zero_score"] + counts["matched"] + sum( + counts["excluded"][name] for name in ("filters", "ranking_scope", "zero_score") + ) ) @@ -101,8 +105,6 @@ def test_find_with_custom_library_keeps_packaged_projections(tmp_path): def test_find_preserves_required_case_readset(tmp_path): - from inferrsi.knowledge.knowledge_find import installed_skill_root - create_campaign(tmp_path, "target") path, _ = capture_context( tmp_path, context_id="r1", reader=KnowledgeReader(), kind="case", @@ -118,3 +120,59 @@ def test_find_preserves_required_case_readset(tmp_path): assert resource["content"].encode("utf-8") == KnowledgeReader( skill_root=installed_skill_root() ).resource(context["ref"], resource["path"], revision=context["revision"]) + + +def test_find_collection_uses_the_case_only_corpus(): + query = "cache per-step masks ran out of device memory" + discovery = find(query, load_corpus(kinds=("case",)), limit=3) + delivery = KnowledgeReader().collect( + query, kind="case", relevance_method="find-lexical-v1", limit=3, + ) + observed = delivery["retrieval"] + assert [(item["ref"], item["score"]) for item in observed["items"]] == [ + (f"checkout:{item['id']}", item["score"]) for item in discovery["items"] + ] + assert observed["counts"]["considered"] == discovery["corpus_size"] + assert observed["counts"]["excluded"]["filters"] == 0 + + +def test_find_collection_does_not_repeat_custom_skills_as_package_projections(tmp_path): + library = tmp_path / "copy" + shutil.copytree(installed_skill_root(), library) + reader = KnowledgeReader(skill_root=library) + result = reader.collect( + "weight loading optimization", relevance_method="find-lexical-v1", limit=6, + ) + refs = {item["ref"] for item in result["items"]} + assert "checkout:weight-loading-optimization" in refs + assert "package:inferrsi.common.skills.weight_loading_optimization" not in refs + + +def test_find_does_not_infer_case_backend_from_applicability(tmp_path): + owner = tmp_path / "owner" + cases = owner / "cases" + cases.mkdir(parents=True) + (owner / "SKILL.md").write_text( + "---\nname: owner\ndescription: Local test\n---\nCore", encoding="utf-8", + ) + (cases / "body.md").write_text("B300 queue tuning.", encoding="utf-8") + (cases / "case.json").write_text(json.dumps({ + "id": "case.legacy-b300", + "kind": "case", + "content": "body.md", + "state": "candidate", + "applicability": ["A B300 run"], + "sources": [{"type": "internal", "issue": "test", "artifact": "case"}], + "limitations": [], + "attribution": [], + }), encoding="utf-8") + reader = KnowledgeReader(skill_root=tmp_path) + options = {"kind": "case", "relevance_method": "find-lexical-v1"} + result = reader.collect("B300", **options) + assert [item["ref"] for item in result["items"]] == ["checkout:case.legacy-b300"] + scoped = reader.collect("B300", backend="cuda", **options) + assert scoped["no_match"] + counts = scoped["retrieval"]["counts"] + assert counts["considered"] == 1 + assert counts["excluded"]["ranking_scope"] == 1 + assert counts["excluded"]["filters"] == 0 diff --git a/tests/test_knowledge_cli.py b/tests/test_knowledge_cli.py index 9461eaaac..827d55e65 100644 --- a/tests/test_knowledge_cli.py +++ b/tests/test_knowledge_cli.py @@ -55,6 +55,20 @@ def test_collect_defaults_to_package_with_query_and_limit(): assert all(item["authority"] == "package" for item in envelope["items"]) +def test_collect_accepts_find_as_an_explicit_ranking_method(): + query = "cache per-step masks ran out of device memory" + method = ("--relevance-method", "find-lexical-v1") + code, stdout, _ = run_cli("collect", query, *method, "--kind", "case", "--json") + assert code == 0 + result = json.loads(stdout) + assert result["selection"]["method"] == "find-lexical-v1" + assert result["items"][0]["ref"] == "checkout:case.qwen-ascend-rejected-directions" + stderr = run_cli_error("collect", query, *method, "--id", f"package:{SOURCE}") + assert "explicit IDs" in stderr + stderr = run_cli_error("collect", query, *method, "--relevance", "cache") + assert "one query or relevance cue" in stderr + + def test_collect_reports_explicit_no_match_with_omission_reason(): code, stdout, _ = run_cli("collect", "--id", "not-a-real-id", "--json") assert code == 0 diff --git a/tests/test_knowledge_install.py b/tests/test_knowledge_install.py index d6880788e..be7b0117d 100644 --- a/tests/test_knowledge_install.py +++ b/tests/test_knowledge_install.py @@ -90,6 +90,29 @@ def test_packaged_skill_search_and_resource_work_without_pyyaml(without_yaml, tm assert output.read_bytes() == expected +def test_capture_scope_skill_delivers_packaged_text_without_pyyaml(without_yaml, tmp_path): + campaign = tmp_path / "campaign" + env = _cli_env(without_yaml) + created = _run(["campaign", "init", "--directory", str(campaign), "--id", "lean"], env) + assert created.returncode == 0, created.stderr + result = _run([ + "campaign", "capture-context", "--directory", str(campaign), "--id", "r1", + "--kind", "skill", "--limit", "1", "ascend graph mode", + ], env) + assert result.returncode == 0, result.stderr + capture = json.loads((campaign / json.loads(result.stdout)["context"]).read_bytes()) + item, = capture["items"] + context, = capture["contexts"] + assert item["ref"] == "package:inferrsi.ascend.skills.ascend_graph_mode" + assert context["revision"] == item["revision"] + original = context["resources"][0] + assert original["path"] == item["skill_documents"]["original"] + assert original["content"].encode() == ( + SKILLS / Path(original["path"]).relative_to("data/skill_library") + ).read_bytes() + assert capture["omissions"] == [] + + def test_custom_skill_does_not_open_the_installed_projection_without_pyyaml(without_yaml, tmp_path): relative = "hardware/ascend/ascend-graph-mode" directory = tmp_path / relative diff --git a/tests/test_optimization_story.py b/tests/test_optimization_story.py index c22048a86..121c3070a 100644 --- a/tests/test_optimization_story.py +++ b/tests/test_optimization_story.py @@ -19,6 +19,7 @@ record_decision, ) from inferrsi.harness.records.optimization_log import LogError, parse_optimization_log, record_log +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE from inferrsi.knowledge.knowledge_operations import KnowledgeReader @@ -149,12 +150,51 @@ def test_capture_keeps_exact_context_and_detects_tampering(tmp_path): def test_capture_matches_real_collect_output(tmp_path): create_campaign(tmp_path, "camp") - path, _ = capture_context(tmp_path, context_id="real", reader=KnowledgeReader(), - relevance="kv cache block size", context_bytes=40000, limit=2) + reader = KnowledgeReader() + options = {"relevance": "kv cache block size", "context_bytes": 40000, "limit": 2} + direct = reader.collect(**options, relevance_method=FIND_RELEVANCE) + path, _ = capture_context(tmp_path, context_id="real", reader=reader, **options) document, state = verify_capture(path.read_bytes()) + assert state == "verified" - assert document["contexts"], "the packaged library should supply a context" - assert document["request"]["sources"] == ["package"] + assert document["contexts"], "the installed library should supply a context" + assert document["request"]["sources"] == direct["retrieval"]["request"]["sources"] + assert document["request"]["sources"] == ["package", "checkout"] + assert document["retrieval"]["calls"] == ["campaign capture-context", "collect", "find"] + assert document["selection"] == direct["selection"] + assert [(item["ref"], item["revision"]) for item in document["items"]] == [ + (item["ref"], item["revision"]) for item in direct["items"] + ] + + +def test_capture_installed_case_at_exact_revision(tmp_path, capsys): + create_campaign(tmp_path, "camp") + ref = "checkout:case.qwen-ascend-rejected-directions" + assert main(["read", ref, "--installed-library", "--material", "--omit-resources"]) == 0 + live = json.loads(capsys.readouterr().out) + revision = live["revision"] + args = ["campaign", "capture-context", "--directory", str(tmp_path), + "--installed-library", "--id-ref", ref] + + def capture(context_id, pin): + assert main([*args, "--id", context_id, "--revision", f"{ref}={pin}"]) == 0 + output = json.loads(capsys.readouterr().out) + document, state = verify_capture((tmp_path / output["context"]).read_bytes()) + assert state == "verified" + assert document["request"]["sources"] == ["package", "checkout"] + assert document["request"]["revisions"] == {ref: pin} + return document + + exact = capture("exact", revision) + context, = exact["contexts"] + assert (context["ref"], context["revision"]) == (ref, revision) + assert any(resource["content"] == live["content"] for resource in context["resources"]) + + stale = capture("stale", "sha256:" + "0" * 64) + assert stale["items"] == stale["contexts"] == [] + assert [(item["id"], item["reason"]) for item in stale["omissions"]] == [ + (ref, "unavailable_revision") + ] def test_capture_packaged_skills_does_not_repeat_bundle_manifest(tmp_path): From 14e6f17bd254aaa05da0d5a2e3aea35ed337b139 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Wed, 18 Nov 2026 17:07:27 +0800 Subject: [PATCH 04/15] Route remaining knowledge discovery through Find Reuse the Find corpus and ranker for package catalog queries, grouped research, Collect and optional context files. Preserve exact inventory, pinned reads, budgeted delivery and historical archives. Document legacy input migration and cover external-only public results. Agent-assisted-by: Codex / gpt-6.2-sol [effort=max] (catalog, Collect and protected guidance) Agent-assisted-by: Codex (CLI, applications and verification) Signed-off-by: Zhou ziheng --- docs/dashboard.md | 5 +- docs/knowledge-base.md | 10 +- docs/knowledge-operations.md | 183 +++++------ docs/knowledge-retrieval-trace.md | 24 +- docs/knowledge-store.md | 7 + docs/operations.md | 47 ++- plugins/inferrsi/skills/inferrsi/SKILL.md | 2 +- .../inferrsi/skills/inferrsi/SKILL.md | 5 +- src/inferrsi/apps/campaign.py | 25 +- src/inferrsi/apps/cli/__init__.py | 11 + src/inferrsi/apps/cli/arguments.py | 28 +- src/inferrsi/apps/context_provider.py | 3 - src/inferrsi/apps/views/capture_delivery.py | 7 +- src/inferrsi/knowledge/_skill_selection.py | 128 +++----- src/inferrsi/knowledge/catalog.py | 136 +++++--- .../inference-engine-research/SKILL.md | 7 +- .../references/model-aware-preparation.md | 4 +- src/inferrsi/knowledge/knowledge_find.py | 45 +-- .../knowledge/knowledge_operations.py | 294 +++++++++--------- src/inferrsi/knowledge/research.py | 23 +- tests/check_installed.py | 25 +- tests/fixtures/indexer/README.md | 5 +- tests/fixtures/knowledge_retrieval.py | 58 ++-- tests/test_campaign_record_injection.py | 106 ++++--- tests/test_campaign_task_selection.py | 11 +- tests/test_cannbot_boundaries.py | 35 +++ tests/test_collect_selection.py | 74 +++-- tests/test_knowledge_cli.py | 34 +- tests/test_knowledge_find.py | 28 ++ tests/test_knowledge_install.py | 12 + tests/test_knowledge_navigation_browser.py | 5 +- tests/test_knowledge_operations.py | 7 +- tests/test_knowledge_store.py | 41 +++ tests/test_optimization_story.py | 2 +- tests/test_raw_string_selection.py | 190 +++-------- tests/test_research.py | 22 +- tests/test_retrieval_projection.py | 78 +++-- tests/test_search_archive.py | 43 ++- tests/test_skill_library.py | 2 + tools/evaluate_indexer.py | 20 +- 40 files changed, 963 insertions(+), 829 deletions(-) diff --git a/docs/dashboard.md b/docs/dashboard.md index 3bbda6443..7af0163a7 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -540,8 +540,9 @@ declared search space and how it was narrowed, each round's hypothesis and reason, the decision, and the success and failure summaries. It has two parts. **1. Capture the knowledge context while working.** Run the same retrieval the -agent uses through `campaign capture-context`. It calls `collect` with a context -budget and keeps the complete envelope: the request, everything search selected, +agent uses through `campaign capture-context`. It calls `collect` to rank the +question with Find and assemble resources within a context budget. The capture +keeps the request, the selected items, everything left out and why, and every resource body with its SHA-256. The dashboard re-derives the assembled text from those resources and withholds a capture whose bytes or hashes no longer match. diff --git a/docs/knowledge-base.md b/docs/knowledge-base.md index f58382d85..d3511c1aa 100644 --- a/docs/knowledge-base.md +++ b/docs/knowledge-base.md @@ -270,10 +270,12 @@ retained Apple feasibility/header-inventory research. Its source-derived candidate state and imported header observations remain separate from native execution, media quality, target memory and performance qualification. -Use `inferrsi knowledge search --repo-type inferrsi --limit 100` in an ordinary -installation. Catalog JSON and full package bundle reads need no YAML parser: -`inferrsi read inferrsi.ascend.skills.ascend_graph_mode --material` reads the -record with its retained library dependency resources. The optional `skills` +Use `inferrsi knowledge search --repo-type inferrsi --limit 100` to list installed +package records in stable ID order. A text query uses Find to rank eligible +package records and returns the original catalog JSON. `knowledge show` reads +that JSON by exact ID. Catalog JSON and full package bundle reads need no YAML +parser. `inferrsi read inferrsi.ascend.skills.ascend_graph_mode --material` reads +the record with its retained library dependency resources. The optional `skills` extra enables frontmatter-based `skills list`, `show`, `resource` and `reference`. An explicit `--skill-root src/inferrsi/knowledge/data/skill_library` exposes the checkout adapter's original skill and case identities for normalized reads. diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md index 59b672a99..cabe382c4 100644 --- a/docs/knowledge-operations.md +++ b/docs/knowledge-operations.md @@ -45,35 +45,35 @@ one authority is never resolved by precedence: `read` raises an ambiguous-identi error naming the qualified choices, explicit Collect reports it as an omission, and a full Collect returns each qualified item and lists the ID under `collisions`. -Omit `skill_root` for installed package-only use, including full migrated -procedures through IDs such as `inferrsi.ascend.skills.ascend_graph_mode`. A reader reads the installed -catalog and complete skill library once, so one reader is one package snapshot. Query -words are matched against normalized metadata and the entry content, not resource -hashes. `repo`, `tier`, `backend`, `scenario` -retain existing catalog filter semantics; those package-only filters exclude local -skills/cases. `kind`, explicit `ids`, per-ID `revisions`, item `limit` and -`max_bytes` are available. Filters apply before output limits; package metadata -filters run before any resource bytes are loaded. `max_bytes` bounds -returned item payloads, not the omission/control envelope. Each omission carries a -`reason` code (`unknown_identity`, `ambiguous_identity`, `invalid_identity`, -`duplicate_identity`, `unavailable_revision`, `invalid`, `filters`, `query`, -`output_limit`, `index_unavailable`) and, for errors, a `detail` message. No-match -is explicit. -Per-item omission applies after identity discovery: a checkout skill or case -record that cannot be parsed or validated during discovery (for example invalid -frontmatter or JSON, a name/directory mismatch, conflicting metadata, a missing -or duplicate case ID) fails the whole reader call explicitly, because -identities cannot be trusted. -Deterministic identity order is not a learned relevance ranking. - -## Optional packaged skill selection and context - -Standalone Collect supports legacy query-based selection and budgeted -assembly. The explicit-ID path can assemble context without a ranking query: +Omitting `skill_root` gives exact inventory and reads over the installed +package, including procedures such as +`inferrsi.ascend.skills.ascend_graph_mode`. A reader loads the installed catalog +and complete skill library once per snapshot. Inventory and exact selection apply +the `repo`, `tier`, `backend` and `scenario` filters to package items. +Filters apply before output limits. Package metadata filters run before loading +resource bytes. Each item omission has a reason code such as +`unknown_identity`, `ambiguous_identity`, `invalid_identity`, +`duplicate_identity`, `unavailable_revision`, `invalid`, `filters`, +`output_limit` or `index_unavailable`. Errors also include a `detail` message. +An empty result reports `no_match`. + +Per-item omissions apply after identity discovery. A checkout whose registered +names or case identities cannot be trusted fails discovery explicitly. Examples +include a skill registration whose path disagrees with its name and a missing +or duplicate Case ID. + +## Ranked discovery and budgeted context + +Collect with one text input uses the local Find ranker. +Pass a positional query or use `relevance` (`--relevance` on the CLI). +Existing callers can explicitly pass `relevance_method="find-lexical-v1"`. +For a retired method, omit the method argument and supply one question. If you +used a positional filter and a relevance cue together, combine them into one +question and pass scope constraints through the structured filters. [Operation guide](operations.md) ```python reader = KnowledgeReader() -legacy = reader.collect("cpu.stat.local", material=True, limit=3) +matches = reader.collect("cgroupv2 cpu.stat.local", material=True, limit=3) selected = reader.collect( ids=["package:inferrsi.ascend.skills.ascend_graph_mode", "package:inferrsi.ascend.skills.ascend_device_health"], @@ -87,77 +87,42 @@ inferrsi collect --id package:inferrsi.ascend.skills.ascend_graph_mode \ --context-bytes 65536 --json ``` -`relevance` is a separate lexical cue, not a replacement for `query`. Discovery -with this cue considers generated `index_type=packaged-skill` package entries -only; they retain `kind=knowledge`. Existing filters and the whitespace-token -AND substring query run first. Each distinct case-folded word (`\w+`) contributes -4 for substring presence in the title, 3 in the description, 2 in -tags/applicability and 1 in the complete core. Repetition within a field adds nothing. Candidates with -positive overlap sort by decreasing score, then qualified identity, before -`limit` and `max_bytes`. This is deterministic lexical matching, not a learned -ranking or a task-quality guarantee. Explicit `ids` retain membership and caller -order regardless of the cue; `selection.ranking_applied` is false for those calls. -Ordinary collection of other authorities and records remains available without -the relevance opt-in. `KnowledgeStore.collect` delegates these same options. - -The default `relevance_method="weighted-lexical-v1"` searches the JSON -representation of every field, including strings. Explicitly select -`relevance_method="weighted-raw-string-v1"` (CLI: -`--relevance-method weighted-raw-string-v1`) to match original string values -instead: an actual newline or tab no longer creates a literal `n` or `t` match -from its JSON escape. Literal backslash-letter text remains searchable. -Nonstring fields, including the combined tags/applicability value, retain their -legacy JSON representation. Both methods keep the same substring matching, -distinct cue words, weights, positive-score admission and identity tie order. -The raw-string method requires `relevance`. Unsupported methods are rejected. -Explicit IDs still bypass ranking. This option changes only owner discovery -ranking, not query matching or optional context-resource scoring. It is a -lexical-presence contract option, not evidence of general retrieval quality. - -For a single question in `campaign knowledge`, `benchmarks session knowledge` -or `campaign capture-context`, the default ranking method is `find-lexical-v1`. -It uses the local `find` ranker to select Cases, Skills and Records before -assembling the material with revisions and budget omissions. To use it in -standalone Collect, pass `--relevance-method find-lexical-v1` with either a -positional query or `--relevance`. The existing selectors support an exact -query filter plus a separate relevance cue. Explicit IDs bypass discovery -ranking. Store collection uses the existing store selection. - -The Find selector limits the scoring corpus to the requested `--kind`. Installed -Skills can be selected with `--kind skill` and delivered through their package -records, which have bundle kind `knowledge` and provide the original `SKILL.md` -through assembled context or `read --resource`. This works on a base install. -In a mixed search with a custom `--skill-root`, the custom library supplies its -registered Skills and installed projections supply names absent from that root. +An empty query and no `relevance` give inventory in qualified identity order. +Explicit `ids` retain caller order and are subject to filters, revision +availability and output limits. `limit` caps the number of returned items. +For explicit IDs, omit the positional query +and optionally pass `relevance` to guide selection of optional context files. +`selection.ranking_applied` records whether the items themselves were ranked. + +Text discovery scores local Cases, Skills and Records within the requested kind +and filters. Installed Skills selected with `--kind skill` can be delivered +through package records with bundle kind `knowledge`. Their original `SKILL.md` +is available through assembled context or `read --resource`. This works on a +base install. In a mixed search using a custom `--skill-root`, that library +provides its registered Skills. Installed projections supply missing names. Each version remains available by its qualified reference. -`task-lexical-v1` remains available for package Skill and Record searches. -Whole words in titles, summaries, tags, repository/scenario labels, -applicability and body text contribute to rank. A portable owner or one -matching the backend receives twice the lexical score of an unscoped -specialization. Without an explicit backend, platform words in the cue -can supply this preference. Qualification by identity breaks ties. - -Use `campaign init --backend ` to retain the runtime backend for -later searches. An explicit capture backend takes precedence. Existing -campaigns can pass `--backend` directly on the capture. For scoped Find selection, -a Case's optional top-level `backend` can name a supported backend or `common`. -Case applicability text stays searchable. Cases without a backend declaration -remain in unscoped discovery and count toward `ranking_scope` exclusions in -scoped selection. The search returns the best matches up to the requested limit -and records the matched and returned counts. For engine Records it captures the -complete `record.json`, including the source, applicable geometry and limitations. For Cases it can -capture the complete entrypoint. Linked documents can be read at the returned -revision. Indexed Skills keep the resource rules below. In Find and task -discovery, `--scenario` admits package Records matching the workload and -packaged Skill records tagged `skills.discovery`. Check each procedure's own -conditions before using it. Captured hits retain the scenario declared by the -owner. Explicit IDs and legacy catalog selectors filter on the declared scenario. - -`context_bytes` requests a separate context view. It takes a positive integer -byte budget. It can be used without `relevance`. For indexed Skills, optional -resources then match the existing query, or use path order for an empty query. -`material=True` on a package Skill record returns its `record.json` entrypoint. +Case applicability text stays searchable. For backend-scoped discovery, a Case's +top-level `backend` names a supported backend or `common`. Discovery without a +backend also includes Cases without that declaration. Scoped selection counts +those Cases under `ranking_scope`. With a scenario, ranked discovery includes +package Records matching that workload and packaged Skills tagged +`skills.discovery`. Captured hits keep the scenario declared by their owner. +Inventory and explicit IDs filter on that declared scenario. + +Set `campaign init --backend ` to retain the runtime backend for +ranked campaign capture. A backend supplied on a capture takes precedence. +Existing campaigns can also supply `--backend` directly on each capture. + +`KnowledgeStore.collect` delegates to the same selector. With a store attached, +discovery includes its candidate bundles under qualified `store:` +references. See the [store read and revision rules](knowledge-store.md). + +`context_bytes` requests a separate context view with a positive byte budget. +Ranked discovery can capture a complete engine `record.json` or Case entrypoint +and its required resources. The assembled context for indexed Skills follows +the resource rules below. `material=True` on a package Skill record returns its +`record.json` entrypoint. | Additional result | Meaning | | --- | --- | @@ -183,15 +148,12 @@ If that set does not fit, the owner's entire context is omitted, with required and available bytes, and a later smaller context can still fit. The owning item can still be returned. Mandatory instructions are never truncated. -Remaining owner-local Markdown/JSON/text/CSV resources are optional. Their -filename contributes 2 per matching word and their complete text contributes 1; -ties use resource path. An empty cue includes them in path order. Nonmatching, -oversized, non-UTF-8 and non-context resources (including owner `metadata.json` -and scripts) are reported under the context's -`omissions`, not silently discarded. Selected files retain their prerequisites -and limitations; links to further material must still be followed through the -owning revision. The context view is not a standalone export or proof that all -linked dependencies have been read. +Remaining owner-local Markdown, JSON, text and CSV resources are optional. With +a cue, Find ranks their filenames and text within the owner and only positive +matches are eligible. An empty cue uses resource path order. The context's +`omissions` list nonmatching, oversized, non-UTF-8 and ineligible resources +such as `metadata.json` and scripts. Follow links to further material through +the owning revision. Use exact `read` for the complete Bundle. The context budget sums exact UTF-8 resource-body bytes across owners in selected item order, counting a shared resource again if returned for another owner. @@ -201,14 +163,11 @@ adding context does not change those item payloads. Use `--json` for full provenance/accounting. CLI text mode prints complete context bodies in delimited blocks plus resource/owner omission codes. -The source-pinned generated projection is validated against the packaged -registry, owner mapping and resource hashes on each optimized read. Missing or -stale bindings produce `index_unavailable` (a discovery item omission or a -context omission); no ordinary-mode fallback occurs. Refresh projections with -the existing `tools/build_skill_index.py` after committing their inputs. No new -registry, persistent cache, service or optional parser dependency is introduced. -Bundle resources, complete library dependency closure and revision calculations -remain unchanged. Collection never executes a skill or its scripts. +Collect validates a generated Skill projection against the packaged registry, +owner mapping and resource hashes before returning a ranked hit or building an +exact context view. Missing or stale bindings produce `index_unavailable` in +the corresponding item or context omissions. Refresh projections with the +existing `tools/build_skill_index.py` after committing their inputs. Offline fixtures establish compatibility and bounded deterministic selection. Task outcomes, tokens and latency require a separately authorized comparison diff --git a/docs/knowledge-retrieval-trace.md b/docs/knowledge-retrieval-trace.md index 922c82f69..06c55aa11 100644 --- a/docs/knowledge-retrieval-trace.md +++ b/docs/knowledge-retrieval-trace.md @@ -15,10 +15,9 @@ interfaces do not depend on unfinished general statistics or workflow proposals. `campaign capture-context` uses Find ranking for a single free-text question, then assembles selected material through Collect. The capture records both -operations. Explicit IDs select material in caller order. The legacy form that -supplies both a positional query and `--relevance` keeps the exact query filter -and uses the separate cue for task ranking. Exact catalog search and explicit -legacy ranking methods remain available for existing workflows. +operations. Pass one question as a positional query or `--relevance`. Explicit +IDs select material in caller order. See the [migration guide](operations.md#migrate-old-text-queries) +for callers that previously passed a separate query filter and ranking cue. ```sh inferrsi find 'decode launch' --explain --json @@ -55,17 +54,16 @@ budgeted item and resource payloads. | `assembly` | `performed` or `not_requested`; old missing evidence is `unknown` | | `candidate_details` | `not_recorded`: no reconstruction of a full rejected candidate set | -Catalog scores count query-term occurrences in the searched JSON; repeated terms -remain repeated components. Ties use ascending ID. A catalog observation's -`catalog-sha256:` revision hashes the returned catalog record using sorted, -UTF-8 JSON. It identifies that record, not a collect bundle or an upstream commit. +A catalog text query records `knowledge search → find` and returns catalog +records. Its `catalog-sha256:` revision hashes the returned record using sorted, +UTF-8 JSON. It identifies those catalog bytes. A Collect revision identifies the +normalized bundle used to deliver resources. Find records each matching term's contribution and uses recorded state, -identity and qualified reference to break score ties. The two legacy weighted -methods record title, summary, tags/applicability and body contributions. Task relevance records title, summary, tags, repo/scenario, -applicability and body contributions plus its platform multiplier. Ties use -ascending qualified ref. Explicit IDs preserve caller order and skip ranking. -A plain inventory records `not_requested`. Filters use first-exclusion +identity and qualified reference to break score ties. Explicit IDs preserve +caller order and skip item ranking. A plain inventory records `not_requested`. +Older captures retain the scores, components and method identifiers recorded in +their original run. Filters use first-exclusion counts; these are not overlapping marginal totals. Per-item explanations cover returned items only. Existing omissions and resource hashes remain available. diff --git a/docs/knowledge-store.md b/docs/knowledge-store.md index fac03b77e..6956e95c7 100644 --- a/docs/knowledge-store.md +++ b/docs/knowledge-store.md @@ -36,6 +36,13 @@ or `None` to declare package-only use. `store.reader()` / `store.collect` combin package, checkout and store items with qualified `store:` refs. An ID owned by the store and a read-only adapter is reported as ambiguous, never shadowed. +Pass a query or `relevance` to `store.collect` for local Find discovery. +Store-owned candidates participate using their metadata and entrypoint text. +An unpinned query ranks the current revision. A query with a revision pin ranks +the retained entrypoint at that revision and returns that same revision if it +matches. Use explicit IDs and revisions for exact collection of historical +bundles and `resource` for their contents. + ## Create and Update - Create registers a supplied **candidate** and rejects duplicate identity. It diff --git a/docs/operations.md b/docs/operations.md index 8ddfaf368..21e105216 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -112,10 +112,11 @@ are distinct from `knowledge search`, whose package catalog filters ### Search packaged records — `knowledge search` / `knowledge show` -Use `find` for ordinary discovery. The package catalog retains its own exact -query behavior for filtered inventory and existing scripts. Its -`catalog-occurrences-v1` ranking requires every query token to match and orders -hits by total occurrences. The `context` export shares this selector. +Use `find` for ordinary discovery. With no query, `knowledge search` lists +matching package records in ID order. With a query, it uses Find ranking over +the filtered package catalog and still returns complete catalog records. The +`context` export shares this selector. Text query results can change in both +membership and order. Use `--explain` for the recorded ranking and counts. - **Inputs:** optional query words and `--tier`, `--repo-type`, `--backend`, `--scenario`, `--limit`, `--json`, `--explain`; `show` takes one record ID. - **Scope:** the installed package catalog (`repo_packs`, `knowledge_packs`) only. @@ -138,7 +139,7 @@ result does not mean that no checkout or store knowledge exists. See the - **Example:** `inferrsi knowledge research MiniMax-H3 --framework mlx --target-backend metal --target-device "M5 Mac Studio" --target-memory-gib 64 --limit 12` - **Python:** `from inferrsi.knowledge.research import research_knowledge`; `research_knowledge("MiniMax-H3", frameworks=["mlx"], modalities=["video"], limit=12)`. -Each research query uses the exact catalog selector described above. Querying +Each research query uses Find over the same filtered package catalog. Querying separate dimensions broadens discovery while preserving the active package filters. Keep those filters explicit when transferring mechanisms from other backends; target intent does not qualify their applicability. Read the canonical @@ -304,22 +305,36 @@ from this integration mechanism. Use Collect to take known material in caller-specified order. With `--context-bytes `, it assembles complete files up to a UTF-8 body budget and -reports unavailable or oversized context. Existing queries and ranking methods -remain available for compatibility. See [selection and context](knowledge-operations.md#optional-packaged-skill-selection-and-context) -for scoring, provenance and accounting. - -`campaign knowledge`, `benchmarks session knowledge` and `campaign capture-context` -use Find for single questions by default. They then assemble and retain the -selected context with its revisions and byte budget. Standalone Collect can -opt in with `--relevance-method find-lexical-v1`. To capture an installed Case -exactly, follow the `find` result to `read`, then pass its ref and revision to +reports unavailable or oversized context. With no IDs and one question, Collect +uses the same local Find ranking as `campaign knowledge`, `benchmarks session +knowledge`, and `campaign capture-context`. See [selection and context](knowledge-operations.md#optional-packaged-skill-selection-and-context) +for provenance and accounting. To capture an installed Case exactly, follow the +`find` result to `read`, then pass its ref and revision to `campaign capture-context --installed-library --id-ref REF --revision REF=REV`. +#### Migrate old text queries + +- `knowledge search QUERY` and `context QUERY` keep their package scope and + output format. Text queries now use Find and can match entries that contain + only part of the query. Omit `QUERY` for filtered inventory or use + `knowledge show ID` for one known record. +- Pass one question as `collect QUERY` or `collect --relevance QUERY`. If an + old script combines a positional query and a separate cue, write one question + and express hard constraints with structured flags such as `--kind` and + `--backend`. Use `--kind knowledge` to search only the package catalog. +- Remove old `--relevance-method` values from scripts and managed calls. Find + is the default and `find-lexical-v1` is the only accepted explicit value. + Check the new rank order before using it to select material in automation. +- For exact collection and replay, use the retained IDs and revisions without + a positional query. With explicit IDs, `--relevance` can order optional + context resources while membership and caller order remain fixed. Read + historical captures for their original selected and delivered bytes. + - **Inputs:** optional query, `--kind`, repeatable explicit `--id`, repeatable `--revision ID=REV`, `--limit`, `--max-bytes`, `--material`; `--skill-root` selects an explicit skills root, `--store` opens an existing store read-only (the store records its reserved skills root and rejects a mismatched or missing `--skill-root`). -- **Scope:** installed package records plus (with `--skill-root`) skills and cases from the explicitly selected root, plus (with `--store`) store candidates; bounded output. +- **Scope:** installed package records, the selected library, and any explicitly opened store. A text query without a custom library or store can discover bundled Cases and Skills. Exact inventory selects the bundled library with `--installed-library`. - **Output:** with `--json`, the complete **normalized** envelope — items carrying the bundle's normalized metadata, resources and dependencies, plus `no_match`, `omissions`, `collisions`, `material_bytes` and the recorded limitations; fields that exist only in a package record's original entrypoint document are not projected into the normalized metadata and are read via `--material` or `resource`. Without `--json`, the envelope's selection is projected line-wise: one tab-separated line per item (`ref`, `revision`, `kind`, `state`, recorded `applicability`; free-text fields are control-character-escaped so one item stays one line), `collision` lines for multi-authority IDs, `omitted` lines with reason codes, an explicit `no_match` line when nothing was selected (query, filters or limits; when items were skipped the `omitted` lines above it carry the per-item causes), and — with `--material` — the entrypoint content in delimited blocks. The text mode does not print `material_bytes` or the envelope limitations; use `--json` for the complete envelope. The recorded `state` and applicability are printed as recorded and no validation status is invented. Exit 0 covers a result envelope, including no-match and omissions. - **Side effects:** none; both adapters and the read-only store connection are read-only. -- **Failure behavior:** discovery-wide invalid identities fail the call; per-item omissions are reported individually. Package repo/tier/backend/scenario filters exclude checkout items. +- **Failure behavior:** discovery-wide invalid identities fail the call. Per-item omissions are reported individually. Repository, tier and scenario filters use the package catalog. A backend filter scopes ranked discovery to the requested backend and shared material. - **Example:** `inferrsi collect --id package:inferrsi.ascend.skills.ascend_graph_mode --id package:inferrsi.ascend.skills.ascend_device_health --context-bytes 65536 --json` ### Read exact material/resources — `read` / `resource` / `KnowledgeReader.read` / `.resource` diff --git a/plugins/inferrsi/skills/inferrsi/SKILL.md b/plugins/inferrsi/skills/inferrsi/SKILL.md index 02a3583e4..b4cb552e2 100644 --- a/plugins/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/inferrsi/skills/inferrsi/SKILL.md @@ -165,7 +165,7 @@ Follow the selected canonical procedure rather than inventing a parallel tuning recipe. Imported source material supplies evidence, not execution authority. The `skills` commands and direct checkout Skill reads use the optional PyYAML dependency. Use `collect --id` for a batch of known materials and -`knowledge search` for package inventory queries. +`knowledge search` without a query for filtered package inventory. ### Consult MindIE for unresolved Ascend questions diff --git a/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md b/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md index a9e0b224d..ddf1c6dfd 100644 --- a/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/workswarm/inferrsi/skills/inferrsi/SKILL.md @@ -55,8 +55,9 @@ resource's text and identifies it in `content_resource`. Use `resource` with the same source flag and revision to extract the exact bytes. Use `collect --id` to assemble a batch of known materials. Use -`knowledge search` for filtered package inventory and `knowledge research` -when you need separate research dimensions for a model. +`knowledge search` without a query for filtered package inventory. For a +model with several research dimensions, `knowledge research` uses Find within +the filtered package catalog for each dimension. Retain qualified identities, exact revisions, source attribution, applicability, candidate/verification state and omissions. Package search does not inspect live diff --git a/src/inferrsi/apps/campaign.py b/src/inferrsi/apps/campaign.py index 33795fe1d..ccb45d236 100644 --- a/src/inferrsi/apps/campaign.py +++ b/src/inferrsi/apps/campaign.py @@ -12,7 +12,7 @@ from inferrsi import __version__ from inferrsi.harness.records import campaign_records, knowledge_capture -from inferrsi.knowledge._skill_selection import FIND_RELEVANCE, TASK_RELEVANCE, single_query +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE, validate_relevance_method from inferrsi.knowledge.catalog import BACKENDS from inferrsi.knowledge.context_projection import ( PROVENANCE_FIELDS, @@ -47,18 +47,12 @@ def capture_context(root: Path, *, context_id: str, reader, query: str = "", """ if revisions and not ids: raise ValueError("Revision pins require explicit IDs (--id-ref)") + validate_relevance_method(relevance_method) campaign = knowledge_capture.capture_campaign(root, context_id) - if relevance_method is None: - has_store = "store" in sources or getattr(reader, "store", None) is not None - if ids is None and not has_store and single_query(query, relevance): - relevance_method = FIND_RELEVANCE - elif ids is None and relevance: - # Preserve the exact query filter and separate cue for dual-input callers. - relevance_method = TASK_RELEVANCE - else: - relevance_method = "weighted-lexical-v1" - if relevance_method in (FIND_RELEVANCE, TASK_RELEVANCE) and ids is None and backend is None: - backend = campaign.get("backend") + if ids is None and (query.strip() or relevance): + relevance_method = FIND_RELEVANCE + if backend is None: + backend = campaign.get("backend") pins = {"revisions": revisions} if revisions else {} result = reader.collect( query, ids=ids, kind=kind, repo=repo, tier=tier, backend=backend, @@ -67,13 +61,12 @@ def capture_context(root: Path, *, context_id: str, reader, query: str = "", ) observation = result.get("retrieval") source_types = list(sources) - if relevance_method == FIND_RELEVANCE and isinstance(observation, dict): + if isinstance(observation, dict): source_types = observation.get("request", {}).get("sources", source_types) + selection = result.get("selection") or {} request = { "query": query, "relevance": relevance, - "relevance_method": ( - relevance_method if relevance or relevance_method == FIND_RELEVANCE else None - ), + "relevance_method": selection.get("method") or relevance_method, "ids": ids, "kind": kind, "repo": repo, "tier": tier, "backend": backend, "scenario": scenario, "limit": limit, "context_bytes": context_bytes, "sources": source_types, **pins, diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index c376d15c3..d3e771eec 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -443,6 +443,9 @@ def main(argv: list[str] | None = None) -> int: limit=args.limit, explain=args.explain, ) + if args.query.strip(): + print("knowledge search text queries use Find ranking in the package catalog. " + "Omit QUERY for filtered inventory.", file=sys.stderr) if args.json or args.explain: _print(records) else: @@ -477,6 +480,8 @@ def main(argv: list[str] | None = None) -> int: ids=args.ids, revisions=_revisions(args.revisions or []), kind=args.kind, + repo=args.repo_type, + tier=args.tier, backend=args.backend, scenario=args.scenario, limit=args.limit, @@ -486,6 +491,9 @@ def main(argv: list[str] | None = None) -> int: relevance_method=args.relevance_method, context_bytes=args.context_bytes, ) + if args.relevance_method is None and result.get("selection", {}).get("ranking_applied"): + print("collect text queries use Find ranking. " + "Use --id and --revision for exact material.", file=sys.stderr) if args.json: _print(result) else: @@ -559,6 +567,9 @@ def main(argv: list[str] | None = None) -> int: scenario=args.scenario, limit=args.limit, ) + if args.query.strip(): + print("context text queries use Find ranking in the package catalog. " + "Omit QUERY for filtered inventory.", file=sys.stderr) text = _context(records) if args.output: destination = Path(args.output) diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 58619a75a..8729dfa0d 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -9,7 +9,6 @@ from ... import __version__ from ...contracts import RunSpec -from ...knowledge._skill_selection import RELEVANCE_METHODS from ...knowledge.catalog import BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES @@ -111,8 +110,8 @@ def _optimization_actions(stages, *, project=False): stage.add_argument("--usage-export", type=Path) if name == "knowledge": stage.add_argument("--query", required=True) - stage.add_argument("--relevance-method", choices=RELEVANCE_METHODS, - help="Ranking method, defaults to find-lexical-v1") + stage.add_argument("--relevance-method", metavar="find-lexical-v1", + help="Find is the default and only ranking method") stage.add_argument("--context-bytes", type=int, required=True) stage.add_argument("--limit", type=int, default=5) @@ -363,9 +362,9 @@ def parser() -> argparse.ArgumentParser: capture.add_argument("--directory", type=Path, required=True) capture.add_argument("--id", required=True, help="New context id, e.g. r1-diagnosis") capture.add_argument("query", nargs="?", default="") - capture.add_argument("--relevance", help="Task cue for package knowledge and skill discovery") - capture.add_argument("--relevance-method", choices=RELEVANCE_METHODS, - help="Defaults to Find for one query or cue without IDs or a store") + capture.add_argument("--relevance", help="Alternate spelling for one Find query") + capture.add_argument("--relevance-method", metavar="find-lexical-v1", + help="Find is the default and only ranking method") capture.add_argument("--id-ref", action="append", dest="ids", metavar="ID", help="Exact knowledge identity to include (repeatable)") capture.add_argument("--revision", action="append", metavar="ID=REV", @@ -435,7 +434,7 @@ def parser() -> argparse.ArgumentParser: actions = knowledge.add_subparsers(dest="knowledge_action", required=True) search = actions.add_parser( "search", - help="Filter or search the installed record catalog. Use find for general discovery.", + help="List filtered package records, or search that catalog with Find ranking", ) search.add_argument("query", nargs="?", default="") search.add_argument("--repo-type", choices=KNOWLEDGE_REPOS) @@ -449,7 +448,7 @@ def parser() -> argparse.ArgumentParser: show = actions.add_parser("show") show.add_argument("id") research = actions.add_parser( - "research", help="Union model/framework/task/modality/mechanism queries for preparation" + "research", help="Combine model research dimensions with package-scoped Find queries" ) research.add_argument("model") for dimension in ("framework", "task", "modality", "mechanism"): @@ -510,13 +509,14 @@ def parser() -> argparse.ArgumentParser: collect = commands.add_parser( "collect", - help="Collect explicitly selected sources as a revision-pinned envelope; " - "package-only unless --skill-root or --store adds them", + help="Collect exact IDs, list material, or use Find to assemble context for a question", ) collect.add_argument("query", nargs="?", default="") collect.add_argument("--kind", choices=("skill", "knowledge", "case")) + collect.add_argument("--repo-type", choices=KNOWLEDGE_REPOS) + collect.add_argument("--tier", choices=KNOWLEDGE_SCOPES) collect.add_argument("--backend", choices=BACKENDS, help="Include this backend and common") - collect.add_argument("--scenario", help="Workload scenario; task ranking also considers skills") + collect.add_argument("--scenario", help="Workload scenario (also admits packaged skills)") collect.add_argument( "--id", action="append", dest="ids", metavar="ID", help="Exact requested identity" ) @@ -530,11 +530,11 @@ def parser() -> argparse.ArgumentParser: collect.add_argument("--limit", type=int, default=10) collect.add_argument("--max-bytes", type=int) collect.add_argument("--material", action="store_true") - collect.add_argument("--relevance", help="Compatibility ranking for packaged knowledge") collect.add_argument( - "--relevance-method", choices=RELEVANCE_METHODS, default="weighted-lexical-v1", - help="Keep the packaged ranking or opt into Find with one question or cue", + "--relevance", help="Alternate spelling for a Find query, or a resource cue with --id", ) + collect.add_argument("--relevance-method", metavar="find-lexical-v1", + help="Find is the default and only ranking method") collect.add_argument( "--context-bytes", type=int, help="Opt-in context view; total UTF-8 resource-body byte budget", diff --git a/src/inferrsi/apps/context_provider.py b/src/inferrsi/apps/context_provider.py index bcee5b520..559a3ed35 100644 --- a/src/inferrsi/apps/context_provider.py +++ b/src/inferrsi/apps/context_provider.py @@ -5,7 +5,6 @@ from hashlib import sha256 from inferrsi.contracts.json_v1 import MAX_SAFE_INTEGER -from inferrsi.knowledge._skill_selection import FIND_RELEVANCE, single_query from inferrsi.knowledge.knowledge_find import installed_skill_root from inferrsi.knowledge.knowledge_operations import KnowledgeReader @@ -59,8 +58,6 @@ def collect(self, request: dict) -> DeliveredContextDraft: ): raise ValueError("invalid Knowledge selection option value") - if not ids and self.reader.store is None and single_query(query, limits.get("relevance")): - limits = {"relevance_method": FIND_RELEVANCE, **limits} reader = self.reader if reader.skill_root is None and reader.store is None and any( identity.startswith("checkout:") for identity in ids diff --git a/src/inferrsi/apps/views/capture_delivery.py b/src/inferrsi/apps/views/capture_delivery.py index 99851bb43..d1966eb4d 100644 --- a/src/inferrsi/apps/views/capture_delivery.py +++ b/src/inferrsi/apps/views/capture_delivery.py @@ -158,8 +158,11 @@ def capture_event(context_id, captured, *, archive=False, return_chunks=None, materials.append({"ref": ref, "revision": revision, "resources": resources.get(ref, [])}) observation = doc.get("retrieval") or {} calls = observation.get("calls", []) - recorded_tool = calls[-1] if calls else None - tool = recorded_tool if recorded_tool in ("collect", "find", "knowledge search") else "collect" + if "knowledge search" in calls: + tool = "knowledge search" + else: + recorded_tool = calls[-1] if calls else None + tool = recorded_tool if recorded_tool in ("collect", "find") else "collect" event = {"action": "search", "tool": tool, "context_id": context_id, "source": source, "sha256": digest, "request": doc["request"], "selection": doc.get("selection"), "materials": materials} diff --git a/src/inferrsi/knowledge/_skill_selection.py b/src/inferrsi/knowledge/_skill_selection.py index 2e464628e..b7a6b74a3 100644 --- a/src/inferrsi/knowledge/_skill_selection.py +++ b/src/inferrsi/knowledge/_skill_selection.py @@ -5,12 +5,9 @@ import re from ._source_snapshot import source_entry +from .catalog import packaged_skill_id -TASK_RELEVANCE = "task-lexical-v1" FIND_RELEVANCE = "find-lexical-v1" -RELEVANCE_METHODS = ( - "weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE, FIND_RELEVANCE, -) TASK_STOP_WORDS = frozenset({"a", "an", "and", "for", "in", "of", "on", "the", "to", "with"}) _BACKEND_TERMS = { @@ -23,40 +20,21 @@ class SkillIndexError(ValueError): pass -def single_query(query: str | None, relevance: str | None) -> str | None: - """Find takes one question; the legacy selectors can also take a filter and cue.""" - if query and query.strip(): - return None if relevance else query - return relevance or None - - -def words(cue: str) -> set[str]: - return set(re.findall(r"\w+", cue.casefold())) +def validate_relevance_method(method: str | None) -> None: + if method not in (None, FIND_RELEVANCE): + raise ValueError( + f"Unsupported relevance_method {method!r}. " + f"Omit the method or use {FIND_RELEVANCE!r} with one query or relevance cue." + ) def named_backends(terms: set[str]) -> set[str]: return {backend for backend, names in _BACKEND_TERMS.items() if names & terms} -def lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: - tokens = words(cue) - return sum( - weight * sum(token in json.dumps(value, ensure_ascii=False).casefold() for token in tokens) - for weight, value in fields - ) - - -def raw_string_lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: - """Match original strings; retain legacy JSON matching for nonstring fields.""" - tokens = words(cue) - score = 0 - for weight, value in fields: - text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) - score += weight * sum(token in text.casefold() for token in tokens) - return score - - -def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: +def indexed_skill( + record: dict, view: dict, content: dict[str, bytes], +) -> tuple[str, list[str], list[str]]: """Validate the existing projection against this exact canonical bundle.""" registry_path = "data/skill_library/skill-index.json" registry = json.loads(content[registry_path]) @@ -64,7 +42,7 @@ def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: entry = registry["skills"][name] prefix = f"data/skill_library/{entry['path']}/" core = prefix + "SKILL.md" - expected_id = f"inferrsi.{entry['backend']}.skills.{name.replace('-', '_')}" + expected_id = packaged_skill_id(name, entry["backend"]) if ( registry["schema_version"] != 1 or view["authority"] != "package" @@ -117,55 +95,6 @@ def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: return core, required, optional -def skill_score( - cue: str, record: dict, core: bytes, *, method: str = "weighted-lexical-v1", - explanation: dict | None = None, -) -> int: - if method not in ("weighted-lexical-v1", "weighted-raw-string-v1"): - raise ValueError("Unknown relevance method for skill-only ranking") - scorer = raw_string_lexical_score if method == "weighted-raw-string-v1" else lexical_score - fields = [ - (4, record["title"]), - (3, record["summary"]), - (2, [record.get("tags", []), record.get("applicability", {})]), - (1, core.decode("utf-8")), - ] - components = [scorer(cue, [field]) for field in fields] - score = sum(components) - if explanation is not None: - explanation.update(score=score, components=dict(zip( - ("title", "summary", "tags_applicability", "body"), components)), multiplier=1) - return score - - -def task_score(cue: str, record: dict, core: bytes, backend: str | None, - *, explanation: dict | None = None) -> int: - """Rank whole task terms across both skill and repository knowledge owners.""" - def terms(text: str) -> set[str]: - return set(re.findall(r"[^\W_]+", text.casefold())) - - tokens = terms(cue) - TASK_STOP_WORDS - fields = ( - (4, record["title"]), - (3, record["summary"]), - (2, " ".join(record.get("tags", []))), - (2, record["repo"] + " " + record["scenario"]), - (1, json.dumps(record.get("applicability", {}), ensure_ascii=False)), - (1, core.decode("utf-8")), - ) - components = [weight * len(tokens & terms(text)) for weight, text in fields] - score = sum(components) - # With no platform in scope prefer portable procedures to a specialization - # that happens to repeat the same task words in its description. - in_scope = {backend} if backend else named_backends(tokens) - multiplier = 2 if record["backend"] == "common" or record["backend"] in in_scope else 1 - if explanation is not None: - explanation.update(score=score * multiplier, components=dict(zip( - ("title", "summary", "tags", "repo_scenario", "applicability", "body"), - components)), multiplier=multiplier) - return score * multiplier - - def record_context(record: dict, view: dict, content: dict[str, bytes], available: int) -> tuple: """Deliver whole files, including the declared closure of a Case.""" paths = {view["entrypoint"]: "record" if view["kind"] == "knowledge" else "core"} @@ -202,9 +131,10 @@ def record_context(record: dict, view: dict, content: dict[str, bytes], availabl def context_view( - record: dict, view: dict, content: dict[str, bytes], cue: str, available: int + index: tuple[str, list[str], list[str]], view: dict, content: dict[str, bytes], + cue: str | None, available: int, ) -> tuple[dict | None, dict | None]: - core, required, optional = indexed_skill(record, view, content) + core, required, optional = index required_bytes = sum(len(content[path]) for path in required) if required_bytes > available: return None, { @@ -226,7 +156,7 @@ def include(path: str) -> None: for path in required: include(path) - candidates = [] + candidates = {} for path in optional: if ( posixpath.basename(path) == "metadata.json" @@ -235,16 +165,30 @@ def include(path: str) -> None: omissions.append({"path": path, "reason": "not_context_resource"}) continue try: - text = content[path].decode("utf-8") + candidates[path] = content[path].decode("utf-8") except UnicodeDecodeError: omissions.append({"path": path, "reason": "not_utf8"}) continue - score = lexical_score(cue, [(2, path.rpartition("/")[2]), (1, text)]) - if words(cue) and score == 0: - omissions.append({"path": path, "reason": "not_relevant"}) - continue - candidates.append((score, path)) - for _, path in sorted(candidates, key=lambda item: (-item[0], item[1])): + paths = list(candidates) + if cue and candidates: + from .knowledge_find import Corpus, Document, rank + + documents = [ + Document( + identity=path, kind="record", state="unspecified", path=path, + title=posixpath.basename(path), summary="", + fields={"title": posixpath.basename(path), "body": text}, + ) + for path, text in candidates.items() + ] + _, _, ranked = rank(cue, Corpus(documents)) + paths = [documents[index].identity for _, _, index, _ in ranked] + matched = set(paths) + omissions.extend( + {"path": path, "reason": "not_relevant"} + for path in candidates if path not in matched + ) + for path in paths: if used + len(content[path]) > available: omissions.append({"path": path, "reason": "context_limit"}) continue diff --git a/src/inferrsi/knowledge/catalog.py b/src/inferrsi/knowledge/catalog.py index a8c1fc4a3..86e597647 100644 --- a/src/inferrsi/knowledge/catalog.py +++ b/src/inferrsi/knowledge/catalog.py @@ -6,10 +6,14 @@ import json import re from importlib.resources import files -from typing import Any +from typing import TYPE_CHECKING, Any from ..resources.repository_packs import REPO_TYPES from . import _source_snapshot +from .retrieval import observation + +if TYPE_CHECKING: + from .knowledge_find import Corpus # Learning scope does not imply that a repository has a runnable target adapter. # Legacy repository-pack records retain their contribution tier; an independent @@ -33,6 +37,11 @@ } +def packaged_skill_id(name: str, backend: str) -> str: + """Return the stable catalog ID for a skill registered by name and backend.""" + return f"inferrsi.{backend}.skills.{name.replace('-', '_')}" + + def _walk(root, prefix=()): for child in sorted(root.iterdir(), key=lambda item: item.name): parts = (*prefix, child.name) @@ -130,27 +139,19 @@ def knowledge_records() -> list[dict[str, Any]]: return records -def search_knowledge( - query: str = "", +def _scoped_records( *, repo: str | None = None, tier: str | None = None, backend: str | None = None, scenario: str | None = None, - limit: int = 10, - explain: bool = False, -) -> list[dict[str, Any]] | dict[str, Any]: - if limit < 1: - raise ValueError("limit must be positive") +) -> tuple[list[dict[str, Any]], dict[str, int]]: if tier is not None and tier not in KNOWLEDGE_SCOPES: raise ValueError(f"Unknown knowledge tier: {tier}") scenario = SCENARIO_ALIASES.get(scenario, scenario) - tokens = re.findall(r"[\w.-]+", query.casefold()) - scored = [] - records = knowledge_records() - excluded = {"tier": 0, "repo": 0, "backend": 0, "scenario": 0, "query": 0} - explanations = {} - for record in records: + eligible = [] + excluded = {"tier": 0, "repo": 0, "backend": 0, "scenario": 0} + for record in knowledge_records(): if tier and record.get("tier", "contribution") != tier: excluded["tier"] += 1 continue @@ -167,36 +168,95 @@ def search_knowledge( ): excluded["scenario"] += 1 continue - document = json.dumps(record, ensure_ascii=False).casefold() - if any(token not in document for token in tokens): - excluded["query"] += 1 - continue - score = sum(document.count(token) for token in tokens) - if explain: - explanations[record["id"]] = [ - {"term": token, "occurrences": document.count(token)} for token in tokens] - scored.append((score, record["id"], record)) - scored.sort(key=lambda item: (-item[0], item[1])) - items = [item[2] for item in scored[:limit]] + eligible.append(record) + return eligible, excluded + + +def _catalog_corpus(records: list[dict[str, Any]]) -> tuple[Corpus, list[dict[str, Any]]]: + from .knowledge_find import Corpus, installed_skill_projection, load_documents + + by_id = {record["id"]: record for record in records} + kinds = ("record",) + if any(record.get("index_type") == "packaged-skill" for record in records): + kinds = ("skill", "record") + documents, document_records = [], [] + for document in load_documents(kinds=kinds, records=records): + record = ( + by_id[document.identity] if document.kind == "record" + else installed_skill_projection(document, by_id) + ) + if record is not None: + # Keep the native identity and source state for Find's ties. The + # corresponding record supplies the exact package ID and payload. + documents.append(document) + document_records.append(record) + return Corpus(documents), document_records + + +def search_knowledge( + query: str = "", + *, + repo: str | None = None, + tier: str | None = None, + backend: str | None = None, + scenario: str | None = None, + limit: int = 10, + explain: bool = False, +) -> list[dict[str, Any]] | dict[str, Any]: + """List exact package records, using Find when a text query is supplied.""" + if limit < 1: + raise ValueError("limit must be positive") + records, excluded = _scoped_records(repo=repo, tier=tier, backend=backend, scenario=scenario) + ranking = bool(query.strip()) + if ranking: + from .knowledge_find import rank + + corpus, document_records = _catalog_corpus(records) + _, _, scored = rank(query, corpus) + matches = [ + (document_records[index], score, components) for score, _, index, components in scored + ] + else: + matches = [ + (record, None, None) for record in sorted(records, key=lambda record: record["id"]) + ] + + returned = matches[:limit] + items = [record for record, _, _ in returned] if not explain: return items - from .retrieval import observation + from ._skill_selection import FIND_RELEVANCE - return {"items": items, "retrieval": observation( + observed = [] + for position, (record, score, components) in enumerate(returned, 1): + data = json.dumps(record, sort_keys=True, ensure_ascii=False).encode() + item = {"ref": "package:" + record["id"], "rank": position, + "revision": "catalog-sha256:" + hashlib.sha256(data).hexdigest()} + if score is not None: + item.update(score=round(score, 2), components=components) + observed.append(item) + retrieval = observation( entrypoint="knowledge search", request={"query": query, "repo": repo, "tier": tier, "backend": backend, - "scenario": scenario, "limit": limit, "sources": ["package"]}, - selection={"method": "catalog-occurrences-v1", "scope": "package-records", - "ranking_applied": True, "state": "applied", "tie_break": "id_ascending"}, - items=[{"ref": "package:" + identity, "rank": rank, "score": score, - "revision": "catalog-sha256:" + hashlib.sha256(json.dumps( - record, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), - "components": explanations[identity]} - for rank, (score, identity, record) in enumerate(scored[:limit], 1)], - counts={"considered": len(records), "matched": len(scored), "returned": len(items), - "excluded": {**excluded, "item_limit": max(0, len(scored) - limit)}}, + "scenario": SCENARIO_ALIASES.get(scenario, scenario), "limit": limit, + "sources": ["package"]}, + selection={ + "method": FIND_RELEVANCE if ranking else None, "scope": "package-records", + "ranking_applied": ranking, "state": "applied" if ranking else "not_requested", + "tie_break": "state_then_id_then_ref" if ranking else None, + }, + items=observed, + counts={ + "considered": len(records) + sum(excluded.values()), + "matched": len(matches), "returned": len(items), + "excluded": {**excluded, "query": len(records) - len(matches), + "item_limit": len(matches) - len(items)}, + }, assembly="not_requested", - )} + ) + if ranking: + retrieval["calls"].append("find") + return {"items": items, "retrieval": retrieval} def get_knowledge(record_id: str) -> dict[str, Any]: diff --git a/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/SKILL.md b/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/SKILL.md index ee8ef0219..afd0f6d69 100644 --- a/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/SKILL.md +++ b/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/SKILL.md @@ -24,7 +24,12 @@ Use this collection to prepare a dedicated engine, compare architectures or desi ## Comparison procedure 1. Fix the model/checkpoint/tokenizer, tasks/modalities, component graph, target hardware/memory, software and quality contract. Ask only for missing facts that alter a decision; otherwise state reversible assumptions. Do not infer architecture from a family name. -2. Search model, framework, task, modality and mechanism dimensions independently and union the results. Use `inferrsi knowledge research --framework ` with explicit `--task`, `--modality` or `--mechanism` additions as needed; known profiles supply grounded terms, unknown models require source inspection. Exact `knowledge search` still ANDs all tokens. Retain hard filters and record empty dimensions; broaden an over-restrictive scope only explicitly. +2. Search model, framework, task, modality and mechanism dimensions independently and union the results. + Use `inferrsi knowledge research --framework ` with explicit `--task`, + `--modality` or `--mechanism` additions as needed. Known profiles supply grounded terms. + Inspect sources for unknown models. Each query uses Find's lexical ranking over the eligible + package records. Retain hard filters and record empty dimensions. Change the scope explicitly + if it is too restrictive. 3. Compare eligible native implementations at pinned revisions. If the user requests the most-starred reference, record the candidate set, eligibility, observation date and counts; choose the largest count within that set, then check license and compatibility. Read that source directly or distill its relevant paths with an available repo-to-skill workflow before engine generation. Popularity is a reference-selection input, not a quality or hardware guarantee. 4. Inventory every selected encoder, denoiser, VAE/decoder, conditioning path, adapter and auxiliary asset. Reconcile tensor shapes, dtypes, quantization metadata, files and actual execution paths. Separate full checkpoint weights, required subsets, phase and cold-start peaks, activations/workspace, unified/host memory and storage. Do not count alternative task checkpoints twice or equate a DiT-only quant with a complete pipeline. Follow the [preparation procedure](references/model-aware-preparation.md) before deciding target feasibility. 5. Load the smallest relevant repository branches. Trace work units, state ownership, scheduling, memory, communication, output and cleanup. Produce supported settings, semantics-preserving implementation changes and research extensions as separate candidate classes, with explicit approximation budgets. diff --git a/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/references/model-aware-preparation.md b/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/references/model-aware-preparation.md index f747ca52b..2f49d11fb 100644 --- a/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/references/model-aware-preparation.md +++ b/src/inferrsi/knowledge/data/skill_library/workflows/inference-engine-research/references/model-aware-preparation.md @@ -42,8 +42,8 @@ their linked repository skills own the detailed source map and inventories. The MiniMax-H3 profile points to the [h3.c residency audit](../../../repositories/h3-c/audit-h3-runtime-residency/SKILL.md). For an unknown model, supply `--task` and `--modality` after inspecting its sources; no architecture is guessed. Repeat dimensions to add queries. -`knowledge search 'model mlx diffusion video'` retains token-AND semantics and -can return less knowledge than the independent union. +A text query to `knowledge search` and each research query use the same Find +lexical ranking over the eligible package records. Tier/repository/backend/scenario flags remain hard catalog filters. An empty dimension can mean missing or filtered knowledge: report it, inspect the missing diff --git a/src/inferrsi/knowledge/knowledge_find.py b/src/inferrsi/knowledge/knowledge_find.py index 431033ee2..d2d56b6cb 100644 --- a/src/inferrsi/knowledge/knowledge_find.py +++ b/src/inferrsi/knowledge/knowledge_find.py @@ -5,11 +5,11 @@ caller can read, grep around, or stop. The audit half lives in ``read`` / ``resource``: exact identities, pinned revisions and content hashes. -Three properties separate this from the existing ``collect`` selection path: +Catalog search, model research and Collect reuse these documents and ranking +over the corpus eligible for their operation. -* Cases participate. A rejected direction is the material a retrieval system - can supply that re-reading the current source cannot, so it is ranked beside - skills rather than excluded until its owner is already chosen. +* Standalone Find ranks cases beside skills, including rejected directions + and the conditions behind them. * Each distinct matching term contributes to the score. One match is enough to be eligible, and ``limit`` caps the ranked results. * Scoring reads the entries directly. Bundle closures may contain unrelated @@ -27,6 +27,7 @@ import shlex import sys from collections import Counter +from collections.abc import Iterable, Mapping from dataclasses import dataclass from functools import lru_cache from importlib.resources import files @@ -40,8 +41,7 @@ from ._knowledge_errors import MissingSkillsExtra from ._skill_selection import TASK_STOP_WORDS, indexed_skill, named_backends -from ._source_snapshot import BUNDLE_LIBRARY -from .catalog import BACKENDS +from .catalog import BACKENDS, packaged_skill_id # Author-written intent outranks bulk prose: a term in a title is evidence the # document is about it, the same term in a long body often is not. @@ -507,14 +507,12 @@ def _skill_documents(root: Path, *, source_location: str) -> list[Document]: return documents -def _record_documents() -> list[Document]: +def _record_documents(records: Iterable[dict]) -> list[Document]: """Imported engine records. Skill projections are skipped: the checkout documents above already cover them, and counting both double-ranks one owner under two identities.""" - from .catalog import knowledge_records - documents = [] - for record in knowledge_records(): + for record in records: if record.get("index_type") == "packaged-skill": continue identity = record["id"] @@ -615,17 +613,24 @@ def installed_skill_root() -> Path: return Path(files("inferrsi").joinpath("knowledge", "data", "skill_library")).resolve() -def packaged_skill_projection(document: Document, reader) -> dict | None: - """Find the generated record for this exact installed Skill path.""" - if document.kind != "skill" or document.source_location != "installed" or not document.path: +def installed_skill_projection(document: Document, records: Mapping[str, dict]) -> dict | None: + """Find the generated record for this registered installed Skill identity.""" + if ( + document.kind != "skill" or document.source_location != "installed" + or document.backend is None or document.root != installed_skill_root() + ): return None - if document.root != installed_skill_root(): + record = records.get(packaged_skill_id(document.identity, document.backend)) + if record is None or record.get("index_type") != "packaged-skill": return None - return reader.packaged_skill_records.get(f"{BUNDLE_LIBRARY}/{document.path}") + return record def load_documents( - skill_root: Path | None = None, *, kinds: tuple[str, ...] = ("case", "skill", "record") + skill_root: Path | None = None, + *, + kinds: tuple[str, ...] = ("case", "skill", "record"), + records: Iterable[dict] | None = None, ) -> list[Document]: documents: list[Document] = [] if "case" in kinds or "skill" in kinds: @@ -639,7 +644,9 @@ def load_documents( if "skill" in kinds: documents.extend(_skill_documents(root, source_location=source_location)) if "record" in kinds: - documents.extend(_record_documents()) + from .catalog import knowledge_records + + documents.extend(_record_documents(knowledge_records() if records is None else records)) identities = [document.qualified for document in documents] if len(set(identities)) != len(identities): duplicate = next(i for i, c in Counter(identities).items() if c > 1) @@ -725,7 +732,9 @@ def _open_args(document: Document, readers: dict) -> tuple[str, ...]: try: reader.read(document.qualified).view(material=True) except MissingSkillsExtra: - record = packaged_skill_projection(document, reader) + if document.source_location != "installed": + raise + record = installed_skill_projection(document, reader.records()) if record is None: raise package_reader = readers.get(None) diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py index 1edc8410f..062d47796 100644 --- a/src/inferrsi/knowledge/knowledge_operations.py +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -6,25 +6,23 @@ import sqlite3 from dataclasses import dataclass from datetime import date, datetime -from functools import cached_property from importlib.resources import files from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING from ._knowledge_errors import MissingSkillsExtra from ._skill_selection import ( FIND_RELEVANCE, - RELEVANCE_METHODS, - TASK_RELEVANCE, context_view, indexed_skill, record_context, - single_query, - skill_score, - task_score, - words, + validate_relevance_method, ) from .catalog import KNOWLEDGE_SCOPES, SCENARIO_ALIASES, knowledge_records +if TYPE_CHECKING: + from .knowledge_find import Document + AUTHORITIES = ("package", "checkout", "store") IDENTITY = r"[a-z0-9][a-z0-9._-]*" @@ -233,6 +231,34 @@ def view(self, *, material: bool = False) -> dict: return result +def _store_document(reference: str, bundle: Bundle) -> Document: + from .knowledge_find import Document, _flatten + + metadata = bundle.metadata + identity = metadata["id"] + title = _flatten(metadata.get("title", identity)) + summary = _flatten(metadata.get("summary")) + entrypoint = metadata["entrypoint"] + return Document( + identity=identity, + kind="record" if metadata["kind"] == "knowledge" else metadata["kind"], + state=metadata["state"], + path=entrypoint, + title=title, + summary=summary, + qualified=reference, + backend=metadata.get("backend"), + fields={ + "title": f"{title} {identity.replace('.', ' ')}", + "summary": summary, + "tags": _flatten(metadata.get("tags")), + "findings": _flatten(metadata.get("findings")), + "applicability": _flatten(metadata.get("applicability")), + "body": bundle.content[entrypoint].decode("utf-8"), + }, + ) + + INCIDENTAL = ("__pycache__", ".DS_Store") @@ -281,16 +307,6 @@ def records(self) -> dict[str, dict]: self._records = {record["id"]: record for record in knowledge_records()} return self._records - @cached_property - def packaged_skill_records(self) -> dict[str, dict]: - """Index this catalog snapshot by the original library resource.""" - return { - record["skill_documents"]["original"]: record - for record in self.records().values() - if record.get("index_type") == "packaged-skill" - and (record.get("skill_documents") or {}).get("original") - } - def _skill_library(self) -> tuple[dict[str, bytes], str]: if self._library is None: # K02 moved the physical corpus under the Knowledge module. The bundle @@ -307,23 +323,24 @@ def owners(self) -> dict[str, list[str]]: result: dict[str, list[str]] = {} for identity in self.records(): result.setdefault(identity, []).append("package") - if self.skill_root is not None: - # Enumeration reads registered names only. Validating sidecar content - # here would abort the whole collection over one malformed, possibly - # unrequested, skill; Collect reports that per item through _read. - from .skill_catalog import skill_directories - - local = list(skill_directories(self.skill_root)) - local.extend(self._cases()) - if len(set(local)) != len(local): - raise IdentityError("collision", "Duplicate identity inside the checkout") - for identity in local: - result.setdefault(identity, []).append("checkout") + for identity in self._checkout_identities(): + result.setdefault(identity, []).append("checkout") if self.store is not None: for identity in self.store.identities(): result.setdefault(identity, []).append("store") return result + def _checkout_identities(self) -> list[str]: + if self.skill_root is None: + return [] + # Enumeration reads registered names. Collect validates sidecars per item. + from .skill_catalog import skill_directories + + identities = [*skill_directories(self.skill_root), *self._cases()] + if len(set(identities)) != len(identities): + raise IdentityError("collision", "Duplicate identity inside the checkout") + return identities + def identities(self) -> list[str]: return sorted(f"{a}:{i}" for i, authorities in self.owners().items() for a in authorities) @@ -528,12 +545,13 @@ def resource(self, identity: str, path: str, *, revision: str) -> bytes: def _search_order( self, cue: str, matches, *, kind: str | None, package_only: bool, backend: str | None, + store_documents: list[Document], ) -> tuple[dict, int, dict[str, int]]: from .knowledge_find import ( Corpus, + installed_skill_projection, installed_skill_root, load_documents, - packaged_skill_projection, rank, ) @@ -544,7 +562,8 @@ def _search_order( "knowledge": ("skill", "record"), } root = None if package_only else self.skill_root - documents = load_documents(root, kinds=kinds[kind]) + records = self.records() + documents = load_documents(root, kinds=kinds[kind], records=records.values()) if root is not None and root != installed_skill_root() and kind is None: # A selected custom library owns its registered skills. The package # supplies missing skills and remains available for exact reads. @@ -554,21 +573,25 @@ def _search_order( documents.extend( document for document in load_documents(kinds=("skill",)) if document.identity not in custom_skills - and packaged_skill_projection(document, self) is not None + and installed_skill_projection(document, records) is not None ) + documents.extend(store_documents) eligible, references = [], [] excluded = {"filters": 0, "ranking_scope": 0} for document in documents: authority, _, identity = document.qualified.partition(":") - projection = packaged_skill_projection(document, self) + projection = installed_skill_projection(document, records) if projection is not None: authority, identity = "package", projection["id"] if authority == "package": material_kind = "skill" if kind == "skill" else "knowledge" - metadata = {**self.records()[identity], "kind": material_kind} + metadata = {**records[identity], "kind": material_kind} else: - metadata = {"kind": document.kind, "backend": document.backend} + metadata = { + "kind": "knowledge" if document.kind == "record" else document.kind, + "backend": document.backend, + } if ( backend and not package_only and document.kind == "case" and document.backend is None @@ -590,7 +613,7 @@ def _search_order( def collect( self, - query: str = "", + query: str | None = "", *, ids: list[str] | None = None, revisions: dict[str, str] | None = None, @@ -603,7 +626,7 @@ def collect( max_bytes: int | None = None, material: bool = False, relevance: str | None = None, - relevance_method: str = "weighted-lexical-v1", + relevance_method: str | None = None, context_bytes: int | None = None, ) -> dict: if type(limit) is not int or limit < 1: @@ -616,20 +639,24 @@ def collect( raise ValueError("ids must be a list of identity strings") if kind not in (None, "skill", "knowledge", "case"): raise ValueError("Unknown content kind") - if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): - raise ValueError("relevance requires at least one word") - if relevance_method not in RELEVANCE_METHODS: - raise ValueError("Unknown relevance method") - find_ranking = relevance_method == FIND_RELEVANCE - cue = relevance or query - if find_ranking: + validate_relevance_method(relevance_method) + if query is not None and not isinstance(query, str): + raise ValueError("query must be a string or None") + if query and query.strip(): if ids is not None: - raise ValueError("Find ranking cannot be combined with explicit IDs") - cue = single_query(query, relevance) - if cue is None: - raise ValueError("Find selection requires one query or relevance cue") - elif relevance is None and relevance_method != "weighted-lexical-v1": - raise ValueError("relevance_method requires relevance") + raise ValueError( + "Remove query when using explicit IDs. Use relevance to guide context assembly." + ) + if relevance is not None: + raise ValueError("Find takes one query or relevance cue. Supply only one.") + cue = query + else: + cue = relevance + if relevance is not None and ( + not isinstance(relevance, str) or not re.search(r"\w", relevance) + ): + raise ValueError("relevance requires at least one word") + find_ranking = cue is not None and ids is None if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): raise ValueError("context_bytes must be a positive integer") if revisions is None: @@ -641,15 +668,14 @@ def collect( raise ValueError("revisions must map identity strings to revision strings") if ids is not None and len(set(ids)) != len(ids): raise ValueError("Duplicate requested identity") - reader = self - if find_ranking: - if self.store is not None: - raise ValueError("Use explicit IDs or catalog collection for stores") - if self.skill_root is None: - from .knowledge_find import installed_skill_root - - reader = KnowledgeReader(skill_root=installed_skill_root()) - owners = reader.owners() + checkout = self + owners = self.owners() + if find_ranking and self.skill_root is None: + from .knowledge_find import installed_skill_root + + checkout = KnowledgeReader(skill_root=installed_skill_root()) + for identity in checkout._checkout_identities(): + owners.setdefault(identity, []).append("checkout") requested = ids if ids is not None else sorted( f"{authority}:{identity}" for identity, values in owners.items() for authority in values ) @@ -657,7 +683,7 @@ def collect( def qualified(reference: str) -> str | None: # Revision keys may use any spelling the API accepts for an identity. try: - return reader.resolve(reference, owners) + return self.resolve(reference, owners) except IdentityError: return None @@ -679,7 +705,6 @@ def qualified(reference: str) -> str | None: package_filters = any( value is not None for value in (repo, tier, scenario) ) or (backend is not None and not find_ranking) - task_ranking = relevance_method in (TASK_RELEVANCE, FIND_RELEVANCE) def matches(metadata: dict, authority: str) -> bool: declared = metadata.get("scenario") @@ -688,9 +713,9 @@ def matches(metadata: dict, authority: str) -> bool: # not the workload served by a procedure. Let the task cue rank # these procedures alongside records declaring the requested workload. discovery_skill = ( - task_ranking and ids is None and authority == "package" + find_ranking and authority == "package" and declared == "skills.discovery" - and reader.records()[metadata["id"]].get("index_type") == "packaged-skill" + and metadata.get("index_type") == "packaged-skill" ) return not ( (kind and metadata.get("kind") != kind) @@ -711,24 +736,57 @@ def matches(metadata: dict, authority: str) -> bool: "item_limit": 0, "material_bytes": 0, } considered = len(requested) - found = {} + found, store_revisions = {}, {} + omitted = [] if find_ranking: - found, considered, rank_excluded = reader._search_order( + store_documents = [] + for reference in requested: + if not reference.startswith("store:"): + continue + try: + bundle = self._read(reference, pinned.get(reference)) + store_revisions[reference] = bundle.revision + store_documents.append(_store_document(reference, bundle)) + except IdentityError as exc: + omitted.append({"id": reference, "reason": exc.code, "detail": str(exc)}) + except RevisionConflict as exc: + omitted.append({ + "id": reference, "reason": "unavailable_revision", "detail": str(exc), + }) + except (ValueError, KeyError, TypeError, OSError, sqlite3.Error) as exc: + omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) + found, considered, rank_excluded = self._search_order( cue, matches, kind=kind, backend=backend, package_only=package_filters or kind == "knowledge", + store_documents=store_documents, ) + considered += len(omitted) requested = list(found) excluded.update(rank_excluded) - cases = reader._cases() if kind and not find_ranking else {} - selected, omitted, seen = [], [], set() + cases = self._cases() if kind and not find_ranking else {} + selected, seen = [], set() used = 0 - ranked, contexts, context_omissions = [], [], [] + contexts, context_omissions = [], [] context_used = 0 - explanations = found matched_count = 0 def select(reference: str, bundle: Bundle, view: dict) -> None: nonlocal used, context_used, matched_count + packaged = view["authority"] == "package" + record = ( + self.records().get(view["id"], {}) + if packaged and (find_ranking or context_bytes is not None) else {} + ) + is_skill = record.get("index_type") == "packaged-skill" + index = None + if find_ranking and is_skill: + try: + index = indexed_skill(record, view, bundle.content) + except (ValueError, KeyError, TypeError) as exc: + omitted.append({ + "id": reference, "reason": "index_unavailable", "detail": str(exc), + }) + return matched_count += 1 # Normal Bundle views were serialized while computing their revision. # Preserve custom subclasses' serialization/error behavior below. @@ -745,23 +803,20 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: used += size if context_bytes is None: return - packaged = view["authority"] == "package" - record = reader.records().get(view["id"], {}) if packaged else {} - is_skill = record.get("index_type") == "packaged-skill" pinned_item = ids is not None and view["ref"] in pinned checkout_context = ( view["authority"] == "checkout" and view["kind"] in ("case", "skill") and (find_ranking or pinned_item) ) - if not ((packaged and (is_skill or task_ranking or pinned_item)) or checkout_context): + if not ((packaged and (is_skill or find_ranking or pinned_item)) or checkout_context): context_omissions.append({"ref": view["ref"], "reason": "unsupported_context"}) return try: available = context_bytes - context_used if is_skill: - context, omission = context_view( - record, view, bundle.content, cue, available - ) + if index is None: + index = indexed_skill(record, view, bundle.content) + context, omission = context_view(index, view, bundle.content, cue, available) else: context, omission = record_context(record, view, bundle.content, available) except (ValueError, KeyError, TypeError) as exc: @@ -780,7 +835,7 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: excluded["item_limit"] += len(requested) - len(seen) break try: - qualified = reader.resolve(reference, owners) + qualified = self.resolve(reference, owners) if qualified in seen: raise IdentityError("duplicate_identity", f"Already selected: {qualified}") seen.add(qualified) @@ -788,23 +843,19 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: if not find_ranking: # Cheap metadata checks first, before loading bundle resources. if authority == "package": - early = {**reader.records()[identity], "kind": "knowledge"} + early = {**self.records()[identity], "kind": "knowledge"} elif authority == "checkout": early = {"kind": "case" if identity in cases else "skill"} else: early = None - if relevance is not None and ids is None and ( - authority != "package" - or (not task_ranking and early.get("index_type") != "packaged-skill") - ): - excluded["ranking_scope"] += 1 - continue if early is not None and not matches(early, authority): excluded["filters"] += 1 if ids is not None: omitted.append({"id": reference, "reason": "filters"}) continue - bundle = reader._read(qualified, pinned.get(qualified)) + reader = checkout if authority == "checkout" else self + revision = store_revisions.get(qualified, pinned.get(qualified)) + bundle = reader._read(qualified, revision) view = {**bundle.view(material=material), "authority": authority, "ref": qualified} if find_ranking and view["kind"] == "case": view.setdefault("title", identity.removeprefix("case.")) @@ -832,45 +883,6 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: if ids is not None: omitted.append({"id": reference, "reason": "filters"}) continue - entry = bundle.content[bundle.metadata["entrypoint"]] - if not find_ranking and query: - haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() - if any(token not in haystack for token in query.casefold().split()): - excluded["query"] += 1 - if ids is not None: - omitted.append({"id": reference, "reason": "query"}) - continue - if not find_ranking and relevance is not None and ids is None: - record = reader.records().get(identity, {}) - try: - is_skill = record.get("index_type") == "packaged-skill" - if is_skill: - core, _, _ = indexed_skill(record, view, bundle.content) - body = bundle.content[core] - else: - body = entry - explanation = {} - if task_ranking: - score = task_score(relevance, record, body, backend, - explanation=explanation) - else: - score = skill_score(relevance, record, body, method=relevance_method, - explanation=explanation) - except (ValueError, KeyError, TypeError) as exc: - omitted.append({ - "id": reference, "reason": "index_unavailable", "detail": str(exc), - }) - continue - if score: - explanations[view["ref"]] = explanation - ranked.append((score, reference, bundle, view)) - else: - excluded["zero_score"] += 1 - continue - select(reference, bundle, view) - for _, reference, bundle, view in sorted( - ranked, key=lambda item: (-item[0], item[3]["ref"]) - ): select(reference, bundle, view) result = { "items": selected, @@ -879,7 +891,7 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: "collisions": collisions, "material_bytes": used, "limitations": [ - "No crawling, ranking model or conversation state; deterministic identity order.", + "Items follow explicit ID order or inventory identity order.", "max_bytes bounds item payloads, not omission/control metadata.", "Read references must be resolved with their returned exact revision.", ], @@ -887,24 +899,19 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: if find_ranking: result["selection"] = { "method": FIND_RELEVANCE, "cue": cue, - "scope": "package-records-and-library", "ranking_applied": True, + "scope": ("package-records-library-and-store" if self.store is not None + else "package-records-and-library"), + "ranking_applied": True, "matched_count": len(found), "returned_count": len(selected), } result["limitations"][0] = ( - "Find ranks textual matches; check each item's applicability and limitations." + "Find ranks textual matches. Check each item's applicability and limitations." ) elif relevance is not None: result["selection"] = { - "method": relevance_method, "cue": relevance, - "scope": ("package-skills-and-records" if task_ranking - else "generated-packaged-skills"), - "ranking_applied": ids is None, + "method": FIND_RELEVANCE, "cue": relevance, + "scope": "context-resources", "ranking_applied": False, } - if task_ranking and ids is None: - result["selection"].update(matched_count=len(ranked), returned_count=len(selected)) - result["limitations"][0] = ( - "Deterministic lexical selection; explicit IDs retain caller order and membership." - ) if context_bytes is not None: result.update({ "contexts": contexts, "context_omissions": context_omissions, @@ -916,7 +923,6 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: }) from .retrieval import observation - ranking = find_ranking or (relevance is not None and ids is None) selection = result.get("selection", {}) result["retrieval"] = observation( entrypoint="collect", request={ @@ -924,19 +930,17 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: "ids": ids, "revisions": revisions, "kind": kind, "repo": repo, "tier": tier, "backend": backend, "scenario": scenario, "limit": limit, "max_bytes": max_bytes, "context_bytes": context_bytes, - "sources": ["package"] + (["checkout"] if reader.skill_root else []) - + (["store"] if reader.store is not None else []), + "sources": ["package"] + (["checkout"] if checkout.skill_root else []) + + (["store"] if self.store is not None else []), }, selection={ "method": selection.get("method"), "scope": selection.get("scope"), - "ranking_applied": ranking, - "state": "applied" if ranking else "explicit_ids" if ids is not None + "ranking_applied": find_ranking, + "state": "applied" if find_ranking else "explicit_ids" if ids is not None else "not_requested", - "tie_break": ( - "state_then_id_then_ref" if find_ranking else "ref_ascending" - ) if ranking else None, + "tie_break": "state_then_id_then_ref" if find_ranking else None, }, items=[{"ref": item["ref"], "revision": item["revision"], "rank": n, - **explanations.get(item["ref"], {})} + **found.get(item["ref"], {})} for n, item in enumerate(selected, 1)], counts={ "considered": considered, "matched": len(found) if find_ranking else matched_count, diff --git a/src/inferrsi/knowledge/research.py b/src/inferrsi/knowledge/research.py index 97f6f2708..214f1c7b6 100644 --- a/src/inferrsi/knowledge/research.py +++ b/src/inferrsi/knowledge/research.py @@ -7,7 +7,8 @@ from collections.abc import Sequence from importlib.resources import files -from .catalog import knowledge_records, search_knowledge +from .catalog import _catalog_corpus, _scoped_records +from .knowledge_find import rank PROCEDURE = ( "data/skill_library/workflows/inference-engine-research/" @@ -66,9 +67,10 @@ def research_knowledge( } filters = {"repo": repo, "tier": tier, "backend": backend, "scenario": scenario} queries, streams, matches = [], [], {} - # Retrieve every scoped match before applying the shared output limit; an - # early per-query limit would lose attribution for overlapping records. - catalog_size = max(1, len(knowledge_records())) + scoped, _ = _scoped_records(**filters) + corpus, document_records = _catalog_corpus(scoped) + # Retain every query's matches before capping the union so overlapping + # records keep all their query attributions. for group, terms in groups.items(): seen = set() for term in terms: @@ -79,16 +81,17 @@ def research_knowledge( continue seen.add(term.casefold()) query = {"group": group, "query": term} - records = search_knowledge(term, **filters, limit=catalog_size) + _, _, scored = rank(term, corpus) + records = [document_records[index] for _, _, index, _ in scored] queries.append({**query, "matches": len(records)}) streams.append(records) for record in records: matches.setdefault(record["id"], []).append(query) selected = {} - for rank in range(max((len(stream) for stream in streams), default=0)): + for position in range(max((len(stream) for stream in streams), default=0)): for stream in streams: - if rank < len(stream): - record = stream[rank] + if position < len(stream): + record = stream[position] selected.setdefault(record["id"], record) if len(selected) == limit: break @@ -116,8 +119,8 @@ def research_knowledge( "inventory from the selected checkpoint before judging feasibility." ], "limitations": [ - "Queries search installed package records only; empty results do not rule out " - "external sources. Existing token-AND search and scope filters are unchanged.", + "Each query uses Find's lexical ranking over installed package records eligible " + "under the scope filters. Other sources require separate discovery.", "Mechanism queries are research leads, not claims of model/backend support. " "Retain each record's applicability, source revision and verification state.", "Target annotations do not inspect hardware or add a runtime backend. Full weight " diff --git a/tests/check_installed.py b/tests/check_installed.py index 1d11b1d85..ad3464586 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -105,9 +105,6 @@ def resource(logical): assert indexed.dependencies assert json.loads(indexed.content["record.json"])["skill_metadata"]["license"] == "UNKNOWN" h3_identity = "inferrsi.common.skills.audit_h3_runtime_residency" -assert any(item["id"] == h3_identity for item in search_knowledge( - "MiniMax-H3", repo="inferrsi", limit=100 -)) h3_bundle = KnowledgeReader().read(h3_identity) h3_prefix = "data/skill_library/repositories/h3-c/audit-h3-runtime-residency/" h3_inventory = json.loads(h3_bundle.content[ @@ -148,18 +145,12 @@ def resource(logical): assert h3_metal_audit["layout"]["topk_double"] == 12 assert h3_metal_audit["native_execution"] == "not_run" afd_identity = "inferrsi.cuda.skills.audit_afd_p2p_exchanges" -assert any(item["id"] == afd_identity for item in search_knowledge( - "AFD", repo="inferrsi", backend="cuda", limit=100 -)) afd_bundle = KnowledgeReader().read(afd_identity) afd_prefix = "data/skill_library/repositories/afd-plugin/audit-afd-p2p-exchanges/" for relative in ("SKILL.md", "metadata.json", "references/source-notes.md", "references/afd-plugin-LICENSE.md"): assert afd_prefix + relative in afd_bundle.content hopper_identity = "inferrsi.cuda.skills.reason_hopper_kernels" -assert any(item["id"] == hopper_identity for item in search_knowledge( - "WGMMA", repo="inferrsi", backend="cuda", limit=100 -)) hopper_bundle = KnowledgeReader().read(hopper_identity) hopper_prefix = "data/skill_library/hardware/cuda/reason-hopper-kernels/" hopper_metadata = json.loads(hopper_bundle.content[hopper_prefix + "metadata.json"]) @@ -211,9 +202,18 @@ def resource(logical): assert lithos_owner + relative in lithos.content, relative assert json.loads(lithos.content["record.json"])["skill_metadata"]["status"] == "candidate" assert "monolith" not in sys.modules, "Skill reads must not import the source engine" + +# Text ranking uses Find's frontmatter reader when the optional parser is installed. +for query, identity, backend in ( + ("MiniMax-H3", h3_identity, None), ("AFD", afd_identity, "cuda"), + ("WGMMA", hopper_identity, "cuda"), +): + assert any(item["id"] == identity for item in search_knowledge( + query, repo="inferrsi", backend=backend, limit=100, + )) selection_reader = KnowledgeReader() selection = selection_reader.collect( - "cpu.stat.local", relevance="cgroupv2", context_bytes=65536, limit=1, + "cgroupv2 cpu.stat.local", kind="knowledge", context_bytes=65536, limit=1, ) assert selection["items"][0]["id"] == "inferrsi.common.skills.cgroup_resource_controls" assert not selection["context_omissions"] @@ -224,7 +224,6 @@ def resource(logical): selection_context["ref"], selection_core["path"], revision=selection_context["revision"], ) assert selection["context_bytes"] <= 65536 -assert "yaml" not in sys.modules, "Packaged selection must not require the skills extra" assert resource("data/knowledge_packs/README.md").is_file() for tier, count in (("dependency", 6), ("peer", 5)): imported = search_knowledge(tier=tier, limit=100) @@ -329,9 +328,7 @@ def resource(logical): ascend_image = search_knowledge( "Qwen-Image-2.1", repo="vllm_omni", backend="ascend", scenario="diffusion.image_generation" ) -assert [item["id"] for item in ascend_image] == [ - "vllm_omni.ascend.diffusion.qwen_image_21_eager_t2i" -] +assert ascend_image[0]["id"] == "vllm_omni.ascend.diffusion.qwen_image_21_eager_t2i" assert resource( "data/repo_packs/vllm_omni/knowledge/ascend/diffusion/qwen-image-21-eager-t2i.json" ).is_file() diff --git a/tests/fixtures/indexer/README.md b/tests/fixtures/indexer/README.md index 26229369e..cc6ec7ff0 100644 --- a/tests/fixtures/indexer/README.md +++ b/tests/fixtures/indexer/README.md @@ -2,7 +2,10 @@ - `development.json`: 8 source-reviewed partial-positive queries, visible before the candidate. - `heldout.json`: 4 independently source-reviewed unseen queries/labels, released only after runtime candidate freeze. -- Reproduce with `tools/evaluate_indexer.py`; issue-scoped gates, exact revisions, raw data and the efficiency/accuracy table live under `evidence/indexer-255-20261029/`. +- Reproduce the historical comparison with the runner, source revisions and + protocol pinned in [the campaign evidence](../../../evidence/indexer-255-20261029/). + Current text retrieval uses Find. Use the pinned runtime to reproduce the + recorded ranking and output digests. These are not an exhaustive question bank or complete relevance ground truth. Partial-label Hit@5/MRR@5 measure whether a known positive appears, not full Recall/Precision or an agent's task success. Explicit-ID/resource-budget/no-match/stale-projection contracts are checked separately. diff --git a/tests/fixtures/knowledge_retrieval.py b/tests/fixtures/knowledge_retrieval.py index 0c341621e..c892fcaf1 100644 --- a/tests/fixtures/knowledge_retrieval.py +++ b/tests/fixtures/knowledge_retrieval.py @@ -7,6 +7,7 @@ import json import sys +from contextlib import contextmanager from pathlib import Path from unittest.mock import patch @@ -15,12 +16,15 @@ from inferrsi.apps.views.search_attribution import import_search from inferrsi.harness.records.campaign_records import record_candidate from inferrsi.harness.records.optimization_log import record_log +from inferrsi.knowledge import knowledge_find from inferrsi.knowledge._source_snapshot import source_entry from inferrsi.knowledge.catalog import search_knowledge from inferrsi.knowledge.knowledge_operations import KnowledgeReader, _hash, _json -def reader(): +@contextmanager +def reader(root): + """Use an isolated on-disk installed library and its matching catalog snapshot.""" registry_path = "data/skill_library/skill-index.json" registry, library, records = {"schema_version": 1, "skills": {}}, {}, {} for name in ("a", "b", "c", "d", "e"): @@ -39,6 +43,10 @@ def reader(): "skill_metadata": {"name": "synthetic-" + name, "references": []}, } library[registry_path] = _json(registry) + for path, content in library.items(): + target = root / path.removeprefix("data/skill_library/") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) for record in records.values(): core = record["skill_documents"]["original"] record["source_files"] = {source_entry(p): _hash(library[p]) @@ -46,7 +54,8 @@ def reader(): result = KnowledgeReader() result._records = records result._library = library, "sha256:" + _hash(_json({p: _hash(b) for p, b in library.items()})) - return result + with patch.object(knowledge_find, "installed_skill_root", return_value=root.resolve()): + yield result def write(path, value): @@ -66,28 +75,33 @@ def build(root, *, runs=DEFAULT_RUNS): create_campaign(root, "synthetic-retrieval") manifest = json.loads((root / "campaign.json").read_text()) write(root / "campaign.json", {**manifest, "synthetic": True}) - source = reader() budget = 2 * len(b"Synthetic launch guidance.\n") captures = {} - path, captures["capture"] = capture_context( - root, context_id="capture", reader=source, relevance="launch", - relevance_method="task-lexical-v1", context_bytes=budget, - ) - captured = json.loads(path.read_text()) - with patch("inferrsi.knowledge.catalog.knowledge_records", - return_value=list(source.records().values())): - _, captures["catalog"] = save_retrieval(root, context_id="catalog", - result=search_knowledge("launch", explain=True)) - for name, options in { - "collect": {"relevance": "launch", "context_bytes": budget}, - "explicit": {"ids": list(reversed(source.records())), "relevance": "launch"}, - "no-ranking": {}, "empty": {"query": "no-such-term"}, - "legacy": {"relevance": "launch"}, - }.items(): - result = source.collect(**options) - if name == "legacy": - result.pop("retrieval") - _, captures[name] = save_retrieval(root, context_id=name, result=result) + with reader(root / "corpus") as source: + path, captures["capture"] = capture_context( + root, context_id="capture", reader=source, relevance="launch", context_bytes=budget, + ) + captured = json.loads(path.read_text()) + with patch("inferrsi.knowledge.catalog.knowledge_records", + return_value=list(source.records().values())): + _, captures["catalog"] = save_retrieval( + root, context_id="catalog", result=search_knowledge("launch", explain=True), + ) + for name, options in { + "collect": {"relevance": "launch", "context_bytes": budget}, + "explicit": {"ids": list(reversed(source.records())), "relevance": "launch"}, + "no-ranking": {}, "empty": {"query": "no-such-term"}, + }.items(): + result = source.collect(**options) + _, captures[name] = save_retrieval(root, context_id=name, result=result) + + # Keep a synthetic pre-observation capture in the dashboard fixture. Its + # payload is fixed by ID and its recorded selection is historical metadata. + legacy = source.collect(ids=list(source.records())) + legacy.pop("retrieval") + legacy["selection"] = {"method": "weighted-lexical-v1", "cue": "launch", + "scope": "generated-packaged-skills", "ranking_applied": True} + _, captures["legacy"] = save_retrieval(root, context_id="legacy", result=legacy) for candidate, run, number, verdict, gain, quality in runs: directory = root / "runs" / run write(directory / "run.lock.json", {"workload_id": "synthetic", "scenario": "http"}) diff --git a/tests/test_campaign_record_injection.py b/tests/test_campaign_record_injection.py index 1f7761249..b6543182b 100644 --- a/tests/test_campaign_record_injection.py +++ b/tests/test_campaign_record_injection.py @@ -5,14 +5,16 @@ import os import subprocess import sys -from datetime import datetime, timezone import pytest from inferrsi import __version__ from inferrsi.apps import campaign +from inferrsi.apps.views.search_attribution import import_search from inferrsi.harness.records import campaign_records, knowledge_capture +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE from inferrsi.knowledge.context_projection import context_item +from inferrsi.knowledge.knowledge_operations import KnowledgeReader from tests._gate_policy_support import ( NEW_H_ENTRIES, REPO, @@ -49,44 +51,31 @@ def collect(self, query="", **options): "contexts": [context], "context_bytes": context["content_bytes"]} -# Each row: capture options, the method and backend stored in the record, and -# the method handed to ``collect``. Include the legacy methods explicitly to -# cover both current defaults and older callers. +# Each row checks the options that are passed to the reader and retained in the capture. SELECTION = { - "cue": ({"relevance": "decode launch"}, "find-lexical-v1", "cuda", "find-lexical-v1"), - "cue-and-ids": ({"relevance": "decode launch", "ids": ["package:graphs"]}, - "weighted-lexical-v1", None, "weighted-lexical-v1"), - "query-only": ({"query": "decode"}, "find-lexical-v1", "cuda", "find-lexical-v1"), - "query-and-cue": ({"query": "decode", "relevance": "kernel launch"}, - "task-lexical-v1", "cuda", "task-lexical-v1"), - "explicit-raw": ({"relevance": "decode launch", - "relevance_method": "weighted-raw-string-v1"}, - "weighted-raw-string-v1", None, "weighted-raw-string-v1"), - "task-without-cue": ({"relevance_method": "task-lexical-v1"}, None, "cuda", - "task-lexical-v1"), + "cue": ({"relevance": "decode launch"}, FIND_RELEVANCE, "cuda"), + "cue-and-ids": ({"relevance": "decode launch", "ids": ["package:graphs"]}, None, None), + "query-only": ({"query": "decode"}, FIND_RELEVANCE, "cuda"), + "inventory": ({}, None, None), "explicit-backend": ({"relevance": "decode launch", "backend": "ascend"}, - "find-lexical-v1", "ascend", "find-lexical-v1"), - "explicit-legacy": ({"relevance": "decode launch", - "relevance_method": "weighted-lexical-v1"}, - "weighted-lexical-v1", None, "weighted-lexical-v1"), - "task-and-ids": ({"relevance": "decode launch", "ids": ["package:graphs"], - "relevance_method": "task-lexical-v1"}, - "task-lexical-v1", None, "task-lexical-v1"), + FIND_RELEVANCE, "ascend"), + "explicit-find": ({"query": "decode", "relevance_method": FIND_RELEVANCE}, + FIND_RELEVANCE, "cuda"), } @pytest.mark.parametrize("name", SELECTION) def test_capture_record_keeps_its_relevance_method(tmp_path, name): - options, stored, backend, delivered = SELECTION[name] + options, method, backend = SELECTION[name] campaign.create_campaign(tmp_path, "selection", backend="cuda") reader = Reader() path, digest = campaign.capture_context(tmp_path, context_id=name, reader=reader, **options) raw = path.read_bytes() document = json.loads(raw) - assert document["request"]["relevance_method"] == stored + assert document["request"]["relevance_method"] == method assert document["request"]["backend"] == backend - assert reader.calls[0]["relevance_method"] == delivered + assert reader.calls[0]["relevance_method"] == method assert reader.calls[0]["query"] == options.get("query", "") assert reader.calls[0]["relevance"] == options.get("relevance") assert reader.calls[0]["backend"] == backend @@ -94,6 +83,25 @@ def test_capture_record_keeps_its_relevance_method(tmp_path, name): assert campaign.verify_capture(raw)[1] == "verified" +@pytest.mark.parametrize( + "method", ["weighted-lexical-v1", "weighted-raw-string-v1", "task-lexical-v1"], +) +def test_capture_rejects_legacy_ranking_methods_before_calling_reader(tmp_path, method): + campaign.create_campaign(tmp_path, "selection") + reader = Reader() + with pytest.raises(ValueError, match="Unsupported relevance_method"): + campaign.capture_context(tmp_path, context_id="r1", reader=reader, + relevance="decode launch", relevance_method=method) + assert reader.calls == [] + + +def test_capture_rejects_a_query_and_relevance_together(tmp_path): + campaign.create_campaign(tmp_path, "selection") + with pytest.raises(ValueError, match="Find takes one query or relevance cue"): + campaign.capture_context(tmp_path, context_id="r1", reader=KnowledgeReader(), + query="decode", relevance="launch") + + def test_capture_record_layout_is_unchanged(tmp_path): campaign.create_campaign(tmp_path, "layout") path, _ = campaign.capture_context(tmp_path, context_id="r1", reader=Reader(), @@ -119,14 +127,6 @@ def test_capture_record_layout_is_unchanged(tmp_path): ) -class FixedClock(datetime): - """One capture instant, so stored bytes can be compared exactly.""" - - @classmethod - def now(cls, tz=None): - return datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc) - - # The record ``capture_context`` wrote for ``Reader`` before retrieval observations # existed. Layout, spacing, nulls and order are literal; only the package version and # the one line too long for this file are inserted. @@ -210,25 +210,44 @@ def legacy_record_bytes(version: str) -> bytes: ).encode("utf-8") -def test_reader_without_observation_serializes_the_exact_earlier_bytes(tmp_path, monkeypatch): - monkeypatch.setattr(knowledge_capture, "datetime", FixedClock) +def test_legacy_capture_bytes_are_verified_and_archived_without_search(tmp_path, monkeypatch): + def search_again(*args, **kwargs): + pytest.fail("verification and archival replay must use the original capture") + + monkeypatch.setattr(KnowledgeReader, "collect", search_again) + monkeypatch.setattr("inferrsi.knowledge.knowledge_find.rank", search_again) campaign.create_campaign(tmp_path, "layout") - path, digest = campaign.capture_context(tmp_path, context_id="r1", reader=Reader(), - relevance="decode launch", - relevance_method="task-lexical-v1") expected = legacy_record_bytes(__version__) + path = tmp_path / "knowledge/r1.json" + path.parent.mkdir(exist_ok=True) + path.write_bytes(expected) + doc, state = campaign.verify_capture(expected) + assert state == "verified" + assert doc["request"]["relevance_method"] == "task-lexical-v1" + + review = tmp_path / "review.json" + review.write_text(json.dumps({"task": "Synthetic historical capture", + "corpus": {"repository": "example/synthetic", + "revision": "a" * 40, "root": "."}, + "decisions": []}), encoding="utf-8") + archive = import_search(tmp_path, number=1, context_ids=["r1"], archive_review=review) + event, = json.loads(archive.read_bytes())["events"] assert path.read_bytes() == expected - assert digest == hashlib.sha256(expected).hexdigest() - assert campaign.verify_capture(expected)[1] == "verified" + assert event["tool"] == "collect" + assert event["request"]["relevance_method"] == "task-lexical-v1" + assert event["capture"]["sha256"] == hashlib.sha256(expected).hexdigest() + assert event["materials"][0]["revision"] == doc["items"][0]["revision"] class ObservedReader(Reader): """The same context, with the retrieval observation a current reader adds.""" def collect(self, query="", **options): - observation = {"schema_version": 1, "entrypoint": "collect", "calls": ["collect"], - "request": {}, "selection": {}, "counts": {"matched": 1}, "items": []} + observation = {"schema_version": 1, "entrypoint": "collect", "calls": ["collect", "find"], + "request": {"sources": ["package", "checkout"]}, + "selection": {"method": FIND_RELEVANCE, "state": "applied"}, + "counts": {"matched": 1}, "items": []} return {**super().collect(query, **options), "retrieval": observation} @@ -246,9 +265,10 @@ def test_capture_with_an_observation_keeps_its_retrieval_and_item_provenance(tmp ] assert document["retrieval"]["schema_version"] == 1 assert document["retrieval"]["entrypoint"] == "campaign capture-context" - assert document["retrieval"]["calls"] == ["campaign capture-context", "collect"] + assert document["retrieval"]["calls"] == ["campaign capture-context", "collect", "find"] assert document["captured_by"] == "inferrsi campaign capture-context " + __version__ assert document["request"]["ids"] is None + assert document["request"]["sources"] == ["package", "checkout"] assert list(document["items"][0]) == [ "ref", "revision", "title", "kind", "backend", "scenario", "authority", "sources", "entrypoint", "skill_documents", diff --git a/tests/test_campaign_task_selection.py b/tests/test_campaign_task_selection.py index 4dda1f73b..7218a3d47 100644 --- a/tests/test_campaign_task_selection.py +++ b/tests/test_campaign_task_selection.py @@ -11,16 +11,15 @@ def test_campaign_search_returns_portable_methods_and_records_counts(tmp_path): - create_campaign(tmp_path, "diffusion") + create_campaign(tmp_path, "diffusion", backend="cuda") path, _ = capture_context(tmp_path, context_id="first", reader=KnowledgeReader(), - relevance=BASELINE_CUE, relevance_method="task-lexical-v1", - limit=2, context_bytes=14000) + relevance=BASELINE_CUE, limit=2, context_bytes=14000) document, state = verify_capture(path.read_bytes()) assert state == "verified" assert [item["title"] for item in document["items"]] == [ "inference-benchmark-design", "video-diffusion-optimization", ] - assert document["selection"]["method"] == "task-lexical-v1" + assert document["selection"]["method"] == "find-lexical-v1" assert document["selection"]["matched_count"] > document["selection"]["returned_count"] == 2 assert document["context_omissions"][0]["reason"] == "context_limit" assert document["contexts"][0]["resources"][0]["path"].endswith( @@ -53,8 +52,8 @@ def test_campaign_backend_routes_to_repository_evidence_and_can_be_overridden(tm def test_workload_scenario_discovers_skills_and_compatible_engine_records(): reader = KnowledgeReader() - options = {"relevance": BASELINE_CUE, "relevance_method": "task-lexical-v1", - "backend": "cuda", "scenario": "diffusion.t2i", "limit": 8} + options = {"relevance": BASELINE_CUE, "backend": "cuda", + "scenario": "diffusion.t2i", "limit": 8} items = reader.collect(**options)["items"] ids = [i["id"] for i in items] portable = "inferrsi.common.skills.video_diffusion_optimization" diff --git a/tests/test_cannbot_boundaries.py b/tests/test_cannbot_boundaries.py index bf0648ee6..f4ee21dcf 100644 --- a/tests/test_cannbot_boundaries.py +++ b/tests/test_cannbot_boundaries.py @@ -304,6 +304,41 @@ def test_platform_override_and_cuda_gate(upstream): assert "external" not in json.loads(result.stdout) +def test_public_find_displays_an_external_only_hit_and_managed_capture_stays_local(upstream): + _, script, card = upstream + invoked = script.with_suffix(".invoked") + script.write_text( + "import json, sys\n" + "from pathlib import Path\n" + "Path(__file__).with_suffix('.invoked').touch()\n" + f"print(json.dumps({discovery()!r} if sys.argv[1] == 'discover' " + f"else {search_payload(card)!r}))\n" + ) + query = "zzcanaryexternalonlyqv" + encoded = cli("find", query, "--backend", "ascend", "--json") + assert encoded.returncode == 0, encoded.stderr + result = json.loads(encoded.stdout) + assert result["no_match"] and result["items"] == [] + external, = result["external"] + assert external["state"] == "ok" and external["count"] == 1 + assert external["items"][0]["locator"] == "ops/card" + rendered = cli("find", query, "--backend", "ascend") + assert rendered.returncode == 0 and "ops/card" in rendered.stdout + assert invoked.exists() + invoked.unlink() + + campaign = Path.cwd() / "campaign" + created = cli("campaign", "init", "--directory", str(campaign), "--id", "local-lookup") + assert created.returncode == 0, created.stderr + captured = cli("campaign", "capture-context", "--directory", str(campaign), + "--id", "lookup", "--backend", "ascend", query) + assert captured.returncode == 0, captured.stderr + doc = json.loads((campaign / json.loads(captured.stdout)["context"]).read_text()) + assert doc["retrieval"]["calls"] == ["campaign capture-context", "collect", "find"] + assert doc["items"] == [] and doc["contexts"] == [] + assert not invoked.exists() + + @pytest.mark.skipif(os.name != "posix", reason="POSIX process-group cleanup") @pytest.mark.parametrize("phase", ["discover", "search"]) def test_cli_deadline_kills_upstream_descendants(upstream, phase): diff --git a/tests/test_collect_selection.py b/tests/test_collect_selection.py index 577923284..3d4b18149 100644 --- a/tests/test_collect_selection.py +++ b/tests/test_collect_selection.py @@ -4,8 +4,7 @@ import pytest -from inferrsi.apps.cli import main -from inferrsi.knowledge._skill_selection import lexical_score +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE from inferrsi.knowledge.knowledge_operations import Bundle, KnowledgeReader, _hash, _json from inferrsi.knowledge.knowledge_store import KnowledgeStore @@ -15,34 +14,31 @@ AFD = "inferrsi.cuda.skills.audit_afd_p2p_exchanges" -@pytest.mark.parametrize(("cue", "query", "identity", "first"), [ - ("cgroupv2", "", CGROUP, True), - ("MiniMax-H3 AdaLN SSD", "", H3, False), - ("P2pNccl", "", AFD, True), - ("DLRover replica recovery", "", - "inferrsi.common.skills.checkpoint_recovery_consistency", False), - ("cgroupv2", "cpu.stat.local", CGROUP, False), +@pytest.mark.parametrize(("cue", "identity"), [ + ("cgroupv2", CGROUP), + ("MiniMax-H3 AdaLN SSD", H3), + ("P2pNccl", AFD), + ("DLRover replica recovery", "inferrsi.common.skills.checkpoint_recovery_consistency"), + ("cgroupv2 cpu.stat.local", CGROUP), ]) -def test_frozen_offline_pilot(cue, query, identity, first): - result = KnowledgeReader().collect(query, relevance=cue, context_bytes=65536, limit=3) - identities = [item["id"] for item in result["items"]] - assert identity in identities - if first: - assert identities[0] == identity +def test_find_discovery_returns_known_material_with_a_context_budget(cue, identity): + result = KnowledgeReader().collect(relevance=cue, context_bytes=65536, limit=3) + assert result["items"][0]["id"] == identity assert not [o for o in result["omissions"] if o["reason"] == "index_unavailable"] assert result["context_bytes"] <= 65536 -def test_no_match_and_query_remains_token_and_substring_matching(): +def test_default_query_and_relevance_use_the_same_find_ranking(): reader = KnowledgeReader() assert reader.collect(relevance="floristry orchid")["no_match"] - item = reader.read(CGROUP) - assert "cpu.stat.local" not in json.dumps(item.metadata).casefold() - assert "cpu.stat.local" in item.content["record.json"].decode() - result = reader.collect("cpu.stat.local", relevance="cgroupv2") - assert result["items"][0]["id"] == CGROUP - assert reader.collect("cpu.stat.local impossibleword", relevance="cgroupv2")["no_match"] - assert reader.collect("stat.loc", relevance="cgroupv2")["items"][0]["id"] == CGROUP + cue = "前缀缓存" + result = reader.collect(cue, limit=3) + explicit = reader.collect(relevance=cue, relevance_method=FIND_RELEVANCE, limit=3) + assert result["items"] == explicit["items"] + assert result["selection"] == explicit["selection"] + assert result["items"][0]["id"] == "sglang.cuda.llm.hierarchical_prefix_cache" + assert result["retrieval"]["calls"] == ["collect", "find"] + assert result["retrieval"]["items"] == explicit["retrieval"]["items"] def test_existing_item_payload_and_material_accounting_are_unchanged(): @@ -88,9 +84,14 @@ def test_filters_stay_package_only_and_precede_ranking(): reader = KnowledgeReader(skill_root=ROOT) assert reader.collect(relevance="P2pNccl", backend="ascend")["no_match"] assert reader.collect(relevance="P2pNccl", tier="peer")["no_match"] - assert reader.collect(relevance="cgroupv2", kind="skill")["no_match"] assert reader.collect(relevance="cgroupv2", repo="vllm")["no_match"] - assert reader.collect(relevance="cgroupv2", scenario="diffusion.t2i")["no_match"] + assert reader.collect(relevance="cgroupv2", kind="skill")["items"][0]["id"] == CGROUP + assert reader.collect(relevance="cgroupv2", scenario="diffusion.t2i")[ + "items" + ][0]["scenario"] == "skills.discovery" + assert reader.collect(ids=[CGROUP], scenario="diffusion.t2i")["omissions"] == [ + {"id": CGROUP, "reason": "filters"}, + ] result = reader.collect( relevance="cgroupv2", kind="knowledge", repo="inferrsi", tier="contribution", backend="cpu", scenario="skills.discovery", @@ -98,20 +99,13 @@ def test_filters_stay_package_only_and_precede_ranking(): assert result["items"][0]["id"] == CGROUP -def test_weighted_presence_and_identity_ties_are_deterministic(): - assert lexical_score("MATCH match", [(4, "match match"), (1, "match")]) == 5 +def test_inventory_without_a_cue_uses_identity_order(): reader = KnowledgeReader() - all_items = reader.collect(relevance="candidate", limit=100)["items"] - # With no title/description/tag occurrence, these owners share the core-only score. - tied = [] - for item in all_items: - record = reader.records()[item["id"]] - if "candidate" not in json.dumps([ - record["title"], record["summary"], record["tags"], record["applicability"], - ]).casefold(): - tied.append(item["ref"]) - assert len(tied) > 1 and tied == sorted(tied) - assert reader.collect(relevance="candidate", limit=1)["items"] == all_items[:1] + result = reader.collect(limit=3) + assert [item["ref"] for item in result["items"]] == reader.identities()[:3] + assert result["retrieval"]["calls"] == ["collect"] + assert not result["retrieval"]["selection"]["ranking_applied"] + assert reader.collect(limit=3, relevance_method=FIND_RELEVANCE)["items"] == result["items"] def test_context_keeps_complete_core_prerequisites_provenance_and_exact_bytes(): @@ -182,7 +176,7 @@ def test_source_drift_and_cross_owner_mapping_never_fall_back(): del library["data/skill_library/skill-index.json"] digest = _hash(_json({p: _hash(data) for p, data in sorted(library.items())})) reader._library = library, "sha256:" + digest - result = reader.collect("cpu.stat.local", relevance="cgroupv2") + result = reader.collect("cgroupv2") assert result["no_match"] assert any(o["reason"] == "index_unavailable" for o in result["omissions"]) explicit = reader.collect(ids=[CGROUP], context_bytes=65536) @@ -309,6 +303,8 @@ def test_invalid_opt_ins_fail_at_request_boundary(options): def test_cli_exposes_context_in_json_and_text(capsys): + from inferrsi.apps.cli import main + args = ["collect", "--id", CGROUP, "--relevance", "cgroupv2", "--context-bytes", "65536"] assert main([*args, "--json"]) == 0 result = json.loads(capsys.readouterr().out) diff --git a/tests/test_knowledge_cli.py b/tests/test_knowledge_cli.py index 827d55e65..96823239d 100644 --- a/tests/test_knowledge_cli.py +++ b/tests/test_knowledge_cli.py @@ -47,12 +47,29 @@ def local_candidate(source: Bundle) -> Bundle: ) -def test_collect_defaults_to_package_with_query_and_limit(): - code, stdout, _ = run_cli("collect", "Qwen", "--limit", "3", "--json") - assert code == 0 +def test_collect_inventory_defaults_to_package_with_limit(): + code, stdout, stderr = run_cli("collect", "--limit", "3", "--json") + assert code == 0 and not stderr envelope = json.loads(stdout) - assert envelope["items"] and len(envelope["items"]) <= 3 + assert len(envelope["items"]) == 3 assert all(item["authority"] == "package" for item in envelope["items"]) + assert not envelope["retrieval"]["selection"]["ranking_applied"] + + +def test_catalog_and_collect_share_package_rank_with_the_same_filters(): + query = "graph mode dynamic shapes" + filters = ("--repo-type", "inferrsi", "--backend", "ascend", "--limit", "4") + code, stdout, stderr = run_cli("knowledge", "search", query, *filters, "--explain") + assert code == 0 and "Find ranking" in stderr + catalog = json.loads(stdout) + code, stdout, stderr = run_cli("collect", "--kind", "knowledge", query, *filters, "--json") + assert code == 0 and "Find ranking" in stderr + collect = json.loads(stdout) + expected = [(item["ref"], item["score"]) for item in catalog["retrieval"]["items"]] + assert len(expected) == 4 + assert [(item["ref"], item["score"]) for item in collect["retrieval"]["items"]] == expected + assert catalog["retrieval"]["calls"] == ["knowledge search", "find"] + assert collect["retrieval"]["calls"] == ["collect", "find"] def test_collect_accepts_find_as_an_explicit_ranking_method(): @@ -63,10 +80,15 @@ def test_collect_accepts_find_as_an_explicit_ranking_method(): result = json.loads(stdout) assert result["selection"]["method"] == "find-lexical-v1" assert result["items"][0]["ref"] == "checkout:case.qwen-ascend-rejected-directions" + code, stdout, stderr = run_cli("collect", "--relevance", query, "--kind", "case", "--json") + assert code == 0 and "Find ranking" in stderr + assert json.loads(stdout)["items"] == result["items"] stderr = run_cli_error("collect", query, *method, "--id", f"package:{SOURCE}") assert "explicit IDs" in stderr stderr = run_cli_error("collect", query, *method, "--relevance", "cache") - assert "one query or relevance cue" in stderr + assert "one" in stderr and "relevance" in stderr + stderr = run_cli_error("collect", query, "--relevance-method", "weighted-lexical-v1") + assert "find-lexical-v1" in stderr def test_collect_reports_explicit_no_match_with_omission_reason(): @@ -268,7 +290,7 @@ def test_collect_text_lines_match_the_json_envelope(): def test_collect_reports_no_match_omissions_and_collisions_in_text_mode(tmp_path): - code, stdout, _ = run_cli("collect", "definitely-no-such-knowledge-anywhere") + code, stdout, _ = run_cli("collect", "zzcollectnomatchqvf") assert code == 0 assert stdout.strip() == ( "no_match\tnothing selected from the requested sources; " diff --git a/tests/test_knowledge_find.py b/tests/test_knowledge_find.py index a0ac666c4..f2cd56150 100644 --- a/tests/test_knowledge_find.py +++ b/tests/test_knowledge_find.py @@ -13,6 +13,7 @@ import pytest from inferrsi.apps.cli import main +from inferrsi.knowledge.catalog import search_knowledge from inferrsi.knowledge.knowledge_find import ( FINDING_OUTCOMES, FINDING_SCOPES, @@ -23,6 +24,7 @@ expand, find, load_corpus, + rank, tokens, ) @@ -86,6 +88,32 @@ def test_partial_query_still_ranks_instead_of_filtering(corpus): assert result["matched"] > 1 +@pytest.mark.parametrize(("query", "filters"), [ + ("图模式 qxsearchtoken", {"repo": "inferrsi", "tier": "contribution", "backend": "ascend", + "scenario": "skills.discovery"}), + ("CUDA-Graph capture capture qxsearchtoken", {"backend": "cuda"}), + ("offload graph qxsearchtoken", {"repo": "vllm_omni", "tier": "contribution", + "backend": "ascend", "scenario": "diffusion.t2v"}), +]) +def test_catalog_ranks_find_documents_over_the_filtered_package_scope(corpus, query, filters): + inventory = search_knowledge(**filters, limit=1000) + by_find_ref = {record.get("checkout_ref", f"package:{record['id']}"): record + for record in inventory} + eligible = [document for document in corpus.documents if document.qualified in by_find_ref] + assert len(eligible) == len(inventory) + _, _, scored = rank(query, Corpus(eligible)) + assert scored + + result = search_knowledge(query, **filters, limit=5, explain=True) + assert result["items"] == [ + by_find_ref[eligible[index].qualified] for _, _, index, _ in scored[:5] + ] + assert [(item["score"], item["components"]) for item in result["retrieval"]["items"]] == [ + (round(score, 2), detail) for score, _, _, detail in scored[:5] + ] + assert result["retrieval"]["counts"]["matched"] == len(scored) + + def test_repeated_query_terms_contribute_once(corpus): single = find("vae tiling", corpus, limit=1, explain=True) repeated = find("vae tiling vae tiling", corpus, limit=1, explain=True) diff --git a/tests/test_knowledge_install.py b/tests/test_knowledge_install.py index be7b0117d..407bbb06d 100644 --- a/tests/test_knowledge_install.py +++ b/tests/test_knowledge_install.py @@ -58,6 +58,18 @@ def test_package_only_commands_work_without_pyyaml(without_yaml): assert result.returncode == 0, result.stderr assert json.loads(result.stdout)["id"] == items[0]["id"] + query = "ascend graph mode" + catalog = _run(["knowledge", "search", query, "--limit", "1", "--explain"], env) + assert catalog.returncode == 0, catalog.stderr + searched = json.loads(catalog.stdout) + target = "inferrsi.ascend.skills.ascend_graph_mode" + assert searched["items"][0]["id"] == target + assert searched["retrieval"]["selection"]["method"] == "find-lexical-v1" + + research = _run(["knowledge", "research", query, "--limit", "1"], env) + assert research.returncode == 0, research.stderr + assert json.loads(research.stdout)["items"][0]["record"]["id"] == target + def test_packaged_skill_search_and_resource_work_without_pyyaml(without_yaml, tmp_path): env = _cli_env(without_yaml) diff --git a/tests/test_knowledge_navigation_browser.py b/tests/test_knowledge_navigation_browser.py index 0742394e3..f642186f8 100644 --- a/tests/test_knowledge_navigation_browser.py +++ b/tests/test_knowledge_navigation_browser.py @@ -129,7 +129,7 @@ def test_navigation_keyboard_versions_and_refresh(browser, tmp_path, profile_fir root.get_by_role("button", name="capture", exact=True).click() capture = root.locator('details[data-kt-key="navigation:capture"]') playwright.expect(capture).to_have_attribute("open", "") - assert "campaign capture-context → collect" in capture.inner_text() + assert "campaign capture-context → collect → find" in capture.inner_text() assert "5" in capture.get_by_role("table").first.inner_text() assert "2" in capture.get_by_role("table").first.inner_text() rows = capture.get_by_role("table").nth(1) @@ -197,7 +197,8 @@ def test_navigation_keyboard_versions_and_refresh(browser, tmp_path, profile_fir assert root.get_by_role("button", name="capture", exact=True).count() == 0 root.get_by_role("button", name="collect", exact=True).click() collect = root.locator('details[data-kt-key="navigation:collect"]') - assert "collect" in collect.inner_text() and "weighted-lexical-v1" in collect.inner_text() + assert "collect → find" in collect.inner_text() + assert "find-lexical-v1" in collect.inner_text() assert "use not recorded in this field" in collect.inner_text() assert not errors if not live: diff --git a/tests/test_knowledge_operations.py b/tests/test_knowledge_operations.py index 77ba27841..7835a532c 100644 --- a/tests/test_knowledge_operations.py +++ b/tests/test_knowledge_operations.py @@ -64,9 +64,8 @@ def test_collect_limits_omissions_and_shared_resource_revision(): assert limited["no_match"] and limited["omissions"] missing = reader.collect(ids=["not-a-real-id"]) assert missing["no_match"] and missing["omissions"] - assert reader.collect(query="doesnotmatchanyrecord", ids=["calibrate-serving-simulation"])[ - "no_match" - ] + with pytest.raises(ValueError, match="explicit IDs"): + reader.collect(query="doesnotmatchanyrecord", ids=["calibrate-serving-simulation"]) def test_collect_filters_and_revision_conflicts(): @@ -152,7 +151,7 @@ def test_incidental_files_and_hashes_do_not_affect_reads(tmp_path): (directory / "__pycache__" / "x.pyc").write_bytes(b"\0") reader = KnowledgeReader(skill_root=tmp_path) assert reader.read("selected").revision == before - assert reader.collect("sha256", ids=["selected"])["omissions"][0]["reason"] == "query" + assert reader.collect("sha256", kind="skill")["no_match"] def test_qualified_identity_rules(tmp_path): diff --git a/tests/test_knowledge_store.py b/tests/test_knowledge_store.py index 870434cb4..19683898d 100644 --- a/tests/test_knowledge_store.py +++ b/tests/test_knowledge_store.py @@ -70,6 +70,47 @@ def test_duplicate_stale_update_and_immutable_resources(tmp_path): assert item["ref"] == f"store:{LOCAL}" +def test_store_find_searches_the_delivered_revision(tmp_path): + store = KnowledgeStore(tmp_path, skill_root=None) + ref = "store:local-budget-notes" + metadata = { + "id": "local-budget-notes", + "kind": "knowledge", + "entrypoint": "note.md", + "state": "candidate", + "title": "Local capacity experiment", + "sources": [{"type": "unknown"}], + "applicability": {}, + "limitations": [], + "attribution": [], + } + original = Bundle(metadata, {"note.md": b"borealcapacitymarker"}, {}) + successor = Bundle(metadata, {"note.md": b"alpinecapacitymarker"}, {}) + old_revision = store.create(original) + new_revision = store.update(successor, expected=old_revision, reason="amend", evidence=[]) + options = {"kind": "knowledge", "material": True} + + assert store.collect("borealcapacitymarker", **options)["no_match"] + current = store.collect("alpinecapacitymarker", **options) + item, = current["items"] + assert (item["ref"], item["revision"], item["content"]) == ( + ref, new_revision, "alpinecapacitymarker", + ) + assert current["retrieval"]["calls"] == ["collect", "find"] + + retained = store.collect( + relevance="borealcapacitymarker", revisions={ref: old_revision}, **options, + ) + item, = retained["items"] + assert (item["ref"], item["revision"], item["content"]) == ( + ref, old_revision, "borealcapacitymarker", + ) + assert not retained["omissions"] + assert store.collect("alpinecapacitymarker", revisions={ref: old_revision}, **options)[ + "no_match" + ] + + def test_store_cannot_shadow_read_only_identities(tmp_path): store = store_at(tmp_path) for source in ("calibrate-serving-simulation", "case.admission-drain-occupancy"): diff --git a/tests/test_optimization_story.py b/tests/test_optimization_story.py index 121c3070a..b8b3dd0e3 100644 --- a/tests/test_optimization_story.py +++ b/tests/test_optimization_story.py @@ -201,7 +201,7 @@ def test_capture_packaged_skills_does_not_repeat_bundle_manifest(tmp_path): create_campaign(tmp_path, "camp", noise=1.5) reader = KnowledgeReader() options = {"relevance": "performance kernel optimization", "limit": 40, - "context_bytes": 65536, "relevance_method": "task-lexical-v1"} + "context_bytes": 65536, "kind": "knowledge"} expected = reader.collect(**options) path, _ = capture_context(tmp_path, context_id="many-skills", reader=reader, **options) document, state = verify_capture(path.read_bytes()) diff --git a/tests/test_raw_string_selection.py b/tests/test_raw_string_selection.py index aa8132365..999be76c3 100644 --- a/tests/test_raw_string_selection.py +++ b/tests/test_raw_string_selection.py @@ -1,24 +1,12 @@ -"""Ordinary synthetic compatibility controls, not a retrieval-quality benchmark.""" +"""Collect migration checks for selectors and budgeted optional resources.""" -import copy import hashlib -import json import pytest -from inferrsi.apps.cli import main -from inferrsi.knowledge._skill_selection import ( - indexed_skill, - lexical_score, - raw_string_lexical_score, - skill_score, -) +from inferrsi.knowledge._skill_selection import indexed_skill from inferrsi.knowledge._source_snapshot import source_entry from inferrsi.knowledge.knowledge_operations import KnowledgeReader, _hash, _json -from inferrsi.knowledge.knowledge_store import KnowledgeStore - -RAW = "weighted-raw-string-v1" -LEGACY = "weighted-lexical-v1" def synthetic_reader(entries): @@ -59,85 +47,29 @@ def synthetic_reader(entries): return reader -def without_method(result): - value = copy.deepcopy(result) - value.get("selection", {}).pop("method", None) - value.pop("retrieval", None) # Execution explanation is method-specific. - return value - - -@pytest.mark.parametrize("value", [None, True, 13.5, ["\n\t", None], {"n": "\r"}]) -def test_nonstring_values_retain_exact_legacy_json_matching(value): - cue = "n t r null true 13 5" - assert raw_string_lexical_score(cue, [(2, value)]) == lexical_score(cue, [(2, value)]) - - -def test_raw_string_matching_does_not_change_the_legacy_helper(): - assert raw_string_lexical_score("n t", [(4, "\n\t")]) == 0 - assert lexical_score("n t", [(4, "\n\t")]) == 8 - assert raw_string_lexical_score("n t", [(4, r"\n\t")]) == 8 - assert raw_string_lexical_score("RUN run STRASSE", [(3, "running Straße")]) == 6 - - -def test_skill_adapter_keeps_field_weights_and_combined_nonstring_metadata(): - record = {"title": "n", "summary": "n", "tags": ["\n"], "applicability": {}} - assert skill_score("n N", record, b"n", method=RAW) == 10 - assert skill_score("n N", record, b"n") == 10 - for method in ("unknown", "task-lexical-v1"): - with pytest.raises(ValueError, match="Unknown relevance method"): - skill_score("n", record, b"n", method=method) - - -def test_opt_in_changes_only_raw_string_owner_ranking_and_positive_admission(): - reader = synthetic_reader({ - "a-phantom": {"title": "\n", "summary": "\n", "core": "\n"}, - "b-hit": {"title": "n"}, "c-hit": {"title": "n"}, - }) - legacy = reader.collect(relevance="n") - assert legacy == reader.collect(relevance="n", relevance_method=LEGACY) - assert [r["title"] for r in legacy["items"]] == ["\n", "n", "n"] - raw = reader.collect(relevance="n", relevance_method=RAW) - expected = ["inferrsi.common.skills.b_hit", "inferrsi.common.skills.c_hit"] - assert [r["id"] for r in raw["items"]] == expected - assert raw["selection"] == { - "method": RAW, "cue": "n", "scope": "generated-packaged-skills", "ranking_applied": True, - } - assert reader.collect(relevance="n", relevance_method=RAW, limit=1)["items"] == raw["items"][:1] - assert reader.collect() == reader.collect(relevance_method=LEGACY) - - -@pytest.mark.parametrize("options", [ - {"query": "missing"}, {"query": "synthetic n"}, - {"kind": "skill"}, {"repo": "vllm"}, {"backend": "cpu"}, - {"tier": "peer"}, {"scenario": "diffusion.t2i"}, {"material": True}, +@pytest.mark.parametrize(("options", "message"), [ + ({"relevance_method": "weighted-lexical-v1"}, "Omit the method or use 'find-lexical-v1'"), + ({"relevance_method": "weighted-raw-string-v1", "relevance": "cue"}, "find-lexical-v1"), + ({"relevance_method": "task-lexical-v1", "ids": ["owner"]}, "find-lexical-v1"), + ({"relevance_method": "unknown"}, "Unsupported relevance_method"), + ({"relevance_method": []}, "Unsupported relevance_method"), + ({"query": "filtered", "relevance": "cue"}, "Find takes one query or relevance cue"), + ({"query": "filtered", "ids": ["owner"]}, "Remove query when using explicit IDs"), ]) -def test_query_and_filters_preserve_ordinary_semantics(options): - reader = synthetic_reader({"item": {"title": "running", "core": "n"}}) - legacy = reader.collect(relevance="n", **options) - raw = reader.collect(relevance="n", relevance_method=RAW, **options) - assert without_method(raw) == without_method(legacy) +def test_retired_selectors_and_ambiguous_inputs_fail_before_reader_access( + monkeypatch, options, message, +): + reader = KnowledgeReader() + def unexpected_access(): + raise AssertionError("invalid request accessed owners") -def test_explicit_ids_keep_membership_order_aliases_revisions_and_item_budgets(): - reader = synthetic_reader({"a": {"title": "\n"}, "b": {"title": "n"}}) - a, b = sorted(reader.records()) - options = {"ids": [b, a, "package:" + a], "relevance": "absent", "material": True} - legacy = reader.collect(**options) - raw = reader.collect(**options, relevance_method=RAW) - assert without_method(raw) == without_method(legacy) - assert [r["id"] for r in raw["items"]] == [b, a] - assert raw["selection"]["ranking_applied"] is False - for extra in ( - {"limit": 1}, {"max_bytes": len(_json(raw["items"][0]))}, - {"revisions": {a: reader.read(a).revision}}, {"revisions": {a: "sha256:" + "0" * 64}}, - ): - assert without_method(reader.collect(**options, **extra, relevance_method=RAW)) == ( - without_method(reader.collect(**options, **extra)) - ) + monkeypatch.setattr(reader, "owners", unexpected_access) + with pytest.raises(ValueError, match=message): + reader.collect(**options) -@pytest.mark.parametrize("explicit", [False, True]) -def test_context_required_hashes_atomic_budgets_and_optional_legacy_order(explicit): +def test_optional_context_uses_find_and_required_files_keep_atomic_exact_bytes(): reader = synthetic_reader({"owner": { "title": "n", "core": "Core é", "references": ["refs.md"], "resources": {"refs.md": "Required", "LICENSE": "Legal", "a.md": "\n", "b.md": "n"}, @@ -147,70 +79,32 @@ def test_context_required_hashes_atomic_budgets_and_optional_legacy_order(explic view = {**bundle.view(), "authority": "package"} _, required, _ = indexed_skill(reader.records()[identity], view, bundle.content) required_bytes = sum(len(bundle.content[path]) for path in required) - options = {"relevance": "n"} - if explicit: - options["ids"] = [identity] - for budget in (required_bytes - 1, required_bytes, required_bytes + 1, required_bytes + 2): - legacy = reader.collect(**options, context_bytes=budget) - raw = reader.collect(**options, context_bytes=budget, relevance_method=RAW) - assert without_method(raw) == without_method(legacy) - if budget < required_bytes: - assert not raw["contexts"] - assert raw["context_omissions"][0]["reason"] == "context_limit" - continue - context, = raw["contexts"] + too_small = reader.collect(ids=[identity], relevance="n", context_bytes=required_bytes - 1) + assert not too_small["contexts"] + assert too_small["context_omissions"][0]["reason"] == "context_limit" + + no_cue = reader.collect(ids=[identity], context_bytes=required_bytes + 1) + assert no_cue["contexts"][0]["resources"][-1]["path"].endswith("/a.md") + for budget in (required_bytes, required_bytes + 1, required_bytes + 2): + result = reader.collect( + ids=[identity], revisions={identity: bundle.revision}, + relevance="n", context_bytes=budget, + ) + assert result["items"] == no_cue["items"] + assert result["selection"]["ranking_applied"] is False + context, = result["contexts"] assert context["revision"] == bundle.revision - assert context["content_bytes"] == budget resources = context["resources"] assert [r["path"] for r in resources[:len(required)]] == required for resource in resources: data = reader.resource(identity, resource["path"], revision=context["revision"]) assert resource["content"].encode() == data assert resource["sha256"] == hashlib.sha256(data).hexdigest() - if budget > required_bytes: - # JSON's newline contribution must still admit the optional a.md first. - assert resources[len(required)]["path"].endswith("/a.md") - - -@pytest.mark.parametrize("options", [ - {"relevance_method": RAW}, {"relevance_method": "unknown"}, - {"relevance": "n", "relevance_method": "unknown"}, - {"relevance": "n", "relevance_method": None}, - {"relevance": "n", "relevance_method": []}, - {"relevance": "!!!", "relevance_method": RAW}, -]) -def test_method_validation_happens_before_reader_access(monkeypatch, options): - reader = KnowledgeReader() - - def unexpected_access(): - raise AssertionError("invalid request accessed owners") - - monkeypatch.setattr(reader, "owners", unexpected_access) - with pytest.raises(ValueError): - reader.collect(**options) - - -def test_raw_method_keeps_fail_closed_index_validation(): - reader = synthetic_reader({"owner": {"title": "n"}}) - identity, = reader.records() - reader.records()[identity]["source_files"] = {} - result = reader.collect(relevance="n", relevance_method=RAW) - assert result["no_match"] - assert result["omissions"][0]["reason"] == "index_unavailable" - - -def test_cli_and_store_delegate_the_explicit_method(monkeypatch, capsys, tmp_path): - reader = synthetic_reader({"owner": {"title": "n"}}) - monkeypatch.setattr("inferrsi.apps.cli._knowledge_reader", lambda args: reader) - assert main(["collect", "--relevance", "n", "--relevance-method", RAW, "--json"]) == 0 - assert json.loads(capsys.readouterr().out)["selection"]["method"] == RAW - with pytest.raises(SystemExit): - main(["collect", "--relevance", "n", "--relevance-method", "unknown"]) - with pytest.raises(SystemExit) as error: - main(["collect", "--relevance-method", RAW]) - assert error.value.code == 2 - store = KnowledgeStore(tmp_path, skill_root=None) - monkeypatch.setattr(store, "reader", lambda: reader) - assert store.collect(relevance="n", relevance_method=RAW) == reader.collect( - relevance="n", relevance_method=RAW - ) + omissions = {r["path"].rsplit("/", 1)[-1]: r["reason"] for r in context["omissions"]} + assert omissions["a.md"] == "not_relevant" + if budget == required_bytes: + assert omissions["b.md"] == "context_limit" + assert context["content_bytes"] == required_bytes + else: + assert resources[-1]["path"].endswith("/b.md") + assert context["content_bytes"] == required_bytes + 1 diff --git a/tests/test_research.py b/tests/test_research.py index 95ee1bce2..3beaf8ad6 100644 --- a/tests/test_research.py +++ b/tests/test_research.py @@ -75,10 +75,16 @@ def test_scoped_mlx_research_retains_the_lifecycle_resource(): assert {"group": "mechanism", "query": "lazy evaluation"} in lifecycle["matched_queries"] -def test_independent_queries_are_unioned_fairly_and_exact_search_is_unchanged(monkeypatch): +def _synthetic_record(identity, summary): + return { + "id": identity, "repo": "vllm_omni", "backend": "common", "scenario": "diffusion.*", + "summary": summary, "status": "imported", "verification": {"locally_verified": False}, + } + + +def test_independent_find_queries_are_unioned_fairly(monkeypatch): records = [ - {"id": name, "repo": "vllm_omni", "backend": "common", "scenario": "diffusion.*", - "summary": summary} + _synthetic_record(name, summary) for name, summary in ( ("a-model", "ExampleModel"), ("b-framework", "mlx"), @@ -89,8 +95,9 @@ def test_independent_queries_are_unioned_fairly_and_exact_search_is_unchanged(mo ) ] monkeypatch.setattr(knowledge_catalog, "knowledge_records", lambda: records) - exact = search_knowledge("ExampleModel mlx diffusion video offload") - assert [record["id"] for record in exact] == ["f-overlap"] + compound = search_knowledge("ExampleModel mlx diffusion video offload") + assert compound[0]["id"] == "f-overlap" + assert {record["id"] for record in compound} == {record["id"] for record in records} result = research_knowledge( "ExampleModel", frameworks=["mlx", "MLX"], tasks=["diffusion"], modalities=["video"], mechanisms=["offload"], limit=6, @@ -101,7 +108,6 @@ def test_independent_queries_are_unioned_fairly_and_exact_search_is_unchanged(mo } overlap = next(item for item in result["items"] if item["record"]["id"] == "f-overlap") assert len(overlap["matched_queries"]) == 5 - assert search_knowledge("ExampleModel mlx diffusion video offload") == exact limited = research_knowledge( "ExampleModel", frameworks=["mlx"], tasks=["diffusion"], modalities=["video"], mechanisms=["offload"], limit=5, @@ -114,8 +120,8 @@ def test_independent_queries_are_unioned_fairly_and_exact_search_is_unchanged(mo def test_query_attribution_survives_the_shared_output_limit(monkeypatch): records = [ - {"id": "model", "summary": "SyntheticModel mlx"}, - {"id": "framework", "summary": "mlx mlx"}, + _synthetic_record("model", "SyntheticModel mlx"), + _synthetic_record("framework", "mlx mlx"), ] monkeypatch.setattr(knowledge_catalog, "knowledge_records", lambda: records) result = research_knowledge("SyntheticModel", frameworks=["mlx"], limit=1) diff --git a/tests/test_retrieval_projection.py b/tests/test_retrieval_projection.py index 049d9720f..380855ef9 100644 --- a/tests/test_retrieval_projection.py +++ b/tests/test_retrieval_projection.py @@ -25,44 +25,58 @@ def campaign(tmp_path): return tmp_path -@pytest.mark.parametrize("method", ["weighted-lexical-v1", "weighted-raw-string-v1", - "task-lexical-v1"]) -def test_execution_scores_and_exact_budget_preserve_payloads(method): - reader = FIXTURE["reader"]() - result = reader.collect(relevance="launch", relevance_method=method, context_bytes=54) +@pytest.mark.parametrize("question", [{"query": "launch"}, {"relevance": "launch"}]) +def test_execution_scores_and_exact_budget_preserve_payloads(tmp_path, question): + with FIXTURE["reader"](tmp_path) as reader: + result = reader.collect(**question, context_bytes=54) + # Observations cannot eject a result from the budgeted item bytes. + bounded = reader.collect(**question, context_bytes=54, max_bytes=result["material_bytes"]) + fewer = reader.collect(**question, limit=2) assert len(result["items"]) == 5 and len(result["contexts"]) == 2 assert len(result["context_omissions"]) == 3 and result["context_bytes"] == 54 observation = result["retrieval"] + assert observation["calls"] == ["collect", "find"] + assert observation["selection"]["method"] == "find-lexical-v1" assert observation["counts"]["matched"] == 5 assert observation["assembly"] == "performed" assert [m["ref"] for m in observation["items"]] == [m["ref"] for m in result["items"]] for item in observation["items"]: - assert item["score"] == sum(item["components"].values()) * item["multiplier"] + assert item["score"] == sum(item["components"].values()) assert len({m["score"] for m in observation["items"]}) == 1 assert [m["ref"] for m in observation["items"]] == sorted(m["ref"] for m in result["items"]) - # Observations are outside the budgeted item bytes; adding them cannot eject a result. - budget = result["material_bytes"] - bounded = reader.collect(relevance="launch", relevance_method=method, context_bytes=54, - max_bytes=budget) for field in ("items", "contexts", "context_omissions", "context_bytes", "material_bytes"): assert result[field] == bounded[field] - fewer = reader.collect(relevance="launch", relevance_method=method, limit=2) assert fewer["retrieval"]["counts"]["excluded"]["item_limit"] == 3 assert fewer["retrieval"]["assembly"] == "not_requested" -def test_catalog_observation_does_not_mutate_records_or_return_order(monkeypatch): - records = list(FIXTURE["reader"]().records().values()) - original = copy.deepcopy(records) - monkeypatch.setattr("inferrsi.knowledge.catalog.knowledge_records", lambda: records) - for query in ("launch", "missing", "launch launch", ""): - result = search_knowledge(query, limit=2, explain=True) - assert result["items"] == search_knowledge(query, limit=2) - for item in result["retrieval"]["items"]: - assert item["score"] == sum(term["occurrences"] for term in item["components"]) +def test_catalog_observation_does_not_mutate_records_or_return_order(tmp_path, monkeypatch): + with FIXTURE["reader"](tmp_path) as reader: + records = list(reader.records().values()) + original = copy.deepcopy(records) + monkeypatch.setattr("inferrsi.knowledge.catalog.knowledge_records", lambda: records) + observations = {} + for query in ("guidance missing", "missing", "guidance guidance", ""): + result = search_knowledge(query, limit=2, explain=True) + assert result["items"] == search_knowledge(query, limit=2) + observations[query] = result["retrieval"] + excluded = search_knowledge("guidance", repo="other", explain=True) + assert records == original - result = search_knowledge("launch", repo="other", explain=True) - assert result["retrieval"]["counts"]["excluded"]["repo"] == 5 + ranked = observations["guidance missing"] + assert ranked["calls"] == ["knowledge search", "find"] + assert ranked["selection"]["method"] == "find-lexical-v1" + assert ranked["counts"]["matched"] == 5 # "guidance" is only in the Skill bodies. + assert ranked["items"] == observations["guidance guidance"]["items"] + for item in ranked["items"]: + assert item["score"] == item["components"]["guidance"] + assert observations["missing"]["items"] == [] + inventory = observations[""] + assert inventory["calls"] == ["knowledge search"] + assert inventory["selection"]["state"] == "not_requested" + assert inventory["selection"]["method"] is None + assert all("score" not in item for item in inventory["items"]) + assert excluded["retrieval"]["counts"]["excluded"]["repo"] == 5 def test_shell_search_keeps_its_recorded_query_scope_and_hits(campaign): @@ -81,13 +95,16 @@ def test_actual_paths_ranking_states_and_multi_run_navigation(campaign): data = dashboard_snapshot(campaign)["data"] trace = data["knowledge_trace"] captures = {c["context_id"]: c for c in trace["retrievals"]} - assert captures["catalog"]["trace"]["calls"] == ["knowledge search"] - assert captures["collect"]["trace"]["calls"] == ["collect"] - assert captures["capture"]["trace"]["calls"] == ["campaign capture-context", "collect"] + assert captures["catalog"]["trace"]["calls"] == ["knowledge search", "find"] + assert captures["collect"]["trace"]["calls"] == ["collect", "find"] + assert captures["capture"]["trace"]["calls"] == ["campaign capture-context", "collect", "find"] assert captures["explicit"]["trace"]["selection"]["state"] == "explicit_ids" + assert captures["explicit"]["trace"]["calls"] == ["collect"] + assert all("score" not in item for item in captures["explicit"]["trace"]["items"]) assert captures["no-ranking"]["trace"]["selection"]["state"] == "not_requested" assert captures["legacy"]["trace"]["entrypoint"] == "unknown" assert captures["legacy"]["trace"]["items"] == [] + assert captures["legacy"]["trace"]["selection"]["method"] == "weighted-lexical-v1" assert captures["empty"]["trace"]["counts"]["matched"] == 0 assert len(captures["capture"]["contexts"]) == 2 assert len(captures["capture"]["context_omissions"]) == 3 @@ -99,8 +116,17 @@ def test_actual_paths_ranking_states_and_multi_run_navigation(campaign): assert len(trace["searches"]) == 4 # v1/v2 are kept distinct, never summed. archive = next(s for s in trace["searches"] if s["schema_version"] == 2) assert not archive["issues"] + events = {event["context_id"]: event for event in archive["events"]} + assert events["catalog"]["tool"] == "knowledge search" + assert events["catalog"]["delivery"]["state"] == "unknown" + assert events["capture"]["tool"] == "find" + assert events["capture"]["delivery"]["state"] == "visible" material = next(m for m in archive["materials"] if m["evidence"]) - assert material["evidence"][0]["state"] == "archive_reviewed" + evidence, = material["evidence"] + assert evidence["state"] == "archive_reviewed" + assert evidence["event_id"] == events["capture"]["id"] + assert evidence["passage_path"] == captures["capture"]["contexts"][0]["resources"][0]["path"] + assert evidence["passage_text"] == "Synthetic launch guidance." assert any(d["preparation_state"] == "omitted" for d in archive["declarations"]) assert archive["counts"]["used"] == 1 for number in (2, 3): diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py index e0c29138f..8542d675a 100644 --- a/tests/test_search_archive.py +++ b/tests/test_search_archive.py @@ -11,6 +11,7 @@ from inferrsi.apps.campaign import capture_context, create_campaign from inferrsi.apps.views.search_attribution import import_search +from inferrsi.knowledge._skill_selection import FIND_RELEVANCE from inferrsi.knowledge.context_projection import assemble_context_text REVISION = "f6d200" + "a" * 34 @@ -437,13 +438,19 @@ def __init__(self, text): self.text = text def collect(self, query="", **options): + selection = {"method": options.get("relevance_method"), + "cue": options.get("relevance") or query, + "scope": "package-records-and-library", "ranking_applied": True} return { "items": [{"ref": "package:manual", "revision": "sha256:" + "a" * 64, "title": "manual", "kind": "knowledge"}], "omissions": [], "context_omissions": [], - "selection": {"method": options.get("relevance_method"), - "cue": options.get("relevance") or query, - "scope": "generated-packaged-skills", "ranking_applied": True}, + "selection": selection, + "retrieval": {"schema_version": 1, "entrypoint": "collect", + "calls": ["collect", "find"], + "request": {"sources": ["package", "checkout"]}, + "selection": {**selection, "state": "applied"}, + "counts": {"matched": 1}, "items": [], "assembly": "performed"}, "context_bytes": len(self.text.encode()), "context_budget": {"max_bytes": 4096}, "contexts": [{"ref": "package:manual", "revision": "sha256:" + "a" * 64, "content_bytes": len(self.text.encode()), "resources": [{ @@ -467,10 +474,8 @@ def _capture_trace(path, output, assistant_text="No adoption decision."): def _make_capture(tmp_path): campaign = tmp_path / "campaign" create_campaign(campaign, "example") - # Keep this recorder fixture independent of the default retrieval policy. capture, _ = capture_context(campaign, context_id="r1", reader=_Reader("Manual line.\n"), - relevance="manual", relevance_method="weighted-lexical-v1", - context_bytes=4096) + relevance="manual", context_bytes=4096) document = json.loads(capture.read_text(encoding="utf-8")) output = assemble_context_text(document["contexts"]) return campaign, output @@ -509,11 +514,12 @@ def test_indexer_capture_is_prepared_and_unknown_without_return(tmp_path): record = json.loads(path.read_text(encoding="utf-8")) event, = record["events"] material, = event["materials"] + assert event["tool"] == "find" assert event["capture"] == {"state": "verified", "source": "knowledge/r1.json", "sha256": event["sha256"]} assert event["request"]["relevance"] == "manual" - assert event["selection"] == {"method": "weighted-lexical-v1", "cue": "manual", - "scope": "generated-packaged-skills", + assert event["selection"] == {"method": FIND_RELEVANCE, "cue": "manual", + "scope": "package-records-and-library", "ranking_applied": True} assert material["preparation_state"] == "prepared" assert event["delivery"]["state"] == "unknown" @@ -574,7 +580,7 @@ def test_extended_window_preserves_old_unknown_declaration_with_later_observatio assert (read["tool"], read["trace_line"], read["materials"]) == ( "cat", 6, [{"ref": "package:manual"}], ) - capture, = [event for event in expanded["events"] if event["tool"] == "collect"] + capture, = [event for event in expanded["events"] if event["tool"] == "find"] assert capture.get("trace_line") is None assert expanded["materials"]["package:manual"]["use"] == "unknown" assert expanded["materials"]["package:manual"]["declarations"] == [ @@ -635,14 +641,20 @@ def test_indexer_archive_review_matches_exact_visible_resource_and_return(tmp_pa "passage_path": "skills/manual/SKILL.md", "decision_line": 3, "decision_quote": "I will use Manual line", } - review = _review(tmp_path / "review.json", decisions=[use, use], capture_returns={"r1": 2}) + inputs = {"number": 1, "trace": trace, "lines": [2, 3], + "knowledge_root": tmp_path / "corpus", "context_ids": ["r1"]} + missing_path = {key: value for key, value in use.items() if key != "passage_path"} + review = _review(tmp_path / "review.json", decisions=[missing_path], capture_returns={"r1": 2}) + with pytest.raises(ValueError, match="exact captured resource path"): + import_search(campaign, archive_review=review, **inputs) + assert not (campaign / "search/archive/round-1.json").exists() - path = import_search(campaign, number=1, trace=trace, lines=[2, 3], - knowledge_root=tmp_path / "corpus", context_ids=["r1"], - archive_review=review) + review = _review(review, decisions=[use, use], capture_returns={"r1": 2}) + path = import_search(campaign, archive_review=review, **inputs) record = json.loads(path.read_text(encoding="utf-8")) event, = record["events"] material, = event["materials"] + assert event["tool"] == "find" assert event["delivery"]["state"] == "visible" assert material["delivery_state"] == "visible" assert event["delivery"]["visible_resources"] == [ @@ -652,7 +664,10 @@ def test_indexer_archive_review_matches_exact_visible_resource_and_return(tmp_pa "text": "Manual line.", }] assert record["materials"]["package:manual"]["use"] == "used" - assert record["materials"]["package:manual"]["evidence"][0]["basis"] == "archive_reviewed" + evidence, = record["materials"]["package:manual"]["evidence"] + assert evidence["basis"] == "archive_reviewed" + assert (evidence["passage_path"], evidence["passage_line"], evidence["passage_text"]) == ( + "skills/manual/SKILL.md", 1, "Manual line.") assert str(tmp_path) not in path.read_text(encoding="utf-8") diff --git a/tests/test_skill_library.py b/tests/test_skill_library.py index fb7d2bb9f..bb139cb32 100644 --- a/tests/test_skill_library.py +++ b/tests/test_skill_library.py @@ -49,6 +49,8 @@ def test_imported_repository_cli_and_scope_boundaries(repo, capsys, tmp_path): "cpu", "--scenario", scenario, + "--limit", + "1", "--output", str(destination), ] diff --git a/tools/evaluate_indexer.py b/tools/evaluate_indexer.py index 60099ab5b..0b435b7eb 100644 --- a/tools/evaluate_indexer.py +++ b/tools/evaluate_indexer.py @@ -58,6 +58,8 @@ def options(case, context=False): value = {"query": case.get("query", ""), "limit": case.get("limit", 5)} value.update(case.get("filters", {})) if case.get("relevance") is not None: + if value["query"].strip(): + raise ValueError("Supply one Find question or use the pinned legacy runner.") value["relevance"] = case["relevance"] if context: value["context_bytes"] = case.get("context_bytes", 262144) @@ -78,8 +80,16 @@ def cli(root, mode, case): if mode == "search": args += ["knowledge", "search", "cgroupv2", "--limit", "5", "--json"] else: - args += ["collect", case["query"], "--relevance", case["relevance"], - "--limit", str(case["limit"]), "--json"] + selection = options(case) + args += ["collect", "--limit", str(selection["limit"]), "--json"] + if selection.get("relevance") is not None: + args += ["--relevance", selection["relevance"]] + elif selection["query"]: + args += [selection["query"]] + for field in ("repo", "tier", "backend", "scenario", "kind"): + if field in selection: + flag = "--repo-type" if field == "repo" else "--" + field + args += [flag, selection[field]] start = time.perf_counter_ns() run = subprocess.run(args, capture_output=True, check=True, timeout=180, env={ "PATH": os.defpath, "PYTHONPATH": str(root / "src"), "PYTHONDONTWRITEBYTECODE": "1", @@ -113,7 +123,9 @@ def worker(root, action, payload): case = payload["queries"][0] for mode in ("fresh_relevance", "default", "context", "search"): args = options(case, context=mode == "context") - if mode in ("default", "search"): + if mode == "search": + args = {"query": "cgroupv2", "limit": 5} + elif mode == "default": args.pop("relevance", None) args["query"] = "cgroupv2" reader = KnowledgeReader() @@ -130,7 +142,7 @@ def worker(root, action, payload): "output_sha256": digest(encode(value))} for mode in ("relevance", "search"): rows["cli_" + mode] = cli(root, mode, case) - for api, cli_name in (("hot_relevance/dev01", "cli_relevance"), + for api, cli_name in (("hot_relevance/" + case["id"], "cli_relevance"), ("search", "cli_search")): if rows[api]["output_sha256"] != rows[cli_name]["output_sha256"]: raise ValueError("API/CLI payload mismatch: " + api) From 159a3e3c6843eb06077b3c471c6736d2be7d852e Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Wed, 18 Nov 2026 17:13:11 +0800 Subject: [PATCH 05/15] Honor package-only Store discovery scope Limit package-only Store discovery to reserved package projections and store-owned candidates so every hit can be opened by the same reader. Agent-assisted-by: Codex / gpt-6.2-sol [effort=max] (implementer) Signed-off-by: Zhou ziheng --- docs/knowledge-store.md | 2 ++ .../knowledge/knowledge_operations.py | 14 +++++++++++--- tests/test_knowledge_store.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/knowledge-store.md b/docs/knowledge-store.md index 6956e95c7..3133c84a9 100644 --- a/docs/knowledge-store.md +++ b/docs/knowledge-store.md @@ -37,6 +37,8 @@ package, checkout and store items with qualified `store:` refs. An ID owned the store and a read-only adapter is reported as ambiguous, never shadowed. Pass a query or `relevance` to `store.collect` for local Find discovery. +Discovery follows the store's reserved sources. With `skill_root=None` it +searches package Records, projected Skills and store-owned candidates. Store-owned candidates participate using their metadata and entrypoint text. An unpinned query ranks the current revision. A query with a revision pin ranks the retained entrypoint at that revision and returns that same revision if it diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py index 062d47796..57b40d77e 100644 --- a/src/inferrsi/knowledge/knowledge_operations.py +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -563,7 +563,11 @@ def _search_order( } root = None if package_only else self.skill_root records = self.records() - documents = load_documents(root, kinds=kinds[kind], records=records.values()) + search_kinds = kinds[kind] + package_only_store = self.store is not None and self.skill_root is None + if package_only_store: + search_kinds = tuple(name for name in search_kinds if name != "case") + documents = load_documents(root, kinds=search_kinds, records=records.values()) if root is not None and root != installed_skill_root() and kind is None: # A selected custom library owns its registered skills. The package # supplies missing skills and remains available for exact reads. @@ -579,11 +583,15 @@ def _search_order( eligible, references = [], [] excluded = {"filters": 0, "ranking_scope": 0} + considered = 0 for document in documents: authority, _, identity = document.qualified.partition(":") projection = installed_skill_projection(document, records) if projection is not None: authority, identity = "package", projection["id"] + if package_only_store and authority == "checkout": + continue + considered += 1 if authority == "package": material_kind = "skill" if kind == "skill" else "knowledge" metadata = {**records[identity], "kind": material_kind} @@ -609,7 +617,7 @@ def _search_order( for score, _, index, components in scored } excluded["zero_score"] = len(eligible) - len(results) - return results, len(documents), excluded + return results, considered, excluded def collect( self, @@ -670,7 +678,7 @@ def collect( raise ValueError("Duplicate requested identity") checkout = self owners = self.owners() - if find_ranking and self.skill_root is None: + if find_ranking and self.skill_root is None and self.store is None: from .knowledge_find import installed_skill_root checkout = KnowledgeReader(skill_root=installed_skill_root()) diff --git a/tests/test_knowledge_store.py b/tests/test_knowledge_store.py index 19683898d..f1daf734a 100644 --- a/tests/test_knowledge_store.py +++ b/tests/test_knowledge_store.py @@ -111,6 +111,25 @@ def test_store_find_searches_the_delivered_revision(tmp_path): ] +def test_package_only_store_find_respects_its_reserved_sources(tmp_path): + query = "cache per-step masks ran out of device memory" + options = {"kind": "case", "limit": 1} + ordinary = KnowledgeReader().collect(query, **options) + assert ordinary["items"][0]["authority"] == "checkout" + + store = KnowledgeStore(tmp_path, skill_root=None) + result = store.collect(query, **options) + assert result["no_match"] + assert result["retrieval"]["request"]["sources"] == ["package", "store"] + assert result["retrieval"]["counts"]["considered"] == 0 + + reader = store.reader() + result = reader.collect("cgroupv2", kind="skill", limit=1) + item, = result["items"] + assert item["ref"] == "package:inferrsi.common.skills.cgroup_resource_controls" + assert reader.read(item["ref"], revision=item["revision"]).revision == item["revision"] + + def test_store_cannot_shadow_read_only_identities(tmp_path): store = store_at(tmp_path) for source in ("calibrate-serving-simulation", "case.admission-drain-occupancy"): From 17ede4f4f1e16fc17a15496e966684e6c6da9875 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Wed, 18 Nov 2026 17:19:45 +0800 Subject: [PATCH 06/15] Refresh search compatibility checks and dashboard example Assert the retired scorer names stay absent while preserving the historical move manifest and surviving APIs. Regenerate the synthetic dashboard example with the recorded Find scores and call chain. Agent-assisted-by: Codex (compatibility assertions and fixture regeneration) Signed-off-by: Zhou ziheng --- docs/examples/dashboard-profile-review.html | 2 +- tests/test_knowledge_compatibility_shells.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/examples/dashboard-profile-review.html b/docs/examples/dashboard-profile-review.html index 7d7203bd5..51384bba3 100644 --- a/docs/examples/dashboard-profile-review.html +++ b/docs/examples/dashboard-profile-review.html @@ -835,7 +835,7 @@

}}; - + - + - +