diff --git a/docs/reference-matrix.md b/docs/reference-matrix.md index 62f9f2d39..207a96faf 100644 --- a/docs/reference-matrix.md +++ b/docs/reference-matrix.md @@ -1,5 +1,11 @@ # InferRSI Reference Matrix +## EPD cold-serving allocation case + +| Source | Inspected scope | Extraction and qualification boundary | +| --- | --- | --- | +| Kelvin-Al/epd-provisioning cold comparison and integer allocator | Twelve published files cover the allocator, original comparison and position-balanced follow-up. The [typed source record](../src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.json) owns both immutable source revisions and the rehashed file identities. | [EPD cold-provisioning case](../src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.md). External author-reported four-H800, preprocessed MMMU observation. Formal acceptance INCONCLUSIVE. Answer quality and sustained service remain unqualified. | + **Branch:** `opus/meta-skill-bootstrap` | **Integration date:** 2026-10-16 | **Comparison anchor:** `d96ff8d3` Current skill links resolve to the complete categorized diff --git a/docs/searchspace-matrix.md b/docs/searchspace-matrix.md index 6281badd6..c7556b13c 100644 --- a/docs/searchspace-matrix.md +++ b/docs/searchspace-matrix.md @@ -1,5 +1,11 @@ # InferRSI SearchSpace Matrix +## EPD cold-serving configuration precedent + +| Resource layer | Lifecycle use | Owner | Qualification | +| --- | --- | --- | --- | +| Accelerator / Host | Diagnose / Configure / Verify stage instance counts using independent capacity estimates and matched serving windows | [EPD cold-provisioning case](../src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.md) | External author-reported four-H800 cold MMMU observation. Formal acceptance INCONCLUSIVE. Transfer and queue behavior remain deployment-specific. | + **Branch:** `opus/meta-skill-bootstrap` | **Integration date:** 2026-10-16 | **Comparison anchor:** `d96ff8d3` Current skill links resolve to the complete categorized diff --git a/docs/skill-coverage.md b/docs/skill-coverage.md index ad1b8b1ca..61ac86754 100644 --- a/docs/skill-coverage.md +++ b/docs/skill-coverage.md @@ -4,7 +4,7 @@ Generated from the registry, SKILL.md frontmatter and the two maintained matrices. **Do not edit by hand**; regenerate after any registry, skill, matrix or frozen-plan change. -- Input digest: `sha256:93ad79f2b36af340d3c1c819dcb2f9b651d0f6078913aa726bd9ee1f01e9df54` +- Input digest: `sha256:1b437e45037eced08be2653c0577e1d756c1bf12ac137d074da9a4bc3c005fd1` - Regenerate: `python tools/build_skill_coverage.py` - Verify without writing: `python tools/build_skill_coverage.py --check` diff --git a/src/inferrsi/knowledge/data/skill_library/_compiled/_provenance.json b/src/inferrsi/knowledge/data/skill_library/_compiled/_provenance.json index e9c6505a5..0deb82505 100644 --- a/src/inferrsi/knowledge/data/skill_library/_compiled/_provenance.json +++ b/src/inferrsi/knowledge/data/skill_library/_compiled/_provenance.json @@ -1,7 +1,7 @@ { "schema_version": 1, "repository": "https://github.com/InferMatrixLab/InferRSI", - "source_revision": "217d0b386d9c8d53bc06fc0d485f2e99722d5847", + "source_revision": "55cb1c5a945bae7e9dbd211b9f6662af143472dd", "registry": { "path": "src/inferrsi/knowledge/data/skill_library/skill-index.json", "sha256": "c6c235483034dfce3c0a4ec3bf7d68046ea2cf8c933f069833ecaef65cdf2d70" diff --git a/src/inferrsi/knowledge/data/skill_library/_compiled/inputs.json b/src/inferrsi/knowledge/data/skill_library/_compiled/inputs.json index 8fbac40d7..a40f8aca9 100644 --- a/src/inferrsi/knowledge/data/skill_library/_compiled/inputs.json +++ b/src/inferrsi/knowledge/data/skill_library/_compiled/inputs.json @@ -594,11 +594,13 @@ "description": "How to structure and use a casebook of past optimization decisions as a decision-support tool. Extracts the casebook pattern from TensorRT-LLM's perf-optimization-casebook skill. Use when building a record of optimization experiments or when deciding which optimization to try next based on prior art.\n" }, "files": { - "SKILL.md": "cec75f3a721dd13e6913b597d5373833a14c39d9048951a30c4864dd2c49aac8", + "SKILL.md": "2f6061305991384727d6d0d4519ad9b230e19c49bb784e0bdc2ae5306d66e27d", "cases/admission-drain-occupancy.json": "4f8544292b2cc3391f4d4d75994cb81fdd295eff99e55b7e60374c6c122de09e", "cases/admission-drain-occupancy.md": "e88bbac5d253a8e7e3a3d5e21047f23f2970c11ac23887d4303ab94fca52dd27", "cases/drama3-rejected-directions.json": "a2dc5db8aba62081c5743b12f854e74ae500c53bd57e2486260d0b5ae478638a", "cases/drama3-rejected-directions.md": "6f143bc363a3290fc008c08e57cb075f1776d1c8cca8a6a59297d5b1fbbf61bf", + "cases/epd-cold-provisioning.json": "68e471335193e2027779e1377ee22e3384f6c3c66034f0a7b7631e1b67d2e245", + "cases/epd-cold-provisioning.md": "9d1931e74585a57d824f9c48003aa1ad155d9d7c7d0d605f17c7d807e81233c4", "cases/qwen-ascend-rejected-directions.json": "b2c1808919ae525efb82148a24b465459799422fefc6d420891d7dbeb36cd31a", "cases/qwen-ascend-rejected-directions.md": "be396983812262515899a3e2ba4f01b23ff92e1b836881882c7580372b8ae552", "cases/qwen-b300-rejected-directions.json": "90e3cdca085e2d122e648aae527bd6a61de6d4fddb66c393ca25ac4541cd9122", diff --git a/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/SKILL.md b/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/SKILL.md index d67d1a861..509675c49 100644 --- a/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/SKILL.md +++ b/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/SKILL.md @@ -45,6 +45,7 @@ metadata: "Source-derived candidate; no local efficacy validation.", "Original typed source inventory is not normalized; retain source identities and limits in the procedure/readsets." ], + "references": ["cases/epd-cold-provisioning.md"], "provenance_readsets": [], "consumption_mode": "packaged" } @@ -123,6 +124,11 @@ Source: perf-optimization-casebook @ f33e0e5f, §how-to-use. ## Recorded diagnostic options +The [EPD cold-provisioning case](cases/epd-cold-provisioning.md) records +integer E:PD GPU allocation, applicability and matched cold MMMU comparisons +on four H800 GPUs, including a position-balanced follow-up. Its source record retains external author-reported scope, +published artifact hashes and INCONCLUSIVE formal acceptance. + Current owner-local records retain rejected directions alongside their limits: - [Qwen B300](cases/qwen-b300-rejected-directions.md): full-image rejection, diff --git a/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.json b/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.json new file mode 100644 index 000000000..43f3c8348 --- /dev/null +++ b/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.json @@ -0,0 +1,116 @@ +{ + "id": "case.epd-cold-provisioning", + "kind": "case", + "origin": [ + "external" + ], + "state": "author_reported", + "content": "epd-cold-provisioning.md", + "required_resources": [ + "epd-cold-provisioning.json" + ], + "sources": [ + { + "type": "git", + "url": "https://github.com/Kelvin-Al/epd-provisioning/blob/3ed184e3a83229806a9fa1adb2bc75f1744c6518/planner.py", + "commit": "3ed184e3a83229806a9fa1adb2bc75f1744c6518", + "paths": [ + "planner.py", + "experiments/ec-cache/cold_sweep.py", + "experiments/ec-cache/audit.py", + "experiments/ec-cache/results/cold/plan.json", + "experiments/ec-cache/results/cold/results.json", + "experiments/ec-cache/results/failed-cold-attempt/status.json", + "experiments/four-gpu/results/mmmu/plan.json" + ], + "sha256": { + "planner.py": "6bf4ea101fc12da1ca9166121a09f362b22feff3bf45ccf94b037dec1d93ce68", + "experiments/ec-cache/cold_sweep.py": "fcffb83d0351bf36387a971622edd92667838112fc737f68f1c6acff46082853", + "experiments/ec-cache/audit.py": "56be9d553ba04ac1a279035cea95eabe4892b5225eaabb9d604b20d057350816", + "experiments/ec-cache/results/cold/plan.json": "88941e0f5969ce25f94774a0a4c95c6204b9bc0d48669c4d18547017104c2542", + "experiments/ec-cache/results/cold/results.json": "dc5c1ecb7d3f92708a3ea7078d65ab575854a350235d5d14d7257bda956f4f2b", + "experiments/ec-cache/results/failed-cold-attempt/status.json": "33f44ead48d151e3b2b4527e12c2f89085651ed332ddf5ffd8aac7223c538806", + "experiments/four-gpu/results/mmmu/plan.json": "93a5b729357464f9ae2d0b62c120e5e5f22ff2e82db92a3a6afa10eae773c7d5" + }, + "integrity_status": "published_bytes_retrieved_and_rehashed", + "issue": "https://github.com/Kelvin-Al/epd-provisioning/issues/1", + "evidence_url": "https://github.com/Kelvin-Al/epd-provisioning/blob/3ed184e3a83229806a9fa1adb2bc75f1744c6518/experiments/ec-cache/results/cold/results.json" + }, + { + "type": "git", + "url": "https://github.com/Kelvin-Al/epd-provisioning/blob/a0037f7ddc4b1af5f59fbef0a86b2a5627b69feb/experiments/ec-cache/results/cold-order-followup/plan.json", + "commit": "a0037f7ddc4b1af5f59fbef0a86b2a5627b69feb", + "paths": [ + "experiments/ec-cache/order_followup/cold_sweep.py", + "experiments/ec-cache/order_followup/audit.sbatch", + "experiments/ec-cache/results/cold-order-followup/plan.json", + "experiments/ec-cache/results/cold-order-followup/results.json", + "experiments/ec-cache/results/cold-order-followup/interrupted-results.json" + ], + "sha256": { + "experiments/ec-cache/order_followup/cold_sweep.py": "f031d7c6f354461310eecdf8a74798c31615c4f1aad806187671ac3ba147e32f", + "experiments/ec-cache/order_followup/audit.sbatch": "40c3064acd59f1255c62ed53effb4a7be834d1dbc32546eff59387133288d128", + "experiments/ec-cache/results/cold-order-followup/plan.json": "c7fe3d8cbd2cf4cabccfe38bf9a7428ab980af591ed76e423128a2031b42fb95", + "experiments/ec-cache/results/cold-order-followup/results.json": "dc4f3d4f492f60171a5124e0d97a1a52c8dd6134072251666f55b7ff157a9c24", + "experiments/ec-cache/results/cold-order-followup/interrupted-results.json": "834d616d8b1dbd04bb08a28a8e4b19a9128692b342b8714062d26a9c8a248456" + }, + "integrity_status": "published_bytes_retrieved_and_rehashed", + "issue": "https://github.com/Kelvin-Al/epd-provisioning/issues/1", + "evidence_url": "https://github.com/Kelvin-Al/epd-provisioning/blob/a0037f7ddc4b1af5f59fbef0a86b2a5627b69feb/experiments/ec-cache/results/cold-order-followup/results.json" + } + ], + "applicability": [ + "Cold E+PD serving with one GPU per effective stage instance", + "Four H800, Qwen2.5-VL-3B-Instruct, prepared vLLM 0.33.0 overlay, filesystem EC connector", + "720 single-image preprocessed MMMU validation requests, fixed 64 output tokens, zero embedding reuse" + ], + "tags": [ + "EPD", + "GPU-allocation", + "capacity-planning", + "cold-serving", + "MMMU", + "H800", + "prefill-decode", + "queueing" + ], + "limitations": [ + "External author report. InferRSI campaign acceptance INCONCLUSIVE.", + "Published JSON and code hashes checked. Raw worker traces, image bytes, model weights and container were not independently replayed or verified.", + "Model weight revision absent. Overlay identity is partly represented by recorded script digests.", + "Original reverse-order rounds fixed 2E:2PD in the middle. The nine-wave follow-up balances all positions with one observation per split and position, with limited predecessor and variability coverage.", + "One offered rate in each single-node comparison, with a central arrival window of about 12.5 seconds. The interrupted six-wave follow-up is retained separately from the complete rerun.", + "700 of 720 images resized into calibration range. Fixed-length completion does not evaluate MMMU answer accuracy.", + "First cold attempt failed on inconsistent media keys. Complete rerun uses corrected keys and explicitly forwards seed 9197." + ], + "attribution": "Kelvin-Al/epd-provisioning, Apache-2.0 source and published cold-serving observations.", + "findings": [ + { + "direction": "Choose integer E:PD counts using independently estimated stage capacity", + "outcome": "inconclusive", + "reason": "The minimum of E-pool and PD-pool capacity selected 2E:2PD. It ranked first in the two original cold rounds and all three position-balanced follow-up rounds, with 26.04, 26.44 and 26.67 completions/s at the first, second and third positions.", + "revisit_when": "Re-estimate stage rates and test alternatives when model, workload, hardware, transfer path or worker configuration changes.", + "keywords": [ + "EPD", + "integer allocation", + "stage capacity", + "2E:2PD" + ], + "scope": "regime_bound" + }, + { + "direction": "Use actual visual work and paired queue observations to diagnose the limiting stage", + "outcome": "caveat", + "reason": "Each wave encoded 720 images with zero EC and resident hits. One E completed about 23 requests/s with queueing, while two E workers kept up with arrivals and two PD workers completed about 26 requests/s.", + "revisit_when": "Align encoder, decoder and client counters to the same arrival interval before inferring bottlenecks.", + "keywords": [ + "visual forward", + "EC", + "cold cache", + "encoder queue", + "PD queue" + ], + "scope": "regime_bound" + } + ] +} diff --git a/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.md b/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.md new file mode 100644 index 000000000..55f04708c --- /dev/null +++ b/src/inferrsi/knowledge/data/skill_library/casebooks/inference/inference-optimization-casebook/cases/epd-cold-provisioning.md @@ -0,0 +1,173 @@ +# EPD GPU allocation from cold-stage capacity + +External author-reported cold-serving observation. Formal campaign acceptance +remains INCONCLUSIVE. Source identities, inspected artifact hashes and scope +belong to the [source record](epd-cold-provisioning.json). + +The original six-wave measurement and a nine-wave order follow-up completed. +Formal acceptance is INCONCLUSIVE +because the published evidence lacks a prospectively frozen acceptance +gate, an independently checked model-weight revision and complete +independently verified runtime and raw-trace artifacts. The limited repeats +also leave uncertainty about repeatability. No numerical acceptance +threshold is inferred after observing these results. + +## Decision and mechanism + +For disaggregated multimodal serving, let N be the available GPUs, e the +encoder GPU count, and mu_E and mu_PD the effective per-instance completion +rates for encoding and combined prefill/decode. One GPU serves each instance. +Both rates have units of completed requests per second per instance for +the same request mix and generation policy. Pool capacity and C(e) have +units of completed requests per second. The measured four-GPU feasible +encoder counts are 1, 2 and 3, with 4 - e GPUs assigned to PD. +The capacity proxy is + +```text +C(e) = min(e * mu_E, (N - e) * mu_PD), 1 <= e < N +e_fractional = N * mu_PD / (mu_E + mu_PD) +``` + +The implementation evaluates feasible floor and ceiling neighbors of the +fractional crossing and selects the maximum. It can retain an incumbent that +ties for the maximum. Stage rates are inputs with workload-specific uncertainty. +This rule assumes useful stage capacity scales with instance count under the +deployment's routing and transfer behavior. Verify that assumption with the +complete serving path. Preprocessing, embedding transfer, shared host and +storage contention, routing imbalance and batching effects have no separate +terms in this expression. They must be reflected in matched effective rates +or investigated through a more appropriate model. +The [allocator implementation](https://github.com/Kelvin-Al/epd-provisioning/blob/3ed184e3a83229806a9fa1adb2bc75f1744c6518/planner.py) +provides the exact integer and tie behavior. + +For positive constant stage rates, this minimum is increasing before the +crossing and decreasing after it. Evaluating its feasible floor and ceiling +neighbors therefore finds an integer maximizer within this model for any +integer GPU budget of at least two. The H800 comparisons provide an empirical +check of the capacity estimates and deployment assumptions on one task. + +## Cold observation + +The frozen sizing inputs estimated 25.10 requests/s per E and 13.71 per PD. +For four GPUs, the capacity proxy gives 25.10 for 1E:3PD, 27.43 for 2E:2PD, +and 13.71 for 3E:1PD. It therefore recommends 2E:2PD for those inputs. + +The completed cold comparison ran all three splits in one allocation on +four NVIDIA H800 GPUs. The encoder counts were run in the orders +`[1, 2, 3]` and `[3, 2, 1]`. Reversal swapped the outer splits and kept +2E:2PD in the middle, leaving its effect confounded with second position. + +| Round | 1E:3PD completions/s | 2E:2PD completions/s | 3E:1PD completions/s | +| --- | ---: | ---: | ---: | +| 1 | 23.17 | 25.90 | 13.14 | +| 2 | 23.25 | 26.21 | 13.45 | + +2E:2PD ranked first in both rounds. Relative to second-place 1E:3PD, its +completion rate was 11.8% and 12.7% higher. Its P90 time to first token was +6.48 s and 5.69 s, compared with 9.63 s and 9.68 s for 1E:3PD. +Every wave completed all 720 requests without failures and recorded 720 +visual forward calls, 720 encoded images, zero EC hits and zero resident hits. +These are descriptive observations from the +[published cold results](https://github.com/Kelvin-Al/epd-provisioning/blob/3ed184e3a83229806a9fa1adb2bc75f1744c6518/experiments/ec-cache/results/cold/results.json). + +## Position-balanced cold follow-up + +A [frozen follow-up plan](https://github.com/Kelvin-Al/epd-provisioning/blob/a0037f7ddc4b1af5f59fbef0a86b2a5627b69feb/experiments/ec-cache/results/cold-order-followup/plan.json) +cycled encoder counts as `[1, 2, 3]`, `[2, 3, 1]` and `[3, 1, 2]`. +All nine waves ran in one four-H800 allocation on the original node. +The cohort, seed, cold checks, target arrival rate and measurement window +were held to the original settings. Each split occupied each position once. + +| Round | 2E:2PD position | 1E:3PD completions/s | 2E:2PD completions/s | 3E:1PD completions/s | +| --- | --- | ---: | ---: | ---: | +| 1 | Second | 23.59 | 26.44 | 13.00 | +| 2 | First | 23.52 | 26.04 | 13.55 | +| 3 | Third | 23.44 | 26.67 | 13.31 | + +2E:2PD ranked first in all three rounds, ahead of 1E:3PD by 12.1%, 10.7% +and 13.8%, respectively. All nine waves completed 720 requests with no +failures and recorded 720 visual forwards, 720 encoded images, zero EC +hits and zero resident hits per wave. The ranking thus held when 2E:2PD +ran first, second and third under the tested conditions. +See the [complete follow-up results](https://github.com/Kelvin-Al/epd-provisioning/blob/a0037f7ddc4b1af5f59fbef0a86b2a5627b69feb/experiments/ec-cache/results/cold-order-followup/results.json). + +An [interrupted follow-up attempt](https://github.com/Kelvin-Al/epd-provisioning/blob/a0037f7ddc4b1af5f59fbef0a86b2a5627b69feb/experiments/ec-cache/results/cold-order-followup/interrupted-results.json) +completed six waves before a port conflict on worker startup. The +comparison above uses only the complete rerun, with a launcher precheck +for available serving ports outside the system ephemeral range. + +## Applicability and limits + +The tested deployment used Qwen2.5-VL-3B-Instruct, a prepared vLLM 0.33.0 +overlay, a filesystem EC connector, one GPU per worker, one encoder sequence +at a time, 1 GiB PD KV storage and 512-token PD chunks. The cohort contained +720 single-image MMMU validation questions with fixed 64-token generation. +Of the images, 700 were resized into the encoder calibration range. +The model weight revision is absent from the published plan. + +Each wave started fresh workers and empty EC storage. A request-scoped +media key shared by E and PD required a new encode even for repeated images. +The target arrival rate was 34.27 requests/s. Actual seeded arrival rates +were 34.473–34.474 requests/s initially and 34.471–34.475 requests/s in +the follow-up. Completion counters use the middle 60% of +actual arrivals, about 12.5 seconds. The counts include all completed requests +after drain separately from the central-window rate. + +The follow-up observes each split and position combination once. It balances +positions within this workload and leaves limited evidence about predecessor +effects and between-wave variability. The observations cover a finite +arrival window at one offered rate. Sustained saturation, service SLOs +and MMMU answer accuracy remain unqualified. The first attempt at the +original cold comparison failed because its proxy assigned inconsistent +E/PD media keys. The corrected original run supplied the six-wave comparison. + +Recalibration is required when stage cost ratios, batch/concurrency limits, +hardware, model, transport or preprocessing change. This cold observation +does not establish empirical rankings for other GPU counts, non-multimodal +workloads, different cache policies or heterogeneous workers. Extra stages +and shared bottlenecks need an appropriate capacity model. + +For a conclusive future campaign, first agree and freeze its acceptance +criteria, sample size and repetitions. Bind the exact model and runtime, +retain independently checkable traces and workload bytes, qualify the +specified transport/preprocessing regime, and independently review the +complete serving comparison. More repetitions alone cannot repair missing +identity or acceptance evidence. + +## How to use this case + +1. Freeze GPU budget, model/runtime identity, preprocessing, output length, + routing, EC policy and the workload before selecting a configuration. +2. Estimate both stage rates independently for that workload. Check that + observation windows and calibration conditions match the serving run. +3. Calculate the integer split and record all tied optima. For four GPUs, + 1E:3PD exceeds 2E:2PD in this capacity model when mu_E > 2 * mu_PD. +4. Compare the recommendation and feasible alternatives with the same seeded + requests and arrival schedule. Counterbalance all positions so each split + appears equally often in each position. Counterbalance predecessor pairs + if carryover is a concern, and retain failures. +5. Audit actual visual work and cache decisions. Align E/PD success counters + to the same arrival window and inspect queues alongside TTFT and TPOT. +6. If the recommendation misses, inspect stage estimates and deployment + assumptions. Recalibrate changed conditions before applying the split. + Retain the previous deployment as the rollback configuration. + +The single E completed about 23 requests/s in this cold sweep and queued +173–175 requests at peak. With two E workers the pool kept up with about +34 requests/s, while PD completion remained about 26 requests/s. +This gives a concrete signal for distinguishing insufficient encoder +capacity from PD pressure under the tested conditions. + +## Evidence access + +- [Frozen cold plan](https://github.com/Kelvin-Al/epd-provisioning/blob/3ed184e3a83229806a9fa1adb2bc75f1744c6518/experiments/ec-cache/results/cold/plan.json). +- [External experiment issue](https://github.com/Kelvin-Al/epd-provisioning/issues/1). +- [Knowledge contribution scope](https://github.com/InferMatrixLab/InferRSI/issues/505). + +The source record retains SHA-256 values for the fetched published files. +Both evidence snapshots are pinned. Subsequent upstream changes leave these +snapshots historical. Further changes to the case observations require +rechecking the affected evidence and applicability at a pinned revision. +The underlying worker logs, image bytes, container and model weights were +not independently replayed or verified during extraction. Stage estimates +and measured counts remain bounded by that external evidence. diff --git a/tests/test_knowledge_contract.py b/tests/test_knowledge_contract.py index ba9cda2a1..43f6e2683 100644 --- a/tests/test_knowledge_contract.py +++ b/tests/test_knowledge_contract.py @@ -1,6 +1,7 @@ import json import os import re +import shutil from pathlib import Path import pytest @@ -58,7 +59,7 @@ def _case_records(): def test_case_documents_cite_their_record_without_copying_digests(): cases = list(_case_records()) - assert len(cases) == 7 + assert len(cases) == 8 for path, record in cases: body = (path.parent / record["content"]).read_text(encoding="utf-8") # The JSON record is the single home for source identities and digests; @@ -85,6 +86,43 @@ def test_case_document_links_resolve_inside_the_delivered_bundle(): assert data == (path.parent / link).read_bytes(), (path.name, link) +def test_epd_case_closure_identity_and_revision(tmp_path): + casebook = "inference-optimization-casebook" + source = skill_directory(ROOT, casebook) / "cases" + case_body = skill_reference(ROOT, casebook, "cases/epd-cold-provisioning.md") + assert case_body == source / "epd-cold-provisioning.md" + owner = tmp_path / "owner" + cases = owner / "cases" + cases.mkdir(parents=True) + (owner / "SKILL.md").write_text( + "---\nname: owner\ndescription: Test owner\n---\n", encoding="utf-8" + ) + for name in ("epd-cold-provisioning.md", "epd-cold-provisioning.json"): + shutil.copyfile(source / name, cases / name) + reader = KnowledgeReader(skill_root=tmp_path) + identity = "case.epd-cold-provisioning" + original = reader.read(identity) + record_path = cases / "epd-cold-provisioning.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + assert original.metadata["id"] == identity + assert original.metadata["sources"] == [ + {**source, "revision_precision": "full"} for source in record["sources"] + ] + assert original.content["epd-cold-provisioning.json"] == record_path.read_bytes() + assert original.content["case.json"] == record_path.read_bytes() + content_path = cases / record["content"] + content_path.write_bytes(content_path.read_bytes() + b"\n") + assert reader.read(identity).revision != original.revision + with pytest.raises(ValueError): + reader.resource(identity, record_path.name, revision=original.revision) + record_path.unlink() + # Keep the authority record discoverable, while removing its declared copy. + authority = cases / "authority.json" + authority.write_text(json.dumps(record), encoding="utf-8") + with pytest.raises(ValueError, match="Missing case resource"): + reader.read(identity) + + def test_contradictory_legacy_metadata_fails(tmp_path): directory = tmp_path / "selected" directory.mkdir() diff --git a/tests/test_knowledge_find.py b/tests/test_knowledge_find.py index 3b769cf4e..fd206e76b 100644 --- a/tests/test_knowledge_find.py +++ b/tests/test_knowledge_find.py @@ -63,7 +63,7 @@ def unexpected_view(*args, **kwargs): monkeypatch.setattr(catalog, "skill_views", unexpected_view) corpus = load_corpus(ROOT) - assert len(corpus.documents) == len({doc.identity for doc in corpus.documents}) == 135 + assert len(corpus.documents) == len({doc.identity for doc in corpus.documents}) == 136 def test_corpus_covers_all_three_kinds_without_double_counting(corpus): @@ -77,7 +77,7 @@ def test_corpus_covers_all_three_kinds_without_double_counting(corpus): assert navigation.isdisjoint(document.identity for document in corpus.documents) day0 = find("day-0 model integration on Ascend", corpus, limit=3) assert day0["items"] and _ranked(day0, "inference-task-router") is None - assert kinds["case"] == 7 + assert kinds["case"] == 8 assert kinds["record"] == 48 # Packaged-skill projections are excluded so one owner is not ranked twice. assert all( @@ -93,6 +93,15 @@ def test_cases_are_ranked_beside_skills(corpus): assert _ranked(result, "case.drama3-rejected-directions") == 1 +def test_epd_free_text_query_returns_an_openable_case(corpus, capsys): + result = find("EPD cold MMMU GPU allocation stage capacity", corpus, limit=5) + hit = next(item for item in result["items"] if item["id"] == "case.epd-cold-provisioning") + assert main(hit["read_args"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["id"] == "case.epd-cold-provisioning" + assert "## Position-balanced cold follow-up" in payload["content"] + + def test_findings_keywords_reach_a_problem_phrased_query(corpus): """The case title names a campaign; the query names the symptom.""" result = find("average batch size over the whole trace looks healthy", corpus, limit=5) @@ -283,7 +292,7 @@ def test_packaged_cases_declare_known_outcomes(): assert finding["direction"].strip() and finding["reason"].strip() assert finding["keywords"] total += 1 - assert total == 35 + assert total == 37 def test_result_carries_a_reference_the_caller_can_open(corpus):