From b8a342425e7979c24dd2c4b7859026c6c8fd9ed6 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Mon, 4 May 2026 20:06:58 +0200 Subject: [PATCH 01/22] docs: use dark README logo --- docs/assets/vaner-lockup-animated.svg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/assets/vaner-lockup-animated.svg b/docs/assets/vaner-lockup-animated.svg index f7c88d0..8946aa5 100644 --- a/docs/assets/vaner-lockup-animated.svg +++ b/docs/assets/vaner-lockup-animated.svg @@ -1,12 +1,12 @@ - - - - + + + + vaner_ From 36e67df2c38a7db769fbbe482f43d4bae28fe823 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Wed, 6 May 2026 23:02:09 +0200 Subject: [PATCH 02/22] Strengthen context preparation engine --- ...026-05-05-0.9.1-medium-release-canvas.html | 229 +++++ ...-05-05-medium-2h-prep-recovery-canvas.html | 323 +++++++ src/vaner/broker/answerable.py | 91 +- src/vaner/broker/assembler.py | 33 +- src/vaner/broker/compressor.py | 601 ++++++++++++- src/vaner/broker/context_preparation.py | 342 +++++++ src/vaner/broker/selector.py | 345 +++++++- src/vaner/cli/commands/config.py | 4 +- src/vaner/cli/commands/init.py | 2 +- src/vaner/cli/commands/setup.py | 3 +- src/vaner/clients/endpoint_pool.py | 13 +- src/vaner/clients/ollama.py | 12 +- src/vaner/daemon/engine/generator.py | 76 +- src/vaner/daemon/http.py | 86 ++ src/vaner/daemon/precompute_worker.py | 13 +- src/vaner/defaults/catalog_seed.json | 618 ++++++++++--- src/vaner/defaults/model_registry.json | 574 +++++++++--- src/vaner/engine.py | 441 +++++++++- src/vaner/intent/drafter.py | 3 + src/vaner/intent/evidence_resolver.py | 2 + src/vaner/intent/symbol_index.py | 73 +- src/vaner/models/__init__.py | 10 + src/vaner/models/config.py | 30 +- src/vaner/models/context.py | 4 + src/vaner/models/context_preparation.py | 84 ++ src/vaner/models/decision.py | 3 + src/vaner/policy/internal_llm.py | 43 + src/vaner/router/proxy.py | 24 +- src/vaner/semantic_aliases.py | 59 ++ src/vaner/setup/catalog_refresh.py | 114 ++- src/vaner/setup/config_io.py | 13 +- src/vaner/setup/hardware.py | 32 +- src/vaner/setup/model_recommendation.py | 378 +++++++- src/vaner/setup/serializers.py | 1 + src/vaner/store/artefacts.py | 82 +- src/vaner/store/scenarios/sqlite.py | 186 ++++ tests/test_broker/test_answerable.py | 65 ++ tests/test_broker/test_compressor.py | 138 ++- tests/test_broker/test_selector.py | 255 ++++++ tests/test_cli/test_config.py | 2 +- tests/test_clients/test_structured_output.py | 25 + tests/test_daemon/test_generator.py | 49 +- tests/test_daemon/test_http.py | 77 ++ tests/test_engine/test_context_budget.py | 34 + tests/test_engine/test_deep_drill.py | 4 + .../test_exploration_parallelism.py | 42 + .../test_structured_evidence_seeding.py | 80 ++ tests/test_intent/test_drafter.py | 4 + tests/test_intent/test_evidence_resolver.py | 56 ++ tests/test_intent/test_symbol_index.py | 47 + tests/test_policy/test_internal_llm.py | 35 + tests/test_setup/test_catalog_refresh.py | 56 +- tests/test_setup/test_hardware.py | 41 + tests/test_setup/test_model_recommendation.py | 270 +++++- tests/test_store/test_artefacts.py | 37 + tests/test_store/test_store_staleness.py | 50 ++ ui/cockpit/src/App.tsx | 246 +++++- ui/cockpit/src/api/adapt.ts | 2 + ui/cockpit/src/api/client.ts | 16 + ui/cockpit/src/components/CockpitViews.tsx | 52 ++ .../src/components/HeatmapReplayView.tsx | 831 ++++++++++++++++++ ui/cockpit/src/components/Inspector.tsx | 41 + .../src/components/LiveWorkInspector.tsx | 23 +- .../src/components/PipelineCanvas.test.tsx | 30 +- ui/cockpit/src/components/PipelineCanvas.tsx | 60 ++ .../src/components/ScenarioCluster.test.ts | 16 +- ui/cockpit/src/components/ScenarioCluster.tsx | 226 +++-- ui/cockpit/src/components/chrome.tsx | 9 +- ui/cockpit/src/lib/heatmap.test.ts | 179 ++++ ui/cockpit/src/lib/heatmap.ts | 394 +++++++++ ui/cockpit/src/styles/tokens.css | 11 + ui/cockpit/src/types.ts | 39 +- 72 files changed, 7996 insertions(+), 493 deletions(-) create mode 100644 docs/benchmarks/2026-05-05-0.9.1-medium-release-canvas.html create mode 100644 docs/benchmarks/2026-05-05-medium-2h-prep-recovery-canvas.html create mode 100644 src/vaner/broker/context_preparation.py create mode 100644 src/vaner/models/context_preparation.py create mode 100644 src/vaner/policy/internal_llm.py create mode 100644 src/vaner/semantic_aliases.py create mode 100644 tests/test_engine/test_context_budget.py create mode 100644 tests/test_policy/test_internal_llm.py create mode 100644 ui/cockpit/src/components/HeatmapReplayView.tsx create mode 100644 ui/cockpit/src/lib/heatmap.test.ts create mode 100644 ui/cockpit/src/lib/heatmap.ts diff --git a/docs/benchmarks/2026-05-05-0.9.1-medium-release-canvas.html b/docs/benchmarks/2026-05-05-0.9.1-medium-release-canvas.html new file mode 100644 index 0000000..1a59f15 --- /dev/null +++ b/docs/benchmarks/2026-05-05-0.9.1-medium-release-canvas.html @@ -0,0 +1,229 @@ + + + + + + Vaner 0.9.1 Benchmark Canvas + + + +
+
+
+
0.9.1 release validation · public canvas
+

Exact targeting is strong. Broad lifecycle prompts are the remaining bottleneck.

+

+ This run combines the 0.9.1 scenario benchmark, a broad public naked/RAG/Vaner comparison, + and the deep-run maturation suite. It uses public corpora or public-layout fixtures and reports + aggregate benchmark evidence without exposing local workspace details. +

+
+
+
Run date
+
2026-05-05
+
Models
+
Vaner: ollama:qwen3.6:27b
Answer: claude:sonnet
Judge: claude:opus
+
+
+ +
+ +
+
20/20
+
Scenario related coverage
100% related, 95% exact; promotion gate passed.
+
+ +
+
5/8
+
Quality A/B
Vaner-context answers won 62% of judged pairs.
+
+ +
+
8.04
+
Broad public quality
+3.24 vs naked, +0.41 vs RAG across 125 cases.
+
+ +
+
+0.842
+
Deep-run delta
effective pass; raw 3/5 gates passed with scaffold-only non-blocking gates
+
+
+ +
+
+

Layer Results

+ + + + + + + + + +
LayerBenchmarkResultRead
Prediction / exact targeting20 Vaner repo mechanism queries19 exact, 1 partial, 0 no-scenario cyclesExact targeting is now strong on named components; remaining partials are mostly broad process questions.
Answer quality8 Claude-judged pairs5 Vaner wins, 3 cold wins, 0 tiesWhen useful context is present, the answer improves; weak/empty context can lose.
Naked vs RAG vs Vaner125 public casesVaner quality 8.04 vs RAG 7.62 vs naked 4.80Vaner beats naked and edges RAG in this deterministic public harness, with similar hit/recall and lower estimated cost than RAG.
Deep-run maturation80 sessions / 80 outcomesMean improvement +0.842; stale rate 0%Strong reference-ceiling maturation signal; raw anti-self-judging gates remain visible and are non-blocking only in this scaffold mode.
Agent-with-tools layerHarness restored and CLI import validatedNot included as a scored release claimNeeds a configured OpenAI-compatible tool-calling answer endpoint and running Vaner daemon; not faked in this pass.
+
+ +
+

Answer The Question

+
+
1
When Vaner is betterBest on codebase tasks where the needed object/path is present in prepared context. The quality judge favored Vaner in 5 of 8 paired scenario answers, and the broad public harness shows +3.24 quality over naked and +0.41 over RAG.
+
2
Why it worksIt front-loads context selection before the final answer: exact component hints, symbol/path signals, and cached packages reduce answer-time search and can correct cold-answer misunderstandings.
+
3
Where it is weakerWeakness is precision under broad lifecycle prompts: 1 of 20 turns were only partial, with cold-start still under-specified. No turns lacked relevant context.
+
4
Cost and performanceScenario precompute averaged 46.3s; cache lookup averaged 6.0s. Warm answer latency was 61.0s vs 169.0s cold in the judged sample. Broad public estimated cost was $1.155 for Vaner vs $1.230 for RAG and $0.883 naked.
+
+
+
+ +
+
+

Scenario Gate

+ + + + + + + +
Engine LLMwired via ollama:qwen3.6:27b
Mean relevance0.87
No-scenario cycles0/20 (0%)
Gate status5/5 passed; PASS
+
+ +
+

Broad Public Arms

+ + + + + + + +
ArmHit@3Recall@5QualityCost
Naked0%0%4.80$0.883
RAG64%68%7.62$1.230
Vaner64%69%8.04$1.155
+
+ +
+

Eval/Fix Loop

+
+
A
Promote only with gatesSeparate coverage, quality, latency, cost, and leak gates. Failed gates become tracked fixes, not rewritten conclusions.
+
B
Debug by failure sliceFirst slice for 0.9.2: named-component queries that produced zero scenarios or weak primary evidence.
+
C
Publish clean evidencePublic reports should include methodology, aggregate results, limitations, and leak scans without naming local paths or internal workspaces.
+
+
+
+ +
+
+

Broad Public Archetypes

+ + + + + +
ArchetypeCasesVaner hit@3Vaner qualityLift
codebase navigation2532%7.030.40 vs RAG
developer swe2560%7.810.42 vs RAG
learner2568%8.110.40 vs RAG
researcher2564%8.170.45 vs RAG
writer2596%9.060.40 vs RAG
+
+
+

Deep-Run Archetypes

+ + + + + +
ArchetypeMean improvement delta
developer0.846
planner0.837
researcher0.859
writer0.825
+ +
+
+ + +
+ + diff --git a/docs/benchmarks/2026-05-05-medium-2h-prep-recovery-canvas.html b/docs/benchmarks/2026-05-05-medium-2h-prep-recovery-canvas.html new file mode 100644 index 0000000..12b49cf --- /dev/null +++ b/docs/benchmarks/2026-05-05-medium-2h-prep-recovery-canvas.html @@ -0,0 +1,323 @@ + + + + + + Vaner Medium 2h Preparation Recovery Canvas + + + +
+
+
+
Preparation engine validation · recovery canvas
+

Vaner helps AI stop starting from scratch. The reactive floor is strong; continuation still needs proof.

+

+ This canvas reframes the medium two-hour run around Vaner's real product claim: preparation before the prompt. + The corrected A/B quality slice is strong, but the telemetry also shows that normal continuation did not activate + during this run. +

+
+
+
Run date
+
2026-05-05
+
Models
+
Vaner prep: local qwen
Answer: claude:sonnet
Judge: claude:opus
+
+
+ +
+
+
4/4
+
Corrected quality A/B
Vaner-context answers won every recovered medium case.
+
+
+
7189s
+
Preparation observed
The two-hour window reached 99.8% before the original quality phase crashed.
+
+
+
0
+
Continuation rounds
No normal continuation activity was recorded in telemetry.
+
+
+
21
+
Ready predictions
73 total predictions at the end: 21 ready, 52 queued.
+
+
+ +
+
+

Layer Results

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LayerBenchmarkResultRead
Reactive floor4 Claude-judged repo mechanism questions4 Vaner wins, 0 naked wins, 0 tiesWhen prepared context is available, the primary AI answers with more exact files, identifiers, and mechanisms.
Preparation windowNormal background run, target 7200sLast sample at 7188.9s; 236 telemetry samplesVaner stayed active through almost the whole preparation window.
Prepared statePrediction snapshot at final sample73 predictions, 21 ready, 52 queued, 8 prepared-work cardsThe system accumulated useful prepared context, but visible prepared work did not grow during the run.
ContinuationNormal-background continuation profile0 nonzero continuation samplesThis run does not prove that Vaner kept branching, revisiting, or hardening likely next scenarios.
Harness reliabilityOriginal quality phaseCrashed before final quality outputThe recovered A/B is valid, but final release claims need the versioned release harness fixed.
+
+ +
+

Is Vaner Worth It?

+
+
1
Yes, when work has continuityVaner is useful when the same project, repo, research area, writing workflow, or team knowledge base comes up repeatedly.
+
2
Why this slice mattersEven when tested like reactive RAG, Vaner helped Sonnet avoid generic answers and use concrete repo evidence.
+
3
Where it is not enoughThe run does not yet demonstrate the larger product claim: that Vaner keeps preparing many plausible next scenarios over time.
+
4
Next proof neededRun a preparation-over-time benchmark: 0m, 5m, 15m, 60m, 2h, and overnight, with prepared-work hit rate and stale rate.
+
+
+
+ +
+
+

Corrected A/B Cases

+ + + + + + + + +
CaseWinnerJudge read
Reward computationVanerConcrete files and signal mappings; medium hallucination risk for naming variance.
IntentScorer GBDTVanerStrong file paths, feature names, fallback, calibration, and blend details.
LLM exploration flowVanerConcrete JSON contract, parser, ranking, and follow-on behavior.
ArtefactStore schemaVanerConcrete schema and methods; medium hallucination risk around one uncertain cache-table detail.
+
+ +
+

Quality Comparison

+
+
+ Naked Sonnet wins +
+ 0 +
+
+ Ties +
+ 0 +
+
+ Vaner-context wins +
+ 4 +
+
+ +
+ +
+

Preparation Spend

+ + + + + + + + +
Model starts118
Model completions118
Max GPU utilization sample1.00
Queue coalesced53
Queue dropped11
+ +
+
+ +
+
+

Before / After Pattern

+ + + + + + + + + + + + + + + + + + + +
Without VanerWith VanerUser-visible improvement
Generic answer or caveat that repo details are unknown.Specific files such as learning/reward.py, intent/scorer.py, engine.py, and store/artefacts.py.Less setup, fewer clarifying turns, better grounded follow-up work.
Broad architecture pattern.Exact constants, methods, schema fields, and flow stages when evidence was present.The assistant starts from project evidence instead of rediscovery.
Safer but often non-actionable uncertainty.More useful specificity, but still needs hallucination-risk checks in public claims.Better answer quality with a clear need for evidence-precision gates.
+
+ +
+

Release-safe Language

+
+
OK
Safe claimIn a recovered medium quality slice after background preparation, Vaner context helped Claude Sonnet produce better answers than naked prompting on all four tested mechanism questions.
+
NO
Do not claim yetDo not say this run proves Vaner continuously explores and hardens many future scenarios over a two-hour idle window.
+
FIT
Best fitRecurring project work, codebases, research, writing, planning, and team knowledge where prepared context can be reused.
+
LOW
Weak fitGeneric one-off prompts, vague unrelated asks, and tasks where the base model already knows enough.
+
+
+
+ +
+
+

Needed Harness Fixes

+
+
A
Normalize context payloadsStructured Vaner payloads must be converted to text before prompt assembly.
+
B
Keep A/B artifacts explicitMark recovered, failed, and release-harness files separately.
+
C
Gate hallucination riskMedium-risk wins need identifier checks before marketing copy.
+
+
+ +
+

Preparation Benchmark Next

+ + + + + + + + + + +
WindowMeasure
0mCold reactive floor.
5mFast preparation payoff.
15mShort work-session continuity.
60mIdle lunch-break preparation.
2hNormal background exploration, not deep-run.
OvernightDeep preparation and stale-rate pressure.
+
+ +
+

Metrics To Add

+
+
H
Prepared Work Hit RateHow often useful work was ready before the user asked.
+
E
Evidence PrecisionWhether prepared evidence was actually the decisive evidence.
+
S
Stale RateHow often prepared work was outdated or irrelevant.
+
C
Compounding ScoreWhether usefulness improves over repeated sessions.
+
+
+
+ + +
+ + diff --git a/src/vaner/broker/answerable.py b/src/vaner/broker/answerable.py index 19417a9..903ca09 100644 --- a/src/vaner/broker/answerable.py +++ b/src/vaner/broker/answerable.py @@ -5,6 +5,7 @@ import re from pathlib import Path +from vaner.broker.compressor import EvidenceSpan, extract_evidence_spans from vaner.broker.selector import _prompt_terms from vaner.models.answerable import ( Answerability, @@ -43,9 +44,9 @@ def build_answerable_briefing( cost_sensitivity: str = "balanced", ) -> AnswerableBriefing: terms = _prompt_terms(query) - scored = [ - _build_item(query, terms, artefact, repo_root=repo_root, channel=_channel_for(artefact, channels_by_key)) for artefact in artefacts - ] + scored: list[tuple[float, int, AnswerableEvidenceItem]] = [] + for artefact in artefacts: + scored.extend(_build_items(query, terms, artefact, repo_root=repo_root, channel=_channel_for(artefact, channels_by_key))) scored.sort(key=lambda row: (row[0], -row[1], row[2].path), reverse=True) direct: list[AnswerableEvidenceItem] = [] @@ -170,6 +171,8 @@ def build_answerable_briefing( } ) transport_limited_count = dropped_direct + dropped_supporting + dropped_lower + if assembly_mode == "safe" and (direct_truncated or supporting_truncated or lower_truncated): + transport_limited_count = max(1, transport_limited_count) if transport_limited_count: assembly_plan = _mark_transport_limited(assembly_plan, direct + supporting + lower, transport_limited_count) truncation_applied = direct_truncated or supporting_truncated or lower_truncated or dropped_direct > 0 @@ -280,35 +283,84 @@ def build_answerable_briefing_from_text( return AnswerableBriefing(text=text, answer_plan=answer_plan, sections=sections, metadata=metadata) -def _build_item( +def _build_items( query: str, terms: list[str], artefact: Artefact, *, repo_root: Path | None, channel: EvidenceChannel, -) -> tuple[float, int, AnswerableEvidenceItem]: +) -> list[tuple[float, int, AnswerableEvidenceItem]]: source_text = _source_text(artefact, repo_root) - excerpt, truncated = _best_excerpt(source_text or artefact.content, terms) + source_artefact = artefact.model_copy(update={"content": source_text}) if source_text else artefact path = artefact.source_path + spans = extract_evidence_spans(source_artefact, query, base_score=float(artefact.relevance_score or 0.0), max_spans_per_file=7) + if spans: + return [_item_from_span(query, terms, artefact, span, channel=channel) for span in spans] + + excerpt, truncated = _best_excerpt(source_text or artefact.content, terms) score = _score_path_and_excerpt(query, path.lower(), excerpt.lower(), terms) title = _title_for(path, excerpt) confidence = max(0.0, min(1.0, score / 24.0)) + return [ + ( + score, + -len(path), + AnswerableEvidenceItem( + path=path, + title=title, + source=str(artefact.metadata.get("corpus_id", "default")), + channel=channel, + why_selected=_why_selected(path, terms, score), + excerpt=excerpt, + relevance_to_query=_relevance(query, path, terms, score), + confidence=confidence, + token_count=count_tokens(excerpt), + revision_or_hash=str(artefact.metadata.get("revision") or artefact.metadata.get("hash") or "") or None, + truncated=truncated, + ), + ) + ] + + +def _item_from_span( + query: str, + terms: list[str], + artefact: Artefact, + span: EvidenceSpan, + *, + channel: EvidenceChannel, +) -> tuple[float, int, AnswerableEvidenceItem]: + path = artefact.source_path + role_bonus = { + "direct": 8.0, + "constant_or_default": 6.0, + "schema_or_storage": 6.0, + "caller_or_downstream": 5.0, + "test_or_example": 4.0, + "supporting": 1.0, + "provenance": 0.5, + }[span.role] + score = span.score + role_bonus + _score_path_and_excerpt(query, path.lower(), span.excerpt.lower(), terms) + confidence = max(0.0, min(1.0, score / 32.0)) + symbol = f" {span.symbol}" if span.symbol else "" + title = f"{span.role}{symbol} L{span.start_line}-{span.end_line}" + why = f"{_why_selected(path, terms, score)}; role={span.role}; lines={span.start_line}-{span.end_line}; {span.reason}" return ( score, - -len(path), + -span.start_line, AnswerableEvidenceItem( path=path, title=title, source=str(artefact.metadata.get("corpus_id", "default")), channel=channel, - why_selected=_why_selected(path, terms, score), - excerpt=excerpt, + why_selected=why, + excerpt=span.excerpt, relevance_to_query=_relevance(query, path, terms, score), confidence=confidence, - token_count=count_tokens(excerpt), + token_count=count_tokens(span.excerpt), revision_or_hash=str(artefact.metadata.get("revision") or artefact.metadata.get("hash") or "") or None, - truncated=truncated, + truncated=False, ), ) @@ -440,6 +492,15 @@ def _mark_transport_limited( marked += 1 else: updated.append(decision) + if transport_limited_count and marked == 0 and updated: + updated[-1] = updated[-1].model_copy( + update={ + "decision": "transport_limited", + "reason": "evidence was constrained by the caller transport limit", + "would_change_output_in_shadow": True, + } + ) + marked = 1 return assembly.model_copy( update={ "items_transport_limited": assembly.items_transport_limited + max(transport_limited_count, marked), @@ -465,6 +526,14 @@ def _dedupe_key(item: AnswerableEvidenceItem) -> str: def _protected_reason(item: AnswerableEvidenceItem, role: str, conflict_paths: set[str]) -> str: if role == "direct_answer_evidence": return "direct answer evidence" + for evidence_role, reason in ( + ("role=constant_or_default", "constant/default evidence"), + ("role=schema_or_storage", "schema/storage evidence"), + ("role=caller_or_downstream", "caller/downstream evidence"), + ("role=test_or_example", "test/example evidence"), + ): + if evidence_role in item.why_selected: + return reason if item.path in conflict_paths: return "conflict evidence" if item.channel == "prediction": diff --git a/src/vaner/broker/assembler.py b/src/vaner/broker/assembler.py index 9f0aa81..43d6b96 100644 --- a/src/vaner/broker/assembler.py +++ b/src/vaner/broker/assembler.py @@ -12,6 +12,7 @@ from vaner.models.answerable import EvidenceAssemblyMode from vaner.models.artefact import Artefact from vaner.models.context import ContextPackage, ContextSelection +from vaner.models.context_preparation import PreparedContextDiagnostics from vaner.models.decision import DecisionRecord, ScoreFactor, SelectionDecision from vaner.policy.staleness import is_stale_timestamp @@ -42,6 +43,7 @@ def assemble_context_package( evidence_assembly_mode: EvidenceAssemblyMode = "shadow", evidence_assembly_quality_bias: str = "protect_recall", evidence_assembly_cost_sensitivity: str = "balanced", + prepared_context_diagnostics: PreparedContextDiagnostics | None = None, return_decision: bool = False, ) -> ContextPackage | tuple[ContextPackage, DecisionRecord]: resolved_score_map = score_map or {artefact.key: score_artefact(prompt, artefact) for artefact in artefacts} @@ -49,6 +51,7 @@ def assemble_context_package( artefacts, max_tokens=max_tokens, score_by_key=resolved_score_map, + query=prompt, ) selection_decisions: list[SelectionDecision] = [] context_selections: list[ContextSelection] = [] @@ -104,7 +107,7 @@ def assemble_context_package( ) answerable = build_answerable_briefing( prompt, - [artefact for artefact in artefacts if artefact.key in kept_keys], + artefacts, repo_root=repo_root, max_tokens=max_tokens, channels_by_key={ @@ -121,6 +124,12 @@ def assemble_context_package( ) context_package.answerable_briefing = answerable context_package.answerability_metadata = answerable.metadata + context_package.prepared_context_briefing = answerable.text + context_package.prepared_context_mode = _prepared_context_mode(prepared_context_diagnostics, answerable.metadata.answerability) + if prepared_context_diagnostics is not None: + prepared_context_diagnostics.token_used = used + prepared_context_diagnostics.truncation_risk = answerable.metadata.truncation_risk + context_package.prepared_context_diagnostics = prepared_context_diagnostics decision_record = DecisionRecord( id=context_package.id, prompt=prompt, @@ -129,7 +138,29 @@ def assemble_context_package( token_budget=max_tokens, token_used=used, selections=selection_decisions, + prepared_context_diagnostics=prepared_context_diagnostics, ) if return_decision: return context_package, decision_record return context_package + + +def _prepared_context_mode(diagnostics: PreparedContextDiagnostics | None, answerability: str) -> str: + if diagnostics is None: + return "answerable" + need = diagnostics.profile.need + if need == "implementation_support": + return "implementation" + if need == "research_mapping": + return "research" + if need == "creative_grounding": + return "writing" + if need == "decision_support": + return "decision" + if need == "conflict_resolution": + return "conflict_resolution" + if need == "absence_check": + return "absence_check" + if answerability in {"full", "weak", "conflict"}: + return "answerable" + return "planning" diff --git a/src/vaner/broker/compressor.py b/src/vaner/broker/compressor.py index f78e35f..4a6bb79 100644 --- a/src/vaner/broker/compressor.py +++ b/src/vaner/broker/compressor.py @@ -2,23 +2,613 @@ from __future__ import annotations +import ast +import re +from dataclasses import dataclass +from typing import Literal + +from vaner.broker.selector import _prompt_terms from vaner.models.artefact import Artefact from vaner.policy.budget import count_tokens +EvidenceSpanRole = Literal[ + "direct", + "constant_or_default", + "schema_or_storage", + "caller_or_downstream", + "test_or_example", + "supporting", + "provenance", +] + + +@dataclass(frozen=True) +class EvidenceSpan: + path: str + start_line: int + end_line: int + role: EvidenceSpanRole + symbol: str + excerpt: str + score: float + protected: bool + reason: str + + +def _trim_chunk_to_budget(chunk: str, max_tokens: int) -> str: + if max_tokens <= 0: + return "" + if count_tokens(chunk) <= max_tokens: + return chunk + first_line, _, remainder = chunk.partition("\n") + marker = "\n...[trimmed]\n" + header = first_line.strip() + if count_tokens(header + marker) > max_tokens: + max_chars = max(1, max_tokens * 4) + return (header + "\n")[:max_chars].rstrip() + max_chars = max(120, max_tokens * 4) + trimmed = (header + "\n" + remainder[:max_chars]).rstrip() + while trimmed and count_tokens(trimmed + marker) > max_tokens: + trimmed = trimmed[: max(0, len(trimmed) - 120)].rstrip() + return trimmed + marker if trimmed else header + marker + + +def _chunk_for_artefact(artefact: Artefact, *, max_chunk_tokens: int) -> str: + header = f"### {artefact.source_path}\n" + content = artefact.content or "" + if max_chunk_tokens <= 0: + return header.rstrip() + full = f"{header}{content}\n" + if count_tokens(full) <= max_chunk_tokens: + return full + sections = _important_lines(content) + if sections: + candidate = f"{header}" + "\n".join(sections) + "\n...[compacted to important implementation anchors]\n" + if count_tokens(candidate) <= max_chunk_tokens: + return candidate + return _trim_chunk_to_budget(full, max_chunk_tokens) + + +def _important_lines(content: str) -> list[str]: + important: list[str] = [] + patterns = ( + r"^(Classes|Functions|Constants|Limits|Schema):", + r"\b(CREATE TABLE|CREATE INDEX|USING fts5|INSERT INTO|SELECT .* FROM)\b", + r"\b(class|def|async def)\s+[A-Za-z_][A-Za-z0-9_]*", + r"\b(max_|min_|limit|threshold|timeout|ttl|budget|window|top_n|top_k)\w*\b", + ) + for line in content.splitlines(): + stripped = line.strip() + if not stripped: + continue + if any(re.search(pattern, stripped, flags=re.IGNORECASE) for pattern in patterns): + important.append(stripped[:240]) + if len(important) >= 24: + break + return important + + +def extract_evidence_spans( + artefact: Artefact, + query: str, + *, + base_score: float = 0.0, + max_spans_per_file: int = 8, +) -> list[EvidenceSpan]: + """Return query-aware, line-exact spans for one artefact. + + The extractor is deliberately deterministic. It preserves code and config + ingredients as copied source spans instead of asking a model to summarize + them, which keeps identifiers, constants, table names, and parser contracts + exact under tight context budgets. + """ + + text = artefact.content or "" + lines = text.splitlines() + if not lines: + return [] + terms = _prompt_terms(query) + if not terms: + terms = [part for part in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", query.lower()) if len(part) > 2] + candidates = _structured_candidates(artefact.source_path, lines, terms) + candidates.extend(_line_candidates(artefact.source_path, lines, terms)) + candidates = _merge_candidate_windows(candidates) + + spans: list[EvidenceSpan] = [] + for start, end, role, symbol, score, reason in candidates: + excerpt = _numbered_excerpt(lines, start, end) + if not excerpt.strip(): + continue + adjusted = score + (base_score * 0.08) + spans.append( + EvidenceSpan( + path=artefact.source_path, + start_line=start, + end_line=end, + role=role, + symbol=symbol, + excerpt=excerpt, + score=adjusted, + protected=role in {"direct", "constant_or_default", "schema_or_storage", "caller_or_downstream", "test_or_example"}, + reason=reason, + ) + ) + spans.sort(key=lambda span: (_role_priority(span.role), span.protected, span.score, -span.start_line), reverse=True) + return _diverse_spans(spans, max_spans_per_file=max_spans_per_file) + + +def _structured_candidates( + path: str, + lines: list[str], + terms: list[str], +) -> list[tuple[int, int, EvidenceSpanRole, str, float, str]]: + if not path.endswith(".py"): + return [] + text = "\n".join(lines) + try: + tree = ast.parse(text) + except SyntaxError: + return [] + + candidates: list[tuple[int, int, EvidenceSpanRole, str, float, str]] = [] + imports = [ + node.lineno + for node in tree.body + if isinstance(node, (ast.Import, ast.ImportFrom)) and _node_line_score(lines[node.lineno - 1], terms, path) > 0 + ] + if imports: + candidates.append( + (max(1, min(imports) - 1), min(len(lines), max(imports) + 1), "provenance", "imports", 4.0, "matched import provenance") + ) + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + symbol = node.name + start = node.lineno + end = min(len(lines), getattr(node, "end_lineno", node.lineno)) + block_text = "\n".join(lines[start - 1 : end]).lower() + score = _symbol_score(symbol, terms) + _text_term_score(block_text, terms) + anchor = _block_anchor_score(symbol, block_text) + if score <= 0 and anchor <= 0: + continue + semantic_role = _role_for_block(path, symbol, block_text) + role = semantic_role if semantic_role != "supporting" else "direct" + max_lines = 3 if isinstance(node, ast.ClassDef) else 42 + span_start, span_end = _focused_block_window(lines, start, end, terms, max_lines=max_lines) + candidates.append((span_start, span_end, role, symbol, score + anchor + 3.0, f"matched {symbol} block")) + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + target_names = _assignment_targets(node) + if not target_names: + continue + start = node.lineno + end = min(len(lines), getattr(node, "end_lineno", node.lineno)) + line_text = "\n".join(lines[start - 1 : end]) + score = max((_symbol_score(name, terms) for name in target_names), default=0.0) + _node_line_score(line_text, terms, path) + if score <= 0 and not _constantish(line_text): + continue + candidates.append( + ( + max(1, start - 1), + min(len(lines), end + 1), + "constant_or_default", + ", ".join(target_names[:3]), + score + 5.0, + "preserved constant/default assignment", + ) + ) + return candidates + + +def _line_candidates( + path: str, + lines: list[str], + terms: list[str], +) -> list[tuple[int, int, EvidenceSpanRole, str, float, str]]: + candidates: list[tuple[int, int, EvidenceSpanRole, str, float, str]] = [] + for index, line in enumerate(lines, start=1): + stripped = line.strip() + if path.endswith(".py") and stripped.startswith("#"): + continue + score = _node_line_score(line, terms, path) + if score <= 0: + continue + role = _role_for_text(path, line) + if path.endswith(".py"): + radius = 0 if re.search(r"\b(class|def|async def)\s+[A-Za-z_][A-Za-z0-9_]*", line) else 1 + else: + radius = 1 if role in {"constant_or_default", "schema_or_storage"} else 2 + symbol = _symbol_from_line(line) + candidates.append( + ( + max(1, index - radius), + min(len(lines), index + radius), + role, + symbol, + score, + f"matched {role.replace('_', ' ')} line", + ) + ) + return candidates + + +def _node_line_score(text: str, terms: list[str], path: str) -> float: + lowered = text.lower() + path_lowered = path.lower() + score = 0.0 + content_hit_score = 0.0 + for term in terms: + if len(term) < 3: + continue + if term in lowered: + content_hit_score += 4.0 if "_" not in term else 7.0 + if term in path_lowered: + score += 1.5 + if content_hit_score <= 0: + score = 0.0 + score += content_hit_score + if re.search(r"\b(class|def|async def)\s+[A-Za-z_][A-Za-z0-9_]*", text): + score += 2.0 + if _constantish(text): + score += 4.0 + if re.search(r"\b(CREATE TABLE|CREATE INDEX|USING fts5|ALTER TABLE|INSERT INTO|SELECT\b.+\bFROM)\b", text, re.IGNORECASE): + score += 7.0 + if re.search(r"\b(json\.loads|json\.dumps|JSON_CONTRACT|ranked_files|follow_on|response_format|schema)\b", text): + score += 5.0 + if re.search(r"\b(reward_total|reward_components|cache_tier|quality_lift|judge_score|similarity|raw_reward)\b", text): + score += 5.0 + return score + + +def _text_term_score(text: str, terms: list[str]) -> float: + return sum(2.0 if term in text else 0.0 for term in terms if len(term) > 3) + + +def _symbol_score(symbol: str, terms: list[str]) -> float: + symbol_terms = set(_identifier_parts(symbol)) + score = 0.0 + for term in terms: + if term == symbol.lower(): + score += 10.0 + elif term in symbol_terms: + score += 5.0 + elif len(term) > 3 and term in symbol.lower(): + score += 3.0 + return score + + +def _block_anchor_score(symbol: str, block_text: str) -> float: + score = 0.0 + symbol_lower = symbol.lower() + if any(part in symbol_lower for part in ("reward", "feature", "train", "blend", "store", "retrieve", "persist", "schema", "cache")): + score += 2.0 + if any(anchor in block_text for anchor in ("create table", "json.loads", "reward_total", "threshold", "default", "ranked_files")): + score += 2.0 + return score + + +def _assignment_targets(node: ast.Assign | ast.AnnAssign) -> list[str]: + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + names: list[str] = [] + for target in targets: + if isinstance(target, ast.Name): + names.append(target.id) + elif isinstance(target, ast.Attribute): + names.append(target.attr) + elif isinstance(target, (ast.Tuple, ast.List)): + names.extend(elt.id for elt in target.elts if isinstance(elt, ast.Name)) + return names + + +def _constantish(text: str) -> bool: + return bool( + re.search( + r"\b([A-Z][A-Z0-9_]{2,}|default|defaults|threshold|timeout|ttl|limit|weight|weights|ratio|budget|target|schema|contract)\b", + text, + re.IGNORECASE, + ) + and "=" in text + ) + + +def _role_for_text(path: str, text: str) -> EvidenceSpanRole: + lowered = text.lower() + if path.startswith(("tests/", "test/")) or "/tests/" in path or "assert " in lowered: + return "test_or_example" + if re.search(r"\b(create table|create index|using fts5|alter table|schema|select\b.+\bfrom|insert into)\b", lowered): + return "schema_or_storage" + if re.search(r"\b(default|threshold|timeout|ttl|limit|weight|weights|ratio|budget|target|contract)\b", lowered) and "=" in text: + return "constant_or_default" + if re.search(r"\b(call|caller|consumer|downstream|insert|select)\b|retrieve|persist|compute_|load_|save_|train|blend|parse", lowered): + return "caller_or_downstream" + if re.search(r"\b(class|def|async def)\s+[A-Za-z_][A-Za-z0-9_]*", text): + return "direct" + return "supporting" + + +def _role_for_block(path: str, symbol: str, block_text: str) -> EvidenceSpanRole: + semantic = _role_for_text(path, block_text) + if semantic in {"schema_or_storage", "test_or_example"}: + return semantic + lowered_symbol = symbol.lower() + if re.search(r"\b(downstream|consumer|caller|train|target|blend|parse|json\.loads|retrieve|persist|insert|select)\b", block_text): + return "caller_or_downstream" + if any(part in lowered_symbol for part in ("train", "target", "blend", "parse", "retrieve", "persist")): + return "caller_or_downstream" + if semantic == "constant_or_default": + return semantic + return "direct" if block_text.strip() else "supporting" + + +def _role_priority(role: EvidenceSpanRole) -> int: + return { + "direct": 70, + "constant_or_default": 60, + "schema_or_storage": 55, + "caller_or_downstream": 50, + "test_or_example": 45, + "supporting": 25, + "provenance": 10, + }[role] + + +def _focused_block_window(lines: list[str], start: int, end: int, terms: list[str], *, max_lines: int) -> tuple[int, int]: + if end - start + 1 <= max_lines: + return start, end + best_line = start + best_score = -1.0 + for index in range(start, end + 1): + score = _node_line_score(lines[index - 1], terms, "") + if score > best_score: + best_line = index + best_score = score + half = max_lines // 2 + window_start = max(start, best_line - half) + window_end = min(end, window_start + max_lines - 1) + window_start = max(start, window_end - max_lines + 1) + return window_start, window_end + + +def _merge_candidate_windows( + candidates: list[tuple[int, int, EvidenceSpanRole, str, float, str]], +) -> list[tuple[int, int, EvidenceSpanRole, str, float, str]]: + ordered = sorted(candidates, key=lambda row: (row[0], row[1], -row[4])) + merged: list[tuple[int, int, EvidenceSpanRole, str, float, str]] = [] + for candidate in ordered: + start, end, role, symbol, score, reason = candidate + if not merged or start > merged[-1][1] + 1: + merged.append(candidate) + continue + prev_start, prev_end, prev_role, prev_symbol, prev_score, prev_reason = merged[-1] + best_role = role if _merge_role_priority(role) > _merge_role_priority(prev_role) else prev_role + best_symbol = symbol or prev_symbol + merged[-1] = ( + prev_start, + max(prev_end, end), + best_role, + best_symbol, + max(prev_score, score), + prev_reason if prev_score >= score else reason, + ) + return merged + + +def _merge_role_priority(role: EvidenceSpanRole) -> int: + return { + "schema_or_storage": 80, + "constant_or_default": 75, + "caller_or_downstream": 70, + "test_or_example": 65, + "direct": 55, + "supporting": 25, + "provenance": 10, + }[role] + + +def _diverse_spans(spans: list[EvidenceSpan], *, max_spans_per_file: int) -> list[EvidenceSpan]: + kept: list[EvidenceSpan] = [] + seen_roles: set[str] = set() + for span in spans: + if span.role in seen_roles and len([item for item in kept if item.role == span.role]) >= 3: + continue + kept.append(span) + seen_roles.add(span.role) + if len(kept) >= max_spans_per_file: + break + kept.sort(key=lambda span: (_role_priority(span.role), span.score), reverse=True) + return kept + + +def _numbered_excerpt(lines: list[str], start: int, end: int) -> str: + width = len(str(end)) + return "\n".join(f"L{line_no:0{width}d}: {lines[line_no - 1]}" for line_no in range(start, end + 1)) + + +def _symbol_from_line(line: str) -> str: + match = re.search(r"\b(?:class|def|async def)\s+([A-Za-z_][A-Za-z0-9_]*)", line) + if match: + return match.group(1) + match = re.search(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*=", line) + if match: + return match.group(1) + return "" + + +def _identifier_parts(identifier: str) -> list[str]: + pieces = re.split(r"[^A-Za-z0-9]+", identifier) + parts: list[str] = [] + for piece in pieces: + if not piece: + continue + parts.append(piece.lower()) + parts.extend(part.lower() for part in re.findall(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)|\d+", piece)) + return [part for part in parts if len(part) > 2] + + +def _chunk_for_spans(path: str, spans: list[EvidenceSpan]) -> str: + parts = [f"### {path}"] + for span in spans: + parts.append(_span_entry(span)) + return "\n".join(parts).rstrip() + "\n" + + +def _span_entry(span: EvidenceSpan) -> str: + symbol = f" symbol={span.symbol}" if span.symbol else "" + protected = " protected" if span.protected else "" + return f"@@ lines {span.start_line}-{span.end_line} role={span.role}{symbol}{protected}\n{span.excerpt}" + + +def _compress_context_with_spans( + artefacts: list[Artefact], + max_tokens: int, + query: str, + score_by_key: dict[str, float] | None, +) -> tuple[str, dict[str, int], int, set[str]]: + token_map: dict[str, int] = {} + kept_spans_by_key: dict[str, list[EvidenceSpan]] = {artefact.key: [] for artefact in artefacts} + spans_by_key: dict[str, list[EvidenceSpan]] = {} + ordered_keys = [artefact.key for artefact in artefacts] + if score_by_key is not None: + ordered_keys.sort(key=lambda key: score_by_key.get(key, 0.0), reverse=True) + artefacts_by_key = {artefact.key: artefact for artefact in artefacts} + for artefact in artefacts: + spans = extract_evidence_spans(artefact, query, base_score=(score_by_key or {}).get(artefact.key, 0.0)) + if not spans: + fallback = _chunk_for_artefact( + artefact, + max_chunk_tokens=max(96, min(max_tokens, max_tokens // max(1, min(3, len(artefacts) or 1)))), + ) + pseudo_span = EvidenceSpan( + path=artefact.source_path, + start_line=1, + end_line=max(1, len((artefact.content or "").splitlines())), + role="supporting", + symbol="", + excerpt=fallback, + score=(score_by_key or {}).get(artefact.key, 0.0), + protected=False, + reason="fallback compacted artefact", + ) + spans = [pseudo_span] if fallback.strip() else [] + spans_by_key[artefact.key] = spans + + role_limits = _role_token_limits(max_tokens) + role_used = {role: 0 for role in role_limits} + role_kept = {role: 0 for role in role_limits} + used = 0 + seen_signatures: set[str] = set() + all_spans: list[tuple[str, EvidenceSpan]] = [] + for key in ordered_keys: + for span in spans_by_key.get(key, []): + all_spans.append((key, span)) + all_spans.sort( + key=lambda row: ( + row[1].protected, + _role_priority(row[1].role), + row[1].score, + (score_by_key or {}).get(row[0], 0.0), + ), + reverse=True, + ) + + for key, span in all_spans: + header_tokens = 0 if kept_spans_by_key[key] else count_tokens(f"### {artefacts_by_key[key].source_path}\n") + span_tokens = header_tokens + count_tokens(_span_entry(span)) + signature = _span_signature(span) + if signature in seen_signatures: + continue + role_limit = role_limits.get(span.role, role_limits["supporting"]) + if role_used.get(span.role, 0) + span_tokens > role_limit: + if not span.protected or role_kept.get(span.role, 0) >= 3: + continue + if used + span_tokens > max_tokens: + if span.protected: + trimmed = _trim_span(span, max(48, max_tokens - used - header_tokens)) + trimmed_tokens = header_tokens + count_tokens(_span_entry(trimmed)) + if used + trimmed_tokens > max_tokens: + continue + span = trimmed + span_tokens = trimmed_tokens + else: + continue + kept_spans_by_key[key].append(span) + seen_signatures.add(signature) + role_used[span.role] = role_used.get(span.role, 0) + span_tokens + role_kept[span.role] = role_kept.get(span.role, 0) + 1 + used += span_tokens + + kept_keys = {key for key, spans in kept_spans_by_key.items() if spans} + if not kept_keys and ordered_keys and max_tokens > 0: + key = ordered_keys[0] + chunk = _trim_chunk_to_budget(_chunk_for_artefact(artefacts_by_key[key], max_chunk_tokens=max_tokens), max_tokens) + if chunk: + token_map[key] = count_tokens(chunk) + return chunk, token_map, token_map[key], {key} + + chunks: list[str] = [] + for key in ordered_keys: + spans = kept_spans_by_key[key] + if not spans: + token_map[key] = count_tokens(_chunk_for_artefact(artefacts_by_key[key], max_chunk_tokens=256)) + continue + spans.sort(key=lambda span: (_role_priority(span.role), span.score), reverse=True) + chunk = _chunk_for_spans(artefacts_by_key[key].source_path, spans) + token_map[key] = count_tokens(chunk) + chunks.append(chunk) + used = sum(token_map[key] for key in kept_keys) + return "\n".join(chunks).strip(), token_map, used, kept_keys + + +def _role_token_limits(max_tokens: int) -> dict[EvidenceSpanRole, int]: + minimum = min(max_tokens, 96) + return { + "direct": max(minimum, int(max_tokens * 0.48)), + "constant_or_default": max(minimum // 2, int(max_tokens * 0.14)), + "schema_or_storage": max(minimum // 2, int(max_tokens * 0.12)), + "caller_or_downstream": max(minimum // 2, int(max_tokens * 0.12)), + "test_or_example": max(minimum // 2, int(max_tokens * 0.08)), + "supporting": max(minimum // 2, int(max_tokens * 0.05)), + "provenance": max(24, int(max_tokens * 0.01)), + } + + +def _span_signature(span: EvidenceSpan) -> str: + normalized = re.sub(r"\s+", " ", span.excerpt.lower()).strip() + return f"{span.role}:{normalized[:220]}" + + +def _trim_span(span: EvidenceSpan, budget: int) -> EvidenceSpan: + excerpt = _trim_chunk_to_budget(span.excerpt, budget) + return EvidenceSpan( + path=span.path, + start_line=span.start_line, + end_line=span.end_line, + role=span.role, + symbol=span.symbol, + excerpt=excerpt, + score=span.score, + protected=span.protected, + reason=f"{span.reason}; trimmed to fit transport budget", + ) + def compress_context( artefacts: list[Artefact], max_tokens: int, score_by_key: dict[str, float] | None = None, + query: str | None = None, ) -> tuple[str, dict[str, int], int, set[str]]: + if query: + return _compress_context_with_spans(artefacts, max_tokens, query, score_by_key) + token_map: dict[str, int] = {} chunk_map: dict[str, str] = {} ordered_keys: list[str] = [] kept_keys: set[str] = set() used = 0 + per_chunk_cap = max(512, min(max_tokens, max_tokens // 3 if max_tokens >= 4096 else max_tokens)) for artefact in artefacts: - chunk = f"### {artefact.source_path}\n{artefact.content}\n" + chunk = _chunk_for_artefact(artefact, max_chunk_tokens=per_chunk_cap) chunk_tokens = count_tokens(chunk) token_map[artefact.key] = chunk_tokens chunk_map[artefact.key] = chunk @@ -29,9 +619,16 @@ def compress_context( # Greedy knapsack approximation: pack highest-score artefacts first while # still trying lower-ranked/smaller chunks that fit remaining budget. - for key in ordered_keys: + for index, key in enumerate(ordered_keys): chunk_tokens = token_map[key] if used + chunk_tokens > max_tokens: + if index == 0 and not kept_keys: + trimmed = _trim_chunk_to_budget(chunk_map[key], max_tokens) + if trimmed: + chunk_map[key] = trimmed + token_map[key] = count_tokens(trimmed) + kept_keys.add(key) + used += token_map[key] continue kept_keys.add(key) used += chunk_tokens diff --git a/src/vaner/broker/context_preparation.py b/src/vaner/broker/context_preparation.py new file mode 100644 index 0000000..becb5b9 --- /dev/null +++ b/src/vaner/broker/context_preparation.py @@ -0,0 +1,342 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import re +from collections import Counter + +from vaner.models.artefact import Artefact +from vaner.models.context_preparation import ( + ContextConstraint, + ContextCoverageReport, + ContextFacet, + ContextPreparationProfile, + ContextSourceStats, + PreparedContextDiagnostics, +) + +_PATH_RE = re.compile(r"\b(?:[\w.-]+/)+[\w.-]+\b") +_QUOTED_RE = re.compile(r"[`'\"]([^`'\"]{3,100})[`'\"]") +_DATE_RE = re.compile( + r"\b(?:\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}/\d{2,4}|" + r"jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|" + r"jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\b", + re.IGNORECASE, +) +_RESTRICTIVE_RE = re.compile( + r"\b(?:only|except|before|after|latest|current|superseded|newest|oldest|no later than|at least|at most)\b", + re.IGNORECASE, +) + + +def infer_context_preparation_profile(prompt: str) -> ContextPreparationProfile: + lowered = prompt.lower() + archetype = _infer_archetype(lowered) + need = _infer_need(lowered, archetype) + constraints = _extract_constraints(prompt) + facets = _extract_facets(prompt, constraints) + source_hints = _extract_source_hints(lowered) + expected_evidence_count = _expected_evidence_count(lowered, need) + confidence = 0.55 + min(0.35, 0.05 * len(facets) + 0.06 * len(constraints)) + notes: list[str] = [] + if any(constraint.kind == "restrictive_language" for constraint in constraints): + notes.append("hard_constraints_detected") + if need in {"multi_source_synthesis", "conflict_resolution", "research_mapping"}: + notes.append("coverage_sensitive") + return ContextPreparationProfile( + need=need, + archetype=archetype, + facets=facets, + constraints=constraints, + source_hints=source_hints, + expected_evidence_count=expected_evidence_count, + confidence=min(0.95, confidence), + notes=notes, + ) + + +def query_variants(prompt: str, profile: ContextPreparationProfile, *, max_variants: int = 6) -> list[str]: + variants = [prompt] + facet_terms = [facet.value for facet in profile.facets if len(facet.value) > 2] + constraint_terms = [constraint.value for constraint in profile.constraints if constraint.kind != "restrictive_language"] + if facet_terms: + variants.append(" ".join(facet_terms[:8])) + if constraint_terms: + variants.append(" ".join([*constraint_terms[:4], *facet_terms[:6]])) + if profile.need in {"multi_source_synthesis", "research_mapping", "decision_support"}: + variants.append(" ".join([*facet_terms[:10], "summary status decision evidence"])) + if profile.need == "conflict_resolution": + variants.append(" ".join([*facet_terms[:8], "current superseded conflict updated latest"])) + if profile.need == "implementation_support": + variants.append(" ".join([*facet_terms[:10], "implementation caller dependency test"])) + if profile.need == "absence_check": + variants.append(" ".join([*facet_terms[:8], "source truth reference policy"])) + deduped = [] + for variant in variants: + normalized = " ".join(variant.split()) + if normalized and normalized.lower() not in {item.lower() for item in deduped}: + deduped.append(normalized) + if len(deduped) >= max(1, max_variants): + break + return deduped + + +def exact_reference_candidates(prompt: str, available_paths: list[str], *, limit: int = 32) -> list[str]: + references = set(_PATH_RE.findall(prompt)) + references.update(match.group(1).strip() for match in _QUOTED_RE.finditer(prompt)) + identifier_terms = { + token.lower() + for token in re.findall(r"\b[A-Za-z_][A-Za-z0-9_]{3,}\b", prompt) + if "_" in token or any(char.isupper() for char in token[1:]) + } + ranked: list[tuple[int, str]] = [] + for path in available_paths: + lowered_path = path.lower() + score = 0 + for reference in references: + ref = reference.strip().lower() + if not ref: + continue + if ref == lowered_path or lowered_path.endswith(ref): + score += 12 + elif ref in lowered_path: + score += 6 + basename = lowered_path.rsplit("/", 1)[-1] + stem = basename.rsplit(".", 1)[0] + for term in identifier_terms: + if term == stem or term in basename: + score += 5 + elif term in lowered_path: + score += 2 + if score: + ranked.append((score, path)) + ranked.sort(key=lambda item: (-item[0], item[1])) + return [path for _, path in ranked[:limit]] + + +def hard_constraints_satisfied(prompt: str, artefact: Artefact, profile: ContextPreparationProfile) -> tuple[bool, list[str], list[str]]: + text = f"{artefact.source_path}\n{artefact.content}".lower() + satisfied: list[str] = [] + missing: list[str] = [] + for constraint in profile.constraints: + value = constraint.value.lower() + if constraint.kind == "restrictive_language": + satisfied.append(constraint.value) + continue + if value in text: + satisfied.append(constraint.value) + elif constraint.required: + missing.append(constraint.value) + if _constraint_terms(prompt) and not satisfied and profile.need in {"direct_reference", "conflict_resolution", "absence_check"}: + return False, satisfied, missing + return not missing, satisfied, missing + + +def competitive_threshold_multiplier(need: str | None) -> float: + if need in {"multi_source_synthesis", "conflict_resolution", "research_mapping", "decision_support"}: + return 0.0 + if need in {"implementation_support", "absence_check", "working_set_extension", "creative_grounding"}: + return 0.20 + if need == "evidence_gathering": + return 0.30 + return 0.45 + + +def build_coverage_report( + profile: ContextPreparationProfile, + selected: list[Artefact], + source_by_key: dict[str, set[str]] | None = None, +) -> ContextCoverageReport: + selected_text = "\n".join(f"{artefact.source_path}\n{artefact.content}" for artefact in selected).lower() + covered_facets = [facet.value for facet in profile.facets if facet.value.lower() in selected_text] + missing_constraints = [ + constraint.value + for constraint in profile.constraints + if constraint.kind != "restrictive_language" and constraint.required and constraint.value.lower() not in selected_text + ] + direct_evidence_count = sum(1 for artefact in selected if _direct_match_count(profile, artefact) > 0) + source_sets = [source_by_key.get(artefact.key, set()) for artefact in selected] if source_by_key else [] + source_agreement = sum(len(sources) for sources in source_sets) / max(1, len(source_sets)) + conflict_pair_coverage = profile.need != "conflict_resolution" or _has_conflict_pair(selected_text) + compactness_risk = "high" if len(selected) > max(8, profile.expected_evidence_count * 3) else "medium" if len(selected) > 6 else "low" + gap_flags: list[str] = [] + if missing_constraints: + gap_flags.append("missing_constraints") + if len(covered_facets) < min(len(profile.facets), profile.expected_evidence_count): + gap_flags.append("facet_coverage_weak") + if profile.need == "conflict_resolution" and not conflict_pair_coverage: + gap_flags.append("conflict_pair_missing") + actionability = "full" + if gap_flags: + actionability = "weak" + if not selected: + actionability = "none" + if profile.need == "conflict_resolution" and conflict_pair_coverage: + actionability = "conflict" + return ContextCoverageReport( + covered_facets=covered_facets, + missing_constraints=missing_constraints, + direct_evidence_count=direct_evidence_count, + conflict_pair_coverage=conflict_pair_coverage, + actionability=actionability, + source_agreement=min(1.0, source_agreement / 2.0), + weak_expansion_dependency=any( + source_by_key and source_by_key.get(artefact.key) == {"generated_query_variants"} for artefact in selected + ), + compactness_risk=compactness_risk, + gap_flags=gap_flags, + ) + + +def build_prepared_context_diagnostics( + profile: ContextPreparationProfile, + *, + source_by_key: dict[str, set[str]], + selected: list[Artefact], + fused_candidate_count: int, + latency_ms: float, +) -> PreparedContextDiagnostics: + selected_keys = {artefact.key for artefact in selected} + source_counter: Counter[str] = Counter() + selected_counter: Counter[str] = Counter() + for key, sources in source_by_key.items(): + for source in sources: + source_counter[source] += 1 + if key in selected_keys: + selected_counter[source] += 1 + constraints = [constraint.value for constraint in profile.constraints] + coverage = build_coverage_report(profile, selected, source_by_key) + return PreparedContextDiagnostics( + profile=profile, + source_counts=[ + ContextSourceStats(source=source, candidate_count=count, selected_count=selected_counter.get(source, 0)) + for source, count in sorted(source_counter.items()) + ], + fused_candidate_count=fused_candidate_count, + selected_count=len(selected), + hard_constraints_extracted=constraints, + hard_constraints_satisfied=[value for value in constraints if value not in coverage.missing_constraints], + hard_constraints_missing=coverage.missing_constraints, + coverage=coverage, + weak_expansion_dependency=coverage.weak_expansion_dependency, + truncation_risk=coverage.truncation_risk, + latency_ms=latency_ms, + compactness_score=1.0 if coverage.compactness_risk == "low" else 0.65 if coverage.compactness_risk == "medium" else 0.35, + provenance_coverage=sum(1 for artefact in selected if source_by_key.get(artefact.key)) / max(1, len(selected)), + ) + + +def _infer_archetype(lowered: str): + if re.search(r"\b(file|symbol|function|class|module|bug|test|diff|pr|commit|implementation|dependency|stack trace)\b", lowered): + return "developer" + if re.search(r"\b(draft|outline|tone|audience|rewrite|section|claim|essay|article|copy)\b", lowered): + return "writer" + if re.search(r"\b(paper|study|dataset|method|citation|literature|hypothesis|experiment|result)\b", lowered): + return "researcher" + if re.search(r"\b(incident|runbook|sla|handoff|customer|support|ops|oncall|status|risk)\b", lowered): + return "operator" + return "general" + + +def _infer_need(lowered: str, archetype: str): + if re.search(r"\b(conflict|contradict|superseded|outdated|latest|current|newer|older|reconcile)\b", lowered): + return "conflict_resolution" + if re.search(r"\b(not found|unavailable|missing|is there|do we have|absence|not available|cannot find)\b", lowered): + return "absence_check" + if archetype == "developer" and re.search(r"\b(implement|fix|debug|change|affected|callers|tests?|dependency|regression)\b", lowered): + return "implementation_support" + if archetype == "writer": + return "creative_grounding" + if archetype == "researcher": + return "research_mapping" + if re.search(r"\b(all|every|list|compare|across|summarize|synthesize|count|which .* and|what .* and)\b", lowered): + return "multi_source_synthesis" + if re.search(r"\b(decide|decision|recommend|tradeoff|risk|should we|plan)\b", lowered): + return "decision_support" + if re.search(r"\b(where|defined|introduced|implemented|exact|which file|what is)\b", lowered): + return "direct_reference" + if re.search(r"\b(continue|next|finish|carry on|pick up)\b", lowered): + return "task_continuation" + return "evidence_gathering" + + +def _extract_constraints(prompt: str) -> list[ContextConstraint]: + constraints: list[ContextConstraint] = [] + for path in _PATH_RE.findall(prompt): + constraints.append(ContextConstraint(kind="path", value=path)) + for quoted in _QUOTED_RE.findall(prompt): + constraints.append(ContextConstraint(kind="quoted_reference", value=quoted.strip())) + for match in _DATE_RE.finditer(prompt): + constraints.append(ContextConstraint(kind="date", value=match.group(0))) + for match in _RESTRICTIVE_RE.finditer(prompt): + constraints.append(ContextConstraint(kind="restrictive_language", value=match.group(0).lower())) + return _dedupe_constraints(constraints) + + +def _extract_facets(prompt: str, constraints: list[ContextConstraint]) -> list[ContextFacet]: + constrained_values = {constraint.value.lower() for constraint in constraints} + facets: list[ContextFacet] = [] + for token in re.findall(r"\b[A-Za-z][A-Za-z0-9_-]{3,}\b", prompt): + lowered = token.lower() + if lowered in constrained_values or lowered in {"what", "where", "when", "which", "does", "with", "from", "that", "this", "should"}: + continue + required = any(char.isupper() for char in token[1:]) or "_" in token or "-" in token + facets.append(ContextFacet(name="term", value=token, required=required)) + return _dedupe_facets(facets[:16]) + + +def _extract_source_hints(lowered: str) -> list[str]: + hints = [] + for hint in ("slack", "gmail", "email", "github", "jira", "linear", "confluence", "docs", "meeting", "runbook", "calendar"): + if hint in lowered: + hints.append(hint) + return hints + + +def _expected_evidence_count(lowered: str, need: str) -> int: + if need in {"multi_source_synthesis", "research_mapping"}: + return 4 + if need == "conflict_resolution": + return 2 + if re.search(r"\b(all|every|complete|completeness|list)\b", lowered): + return 5 + return 1 + + +def _constraint_terms(prompt: str) -> set[str]: + return {match.group(0).lower() for match in _RESTRICTIVE_RE.finditer(prompt)} + + +def _direct_match_count(profile: ContextPreparationProfile, artefact: Artefact) -> int: + text = f"{artefact.source_path}\n{artefact.content}".lower() + return sum(1 for facet in profile.facets if facet.value.lower() in text) + + +def _has_conflict_pair(text: str) -> bool: + current = any(term in text for term in ("current", "latest", "updated", "newer", "now")) + superseded = any(term in text for term in ("superseded", "old", "older", "deprecated", "previous")) + return current and superseded + + +def _dedupe_constraints(constraints: list[ContextConstraint]) -> list[ContextConstraint]: + result = [] + seen = set() + for constraint in constraints: + key = (constraint.kind, constraint.value.lower()) + if key in seen: + continue + seen.add(key) + result.append(constraint) + return result + + +def _dedupe_facets(facets: list[ContextFacet]) -> list[ContextFacet]: + result = [] + seen = set() + for facet in facets: + key = facet.value.lower() + if key in seen: + continue + seen.add(key) + result.append(facet) + return result diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index 2b0ab33..bfe82c6 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -8,8 +8,18 @@ from collections.abc import Callable from fnmatch import fnmatch +from vaner.broker.context_preparation import ( + build_prepared_context_diagnostics, + competitive_threshold_multiplier, + exact_reference_candidates, + hard_constraints_satisfied, + infer_context_preparation_profile, + query_variants, +) from vaner.models.artefact import Artefact +from vaner.models.context_preparation import ContextPreparationProfile, PreparedContextDiagnostics from vaner.models.decision import ScoreFactor +from vaner.semantic_aliases import engineering_semantic_aliases def _recency_bonus(artefact: Artefact, decay_half_life_seconds: int = 1800) -> float: @@ -41,6 +51,7 @@ def _prompt_terms(prompt: str) -> list[str]: terms.extend(part.lower() for part in raw.split("_") if len(part) > 2) if lowered not in {"fastapi", "openapi"}: terms.extend(part.lower() for part in _camel_parts(raw) if len(part) > 2) + terms.extend(engineering_semantic_aliases(prompt, terms, stopwords=_COMMON_WORDS)) return list(dict.fromkeys(term for term in terms if len(term) > 2 and term not in _COMMON_WORDS)) @@ -310,12 +321,18 @@ def _doc_domain_bonus(path_text: str, content_text: str, terms: list[str]) -> fl if path_text in { "src/vaner/intent/features.py", "src/vaner/intent/trainer.py", + "src/vaner/intent/scorer.py", "src/vaner/models/artefact.py", "src/vaner/store/artefacts.py", "tests/test_intent/test_features_follow_up.py", "tests/test_intent/test_trainer_v4_rollover.py", }: bonus += 14.0 + if "intent" in term_set and "scorer" in term_set and path_text in { + "src/vaner/intent/scorer.py", + "src/vaner/intent/features.py", + }: + bonus += 10.0 if any( term in content_text for term in ( @@ -348,6 +365,39 @@ def _doc_domain_bonus(path_text: str, content_text: str, terms: list[str]) -> fl bonus += 2.0 if any(term in content_text for term in ("full_hit", "partial_hit", "warm_start", "cache_full_hit", "cache_partial_hit")): bonus += 8.0 + reward_terms = {"reward", "computation", "compute", "signals", "combine", "final", "value", "quality", "lift", "judge"} + if "reward" in term_set and len(term_set & reward_terms) >= 3: + if path_text in { + "src/vaner/learning/reward.py", + "eval/train_policy.py", + "src/vaner/intent/trainer.py", + "tests/test_learning/test_reward.py", + }: + bonus += 18.0 + if any( + term in content_text + for term in ( + "compute_reward", + "reward_total", + "reward_components", + "quality_lift", + "host_outcome", + "judge_score", + ) + ): + bonus += 10.0 + if path_text == "src/vaner/intent/scoring_policy.py": + bonus -= 4.0 + store_schema_terms = {"artefactstore", "artefact", "artefacts", "persist", "retrieve", "database", "schema", "table", "tables"} + if len(term_set & store_schema_terms) >= 3: + if path_text == "src/vaner/store/artefacts.py": + bonus += 24.0 + if path_text == "src/vaner/models/context.py": + bonus += 8.0 + if "package.json" in path_text: + bonus -= 22.0 + if any(term in content_text for term in ("create table", "from artefacts", "insert into artefacts", "select key")): + bonus += 12.0 llm_exploration_terms = { "llm", "external", @@ -441,8 +491,15 @@ def _origin_bonus(prompt: str, content: str) -> float: def _build_fts_query(prompt: str) -> str: """Build a safe FTS5 query string from a natural-language prompt.""" - filtered = _prompt_terms(prompt)[:15] - return " ".join(filtered) + filtered = _prompt_terms(prompt)[:24] + return " OR ".join(filtered) + + +def _candidate_limit(prompt: str, top_n: int) -> int: + terms = _prompt_terms(prompt) + multi_facet = prompt.count("?") + len(re.findall(r"\b(?:and|what|how|when|which)\b", prompt, flags=re.IGNORECASE)) + requested = max(50, int(top_n) * 12, len(terms) * 8, multi_facet * 12) + return max(50, min(240, requested)) async def select_artefacts_fts( @@ -457,8 +514,16 @@ async def select_artefacts_fts( path_excludes: list[str] | None = None, capture_factors: dict[str, list[ScoreFactor]] | None = None, capture_drop_reasons: dict[str, str] | None = None, + candidate_limit: int | None = None, + full_scan_limit: int = 2000, + context_preparation_mode: str = "balanced", + max_query_variants: int = 6, + max_candidate_keys: int | None = None, + coverage_floor_enabled: bool = True, + max_expansion_passes: int = 1, + capture_prepared_context_diagnostics: list[PreparedContextDiagnostics] | None = None, ) -> list[Artefact]: - """Two-phase selection: FTS candidate retrieval, then scorer re-rank. + """Multi-source context candidate retrieval, then scorer re-rank. Falls back to loading all artefacts when the FTS index returns no hits or when *store* does not expose ``select_artefacts_fts``. @@ -466,37 +531,132 @@ async def select_artefacts_fts( from vaner.store.artefacts import ArtefactStore # avoid circular at module level fts_available = isinstance(store, ArtefactStore) - - # Phase 1: FTS candidate retrieval (gracefully skipped if unavailable) - candidate_keys: set[str] = set() + started = time.monotonic() + profile = infer_context_preparation_profile(prompt) + context_enabled = context_preparation_mode != "legacy" + + source_by_key: dict[str, set[str]] = {} + source_rankings: dict[str, list[str]] = {} + retrieval_limit = candidate_limit if candidate_limit is not None else _candidate_limit(prompt, top_n) + if max_candidate_keys is not None: + retrieval_limit = max(50, min(retrieval_limit, max_candidate_keys)) if fts_available: - fts_query = _build_fts_query(prompt) - if fts_query: + variants = query_variants(prompt, profile, max_variants=max_query_variants) if context_enabled else [prompt] + for index, variant in enumerate(variants): + fts_query = _build_fts_query(variant) + if not fts_query: + continue + source = "lexical_search" if index == 0 else "generated_query_variants" + try: + keys = list(await store.select_artefacts_fts(fts_query, limit=retrieval_limit)) # type: ignore[union-attr] + except Exception: + keys = [] + _record_source_ranking(source_rankings, source_by_key, source, keys) + if context_enabled: try: - candidate_keys = set(await store.select_artefacts_fts(fts_query, limit=50)) # type: ignore[union-attr] + available_paths = await store.list_source_paths(limit=max(5000, retrieval_limit * 10)) # type: ignore[union-attr] except Exception: - candidate_keys = set() + available_paths = [] + exact_paths = set(exact_reference_candidates(prompt, available_paths, limit=min(32, max(8, top_n * 3)))) + else: + exact_paths = set() + else: + exact_paths = set() + + preferred = preferred_keys or set() + preferred_paths_set = preferred_paths or set() + if preferred: + _record_source_ranking(source_rankings, source_by_key, "working_set", list(preferred)) + + fused_keys = _fuse_source_rankings(source_rankings, limit=max_candidate_keys or retrieval_limit) - # Phase 2: load candidates; fall back to full list on FTS miss if fts_available: - all_artefacts: list[Artefact] = await store.list(limit=2000) # type: ignore[union-attr] + if fused_keys or exact_paths or preferred: + loaded: list[Artefact] = [] + load_keys = set(fused_keys) | preferred + loaded.extend(await store.list_by_keys(load_keys, limit=max(retrieval_limit, len(load_keys)))) # type: ignore[union-attr] + path_loads = set(preferred_paths_set) | exact_paths + if path_loads: + path_loaded = await store.list_by_source_paths(path_loads, limit=max(retrieval_limit, len(path_loads))) # type: ignore[union-attr] + loaded.extend(path_loaded) + _record_source_ranking(source_rankings, source_by_key, "exact_reference", [artefact.key for artefact in path_loaded]) + seen: set[str] = set() + candidates = [] + for artefact in loaded: + if artefact.key in seen: + continue + seen.add(artefact.key) + candidates.append(artefact) + selected = select_artefacts( + prompt, + candidates, + top_n=top_n, + preferred_paths=preferred_paths, + preferred_keys=preferred_keys, + scorer=scorer, + exclude_private=exclude_private, + path_bonuses=path_bonuses, + path_excludes=path_excludes, + capture_factors=capture_factors, + capture_drop_reasons=capture_drop_reasons, + context_need=profile.need if context_enabled else None, + context_profile=profile if context_enabled else None, + ) + if context_enabled and coverage_floor_enabled and max_expansion_passes > 0: + diagnostics = build_prepared_context_diagnostics( + profile, + source_by_key=source_by_key, + selected=selected, + fused_candidate_count=len(fused_keys), + latency_ms=(time.monotonic() - started) * 1000.0, + ) + if diagnostics.coverage.gap_flags: + recovery_terms = _coverage_recovery_terms(profile, diagnostics) + recovery_query = _build_fts_query(recovery_terms) + if recovery_query: + try: + recovery_keys = list(await store.select_artefacts_fts(recovery_query, limit=retrieval_limit)) # type: ignore[union-attr] + except Exception: + recovery_keys = [] + new_keys = [key for key in recovery_keys if key not in {candidate.key for candidate in candidates}] + if new_keys: + _record_source_ranking(source_rankings, source_by_key, "coverage_floor", new_keys) + recovered = await store.list_by_keys(set(new_keys), limit=max(retrieval_limit, len(new_keys))) # type: ignore[union-attr] + candidates.extend(recovered) + selected = select_artefacts( + prompt, + candidates, + top_n=top_n, + preferred_paths=preferred_paths, + preferred_keys=preferred_keys, + scorer=scorer, + exclude_private=exclude_private, + path_bonuses=path_bonuses, + path_excludes=path_excludes, + capture_factors=capture_factors, + capture_drop_reasons=capture_drop_reasons, + context_need=profile.need, + context_profile=profile, + ) + _append_source_factors(capture_factors, source_by_key, selected) + if capture_prepared_context_diagnostics is not None: + capture_prepared_context_diagnostics.append( + build_prepared_context_diagnostics( + profile, + source_by_key=source_by_key, + selected=selected, + fused_candidate_count=len(fused_keys), + latency_ms=(time.monotonic() - started) * 1000.0, + ) + ) + return selected + all_artefacts: list[Artefact] = await store.list(limit=full_scan_limit) # type: ignore[union-attr] else: return [] - if candidate_keys: - candidates = [a for a in all_artefacts if a.key in candidate_keys] - # Always include preferred items even if not in FTS results - preferred = preferred_keys or set() - preferred_paths_set = preferred_paths or set() - for a in all_artefacts: - if a.key not in candidate_keys and (a.key in preferred or a.source_path in preferred_paths_set): - candidates.append(a) - else: - candidates = all_artefacts - - return select_artefacts( + selected = select_artefacts( prompt, - candidates, + all_artefacts, top_n=top_n, preferred_paths=preferred_paths, preferred_keys=preferred_keys, @@ -506,7 +666,108 @@ async def select_artefacts_fts( path_excludes=path_excludes, capture_factors=capture_factors, capture_drop_reasons=capture_drop_reasons, + context_need=profile.need if context_enabled else None, + context_profile=profile if context_enabled else None, ) + if capture_prepared_context_diagnostics is not None: + capture_prepared_context_diagnostics.append( + build_prepared_context_diagnostics( + profile, + source_by_key=source_by_key, + selected=selected, + fused_candidate_count=len(all_artefacts), + latency_ms=(time.monotonic() - started) * 1000.0, + ) + ) + return selected + + +def _record_source_ranking( + source_rankings: dict[str, list[str]], + source_by_key: dict[str, set[str]], + source: str, + keys: list[str], +) -> None: + deduped = list(dict.fromkeys(key for key in keys if key)) + if not deduped: + return + source_rankings.setdefault(source, []) + seen = set(source_rankings[source]) + for key in deduped: + source_by_key.setdefault(key, set()).add(source) + if key not in seen: + source_rankings[source].append(key) + seen.add(key) + + +def _fuse_source_rankings(source_rankings: dict[str, list[str]], *, limit: int) -> list[str]: + weights = { + "exact_reference": 1.45, + "prepared_context_cache": 1.25, + "working_set": 1.20, + "lexical_search": 1.0, + "generated_query_variants": 0.72, + "relationship_graph": 0.9, + "semantic_memory": 0.75, + "coverage_floor": 0.85, + } + scores: dict[str, float] = {} + for source, keys in source_rankings.items(): + weight = weights.get(source, 0.8) + for rank, key in enumerate(keys, start=1): + scores[key] = scores.get(key, 0.0) + weight / (60.0 + rank) + return [ + key + for key, _ in sorted( + scores.items(), + key=lambda item: (-item[1], item[0]), + )[: max(1, limit)] + ] + + +def _append_source_factors( + capture_factors: dict[str, list[ScoreFactor]] | None, + source_by_key: dict[str, set[str]], + selected: list[Artefact], +) -> None: + if capture_factors is None: + return + for artefact in selected: + sources = sorted(source_by_key.get(artefact.key, set())) + if not sources: + continue + capture_factors.setdefault(artefact.key, []).append( + ScoreFactor( + name="context_sources", + contribution=min(2.0, 0.25 * len(sources)), + detail="candidate appeared in context sources: " + ", ".join(sources), + ) + ) + + +def _diversity_bucket(artefact: Artefact, context_need: str | None) -> str: + path = artefact.source_path + if context_need == "implementation_support": + parts = path.split("/") + return "/".join(parts[:3]) if len(parts) >= 3 else path + source_type = str(artefact.metadata.get("source_type") or artefact.metadata.get("corpus_id") or "") + if source_type: + return source_type + parts = path.split("/") + return "/".join(parts[:2]) if len(parts) >= 2 else path + + +def _coverage_recovery_terms(profile: ContextPreparationProfile, diagnostics: PreparedContextDiagnostics) -> str: + covered = {item.lower() for item in diagnostics.coverage.covered_facets} + missing_facets = [facet.value for facet in profile.facets if facet.value.lower() not in covered] + terms = [*diagnostics.coverage.missing_constraints, *missing_facets[:10]] + if profile.need == "conflict_resolution": + terms.extend(["current", "superseded", "latest", "updated"]) + elif profile.need == "multi_source_synthesis": + terms.extend(["summary", "status", "evidence"]) + elif profile.need == "implementation_support": + terms.extend(["caller", "dependency", "test", "implementation"]) + return " ".join(term for term in terms if term) def select_artefacts( @@ -521,6 +782,8 @@ def select_artefacts( path_excludes: list[str] | None = None, capture_factors: dict[str, list[ScoreFactor]] | None = None, capture_drop_reasons: dict[str, str] | None = None, + context_need: str | None = None, + context_profile: ContextPreparationProfile | None = None, ) -> list[Artefact]: preferred_paths = preferred_paths or set() preferred_keys = preferred_keys or set() @@ -538,6 +801,13 @@ def select_artefacts( if capture_drop_reasons is not None: capture_drop_reasons[artefact.key] = "path_excluded" continue + satisfied_constraints: list[str] = [] + if context_profile is not None: + constraints_ok, satisfied_constraints, missing_constraints = hard_constraints_satisfied(prompt, artefact, context_profile) + if not constraints_ok: + if capture_drop_reasons is not None: + capture_drop_reasons[artefact.key] = "missing_hard_constraints:" + ",".join(missing_constraints[:3]) + continue factors: list[ScoreFactor] = [] if scorer is not None: score = scorer(prompt, artefact) @@ -605,6 +875,16 @@ def select_artefacts( ) ) score += pinned_path_bonus + if context_profile is not None and satisfied_constraints: + constraint_bonus = min(2.0, 0.35 * len(satisfied_constraints)) + factors.append( + ScoreFactor( + name="hard_constraint_match", + contribution=constraint_bonus, + detail="artefact satisfies extracted hard constraints", + ) + ) + score += constraint_bonus if capture_factors is not None: capture_factors[artefact.key] = factors scored_rows.append((score, artefact)) @@ -612,19 +892,32 @@ def select_artefacts( ranked = sorted(scored_rows, key=lambda item: item[0], reverse=True) selected: list[Artefact] = [] seen_corpora: set[str] = set() - min_competitive_score = ranked[0][0] * 0.45 if ranked else 0.0 + threshold_multiplier = competitive_threshold_multiplier(context_need) + min_competitive_score = ranked[0][0] * threshold_multiplier if ranked else 0.0 + selected_buckets: set[str] = set() for score, artefact in ranked: if score < min_competitive_score: if capture_drop_reasons is not None: capture_drop_reasons[artefact.key] = "below_competitive_threshold" continue corpus_id = str(artefact.metadata.get("corpus_id", "default")) + bucket = _diversity_bucket(artefact, context_need) + if ( + context_need in {"multi_source_synthesis", "conflict_resolution", "research_mapping", "decision_support"} + and bucket in selected_buckets + and len(selected) < max(1, top_n // 2) + ): + if capture_drop_reasons is not None: + capture_drop_reasons[artefact.key] = "deferred_for_context_diversity" + continue if selected and corpus_id not in seen_corpora: selected.append(artefact) seen_corpora.add(corpus_id) + selected_buckets.add(bucket) elif len(selected) < top_n: selected.append(artefact) seen_corpora.add(corpus_id) + selected_buckets.add(bucket) if len(selected) >= top_n: break if capture_drop_reasons is not None: diff --git a/src/vaner/cli/commands/config.py b/src/vaner/cli/commands/config.py index 68e848d..9f85191 100644 --- a/src/vaner/cli/commands/config.py +++ b/src/vaner/cli/commands/config.py @@ -166,6 +166,8 @@ def load_config(repo_root: Path) -> VanerConfig: "exploration_backend": exploration_section.get("backend", "auto"), "embedding_model": exploration_section.get("embedding_model", "all-MiniLM-L6-v2"), "embedding_device": exploration_section.get("embedding_device", "cpu"), + "runtime_options": exploration_section.get("runtime_options", {}), + "sampling_options": exploration_section.get("sampling_options", {}), } exploration = ExplorationConfig(**mapped_exploration) else: @@ -177,7 +179,7 @@ def load_config(repo_root: Path) -> VanerConfig: policy = _build_section(PolicyConfig, "policy", policy_section) max_age_seconds = _coerce_limit(limits_section, "max_age_seconds", 3600) - max_context_tokens = _coerce_limit(limits_section, "max_context_tokens", 4096) + max_context_tokens = _coerce_limit(limits_section, "max_context_tokens", 8192) store_path = repo_root / ".vaner" / "store.db" telemetry_path = repo_root / ".vaner" / "telemetry.db" diff --git a/src/vaner/cli/commands/init.py b/src/vaner/cli/commands/init.py index cbc0cd0..7aeac5e 100644 --- a/src/vaner/cli/commands/init.py +++ b/src/vaner/cli/commands/init.py @@ -121,7 +121,7 @@ [limits] max_age_seconds = 3600 -max_context_tokens = 4096 +max_context_tokens = 8192 [intent] enabled = true diff --git a/src/vaner/cli/commands/setup.py b/src/vaner/cli/commands/setup.py index 5a50510..5595006 100644 --- a/src/vaner/cli/commands/setup.py +++ b/src/vaner/cli/commands/setup.py @@ -1033,7 +1033,8 @@ def catalog_refresh_cmd( f"{len(skipped)} skipped[/dim])" ) for entry in skipped: - _console.print(f" [yellow]skipped[/yellow] {entry['id']} — {entry['reason']}") + family = entry.get("family") or entry.get("id") or "unknown" + _console.print(f" [yellow]skipped[/yellow] {family} — {entry['reason']}") @catalog_app.command( diff --git a/src/vaner/clients/endpoint_pool.py b/src/vaner/clients/endpoint_pool.py index 380bec9..eed1b1b 100644 --- a/src/vaner/clients/endpoint_pool.py +++ b/src/vaner/clients/endpoint_pool.py @@ -138,7 +138,18 @@ def from_endpoints( if ep.weight <= 0: continue if ep.backend == "ollama": - client: LLMCallable = ollama_llm(model=ep.model, base_url=ep.url, timeout=timeout) + options = { + str(key): value + for source in (getattr(ep, "runtime_options", {}) or {}, getattr(ep, "sampling_options", {}) or {}) + for key, value in source.items() + if value is not None and value != "" + } + client: LLMCallable = ollama_llm( + model=ep.model, + base_url=ep.url, + timeout=timeout, + extra_body={"options": options} if options else None, + ) else: api_key = (_env(ep.api_key_env) if ep.api_key_env else None) or "" if not api_key and not cls._is_local_url(ep.url): diff --git a/src/vaner/clients/ollama.py b/src/vaner/clients/ollama.py index a46b904..287fcd6 100644 --- a/src/vaner/clients/ollama.py +++ b/src/vaner/clients/ollama.py @@ -17,6 +17,9 @@ def ollama_llm( model: str, base_url: str = "http://127.0.0.1:11434", timeout: float = 120.0, + max_tokens: int | None = None, + extra_body: dict | None = None, + reasoning_mode: ReasoningMode = "off", ) -> Callable[[str], Awaitable[str]]: """Ollama inference client (bare-string return — legacy contract). @@ -27,7 +30,9 @@ def ollama_llm( model=model, base_url=base_url, timeout=timeout, - reasoning_mode="off", + max_tokens=max_tokens, + extra_body=extra_body, + reasoning_mode=reasoning_mode, ) async def _call(prompt: str) -> str: @@ -79,7 +84,11 @@ async def _call(prompt: str, *, max_tokens: int | None = max_tokens) -> LLMRespo body = {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": False} else: body = {"model": model, "prompt": prompt, "stream": False} + merged_extra = dict(extra_body or {}) + extra_options = merged_extra.pop("options", None) options: dict = {} + if isinstance(extra_options, dict): + options.update(extra_options) if max_tokens is not None and max_tokens > 0: options["num_predict"] = int(max_tokens) if options: @@ -88,7 +97,6 @@ async def _call(prompt: str, *, max_tokens: int | None = max_tokens) -> LLMRespo rf_type = response_format.get("type") if rf_type in ("json_object", "json"): body["format"] = "json" - merged_extra = dict(extra_body or {}) if reasoning_mode == "off": # Modern Ollama reasoning models support a top-level ``think`` # switch and return reasoning in a separate ``thinking`` field. diff --git a/src/vaner/daemon/engine/generator.py b/src/vaner/daemon/engine/generator.py index ddc1d48..441d1b5 100644 --- a/src/vaner/daemon/engine/generator.py +++ b/src/vaner/daemon/engine/generator.py @@ -14,11 +14,14 @@ from vaner.models.artefact import Artefact, ArtefactKind from vaner.models.config import VanerConfig +from vaner.policy.internal_llm import EVIDENCE_SUMMARY_POLICY, internal_llm_policy from vaner.policy.privacy import redact_text logger = logging.getLogger(__name__) -FILE_SUMMARY_PROMPT = """You are generating a precise implementation reference for a context system. +FILE_SUMMARY_PROMPT = ( + internal_llm_policy(EVIDENCE_SUMMARY_POLICY) + + """\n\nYou are generating a precise implementation reference for a context system. A model will later use this summary to answer exact questions about this file. Your summary MUST enable correct answers — not just plausible ones. @@ -42,8 +45,11 @@ {content} --- Implementation reference:""" +) -DIFF_SUMMARY_PROMPT = """Summarize the following workspace change for a context system. +DIFF_SUMMARY_PROMPT = ( + internal_llm_policy(EVIDENCE_SUMMARY_POLICY) + + """\n\nSummarize the following workspace change for a context system. State clearly what changed, which modules/functions were affected, and likely intent. Call out exact limits/guards/conditionals when visible. Be concise. @@ -51,10 +57,33 @@ {diff} --- Summary:""" +) + + +def _format_python_args(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str: + args: list[str] = [] + for arg in list(node.args.posonlyargs) + list(node.args.args): + item = arg.arg + if arg.annotation is not None: + item += f": {ast.unparse(arg.annotation)}" + args.append(item) + if node.args.vararg is not None: + args.append(f"*{node.args.vararg.arg}") + for arg in node.args.kwonlyargs: + item = arg.arg + if arg.annotation is not None: + item += f": {ast.unparse(arg.annotation)}" + args.append(item) + if node.args.kwarg is not None: + args.append(f"**{node.args.kwarg.arg}") + signature = f"{node.name}({', '.join(args)})" + if node.returns is not None: + signature += f" -> {ast.unparse(node.returns)}" + return signature[:180] def _extract_python_shapes(text: str) -> tuple[list[str], list[str]]: - """Extract class names, top-level functions, and class method names from Python sources.""" + """Extract class names, top-level functions, and class method signatures.""" try: tree = ast.parse(text) except SyntaxError: @@ -65,15 +94,15 @@ def _extract_python_shapes(text: str) -> tuple[list[str], list[str]]: for node in tree.body: if isinstance(node, ast.ClassDef): - classes.append(node.name) + bases = [ast.unparse(base) for base in node.bases] + classes.append(f"{node.name}({', '.join(bases)})" if bases else node.name) # Include method names so artefact content matches method-specific queries for item in node.body: if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): if not item.name.startswith("_"): - functions.append(item.name) + functions.append(_format_python_args(item)) elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - args = [arg.arg for arg in node.args.args] - functions.append(f"{node.name}({', '.join(args)})") + functions.append(_format_python_args(node)) return classes[:8], list(dict.fromkeys(functions))[:20] @@ -104,6 +133,30 @@ def _extract_constants(text: str) -> list[str]: return constants[:12] +def _extract_sql_schema(text: str) -> list[str]: + schema_lines: list[str] = [] + collecting = False + buffer: list[str] = [] + for raw_line in text.splitlines(): + stripped = " ".join(raw_line.strip().split()) + lowered = stripped.lower() + if not stripped: + continue + if any(marker in lowered for marker in ("create table", "create index", "using fts5", "insert into", "select ")): + collecting = True + buffer = [stripped] + elif collecting: + buffer.append(stripped) + if collecting and (stripped.endswith(")") or stripped.endswith(')"') or stripped.endswith(";") or len(buffer) >= 8): + joined = " ".join(buffer) + schema_lines.append(joined[:240]) + collecting = False + buffer = [] + if len(schema_lines) >= 16: + break + return schema_lines + + def _summarize_text( text: str, source_path: Path, @@ -124,8 +177,9 @@ def _summarize_text( ast_source = full_text if full_text is not None else text sections: list[str] = [] - constants = _extract_constants(text) - limits = _extract_limits(text) + constants = _extract_constants(ast_source) + limits = _extract_limits(ast_source) + sql_schema = _extract_sql_schema(ast_source) if source_path.suffix == ".py": classes, functions = _extract_python_shapes(ast_source) @@ -138,9 +192,11 @@ def _summarize_text( sections.append("Constants: " + "; ".join(constants[:6])) if limits: sections.append("Limits: " + "; ".join(limits[:6])) + if sql_schema: + sections.append("Schema: " + "; ".join(sql_schema[:8])) sections.append("Snippet: " + " ".join(lines[:max_lines])[:900]) - return "\n".join(sections)[:1600] + return "\n".join(sections)[:2400] def _build_artefact( diff --git a/src/vaner/daemon/http.py b/src/vaner/daemon/http.py index 61ff2d1..f32585f 100644 --- a/src/vaner/daemon/http.py +++ b/src/vaner/daemon/http.py @@ -1421,6 +1421,92 @@ async def list_items(kind: str | None = None, limit: int = 10, visibility: str = ) return JSONResponse({"count": len(rows), "visibility": visibility_mode, "scenarios": [_scenario_payload(row) for row in rows]}) + async def _heatmap_replay_payload( + *, + from_ts: float | None = None, + to_ts: float | None = None, + range_seconds: float | None = None, + limit: int = 100, + ) -> dict[str, Any]: + end_ts = float(to_ts or time.time()) + default_range = float(range_seconds or 15 * 60) + start_ts = float(from_ts if from_ts is not None else end_ts - default_range) + if start_ts > end_ts: + start_ts, end_ts = end_ts, start_ts + capped_limit = max(1, min(int(limit), 200)) + rows = await _best_effort( + scenario_store.list_top(limit=capped_limit, visibility="all"), + [], + label="heatmap scenarios", + timeout=2.0, + ) + scenario_ids = [str(row.id) for row in rows] + samples = await _best_effort( + scenario_store.list_samples(scenario_ids=scenario_ids, start_ts=start_ts, end_ts=end_ts, limit=50_000), + [], + label="heatmap samples", + timeout=2.0, + ) + live_events = [ + row + for row in read_live_work_events(config.repo_root, limit=1000) + if start_ts <= float(row.get("ts") or 0.0) <= end_ts + ] + return { + "from_ts": start_ts, + "to_ts": end_ts, + "scenarios": [_scenario_payload(row) for row in rows], + "samples": [sample.__dict__ for sample in samples], + "events": live_events, + "metadata": { + "sample_source": "scenario_samples", + "event_source": "live_work_events", + "synthetic": False, + "sample_count": len(samples), + "event_count": len(live_events), + "scenario_count": len(rows), + "complete": bool(samples), + }, + } + + @app.get("/heatmap/replay/stream") + async def heatmap_replay_stream( + from_ts: float | None = None, + to_ts: float | None = None, + range_seconds: float | None = None, + limit: int = 100, + ) -> StreamingResponse: + async def event_gen() -> AsyncIterator[str]: + last_fingerprint = "" + last_keepalive = time.monotonic() + sent = 0 + while True: + payload = await _heatmap_replay_payload( + from_ts=from_ts if to_ts is not None else None, + to_ts=to_ts, + range_seconds=range_seconds, + limit=limit, + ) + serialized = json.dumps(payload, sort_keys=True, default=str) + if serialized != last_fingerprint: + yield f"event: replay_snapshot\ndata: {serialized}\n\n" + last_fingerprint = serialized + last_keepalive = time.monotonic() + sent += 1 + if limit is not None and sent >= max(1, int(limit)): + return + now = time.monotonic() + if now - last_keepalive >= 10.0: + yield ": keepalive\n\n" + last_keepalive = now + await asyncio.sleep(1.0) + + return StreamingResponse(event_gen(), media_type="text/event-stream") + + @app.get("/heatmap/replay") + async def heatmap_replay(from_ts: float | None = None, to_ts: float | None = None, limit: int = 100) -> JSONResponse: + return JSONResponse(await _heatmap_replay_payload(from_ts=from_ts, to_ts=to_ts, limit=limit)) + @app.get("/scenarios/{scenario_id}") async def fetch_item(scenario_id: str) -> JSONResponse: row = await scenario_store.get(scenario_id) diff --git a/src/vaner/daemon/precompute_worker.py b/src/vaner/daemon/precompute_worker.py index 1007f29..669de24 100644 --- a/src/vaner/daemon/precompute_worker.py +++ b/src/vaner/daemon/precompute_worker.py @@ -367,6 +367,11 @@ async def _run_job(self, job: WorkerJob, *, engine: Any, focus_manager: FocusMan profile["precompute_ms"] = (time.monotonic() - precompute_start) * 1000.0 profile["total_ms"] = (time.monotonic() - started) * 1000.0 profile["produced"] = float(produced or 0) + if hasattr(engine, "get_last_cycle_profile"): + try: + profile.update(engine.get_last_cycle_profile()) + except Exception: + logger.debug("failed to read engine cycle profile", exc_info=True) self._write_predictions(engine, cycle_id=cycle_id) self._status["profile"] = profile self._set_state("completed", job=job, phase="idle", cycle_id=cycle_id, produced=int(produced or 0)) @@ -533,9 +538,13 @@ def _record_prediction_event(self, event: Any) -> None: stage = "progress" status = "running" if isinstance(payload, dict): + tokens_used = int(payload.get("tokens_used") or 0) + token_budget = int(payload.get("token_budget") or 0) + overage = max(0, tokens_used - token_budget) + budget_label = f"{token_budget} token target" if token_budget else "token target" + overage_label = f" ({overage} over target)" if overage else "" summary = ( - f"Progress: {int(payload.get('tokens_used') or 0)}/" - f"{int(payload.get('token_budget') or 0)} tokens, " + f"Progress: {tokens_used} tokens used / {budget_label}{overage_label}, " f"{int(payload.get('scenarios_complete') or 0)} scenarios complete." ) elif kind == "prediction.artifact_added": diff --git a/src/vaner/defaults/catalog_seed.json b/src/vaner/defaults/catalog_seed.json index 5e740e8..74dce4e 100644 --- a/src/vaner/defaults/catalog_seed.json +++ b/src/vaner/defaults/catalog_seed.json @@ -1,99 +1,305 @@ { "schema_version": 2, - "generated_at": "2026-05-01", - "comment": "Family-level seed for the catalog refresher. The refresher (vaner.setup.catalog_refresh) emits one row per family using the family's configured Ollama tag, derives params/size from the manifest's vnd.ollama.image.model layer bytes, and computes memory budgets. The seed carries family-level metadata that doesn't depend on size — display name, workload tags, ranks, sampling defaults, the model's max_context_window — plus the Ollama family name and an optional HF repo template. context_window here is the architectural ceiling; recommend_local_model picks a hardware/archetype-aware effective_context_window at recommend time, floored at 32K.", + "generated_at": "2026-05-05", + "comment": "Ollama-first, release-curated model recommendation seed. Hugging Face and Ollama public metadata can inform refreshes, but Vaner does not depend on a live recommendation endpoint. The refresher emits one registry row per configured model tag, verifies Ollama tags through registry manifests when online, derives pull size from model layers, and computes memory budgets. context_window is the architectural ceiling; recommend_local_model picks a hardware/archetype-aware effective context window at setup time.", "quantization_profiles": { - "Q4_K_M": { "bytes_per_param": 0.55, "label": "Q4_K_M", "ratio_to_fp16": 0.275 }, - "Q5_K_M": { "bytes_per_param": 0.7, "label": "Q5_K_M", "ratio_to_fp16": 0.35 }, - "Q6_K": { "bytes_per_param": 0.84, "label": "Q6_K", "ratio_to_fp16": 0.42 }, - "Q8_0": { "bytes_per_param": 1.06, "label": "Q8_0", "ratio_to_fp16": 0.53 }, - "FP16": { "bytes_per_param": 2.0, "label": "FP16", "ratio_to_fp16": 1.0 } + "Q4_K_M": { + "bytes_per_param": 0.55, + "label": "Q4_K_M", + "ratio_to_fp16": 0.275 + }, + "Q5_K_M": { + "bytes_per_param": 0.7, + "label": "Q5_K_M", + "ratio_to_fp16": 0.35 + }, + "Q6_K": { + "bytes_per_param": 0.84, + "label": "Q6_K", + "ratio_to_fp16": 0.42 + }, + "Q8_0": { + "bytes_per_param": 1.06, + "label": "Q8_0", + "ratio_to_fp16": 0.53 + }, + "NVFP4": { + "bytes_per_param": 0.63, + "label": "NVFP4", + "ratio_to_fp16": 0.315 + }, + "MXFP4": { + "bytes_per_param": 0.555, + "label": "MXFP4", + "ratio_to_fp16": 0.2775 + }, + "MXFP8": { + "bytes_per_param": 1.1, + "label": "MXFP8", + "ratio_to_fp16": 0.55 + }, + "FP16": { + "bytes_per_param": 2.0, + "label": "FP16", + "ratio_to_fp16": 1.0 + } }, "default_quantization": "Q4_K_M", "families": [ { - "id": "qwen3.6", - "display_name": "Qwen 3.6", + "id": "gpt-oss-120b", + "display_name": "GPT-OSS 120B", + "runtime": "ollama", + "ollama_family": "gpt-oss", + "ollama_tag": "120b", + "hf_org": "openai", + "hf_repo_template": "openai/gpt-oss-120b", + "workload_tags": [ + "general", + "coding", + "summarization", + "reasoning", + "agentic", + "long_context" + ], + "stability_rank": 87, + "quality_rank": 99, + "recency_rank": 98, + "default_params_b": 117, + "default_download_size_gb": 65, + "active_params_b": 5.1, + "architecture": "moe", + "quantization": "MXFP4", + "accelerator_tags": [ + "nvidia", + "blackwell", + "unified_memory", + "dgx_spark", + "moe" + ], + "parameters": { + "context_window": 131072, + "reasoning_mode": "allowed", + "reasoning_effort": "medium", + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 1.0, + "top_p": 0.95, + "top_k": 40, + "repeat_penalty": 1.0, + "min_p": 0.0 + } + }, + { + "id": "qwen3.6-35b-a3b-coding-nvfp4", + "display_name": "Qwen 3.6 35B A3B Coding NVFP4", "runtime": "ollama", "ollama_family": "qwen3.6", - "ollama_tag": "27b", - "hf_org": "Qwen", - "hf_repo_template": "Qwen/Qwen3.6-{size}-Instruct", - "workload_tags": ["general", "coding", "summarization"], - "stability_rank": 85, - "quality_rank": 98, + "ollama_tag": "35b-a3b-coding-nvfp4", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "summarization", + "agentic", + "long_context" + ], + "stability_rank": 82, + "quality_rank": 99, + "recency_rank": 100, + "default_params_b": 35, + "default_download_size_gb": 22, + "active_params_b": 3, + "architecture": "moe", + "quantization": "NVFP4", + "accelerator_tags": [ + "nvidia", + "blackwell", + "moe" + ], + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20, + "repeat_penalty": 1.0, + "min_p": 0.0 + } + }, + { + "id": "qwen3.6-27b-coding-nvfp4", + "display_name": "Qwen 3.6 27B Coding NVFP4", + "runtime": "ollama", + "ollama_family": "qwen3.6", + "ollama_tag": "27b-coding-nvfp4", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "coding", + "agentic", + "general", + "long_context" + ], + "stability_rank": 82, + "quality_rank": 97, "recency_rank": 100, "default_params_b": 27.1, - "default_download_size_gb": 14.9, + "default_download_size_gb": 20, + "architecture": "dense", + "quantization": "NVFP4", + "accelerator_tags": [ + "nvidia", + "blackwell" + ], "parameters": { "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 4096, - "reasoning_token_budget": 8192, - "temperature": 0.7, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, "top_p": 0.95, "top_k": 20, - "min_p": 0.0, - "repeat_penalty": 1.05 + "repeat_penalty": 1.0, + "min_p": 0.0 } }, { - "id": "qwen3.5", - "display_name": "Qwen 3.5", + "id": "qwen3.6-35b-a3b-coding-mxfp8", + "display_name": "Qwen 3.6 35B A3B Coding MXFP8", "runtime": "ollama", - "ollama_family": "qwen3.5", - "hf_org": "Qwen", - "hf_repo_template": "Qwen/Qwen3.5-{size}-Instruct", - "workload_tags": ["general", "coding", "summarization"], - "stability_rank": 90, - "quality_rank": 95, - "recency_rank": 90, + "ollama_family": "qwen3.6", + "ollama_tag": "35b-a3b-coding-mxfp8", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "summarization", + "agentic", + "long_context" + ], + "stability_rank": 84, + "quality_rank": 99, + "recency_rank": 100, + "default_params_b": 35, + "default_download_size_gb": 38, + "active_params_b": 3, + "architecture": "moe", + "quantization": "MXFP8", + "accelerator_tags": [ + "apple_silicon", + "mlx", + "moe" + ], "parameters": { - "context_window": 131072, + "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 4096, - "reasoning_token_budget": 8192, - "temperature": 0.7, - "top_p": 0.8, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, + "top_p": 0.95, "top_k": 20, - "min_p": 0.0, - "repeat_penalty": 1.05 + "repeat_penalty": 1.0, + "min_p": 0.0 } }, { - "id": "qwen3-coder-next", - "display_name": "Qwen 3 Coder Next", + "id": "qwen3.6-27b-coding-mxfp8", + "display_name": "Qwen 3.6 27B Coding MXFP8", "runtime": "ollama", - "ollama_family": "qwen3-coder-next", - "hf_org": "Qwen", - "hf_repo_template": "Qwen/Qwen3-Coder-Next-{size}", - "workload_tags": ["coding", "general"], - "stability_rank": 85, - "quality_rank": 92, - "recency_rank": 95, + "ollama_family": "qwen3.6", + "ollama_tag": "27b-coding-mxfp8", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "coding", + "agentic", + "general", + "long_context" + ], + "stability_rank": 84, + "quality_rank": 97, + "recency_rank": 100, + "default_params_b": 27.1, + "default_download_size_gb": 31, + "architecture": "dense", + "quantization": "MXFP8", + "accelerator_tags": [ + "apple_silicon", + "mlx" + ], "parameters": { "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 4096, - "reasoning_token_budget": 8192, - "temperature": 0.2, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, "top_p": 0.95, "top_k": 20, - "min_p": 0.0, - "repeat_penalty": 1.05 + "repeat_penalty": 1.0, + "min_p": 0.0 } }, { - "id": "qwen3-next", - "display_name": "Qwen 3 Next", + "id": "qwen3.6-35b", + "display_name": "Qwen 3.6 35B A3B", "runtime": "ollama", - "ollama_family": "qwen3-next", - "hf_org": "Qwen", - "hf_repo_template": "Qwen/Qwen3-Next-{size}", - "workload_tags": ["general", "coding"], + "ollama_family": "qwen3.6", + "ollama_tag": "35b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "summarization", + "research", + "long_context" + ], "stability_rank": 85, - "quality_rank": 90, - "recency_rank": 92, + "quality_rank": 98, + "recency_rank": 100, + "default_params_b": 35, + "default_download_size_gb": 24, + "active_params_b": 3, + "architecture": "moe", + "quantization": "Q4_K_M", + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.7, + "top_p": 0.95, + "top_k": 20, + "repeat_penalty": 1.05, + "min_p": 0.0 + } + }, + { + "id": "qwen3.6-27b", + "display_name": "Qwen 3.6 27B", + "runtime": "ollama", + "ollama_family": "qwen3.6", + "ollama_tag": "27b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "summarization", + "long_context" + ], + "stability_rank": 86, + "quality_rank": 97, + "recency_rank": 100, + "default_params_b": 27.1, + "default_download_size_gb": 17, + "architecture": "dense", + "quantization": "Q4_K_M", "parameters": { "context_window": 262144, "reasoning_mode": "allowed", @@ -102,26 +308,37 @@ "temperature": 0.7, "top_p": 0.95, "top_k": 20, - "min_p": 0.0, - "repeat_penalty": 1.05 + "repeat_penalty": 1.05, + "min_p": 0.0 } }, { - "id": "llama4", - "display_name": "Llama 4", + "id": "gemma4-e2b", + "display_name": "Gemma 4 E2B", "runtime": "ollama", - "ollama_family": "llama4", - "hf_org": "meta-llama", - "hf_repo_template": "meta-llama/Llama-4-{size}-Instruct", - "workload_tags": ["general", "summarization"], + "ollama_family": "gemma4", + "ollama_tag": "e2b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "summarization", + "lightweight", + "vision" + ], "stability_rank": 90, - "quality_rank": 94, - "recency_rank": 95, + "quality_rank": 84, + "recency_rank": 96, + "default_params_b": 13, + "default_download_size_gb": 7.2, + "active_params_b": 2, + "architecture": "moe", + "quantization": "Q4_K_M", "parameters": { - "context_window": 1048576, - "reasoning_mode": "none", - "max_response_tokens": 4096, - "reasoning_token_budget": 0, + "context_window": 131072, + "reasoning_mode": "allowed", + "max_response_tokens": 3072, + "reasoning_token_budget": 4096, "temperature": 0.6, "top_p": 0.9, "top_k": 50, @@ -129,71 +346,248 @@ } }, { - "id": "gemma4", - "display_name": "Gemma 4", + "id": "gemma4-e4b", + "display_name": "Gemma 4 E4B", "runtime": "ollama", "ollama_family": "gemma4", - "hf_org": "google", - "hf_repo_template": "google/gemma-4-{size}-it", - "workload_tags": ["general", "summarization", "lightweight"], - "stability_rank": 88, - "quality_rank": 78, - "recency_rank": 92, + "ollama_tag": "e4b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "summarization", + "coding", + "lightweight", + "vision" + ], + "stability_rank": 90, + "quality_rank": 88, + "recency_rank": 96, + "default_params_b": 17, + "default_download_size_gb": 9.6, + "active_params_b": 4, + "architecture": "moe", + "quantization": "Q4_K_M", "parameters": { "context_window": 131072, "reasoning_mode": "allowed", - "max_response_tokens": 3072, + "max_response_tokens": 4096, "reasoning_token_budget": 4096, - "temperature": 0.8, - "top_p": 0.95, - "top_k": 64, - "repeat_penalty": 1.0 + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1 } }, { - "id": "deepseek-r2", - "display_name": "DeepSeek-R2", + "id": "gemma4-26b", + "display_name": "Gemma 4 26B", "runtime": "ollama", - "ollama_family": "deepseek-r2", - "hf_org": "deepseek-ai", - "hf_repo_template": "deepseek-ai/DeepSeek-R2-Distill-Qwen-{size}", - "workload_tags": ["general", "coding", "research"], - "stability_rank": 80, - "quality_rank": 96, + "ollama_family": "gemma4", + "ollama_tag": "26b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "summarization", + "agentic", + "vision", + "long_context" + ], + "stability_rank": 88, + "quality_rank": 94, "recency_rank": 96, + "default_params_b": 26, + "default_download_size_gb": 18, + "active_params_b": 4, + "architecture": "moe", + "quantization": "Q4_K_M", "parameters": { - "context_window": 131072, - "reasoning_mode": "forced", + "context_window": 262144, + "reasoning_mode": "allowed", "max_response_tokens": 4096, + "reasoning_token_budget": 8192, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1 + } + }, + { + "id": "gemma4-31b", + "display_name": "Gemma 4 31B", + "runtime": "ollama", + "ollama_family": "gemma4", + "ollama_tag": "31b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "summarization", + "agentic", + "vision", + "long_context" + ], + "stability_rank": 89, + "quality_rank": 95, + "recency_rank": 96, + "default_params_b": 31, + "default_download_size_gb": 20, + "architecture": "dense", + "quantization": "Q4_K_M", + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 4096, + "reasoning_token_budget": 8192, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1 + } + }, + { + "id": "nemotron-cascade-2-30b", + "display_name": "Nemotron Cascade 2 30B A3B", + "runtime": "ollama", + "ollama_family": "nemotron-cascade-2", + "ollama_tag": "30b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "reasoning", + "agentic", + "long_context" + ], + "stability_rank": 80, + "quality_rank": 93, + "recency_rank": 95, + "default_params_b": 30, + "default_download_size_gb": 24, + "active_params_b": 3, + "architecture": "moe", + "quantization": "Q4_K_M", + "accelerator_tags": [ + "nvidia", + "moe" + ], + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 8192, "reasoning_token_budget": 16384, "temperature": 0.6, "top_p": 0.95, "top_k": 40, - "repeat_penalty": 1.0 + "repeat_penalty": 1.05 + } + }, + { + "id": "nemotron3-33b", + "display_name": "Nemotron 3 Nano Omni 33B", + "runtime": "ollama", + "ollama_family": "nemotron3", + "ollama_tag": "33b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "summarization", + "vision", + "agentic" + ], + "stability_rank": 78, + "quality_rank": 88, + "recency_rank": 100, + "default_params_b": 33, + "default_download_size_gb": 28, + "architecture": "dense", + "quantization": "Q4_K_M", + "accelerator_tags": [ + "nvidia" + ], + "parameters": { + "context_window": 131072, + "reasoning_mode": "allowed", + "max_response_tokens": 4096, + "reasoning_token_budget": 8192, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 40, + "repeat_penalty": 1.05 } }, { - "id": "qwen3", - "display_name": "Qwen 3", + "id": "mistral-medium-3.5-128b", + "display_name": "Mistral Medium 3.5 128B", "runtime": "ollama", - "ollama_family": "qwen3", - "hf_org": "Qwen", - "hf_repo_template": "Qwen/Qwen3-{size}-Instruct", - "workload_tags": ["general", "lightweight"], - "stability_rank": 95, - "quality_rank": 75, - "recency_rank": 80, + "ollama_family": "mistral-medium-3.5", + "ollama_tag": "128b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "coding", + "summarization", + "reasoning", + "agentic", + "vision", + "long_context" + ], + "stability_rank": 82, + "quality_rank": 96, + "recency_rank": 100, + "default_params_b": 128, + "default_download_size_gb": 80, + "architecture": "dense", + "quantization": "Q4_K_M", "parameters": { - "context_window": 32768, + "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 2048, - "reasoning_token_budget": 2048, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, "temperature": 0.7, - "top_p": 0.8, - "top_k": 20, - "min_p": 0.0, + "top_p": 0.95, + "top_k": 40, "repeat_penalty": 1.05 } + }, + { + "id": "llama4-scout", + "display_name": "Llama 4 Scout 16x17B", + "runtime": "ollama", + "ollama_family": "llama4", + "ollama_tag": "16x17b", + "hf_org": "", + "hf_repo_template": "", + "workload_tags": [ + "general", + "summarization", + "vision", + "long_context" + ], + "stability_rank": 88, + "quality_rank": 87, + "recency_rank": 65, + "default_params_b": 109, + "default_download_size_gb": 67, + "active_params_b": 17, + "architecture": "moe", + "quantization": "Q4_K_M", + "parameters": { + "context_window": 10485760, + "reasoning_mode": "none", + "max_response_tokens": 4096, + "reasoning_token_budget": 0, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1 + } } ] } diff --git a/src/vaner/defaults/model_registry.json b/src/vaner/defaults/model_registry.json index ccd167b..0f323a4 100644 --- a/src/vaner/defaults/model_registry.json +++ b/src/vaner/defaults/model_registry.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "verified_at": "2026-05-01", + "verified_at": "2026-05-05", "generator": "vaner.setup.catalog_refresh", "online": true, "sources": [ @@ -9,111 +9,257 @@ ], "models": [ { - "id": "qwen3.6:27b", - "display_name": "Qwen 3.6 27B", + "id": "gpt-oss:120b", + "display_name": "GPT-OSS 120B", "runtime": "ollama", "workload_tags": [ "general", "coding", - "summarization" + "summarization", + "reasoning", + "agentic", + "long_context" ], - "quality_rank": 98, - "stability_rank": 85, + "quality_rank": 99, + "stability_rank": 87, + "recency_rank": 98, + "download_size_gb": 60.9, + "min_effective_memory_gb": 73.8, + "recommended_effective_memory_gb": 73.8, + "parameters": { + "context_window": 131072, + "reasoning_mode": "allowed", + "reasoning_effort": "medium", + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 1.0, + "top_p": 0.95, + "top_k": 40, + "repeat_penalty": 1.0, + "min_p": 0.0, + "num_ctx": 131072 + }, + "family_id": "gpt-oss-120b", + "params_b": 109.7, + "active_params_b": 5.1, + "architecture": "moe", + "quantization": "MXFP4", + "accelerator_tags": [ + "nvidia", + "blackwell", + "unified_memory", + "dgx_spark", + "moe" + ] + }, + { + "id": "qwen3.6:35b-a3b-coding-nvfp4", + "display_name": "Qwen 3.6 35B A3B Coding NVFP4", + "runtime": "ollama", + "workload_tags": [ + "general", + "coding", + "summarization", + "agentic", + "long_context" + ], + "quality_rank": 99, + "stability_rank": 82, "recency_rank": 100, - "download_size_gb": 14.9, - "min_effective_memory_gb": 17.2, - "recommended_effective_memory_gb": 37.9, + "download_size_gb": 22.0, + "min_effective_memory_gb": 31.8, + "recommended_effective_memory_gb": 32.8, "parameters": { "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 4096, - "reasoning_token_budget": 8192, - "temperature": 0.7, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, "top_p": 0.95, "top_k": 20, + "repeat_penalty": 1.0, + "min_p": 0.0, + "num_ctx": 262144 + }, + "family_id": "qwen3.6-35b-a3b-coding-nvfp4", + "params_b": 35.0, + "active_params_b": 3.0, + "architecture": "moe", + "quantization": "NVFP4", + "accelerator_tags": [ + "nvidia", + "blackwell", + "moe" + ] + }, + { + "id": "qwen3.6:27b-coding-nvfp4", + "display_name": "Qwen 3.6 27B Coding NVFP4", + "runtime": "ollama", + "workload_tags": [ + "coding", + "agentic", + "general", + "long_context" + ], + "quality_rank": 97, + "stability_rank": 82, + "recency_rank": 100, + "download_size_gb": 20.0, + "min_effective_memory_gb": 19.6, + "recommended_effective_memory_gb": 43.2, + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20, + "repeat_penalty": 1.0, "min_p": 0.0, - "repeat_penalty": 1.05, "num_ctx": 262144 }, - "family_id": "qwen3.6", + "family_id": "qwen3.6-27b-coding-nvfp4", "params_b": 27.1, - "quantization": "Q4_K_M" + "active_params_b": 0.0, + "architecture": "dense", + "quantization": "NVFP4", + "accelerator_tags": [ + "nvidia", + "blackwell" + ] }, { - "id": "qwen3.5:latest", - "display_name": "Qwen 3.5", + "id": "qwen3.6:35b-a3b-coding-mxfp8", + "display_name": "Qwen 3.6 35B A3B Coding MXFP8", "runtime": "ollama", "workload_tags": [ "general", "coding", - "summarization" + "summarization", + "agentic", + "long_context" ], - "quality_rank": 95, - "stability_rank": 90, - "recency_rank": 90, - "download_size_gb": 6.1, - "min_effective_memory_gb": 7.4, - "recommended_effective_memory_gb": 12.1, + "quality_rank": 99, + "stability_rank": 84, + "recency_rank": 100, + "download_size_gb": 38.0, + "min_effective_memory_gb": 49.6, + "recommended_effective_memory_gb": 51.3, "parameters": { - "context_window": 131072, + "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 4096, - "reasoning_token_budget": 8192, - "temperature": 0.7, - "top_p": 0.8, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, + "top_p": 0.95, "top_k": 20, + "repeat_penalty": 1.0, "min_p": 0.0, - "repeat_penalty": 1.05, - "num_ctx": 131072 + "num_ctx": 262144 }, - "family_id": "qwen3.5", - "params_b": 11.2, - "quantization": "Q4_K_M" + "family_id": "qwen3.6-35b-a3b-coding-mxfp8", + "params_b": 35.0, + "active_params_b": 3.0, + "architecture": "moe", + "quantization": "MXFP8", + "accelerator_tags": [ + "apple_silicon", + "mlx", + "moe" + ] }, { - "id": "qwen3-coder-next:latest", - "display_name": "Qwen 3 Coder Next", + "id": "qwen3.6:27b-coding-mxfp8", + "display_name": "Qwen 3.6 27B Coding MXFP8", "runtime": "ollama", "workload_tags": [ "coding", - "general" + "agentic", + "general", + "long_context" ], - "quality_rank": 92, - "stability_rank": 85, - "recency_rank": 95, - "download_size_gb": 48.2, - "min_effective_memory_gb": 54.5, - "recommended_effective_memory_gb": 119.1, + "quality_rank": 97, + "stability_rank": 84, + "recency_rank": 100, + "download_size_gb": 31.0, + "min_effective_memory_gb": 33.9, + "recommended_effective_memory_gb": 74.2, "parameters": { "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 4096, - "reasoning_token_budget": 8192, - "temperature": 0.2, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, "top_p": 0.95, "top_k": 20, + "repeat_penalty": 1.0, "min_p": 0.0, - "repeat_penalty": 1.05, "num_ctx": 262144 }, - "family_id": "qwen3-coder-next", - "params_b": 87.6, - "quantization": "Q4_K_M" + "family_id": "qwen3.6-27b-coding-mxfp8", + "params_b": 27.1, + "active_params_b": 0.0, + "architecture": "dense", + "quantization": "MXFP8", + "accelerator_tags": [ + "apple_silicon", + "mlx" + ] }, { - "id": "qwen3-next:latest", - "display_name": "Qwen 3 Next", + "id": "qwen3.6:35b", + "display_name": "Qwen 3.6 35B A3B", "runtime": "ollama", "workload_tags": [ "general", - "coding" + "coding", + "summarization", + "research", + "long_context" ], - "quality_rank": 90, + "quality_rank": 98, "stability_rank": 85, - "recency_rank": 92, - "download_size_gb": 46.9, - "min_effective_memory_gb": 53.0, - "recommended_effective_memory_gb": 116.0, + "recency_rank": 100, + "download_size_gb": 22.3, + "min_effective_memory_gb": 32.1, + "recommended_effective_memory_gb": 32.8, + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.7, + "top_p": 0.95, + "top_k": 20, + "repeat_penalty": 1.05, + "min_p": 0.0, + "num_ctx": 262144 + }, + "family_id": "qwen3.6-35b", + "params_b": 40.5, + "active_params_b": 3.0, + "architecture": "moe", + "quantization": "Q4_K_M", + "accelerator_tags": [] + }, + { + "id": "qwen3.6:27b", + "display_name": "Qwen 3.6 27B", + "runtime": "ollama", + "workload_tags": [ + "general", + "coding", + "summarization", + "long_context" + ], + "quality_rank": 97, + "stability_rank": 86, + "recency_rank": 100, + "download_size_gb": 16.2, + "min_effective_memory_gb": 18.7, + "recommended_effective_memory_gb": 41.1, "parameters": { "context_window": 262144, "reasoning_mode": "allowed", @@ -122,108 +268,302 @@ "temperature": 0.7, "top_p": 0.95, "top_k": 20, - "min_p": 0.0, "repeat_penalty": 1.05, + "min_p": 0.0, "num_ctx": 262144 }, - "family_id": "qwen3-next", - "params_b": 85.3, - "quantization": "Q4_K_M" + "family_id": "qwen3.6-27b", + "params_b": 29.5, + "active_params_b": 0.0, + "architecture": "dense", + "quantization": "Q4_K_M", + "accelerator_tags": [] }, { - "id": "llama4:latest", - "display_name": "Llama 4", + "id": "gemma4:e2b", + "display_name": "Gemma 4 E2B", "runtime": "ollama", "workload_tags": [ "general", - "summarization" + "summarization", + "lightweight", + "vision" ], - "quality_rank": 94, + "quality_rank": 84, "stability_rank": 90, - "recency_rank": 95, - "download_size_gb": 62.8, - "min_effective_memory_gb": 70.8, - "recommended_effective_memory_gb": 426.1, + "recency_rank": 96, + "download_size_gb": 6.7, + "min_effective_memory_gb": 15.2, + "recommended_effective_memory_gb": 16.7, "parameters": { - "context_window": 1048576, - "reasoning_mode": "none", - "max_response_tokens": 4096, - "reasoning_token_budget": 0, + "context_window": 131072, + "reasoning_mode": "allowed", + "max_response_tokens": 3072, + "reasoning_token_budget": 4096, "temperature": 0.6, "top_p": 0.9, "top_k": 50, "repeat_penalty": 1.1, - "num_ctx": 1048576 + "num_ctx": 131072 }, - "family_id": "llama4", - "params_b": 114.2, - "quantization": "Q4_K_M" + "family_id": "gemma4-e2b", + "params_b": 12.1, + "active_params_b": 2.0, + "architecture": "moe", + "quantization": "Q4_K_M", + "accelerator_tags": [] }, { - "id": "gemma4:latest", - "display_name": "Gemma 4", + "id": "gemma4:e4b", + "display_name": "Gemma 4 E4B", "runtime": "ollama", "workload_tags": [ "general", "summarization", - "lightweight" + "coding", + "lightweight", + "vision" ], - "quality_rank": 78, - "stability_rank": 88, - "recency_rank": 92, + "quality_rank": 88, + "stability_rank": 90, + "recency_rank": 96, "download_size_gb": 8.9, - "min_effective_memory_gb": 10.5, - "recommended_effective_memory_gb": 16.9, + "min_effective_memory_gb": 17.7, + "recommended_effective_memory_gb": 19.0, "parameters": { "context_window": 131072, "reasoning_mode": "allowed", - "max_response_tokens": 3072, + "max_response_tokens": 4096, "reasoning_token_budget": 4096, - "temperature": 0.8, - "top_p": 0.95, - "top_k": 64, - "repeat_penalty": 1.0, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1, "num_ctx": 131072 }, - "family_id": "gemma4", + "family_id": "gemma4-e4b", "params_b": 16.3, - "quantization": "Q4_K_M" + "active_params_b": 4.0, + "architecture": "moe", + "quantization": "Q4_K_M", + "accelerator_tags": [] + }, + { + "id": "gemma4:26b", + "display_name": "Gemma 4 26B", + "runtime": "ollama", + "workload_tags": [ + "general", + "coding", + "summarization", + "agentic", + "vision", + "long_context" + ], + "quality_rank": 94, + "stability_rank": 88, + "recency_rank": 96, + "download_size_gb": 16.8, + "min_effective_memory_gb": 26.1, + "recommended_effective_memory_gb": 27.9, + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 4096, + "reasoning_token_budget": 8192, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1, + "num_ctx": 262144 + }, + "family_id": "gemma4-26b", + "params_b": 30.5, + "active_params_b": 4.0, + "architecture": "moe", + "quantization": "Q4_K_M", + "accelerator_tags": [] + }, + { + "id": "gemma4:31b", + "display_name": "Gemma 4 31B", + "runtime": "ollama", + "workload_tags": [ + "general", + "coding", + "summarization", + "agentic", + "vision", + "long_context" + ], + "quality_rank": 95, + "stability_rank": 89, + "recency_rank": 96, + "download_size_gb": 18.5, + "min_effective_memory_gb": 21.2, + "recommended_effective_memory_gb": 46.6, + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 4096, + "reasoning_token_budget": 8192, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1, + "num_ctx": 262144 + }, + "family_id": "gemma4-31b", + "params_b": 33.6, + "active_params_b": 0.0, + "architecture": "dense", + "quantization": "Q4_K_M", + "accelerator_tags": [] + }, + { + "id": "nemotron-cascade-2:30b", + "display_name": "Nemotron Cascade 2 30B A3B", + "runtime": "ollama", + "workload_tags": [ + "general", + "coding", + "reasoning", + "agentic", + "long_context" + ], + "quality_rank": 93, + "stability_rank": 80, + "recency_rank": 95, + "download_size_gb": 22.6, + "min_effective_memory_gb": 32.4, + "recommended_effective_memory_gb": 33.2, + "parameters": { + "context_window": 262144, + "reasoning_mode": "allowed", + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, + "temperature": 0.6, + "top_p": 0.95, + "top_k": 40, + "repeat_penalty": 1.05, + "num_ctx": 262144 + }, + "family_id": "nemotron-cascade-2-30b", + "params_b": 41.1, + "active_params_b": 3.0, + "architecture": "moe", + "quantization": "Q4_K_M", + "accelerator_tags": [ + "nvidia", + "moe" + ] + }, + { + "id": "nemotron3:33b", + "display_name": "Nemotron 3 Nano Omni 33B", + "runtime": "ollama", + "workload_tags": [ + "general", + "summarization", + "vision", + "agentic" + ], + "quality_rank": 88, + "stability_rank": 78, + "recency_rank": 100, + "download_size_gb": 25.7, + "min_effective_memory_gb": 29.3, + "recommended_effective_memory_gb": 45.8, + "parameters": { + "context_window": 131072, + "reasoning_mode": "allowed", + "max_response_tokens": 4096, + "reasoning_token_budget": 8192, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 40, + "repeat_penalty": 1.05, + "num_ctx": 131072 + }, + "family_id": "nemotron3-33b", + "params_b": 46.8, + "active_params_b": 0.0, + "architecture": "dense", + "quantization": "Q4_K_M", + "accelerator_tags": [ + "nvidia" + ] }, { - "id": "qwen3:latest", - "display_name": "Qwen 3", + "id": "mistral-medium-3.5:128b", + "display_name": "Mistral Medium 3.5 128B", "runtime": "ollama", "workload_tags": [ "general", - "lightweight" + "coding", + "summarization", + "reasoning", + "agentic", + "vision", + "long_context" ], - "quality_rank": 75, - "stability_rank": 95, - "recency_rank": 80, - "download_size_gb": 4.9, - "min_effective_memory_gb": 5.9, - "recommended_effective_memory_gb": 7.2, + "quality_rank": 96, + "stability_rank": 82, + "recency_rank": 100, + "download_size_gb": 74.7, + "min_effective_memory_gb": 84.2, + "recommended_effective_memory_gb": 183.9, "parameters": { - "context_window": 32768, + "context_window": 262144, "reasoning_mode": "allowed", - "max_response_tokens": 2048, - "reasoning_token_budget": 2048, + "max_response_tokens": 8192, + "reasoning_token_budget": 16384, "temperature": 0.7, - "top_p": 0.8, - "top_k": 20, - "min_p": 0.0, + "top_p": 0.95, + "top_k": 40, "repeat_penalty": 1.05, - "num_ctx": 32768 + "num_ctx": 262144 }, - "family_id": "qwen3", - "params_b": 8.8, - "quantization": "Q4_K_M" - } - ], - "skipped": [ + "family_id": "mistral-medium-3.5-128b", + "params_b": 135.9, + "active_params_b": 0.0, + "architecture": "dense", + "quantization": "Q4_K_M", + "accelerator_tags": [] + }, { - "family": "deepseek-r2", - "reason": "ollama_latest_not_found" + "id": "llama4:16x17b", + "display_name": "Llama 4 Scout 16x17B", + "runtime": "ollama", + "workload_tags": [ + "general", + "summarization", + "vision", + "long_context" + ], + "quality_rank": 87, + "stability_rank": 88, + "recency_rank": 65, + "download_size_gb": 62.8, + "min_effective_memory_gb": 75.8, + "recommended_effective_memory_gb": 609.4, + "parameters": { + "context_window": 10485760, + "reasoning_mode": "none", + "max_response_tokens": 4096, + "reasoning_token_budget": 0, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 50, + "repeat_penalty": 1.1, + "num_ctx": 10485760 + }, + "family_id": "llama4-scout", + "params_b": 114.2, + "active_params_b": 17.0, + "architecture": "moe", + "quantization": "Q4_K_M", + "accelerator_tags": [] } ] } diff --git a/src/vaner/engine.py b/src/vaner/engine.py index 9422859..fa9d9cb 100644 --- a/src/vaner/engine.py +++ b/src/vaner/engine.py @@ -112,6 +112,7 @@ from vaner.models.decision import DecisionRecord, PredictionLink, ScoreFactor from vaner.models.signal import KIND_COMPOSER_LIFECYCLE, SignalEvent from vaner.plan_drafts import TERMINAL_PLAN_DRAFT_STATUSES, list_plan_drafts +from vaner.policy.internal_llm import JSON_CONTRACT_POLICY, PREDICTION_POLICY, internal_llm_policy from vaner.setup.apply import AppliedPolicy, apply_policy_bundle from vaner.setup.catalog import bundle_by_id from vaner.signals.composer import ComposerSignalPump, DraftIntentSnapshot @@ -334,9 +335,36 @@ def _is_cold_start_intent_text(text: str) -> bool: ), ), ) +_CONTINUATION_AGENDAS: tuple[tuple[str, tuple[str, ...], float], ...] = ( + ( + "normal continuation: review quality, risks, policy, and failure modes", + ("review", "risk", "policy", "contract", "error", "failure", "fallback", "timeout", "security"), + 0.78, + ), + ( + "normal continuation: validate behavior, tests, fixtures, and benchmark evidence", + ("test", "spec", "fixture", "benchmark", "eval", "ci", "verify", "check"), + 0.76, + ), + ( + "normal continuation: harden persistence, cache, config, and schema paths", + ("store", "cache", "config", "schema", "sql", "db", "toml", "yaml", "settings"), + 0.74, + ), + ( + "normal continuation: inspect daemon, worker, queue, signal, and event lifecycle", + ("daemon", "worker", "runner", "queue", "signal", "event", "watch", "stream", "background"), + 0.72, + ), + ( + "normal continuation: prepare documentation, handoff, release, and report context", + ("readme", "docs", "guide", "changelog", "release", "report", "handoff", "setup"), + 0.70, + ), +) -def _merge_llm_ranked_with_seed_paths(ranked_files: list[str], seed_paths: list[str]) -> list[str]: +def _merge_llm_ranked_with_seed_paths(ranked_files: list[str], seed_paths: list[str], *, max_paths: int = 8) -> list[str]: """Keep deterministic seed evidence when an LLM rerank is incomplete. The LLM is useful for reordering and adding adjacent files, but it should @@ -345,7 +373,12 @@ def _merge_llm_ranked_with_seed_paths(ranked_files: list[str], seed_paths: list[ paths are appended as a backstop and then filtered/deduped. """ - return list(dict.fromkeys(filter_evidence_paths([*ranked_files, *seed_paths])))[:8] + filtered_ranked = filter_evidence_paths(ranked_files) + filtered_seeds = list(dict.fromkeys(filter_evidence_paths(seed_paths))) + seed_set = set(filtered_seeds) + ranked_without_seeds = [path for path in filtered_ranked if path not in seed_set] + ranked_slots = max(0, int(max_paths) - len(filtered_seeds)) + return list(dict.fromkeys([*ranked_without_seeds[:ranked_slots], *filtered_seeds]))[:max_paths] def _core_group_matches_recent_query(reason: str, recent_queries: list[str]) -> bool: @@ -363,6 +396,24 @@ def _core_group_matches_recent_query(reason: str, recent_queries: list[str]) -> return len(reason_terms & query_terms) >= 2 +def _core_group_paths_for_query(query: str, available_paths: list[str], *, max_paths: int = 12) -> list[str]: + """Return core architecture files whose group labels directly match a query.""" + + if not query: + return [] + available_path_set = set(available_paths) + matched: list[str] = [] + for reason, candidate_paths in _CORE_ARCHITECTURE_GROUPS: + if not _core_group_matches_recent_query(reason, [query]): + continue + for path in candidate_paths: + if path in available_path_set and path not in matched: + matched.append(path) + if len(matched) >= max_paths: + return matched + return matched + + # Phase 4 / WS2: richer LLM callable that returns a structured # ``LLMResponse`` (thinking + content + raw). The engine prefers this when # available so reasoning-model preambles are captured rather than discarded. @@ -639,6 +690,9 @@ def __init__( "hedge_ratio": 0.20, "invest_ratio": 0.10, "no_regret_ratio": 0.20, + "continuation_agenda_cursor": 0.0, + "continuation_rounds_last_cycle": 0.0, + "continuation_admitted_last_cycle": 0.0, } def _refresh_work_style_adjustments(self) -> None: @@ -1066,6 +1120,8 @@ async def predict(self, top_k: int = 5) -> list[IntentPrediction]: async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int = 8) -> ContextPackage: await self.initialize() started_at = time.time() + context_budget = _effective_context_budget(self.config, requested=max_tokens) + selection_top_n = _adaptive_selection_top_n(prompt, requested=top_n, max_context_tokens=context_budget) self._notify_user_request_start() # Fold the freshly-arrived prompt into the timing model so # subsequent precompute cycles size their budgets against the @@ -1084,18 +1140,18 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int prior_prediction_probs = { item.category: max(0.0, float(item.confidence)) / total_conf for item in prior_predictions } - _quick_artefacts = await self.store.list(limit=2000) - _available_quick_paths = sorted({artefact.source_path for artefact in _quick_artefacts if artefact.source_path}) + _available_quick_paths = await self.store.list_source_paths(limit=5000) + _quick_selected = await select_artefacts_fts( + prompt, + self.store, + top_n=selection_top_n, + exclude_private=self.config.privacy.exclude_private, + path_bonuses=self._pinned_focus_paths, + path_excludes=self._pinned_avoid_paths, + ) _quick_paths = { artefact.source_path - for artefact in select_artefacts( - prompt, - _quick_artefacts, - top_n=8, - exclude_private=self.config.privacy.exclude_private, - path_bonuses=self._pinned_focus_paths, - path_excludes=self._pinned_avoid_paths, - ) + for artefact in _quick_selected if artefact.source_path } exact_symbol_paths = set( @@ -1103,7 +1159,7 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int self.config.repo_root, prompt, available_paths=_available_quick_paths, - max_paths=8, + max_paths=min(16, max(8, selection_top_n)), ) ) _quick_paths |= exact_symbol_paths @@ -1111,7 +1167,7 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int # scoring even when the heuristic selector disagrees. Without this # union, the bench finds 96% of precompute entries never get consumed # because their anchor_units don't overlap with the heuristic's picks. - _quick_paths = _quick_paths | await self._cache.candidate_anchor_units(prompt, top_k=3) + _quick_paths = _quick_paths | await self._cache.candidate_anchor_units(prompt, top_k=max(3, min(8, selection_top_n // 2))) cache_result = await self._cache.match(prompt, relevant_paths=_quick_paths) observation = self._arc_model.observe_detail(prompt) category = observation.category @@ -1246,28 +1302,33 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int # Use the LLM-curated files from the precomputed package as primary selection, # then fill remaining top_n slots from the heuristic selector. cache_keys = {s.artefact_key for s in cache_result.package.selections} - all_artefacts = await self.store.list(limit=2000) - artefacts_by_key = {a.key: a for a in all_artefacts} - selected = [artefacts_by_key[k] for k in cache_keys if k in artefacts_by_key] + selected = await self.store.list_by_keys(cache_keys, limit=max(selection_top_n, len(cache_keys))) factor_map: dict[str, list[ScoreFactor]] = {} drop_reasons: dict[str, str] = {} - if len(selected) < top_n: - heuristic_picks = select_artefacts( + prepared_context_diagnostics = [] + if len(selected) < selection_top_n: + heuristic_picks = await select_artefacts_fts( prompt, - all_artefacts, - top_n=top_n, + self.store, + top_n=selection_top_n, exclude_private=self.config.privacy.exclude_private, path_bonuses=self._pinned_focus_paths, path_excludes=self._pinned_avoid_paths, capture_factors=factor_map, capture_drop_reasons=drop_reasons, + context_preparation_mode=self.config.context_preparation.mode, + max_query_variants=self.config.context_preparation.max_query_variants, + max_candidate_keys=self.config.context_preparation.max_candidate_keys, + coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, + max_expansion_passes=self.config.context_preparation.max_expansion_passes, + capture_prepared_context_diagnostics=prepared_context_diagnostics, ) seen = {a.key for a in selected} for pick in heuristic_picks: if pick.key not in seen: selected.append(pick) seen.add(pick.key) - if len(selected) >= top_n: + if len(selected) >= selection_top_n: break source_key = selected[0].key if selected else None features = await extract_hybrid_features(self.store, prompt=prompt, source_key=source_key) @@ -1281,7 +1342,7 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int package, decision_record = assemble_context_package( prompt, selected, - max_tokens if max_tokens is not None else self.config.max_context_tokens, + context_budget, repo_root=self.config.repo_root, max_age_seconds=self.config.max_age_seconds, score_map=score_map, @@ -1290,6 +1351,7 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int evidence_assembly_mode=self.config.evidence_assembly.mode, evidence_assembly_quality_bias=self.config.evidence_assembly.quality_bias, evidence_assembly_cost_sensitivity=self.config.evidence_assembly.cost_sensitivity, + prepared_context_diagnostics=prepared_context_diagnostics[-1] if prepared_context_diagnostics else None, return_decision=True, ) else: @@ -1332,8 +1394,8 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int preferred_keys |= set(str(key) for key in cache_result.enrichment.get("relevant_keys", [])) package, selected, decision_record = await self._build_package_for_prompt( prompt, - max_tokens=max_tokens, - top_n=top_n, + max_tokens=context_budget, + top_n=selection_top_n, preferred_keys=preferred_keys, include_working_set_preferences=not cold_start_prompt, ) @@ -1899,6 +1961,41 @@ async def precompute_cycle( # newly observed user turn can surface as prepared context immediately. await self._prime_latest_history_query_prediction(recent_query_text) + # Latest-query anchors are stronger than broad periodic coverage. Seed + # deterministic exact-symbol paths and matching core groups before the + # heuristic, graph, arc, and rotating-core seeders can crowd them out. + latest_query_anchor_paths: list[str] = [] + if recent_query_text: + latest_query = recent_query_text[-1] + latest_query_anchor_paths.extend( + rank_exact_paths( + self.config.repo_root, + latest_query, + available_paths=available_paths, + max_paths=12, + ) + ) + latest_query_anchor_paths.extend( + _core_group_paths_for_query( + latest_query, + available_paths, + max_paths=12, + ) + ) + latest_query_anchor_paths = list(dict.fromkeys(filter_evidence_paths(latest_query_anchor_paths))) + if latest_query_anchor_paths: + frontier.seed_from_focus_paths( + self._rank_paths_for_recent_intent( + latest_query_anchor_paths, + recent_query_text, + focused_paths=set(latest_query_anchor_paths), + ), + available_paths, + reason="latest query exact symbol/core anchor focus", + priority_floor=1.18, + source="structured_direct", + ) + # Order matters: Jaccard-dedup is first-admitted-wins, and # structured v2 predictions carry the most concrete evidence targets. # Seed them before heuristic/core/arc scenarios so direct evidence @@ -2406,7 +2503,149 @@ async def _process_scenario(scenario: ExplorationScenario) -> None: # frontier empties we still drain in-flight tasks in case LLM branches # push new work back onto the queue. max_inflight = max(2, effective_concurrency * 2) - while governor.should_continue() and not frontier.is_saturated(): + continuation_rounds = 0 + continuation_admitted = 0 + continuation_chunk_limit = max(1, math.ceil(len(available_paths) / 8)) + if self._active_deep_run_session is not None: + continuation_chunk_limit *= 2 + # This is a duplicate/finite-work guard, not a short-run policy cap. + # Normal background prep should spend the cycle budget when useful work + # exists; it just must not spin forever on repeated equivalent chunks. + continuation_chunk_limit = max(len(_CONTINUATION_AGENDAS), min(256, continuation_chunk_limit)) + continuation_cursor = int(self._cycle_policy_state.get("continuation_agenda_cursor", 0.0)) + + def _cycle_has_budget_for_continuation() -> bool: + if not governor.should_continue(): + return False + if cycle_deadline is None: + return True + # Keep enough tail room for persistence, cleanup, and snapshot writes. + return (cycle_deadline - time.monotonic()) > 2.0 + + def _rank_continuation_paths(paths: list[str], keywords: tuple[str, ...]) -> list[str]: + changed_set = set(changed_paths) | set(changed_paths_for_horizon) + keyword_set = {kw.lower() for kw in keywords} + + def score(path: str) -> tuple[int, str]: + lower_path = path.lower() + basename = lower_path.rsplit("/", 1)[-1] + value = 0 + if path in changed_set: + value += 40 + if path not in covered_paths: + value += 18 + if lower_path.endswith(_SOURCE_EXTENSIONS): + value += 8 + if lower_path.startswith(("src/", "lib/", "app/", "packages/")): + value += 6 + if lower_path.startswith(("tests/", "test/")): + value += 5 + for keyword in keyword_set: + if keyword in basename: + value += 12 + elif keyword in lower_path: + value += 5 + if lower_path.startswith(("docs/", "README".lower())): + value += 2 + return value, path + + return sorted(dict.fromkeys(paths), key=lambda p: (-score(p)[0], score(p)[1])) + + def _seed_continuation_agenda() -> int: + nonlocal continuation_rounds, continuation_admitted, continuation_cursor + if continuation_rounds >= continuation_chunk_limit or not _cycle_has_budget_for_continuation(): + return 0 + + agenda_count = len(_CONTINUATION_AGENDAS) + available_set = set(available_paths) + fallback_paths = [ + path + for path in [*core_source_paths, *sorted(core_group_paths), *latest_query_anchor_paths] + if path in available_set + ] + attempts = 0 + while attempts < agenda_count: + agenda_index = (continuation_cursor + attempts) % agenda_count + reason, keywords, priority_floor = _CONTINUATION_AGENDAS[agenda_index] + matched = [ + path + for path in available_paths + if path in available_set and any(keyword in path.lower() for keyword in keywords) + ] + if not matched: + attempts += 1 + continue + ranked = _rank_continuation_paths(matched, keywords) + uncovered = [path for path in ranked if path not in covered_paths] + candidates = uncovered or ranked + stride = 8 + round_offset = (continuation_rounds // max(1, agenda_count)) * stride + if candidates: + offset = round_offset % max(1, len(candidates)) + candidates = [*candidates[offset:], *candidates[:offset]] + admitted = frontier.seed_from_focus_paths( + candidates[:stride], + available_paths, + reason=reason, + priority_floor=priority_floor, + source="horizon", + ) + continuation_cursor = (agenda_index + 1) % agenda_count + if admitted: + continuation_rounds += 1 + continuation_admitted += admitted + self._emit_live_work_event( + { + "entity_type": "worker", + "entity_id": "precompute-cycle", + "stage": "continuation", + "status": "queued", + "summary": reason, + "cycle_id": str(self._precompute_cycles), + "targets": candidates[:stride], + "metadata": { + "round": continuation_rounds, + "normal_background": self._active_deep_run_session is None, + }, + } + ) + return admitted + attempts += 1 + + # If named agendas are exhausted by dedup/coverage, keep a small + # architecture slice moving so normal background prep still broadens + # without requiring a declared Deep-Run session. + fallback_ranked = [path for path in _rank_continuation_paths(fallback_paths, ()) if path not in covered_paths] + if fallback_ranked: + admitted = frontier.seed_from_focus_paths( + fallback_ranked[:8], + available_paths, + reason="normal continuation: broaden uncovered architecture context", + priority_floor=0.68, + source="core_architecture", + ) + if admitted: + continuation_rounds += 1 + continuation_admitted += admitted + self._emit_live_work_event( + { + "entity_type": "worker", + "entity_id": "precompute-cycle", + "stage": "continuation", + "status": "queued", + "summary": "normal continuation: broaden uncovered architecture context", + "cycle_id": str(self._precompute_cycles), + "targets": fallback_ranked[:8], + "metadata": { + "round": continuation_rounds, + "normal_background": self._active_deep_run_session is None, + }, + } + ) + return admitted + return 0 + + while governor.should_continue(): if cycle_deadline is not None and time.monotonic() >= cycle_deadline and self._last_explored_scenarios: break @@ -2416,8 +2655,11 @@ async def _process_scenario(scenario: ExplorationScenario) -> None: if scenario is None: # No pending work right now. If tasks are in flight they may # push follow-ons on completion, so wait for one to finish and - # check again. Otherwise we're done. + # check again. Otherwise refill from normal continuation + # agendas before ending the cycle. if not in_flight: + if _seed_continuation_agenda(): + continue break done, pending = await asyncio.wait(in_flight, return_when=asyncio.FIRST_COMPLETED) in_flight = [t for t in pending] @@ -2459,7 +2701,8 @@ async def _process_scenario(scenario: ExplorationScenario) -> None: "arc": 4, "pattern": 5, "skill": 6, - "llm_branch": 7, + "horizon": 7, + "llm_branch": 8, } self._last_explored_scenarios.sort( key=lambda item: ( @@ -2558,6 +2801,9 @@ async def _process_scenario(scenario: ExplorationScenario) -> None: used_ms=allocation.no_regret_ms * allocation_scale, bucket="no_regret", ) + self._cycle_policy_state["continuation_agenda_cursor"] = float(continuation_cursor) + self._cycle_policy_state["continuation_rounds_last_cycle"] = float(continuation_rounds) + self._cycle_policy_state["continuation_admitted_last_cycle"] = float(continuation_admitted) _record_idle_usage_seconds(self.config, cycle_elapsed_s) if not self._last_explored_scenarios and not self._last_no_scenario_reason: if cycle_deadline is not None and time.monotonic() >= cycle_deadline: @@ -2576,6 +2822,15 @@ def get_last_no_scenario_reason(self) -> str: """Return the latest best-effort reason a cycle explored no scenarios.""" return self._last_no_scenario_reason + def get_last_cycle_profile(self) -> dict[str, float | str]: + """Return compact, non-persistent telemetry for the most recent cycle.""" + return { + "continuation_rounds": float(self._cycle_policy_state.get("continuation_rounds_last_cycle", 0.0)), + "continuation_admitted": float(self._cycle_policy_state.get("continuation_admitted_last_cycle", 0.0)), + "continuation_agenda_cursor": float(self._cycle_policy_state.get("continuation_agenda_cursor", 0.0)), + "no_scenario_reason": self._last_no_scenario_reason, + } + def get_active_predictions(self) -> list[PredictedPrompt]: """Return non-terminal PredictedPrompts for the active cycle. @@ -4121,6 +4376,7 @@ async def _explore_scenario_with_llm( priority_tag = "" prompt = ( + f"{internal_llm_policy(JSON_CONTRACT_POLICY, PREDICTION_POLICY)}\n\n" f"You are a code-context exploration engine.{priority_tag} Evaluate this scenario and decide " "which files are most relevant and what adjacent scenarios are worth exploring next.\n\n" f"Developer context:\n" @@ -4143,7 +4399,7 @@ async def _explore_scenario_with_llm( " need this scenario addresses (e.g. 'authentication middleware, JWT validation').\n" " This is used for matching future queries to this cached context.\n" "4. Set confidence (0.0-1.0): how likely is this area to be needed next?\n\n" - "Return JSON only (no markdown fences):\n" + "Return JSON only (no markdown fences, no extra keys):\n" "{\n" ' "ranked_files": ["path/a.py", "path/b.py"],\n' ' "semantic_intent": "...",\n' @@ -5347,6 +5603,7 @@ async def _run_reasoner_loop_iteration( # LLM focuses purely on the long-tail predictions they can't make. graph_covered = "\n".join(sorted(covered_paths)[:20]) or "none" prompt = ( + f"{internal_llm_policy(JSON_CONTRACT_POLICY, PREDICTION_POLICY)}\n\n" "You are Vaner's speculative prediction engine — the long-tail layer.\n" "The system has ALREADY pre-built context packages for:\n" " • Dependency-graph neighborhoods (structural, high-confidence)\n" @@ -5356,7 +5613,8 @@ async def _run_reasoner_loop_iteration( " – Config or infra files triggered by a specific code change\n" " – Third-party API surfaces the developer will need to read\n" " – Novel debugging paths not implied by recent errors\n\n" - "Return a JSON array with fields: question, file_paths, confidence, rationale.\n" + "Return a JSON array only, with fields: question, file_paths, confidence, rationale.\n" + "Keep rationale as a short observed-signal evidence note.\n" "Limit to 3-5 predictions.\n\n" f"Recent queries:\n{recent_hint or 'none'}\n\n" f"Feedback summary:\n{feedback_summary}\n\n" @@ -5495,7 +5753,7 @@ async def _build_package_for_paths( package, _decision_record = assemble_context_package( question, selected[:8], - self.config.max_context_tokens, + _effective_context_budget(self.config), repo_root=self.config.repo_root, max_age_seconds=self.config.max_age_seconds, score_map=score_map, @@ -5759,6 +6017,7 @@ async def _persist_learning_state(self, *, force: bool = False) -> None: "draft_evidence_threshold", "draft_volatility_ceiling", "draft_budget_min_ms", + "continuation_agenda_cursor", } cycle_state_to_persist = {k: v for k, v in self._cycle_policy_state.items() if k in persistent_cycle_keys} await self.store.upsert_learning_state( @@ -5778,10 +6037,10 @@ async def _build_package_for_prompt( preferred_keys: set[str] | None = None, include_working_set_preferences: bool = True, ) -> tuple[ContextPackage, list, DecisionRecord]: - artefacts = await self.store.list(limit=2000) - if not artefacts: + context_budget = _effective_context_budget(self.config, requested=max_tokens) + top_n = _adaptive_selection_top_n(prompt, requested=top_n, max_context_tokens=context_budget) + if not await self.store.list(limit=1): await self.prepare() - artefacts = await self.store.list(limit=2000) repo_root = self.config.repo_root git_state = read_git_state(repo_root) @@ -5801,6 +6060,7 @@ def _score_with_model(question: str, artefact) -> float: factor_map: dict[str, list[ScoreFactor]] = {} drop_reasons: dict[str, str] = {} + prepared_context_diagnostics = [] selected = await select_artefacts_fts( prompt, self.store, @@ -5813,6 +6073,12 @@ def _score_with_model(question: str, artefact) -> float: path_excludes=self._pinned_avoid_paths, capture_factors=factor_map, capture_drop_reasons=drop_reasons, + context_preparation_mode=self.config.context_preparation.mode, + max_query_variants=self.config.context_preparation.max_query_variants, + max_candidate_keys=self.config.context_preparation.max_candidate_keys, + coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, + max_expansion_passes=self.config.context_preparation.max_expansion_passes, + capture_prepared_context_diagnostics=prepared_context_diagnostics, ) if source_key is None and selected: source_key = selected[0].key @@ -5833,12 +6099,18 @@ def _score_with_source(question: str, artefact) -> float: path_excludes=self._pinned_avoid_paths, capture_factors=factor_map, capture_drop_reasons=drop_reasons, + context_preparation_mode=self.config.context_preparation.mode, + max_query_variants=self.config.context_preparation.max_query_variants, + max_candidate_keys=self.config.context_preparation.max_candidate_keys, + coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, + max_expansion_passes=self.config.context_preparation.max_expansion_passes, + capture_prepared_context_diagnostics=prepared_context_diagnostics, ) score_map = {artefact.key: self._intent_scorer.score(prompt, artefact, features=features) for artefact in selected} package, decision_record = assemble_context_package( prompt, selected, - max_tokens if max_tokens is not None else self.config.max_context_tokens, + context_budget, repo_root=self.config.repo_root, max_age_seconds=self.config.max_age_seconds, score_map=score_map, @@ -5847,6 +6119,7 @@ def _score_with_source(question: str, artefact) -> float: evidence_assembly_mode=self.config.evidence_assembly.mode, evidence_assembly_quality_bias=self.config.evidence_assembly.quality_bias, evidence_assembly_cost_sensitivity=self.config.evidence_assembly.cost_sensitivity, + prepared_context_diagnostics=prepared_context_diagnostics[-1] if prepared_context_diagnostics else None, return_decision=True, ) return package, selected, decision_record @@ -5869,6 +6142,7 @@ def _build_decision_record_from_package( partial_similarity=partial_similarity, token_budget=package.token_budget, token_used=package.token_used, + prepared_context_diagnostics=package.prepared_context_diagnostics, selections=[ { "artefact_key": selection.artefact_key, @@ -5929,7 +6203,16 @@ def _resolve_llm(self, llm: LLMCallable | str | None) -> LLMCallable | None: model = llm.split(":", 1)[1] if not model: return None - return ollama_llm(model=model, timeout=float(self.config.backend.request_timeout_seconds)) + return ollama_llm( + model=model, + timeout=float(self.config.backend.request_timeout_seconds), + max_tokens=int(self.config.backend.max_response_tokens), + extra_body=_ollama_options_extra_body( + self.config.backend.runtime_options, + self.config.backend.sampling_options, + ), + reasoning_mode=self.config.backend.reasoning_mode, + ) if llm.startswith("vllm:"): # vllm: or vllm:@: from vaner.clients.openai import openai_llm @@ -5993,6 +6276,11 @@ def _resolve_structured_llm(self, llm: str) -> StructuredLLMCallable | None: model=model, timeout=timeout, response_format=response_format, + max_tokens=int(self.config.backend.max_response_tokens), + extra_body=_ollama_options_extra_body( + self.config.backend.runtime_options, + self.config.backend.sampling_options, + ), reasoning_mode=reasoning_mode, ) return None @@ -6452,6 +6740,81 @@ def _probe_ollama_endpoint(base_url: str, timeout: float = 2.0) -> tuple[bool, l return False, [] +def _ollama_options_extra_body(runtime_options: dict[str, Any] | None, sampling_options: dict[str, Any] | None) -> dict[str, Any] | None: + options = { + str(key): value + for source in (runtime_options or {}, sampling_options or {}) + for key, value in source.items() + if value is not None and value != "" + } + return {"options": options} if options else None + + +def _adaptive_selection_top_n(prompt: str, *, requested: int, max_context_tokens: int) -> int: + """Expand retrieval breadth for multi-facet questions when context allows.""" + + base = max(1, int(requested)) + if max_context_tokens < 4096: + return base + lowered = prompt.lower() + facet_count = prompt.count("?") + len(re.findall(r"\b(?:and|what|how|when|which|where)\b", lowered)) + identifier_count = len(re.findall(r"\b[A-Za-z_][A-Za-z0-9_]{3,}\b", prompt)) + extra = 0 + if facet_count >= 3: + extra += 2 + if identifier_count >= 8: + extra += 2 + if any(term in lowered for term in ("schema", "pipeline", "flow", "walk me through", "how does")): + extra += 2 + capacity_cap = 16 if max_context_tokens >= 16384 else 12 + return min(capacity_cap, base + extra) + + +def _configured_context_windows(config: VanerConfig) -> list[int]: + windows: list[int] = [] + for source in ( + getattr(config.backend, "runtime_options", {}) or {}, + getattr(config.exploration, "runtime_options", {}) or {}, + ): + if not isinstance(source, dict): + continue + for key in ("num_ctx", "context_window", "max_context_tokens"): + try: + value = int(source.get(key) or 0) + except (TypeError, ValueError): + value = 0 + if value > 0: + windows.append(value) + for endpoint in getattr(config.exploration, "endpoints", []) or []: + try: + value = int(getattr(endpoint, "context_window", 0) or 0) + except (TypeError, ValueError): + value = 0 + if value > 0: + windows.append(value) + return windows + + +def _effective_context_budget(config: VanerConfig, *, requested: int | None = None) -> int: + """Return Vaner's evidence-package budget for the current runtime. + + Explicit per-call budgets stay authoritative. Otherwise use the configured + package budget as a floor, then expand when the local runtime advertises a + large context window. Vaner leaves most of the model window for the user's + prompt, final answer, and reasoning overhead. + """ + + if requested is not None: + return max(1, int(requested)) + configured = max(1, int(getattr(config, "max_context_tokens", 8192) or 8192)) + windows = _configured_context_windows(config) + if not windows: + return max(8192, configured) + largest_window = max(windows) + adaptive = min(262_144, max(16_384, largest_window // 3)) + return max(configured, adaptive) + + def _build_exploration_llm(ecfg: ExplorationConfig) -> LLMCallable | None: """Resolve the exploration LLM from ExplorationConfig, probing endpoints as needed.""" import logging as _logging @@ -6516,7 +6879,11 @@ def _make_ollama(base_url: str, m: str) -> LLMCallable: from vaner.clients.ollama import ollama_llm _log.info("Vaner exploration LLM: Ollama at %s model=%s", base_url, m) - return ollama_llm(model=m, base_url=base_url) + return ollama_llm( + model=m, + base_url=base_url, + extra_body=_ollama_options_extra_body(ecfg.runtime_options, ecfg.sampling_options), + ) # ------------------------------------------------------------------ # Explicit endpoint given: probe or trust based on backend hint diff --git a/src/vaner/intent/drafter.py b/src/vaner/intent/drafter.py index cdff675..dc7db17 100644 --- a/src/vaner/intent/drafter.py +++ b/src/vaner/intent/drafter.py @@ -32,6 +32,7 @@ from vaner.clients.llm_response import approx_tokens from vaner.intent.briefing import Briefing, BriefingAssembler from vaner.intent.prediction import PredictedPrompt +from vaner.policy.internal_llm import DRAFT_POLICY, PREDICTION_POLICY, internal_llm_policy _log = logging.getLogger(__name__) @@ -168,6 +169,7 @@ async def draft_for_prediction( else: recent_hint = "\n".join(recent_queries[-5:]) or "(no recent queries)" rewrite_prompt = ( + f"{internal_llm_policy(PREDICTION_POLICY)}\n\n" "Rewrite the likely next developer prompt as one concrete sentence.\n" "Stay semantically equivalent and concise.\n\n" f"Candidate prompt: {predicted_prompt}\n" @@ -190,6 +192,7 @@ async def draft_for_prediction( summaries_text = "\n".join(file_summaries) or "(no artefact summaries available)" recent_hint = "\n".join(recent_queries[-5:]) or "(no recent queries)" draft_prompt_text = ( + f"{internal_llm_policy(DRAFT_POLICY)}\n\n" "You are Vaner, a context engine drafting a speculative answer for a\n" "prompt the developer is likely to send next. Stay honest: if the\n" "evidence below is insufficient to produce a confident draft, say so\n" diff --git a/src/vaner/intent/evidence_resolver.py b/src/vaner/intent/evidence_resolver.py index 84c4644..427c886 100644 --- a/src/vaner/intent/evidence_resolver.py +++ b/src/vaner/intent/evidence_resolver.py @@ -11,6 +11,7 @@ from vaner.intent.prediction_v2 import StructuredPrediction from vaner.intent.symbol_index import RELATION_PRIORITY, SymbolCandidate from vaner.intent.target_normalization import component_terms, normalize_component +from vaner.semantic_aliases import engineering_semantic_aliases _STOPWORDS = { "about", @@ -131,6 +132,7 @@ def _alias_tokens(text: str) -> set[str]: if token.endswith(suffix) and len(token) > len(suffix) + 2: aliases.add(token[: -len(suffix)]) aliases.add(suffix) + aliases.update(engineering_semantic_aliases(text, tokens, stopwords=_STOPWORDS)) return aliases diff --git a/src/vaner/intent/symbol_index.py b/src/vaner/intent/symbol_index.py index cbdc9da..201b32e 100644 --- a/src/vaner/intent/symbol_index.py +++ b/src/vaner/intent/symbol_index.py @@ -180,15 +180,20 @@ def rank_exact_paths( max_paths: int = 8, ) -> list[str]: candidates = symbol_candidates_for_text(repo_root, text, available_paths=available_paths) - ranked: list[str] = [] - seen: set[str] = set() + if not candidates: + return [] + by_path: dict[str, float] = {} + matched_terms_by_path: dict[str, set[str]] = {} for candidate in candidates: - if candidate.path not in seen: - ranked.append(candidate.path) - seen.add(candidate.path) - if len(ranked) >= max_paths: - break - return ranked + matched_terms_by_path.setdefault(candidate.path, set()).update(candidate.matched_terms) + by_path[candidate.path] = by_path.get(candidate.path, 0.0) + _path_candidate_score(candidate) + for path, terms in matched_terms_by_path.items(): + # Multi-term matches are usually better implementation anchors than + # single generic word hits, especially for prompts like "reward + # computation signals" where many signal modules mention "signal". + by_path[path] = by_path.get(path, 0.0) + min(2.5, max(0, len(terms) - 1) * 0.45) + ranked = sorted(by_path, key=lambda path: (-by_path[path], path)) + return ranked[:max_paths] def _candidate_paths(root: Path, available_paths: list[str] | tuple[str, ...] | None, *, max_files: int) -> list[str]: @@ -259,6 +264,58 @@ def _dedupe_and_sort(candidates: list[SymbolCandidate]) -> list[SymbolCandidate] ) +_LOW_SPECIFICITY_TERMS = { + "combined", + "combine", + "computation", + "context", + "database", + "extract", + "feature", + "final", + "model", + "package", + "persist", + "produce", + "retrieve", + "schema", + "signal", + "signals", + "table", + "value", +} + + +def _path_candidate_score(candidate: SymbolCandidate) -> float: + relation_weight = float(RELATION_PRIORITY[candidate.relation]) * 4.0 + score = relation_weight + candidate.score + normalized_path_terms = set(path_component_terms(candidate.path)) + matched_terms = set(candidate.matched_terms) + basename = Path(candidate.path).stem + normalized_basename = normalize_component(basename) + normalized_symbol = normalize_component(candidate.symbol) + + if matched_terms & normalized_path_terms: + score += 5.0 + if normalized_basename and normalized_basename in matched_terms: + score += 8.0 + if normalized_symbol and normalized_symbol in matched_terms: + score += 4.0 + if candidate.relation == "definition_match" and not candidate.symbol.startswith("_"): + score += 3.0 + if candidate.symbol.startswith("_"): + score -= 2.0 + if matched_terms and matched_terms <= _LOW_SPECIFICITY_TERMS and candidate.relation == "usage_match": + score -= 12.0 + if candidate.relation == "usage_match" and not (matched_terms & normalized_path_terms): + score -= 8.0 + if candidate.path.startswith(("src/", "lib/", "app/", "packages/")): + score += 0.5 + if candidate.path.startswith(("tests/", "test/")): + score -= 0.5 + return score + + def _score_for_relation(relation: SymbolRelation) -> float: return { "definition_match": 1.0, diff --git a/src/vaner/models/__init__.py b/src/vaner/models/__init__.py index 3038931..7f64f3b 100644 --- a/src/vaner/models/__init__.py +++ b/src/vaner/models/__init__.py @@ -12,6 +12,12 @@ from vaner.models.artefact import Artefact, ArtefactKind from vaner.models.config import VanerConfig from vaner.models.context import ContextPackage, ContextSelection +from vaner.models.context_preparation import ( + ContextCoverageReport, + ContextFacet, + ContextPreparationProfile, + PreparedContextDiagnostics, +) from vaner.models.cost import ( CostEstimate, CostLedgerEntry, @@ -38,11 +44,15 @@ "EvidenceAssemblyDecision", "EvidenceAssemblyMetadata", "ContextPackage", + "ContextCoverageReport", + "ContextFacet", + "ContextPreparationProfile", "ContextSelection", "CostEstimate", "CostLedgerEntry", "DecisionRecord", "PredictionLink", + "PreparedContextDiagnostics", "Scenario", "ScoreFactor", "SelectionDecision", diff --git a/src/vaner/models/config.py b/src/vaner/models/config.py index bd7f2c7..5cced02 100644 --- a/src/vaner/models/config.py +++ b/src/vaner/models/config.py @@ -81,6 +81,11 @@ class BackendConfig(BaseModel): reasoning_token_budget: int = 8192 # Try response_format={"type":"json_object"} before tolerant parsing. prefer_structured_output: bool = True + # Provider/runtime knobs for local model calls. For Ollama these are + # forwarded under request ``options`` (for example num_ctx, keep_alive, + # temperature, top_p, top_k, min_p, repeat_penalty). + runtime_options: dict[str, Any] = Field(default_factory=dict) + sampling_options: dict[str, Any] = Field(default_factory=dict) class GenerationConfig(BaseModel): @@ -125,6 +130,18 @@ class EvidenceAssemblyConfig(BaseModel): allow_semantic_compression: bool = False +class ContextPreparationConfig(BaseModel): + mode: Literal["legacy", "balanced", "coverage_plus"] = "balanced" + max_query_variants: int = 6 + max_candidate_keys: int = 640 + max_expansion_passes: int = 1 + model_expansion: Literal["off", "local_first", "cloud_allowed"] = "local_first" + semantic_memory_enabled: bool = False + coverage_floor_enabled: bool = True + hard_constraint_validation_enabled: bool = True + prepared_briefing_injection_enabled: bool = True + + class GatewayConfig(BaseModel): passthrough_enabled: bool = False routes: dict[str, str] = Field(default_factory=dict) @@ -248,6 +265,8 @@ class ExplorationEndpoint(BaseModel): latency_p50_ms: float = 800.0 context_window: int = 8192 + runtime_options: dict[str, Any] = Field(default_factory=dict) + sampling_options: dict[str, Any] = Field(default_factory=dict) reasoning_depth_hint: Literal["low", "medium", "high"] = "medium" structured_output_reliability: float = 0.7 cost_per_1k_tokens: float = 0.0 @@ -378,6 +397,14 @@ class ExplorationConfig(BaseModel): fall back to ``"EMPTY"`` for local endpoints. """ + runtime_options: dict[str, Any] = Field(default_factory=dict) + sampling_options: dict[str, Any] = Field(default_factory=dict) + """Provider/runtime options forwarded to the single exploration endpoint. + + Multi-endpoint routing uses the per-endpoint fields on + ``ExplorationEndpoint`` instead. + """ + endpoints: list[ExplorationEndpoint] = Field(default_factory=list) """Optional pool of exploration endpoints for multi-endpoint routing. @@ -735,7 +762,7 @@ class VanerConfig(BaseModel): store_path: Path telemetry_path: Path max_age_seconds: int = 3600 - max_context_tokens: int = 4096 + max_context_tokens: int = 8192 backend: BackendConfig = Field(default_factory=BackendConfig) privacy: PrivacyConfig = Field(default_factory=PrivacyConfig) generation: GenerationConfig = Field(default_factory=GenerationConfig) @@ -752,3 +779,4 @@ class VanerConfig(BaseModel): policy: PolicyConfig = Field(default_factory=PolicyConfig) cost: CostConfig = Field(default_factory=CostConfig) evidence_assembly: EvidenceAssemblyConfig = Field(default_factory=EvidenceAssemblyConfig) + context_preparation: ContextPreparationConfig = Field(default_factory=ContextPreparationConfig) diff --git a/src/vaner/models/context.py b/src/vaner/models/context.py index fd73f77..3dd5ab8 100644 --- a/src/vaner/models/context.py +++ b/src/vaner/models/context.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field from vaner.models.answerable import AnswerabilityMetadata, AnswerableBriefing +from vaner.models.context_preparation import PreparedContextDiagnostics class ContextSelection(BaseModel): @@ -32,6 +33,9 @@ class ContextPackage(BaseModel): conflict_notes: list[str] = Field(default_factory=list) answerable_briefing: AnswerableBriefing | None = None answerability_metadata: AnswerabilityMetadata | None = None + prepared_context_briefing: str = "" + prepared_context_mode: str = "" + prepared_context_diagnostics: PreparedContextDiagnostics | None = None cache_tier: str = "miss" """How this package was sourced: ``"full_hit"`` | ``"partial_hit"`` | ``"miss"``.""" partial_similarity: float = 0.0 diff --git a/src/vaner/models/context_preparation.py b/src/vaner/models/context_preparation.py new file mode 100644 index 0000000..ba42543 --- /dev/null +++ b/src/vaner/models/context_preparation.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field + +ContextNeed = Literal[ + "direct_reference", + "task_continuation", + "evidence_gathering", + "multi_source_synthesis", + "decision_support", + "conflict_resolution", + "creative_grounding", + "research_mapping", + "implementation_support", + "absence_check", + "working_set_extension", +] + +ContextArchetype = Literal["developer", "writer", "researcher", "operator", "general"] + + +class ContextFacet(BaseModel): + name: str + value: str + required: bool = False + + +class ContextConstraint(BaseModel): + kind: str + value: str + required: bool = True + + +class ContextPreparationProfile(BaseModel): + need: ContextNeed = "evidence_gathering" + archetype: ContextArchetype = "general" + facets: list[ContextFacet] = Field(default_factory=list) + constraints: list[ContextConstraint] = Field(default_factory=list) + source_hints: list[str] = Field(default_factory=list) + expected_evidence_count: int = 1 + confidence: float = 0.5 + notes: list[str] = Field(default_factory=list) + + +class ContextCoverageReport(BaseModel): + covered_facets: list[str] = Field(default_factory=list) + missing_constraints: list[str] = Field(default_factory=list) + direct_evidence_count: int = 0 + conflict_pair_coverage: bool = False + actionability: Literal["full", "weak", "none", "conflict"] = "none" + truncation_risk: Literal["low", "medium", "high"] = "low" + source_agreement: float = 0.0 + weak_expansion_dependency: bool = False + compactness_risk: Literal["low", "medium", "high"] = "low" + gap_flags: list[str] = Field(default_factory=list) + + +class ContextSourceStats(BaseModel): + source: str + candidate_count: int = 0 + selected_count: int = 0 + + +class PreparedContextDiagnostics(BaseModel): + profile: ContextPreparationProfile = Field(default_factory=ContextPreparationProfile) + source_counts: list[ContextSourceStats] = Field(default_factory=list) + fused_candidate_count: int = 0 + selected_count: int = 0 + hard_constraints_extracted: list[str] = Field(default_factory=list) + hard_constraints_satisfied: list[str] = Field(default_factory=list) + hard_constraints_missing: list[str] = Field(default_factory=list) + coverage: ContextCoverageReport = Field(default_factory=ContextCoverageReport) + dropped_direct_evidence: int = 0 + weak_expansion_dependency: bool = False + token_used: int = 0 + truncation_risk: Literal["low", "medium", "high"] = "low" + latency_ms: float = 0.0 + compactness_score: float = 1.0 + provenance_coverage: float = 0.0 + diff --git a/src/vaner/models/decision.py b/src/vaner/models/decision.py index 675f2fc..5ee05a9 100644 --- a/src/vaner/models/decision.py +++ b/src/vaner/models/decision.py @@ -7,6 +7,8 @@ from pydantic import BaseModel, Field +from vaner.models.context_preparation import PreparedContextDiagnostics + class ScoreFactor(BaseModel): name: str @@ -45,6 +47,7 @@ class DecisionRecord(BaseModel): selections: list[SelectionDecision] = Field(default_factory=list) prediction_links: dict[str, PredictionLink] = Field(default_factory=dict) notes: list[str] = Field(default_factory=list) + prepared_context_diagnostics: PreparedContextDiagnostics | None = None def to_legacy_markdown(self) -> str: lines = [ diff --git a/src/vaner/policy/internal_llm.py b/src/vaner/policy/internal_llm.py new file mode 100644 index 0000000..8bd4b3f --- /dev/null +++ b/src/vaner/policy/internal_llm.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Internal prompt policy for Vaner's background LLM pipeline. + +These blocks are for machine-consumed intermediate calls, not user-facing +assistant personas. Keep them short so local models spend context on evidence +and the requested output contract. +""" + +from __future__ import annotations + +CORE_POLICY = """Vaner internal LLM policy: +- Preserve the requested output contract over style. +- Prefer exact supplied evidence over plausible inference. +- Preserve implementation anchors exactly: paths, symbols, constants, config keys, routes, database fields, errors, limits, and conditions. +- Use null, empty arrays, low confidence, or short evidence notes when evidence is missing; do not invent details. +- Do not expose hidden reasoning unless the output schema explicitly requires it.""" + +JSON_CONTRACT_POLICY = """JSON contract: +- Return valid JSON only, with no markdown fences, preamble, commentary, or trailing prose. +- Use only the requested keys and valid JSON values. +- Keep rationale/reason fields to short evidence notes, not chain-of-thought.""" + +EVIDENCE_SUMMARY_POLICY = """Evidence summary policy: +- Summaries must be grounded only in supplied files, diffs, code, or context. +- Do not let predictions or likely intent become factual behavior. +- Prefer exact implementation references over generic descriptions.""" + +PREDICTION_POLICY = """Prediction policy: +- Speculate only because this task asks for prediction. +- Tie predictions to observed signals; label uncertainty through confidence, rationale, evidence, or provenance. +- Keep predictive claims separate from factual summaries.""" + +DRAFT_POLICY = """Draft policy: +- Drafts remain evidence-bound and useful to a downstream AI client. +- Use tentative wording when source evidence is incomplete. +- Avoid overstating what Vaner knows.""" + + +def internal_llm_policy(*blocks: str) -> str: + """Compose the core policy with task-specific overlays.""" + + selected = [CORE_POLICY, *[block for block in blocks if block.strip()]] + return "\n".join(selected) diff --git a/src/vaner/router/proxy.py b/src/vaner/router/proxy.py index 4dcfa81..5ea0ad5 100644 --- a/src/vaner/router/proxy.py +++ b/src/vaner/router/proxy.py @@ -47,6 +47,28 @@ def _inject_context(payload: dict[str, Any], context: str) -> dict[str, Any]: return {**payload, "messages": [system_message, *messages]} +def _prepared_context_for_injection(context_package: Any, config: VanerConfig) -> str: + raw_context = str(getattr(context_package, "injected_context", "") or "") + if not config.context_preparation.prepared_briefing_injection_enabled: + return raw_context + if config.evidence_assembly.mode not in {"safe", "active"}: + return raw_context + briefing = str(getattr(context_package, "prepared_context_briefing", "") or "") + if not briefing: + answerable = getattr(context_package, "answerable_briefing", None) + briefing = str(getattr(answerable, "text", "") or "") if answerable is not None else "" + metadata = getattr(context_package, "answerability_metadata", None) + if metadata is None: + return raw_context + if getattr(metadata, "truncation_risk", "low") == "high": + return raw_context + if int(getattr(metadata, "dropped_direct_evidence_count", 0) or 0) > 0: + return raw_context + if getattr(metadata, "answerability", "none") not in {"full", "weak", "conflict"}: + return raw_context + return briefing or raw_context + + def _normalize_message_content(content: Any) -> str: if isinstance(content, str): return content @@ -536,7 +558,7 @@ async def chat_completions(payload: dict[str, Any], request: Request) -> Any: metrics.partial_similarity = context_package.partial_similarity metrics.context_tokens = context_package.token_used metrics.injected_context_tokens = context_package.token_used - enriched = _inject_context(payload, context_package.injected_context) + enriched = _inject_context(payload, _prepared_context_for_injection(context_package, config)) else: context_package = type("Package", (), {"cache_tier": "disabled", "partial_similarity": 0.0, "token_used": 0})() metrics.t1_context_ready = time.monotonic() diff --git a/src/vaner/semantic_aliases.py b/src/vaner/semantic_aliases.py new file mode 100644 index 0000000..18a1cca --- /dev/null +++ b/src/vaner/semantic_aliases.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Collection + + +def engineering_semantic_aliases(text: str, tokens: Collection[str], *, stopwords: Collection[str] = ()) -> set[str]: + """Expand common engineering paraphrases without model calls. + + These aliases are intentionally broad product vocabulary, not benchmark + fixtures. They help Vaner keep exact evidence in view when users describe + a concept in plain language while code/docs use implementation terms. + """ + + lowered = text.lower().replace("_", " ").replace("-", " ") + normalized_tokens = {token.lower() for token in tokens} + token_text = " ".join(sorted(normalized_tokens)) + haystack = f"{lowered} {token_text}" + aliases: set[str] = set() + + def has_any(*phrases: str) -> bool: + return any(phrase in haystack for phrase in phrases) + + if has_any( + "dry run", + "smoke check", + "smoke policy", + "full traffic", + "candidate release", + "staged promote", + "staged rollout", + ): + aliases.update({"rehearse", "rehearsal", "replay", "canary", "escrow", "promote", "promotion", "traffic"}) + if has_any("rehearse", "rehearsal", "replay", "traffic escrow", "promote", "promotion"): + aliases.update({"dry", "run", "smoke", "canary", "candidate", "release", "rollout", "traffic", "gate", "gating"}) + + if has_any("low bit", "low precision", "numeric mode", "mixed precision", "quantization", "quantized"): + aliases.update( + {"precision", "quantization", "quant", "kernel", "stability", "threshold", "annealing", "int4", "int8", "fp16", "fp32"} + ) + if has_any("precision annealing", "kernel stability", "stability threshold", "int4", "int8", "fp16", "fp32"): + aliases.update({"low", "bit", "numeric", "mode", "quantization", "safest", "pass", "rate"}) + + if has_any("vectorization", "vectorize", "embedding", "embeddings", "embed batch", "batch embeds"): + aliases.update({"embedding", "embeddings", "embed", "vector", "vectorize", "vectorization"}) + + if has_any("western europe", "europe", "european", "eu west", "eu central"): + aliases.update({"eu", "euwest", "eucentral", "europe", "european", "emea"}) + if has_any("southeast asia", "south east asia", "apac", "ap southeast"): + aliases.update({"apac", "asia", "southeast", "apsoutheast"}) + if has_any("egress", "residency", "cross region", "edge fallback", "routing anomaly", "routed"): + aliases.update({"route", "routing", "egress", "residency", "fallback", "failover", "edge", "control", "plane"}) + + if has_any("makes things up", "made things up", "hallucination", "hallucinate", "joke", "meme"): + aliases.update({"hallucination", "hallucinate", "meme", "memes", "satire", "tagging", "joke", "jokes"}) + + stopword_set = {word.lower() for word in stopwords} + return {alias for alias in aliases if len(alias) >= 3 and alias not in stopword_set} diff --git a/src/vaner/setup/catalog_refresh.py b/src/vaner/setup/catalog_refresh.py index 589d82c..2cf9364 100644 --- a/src/vaner/setup/catalog_refresh.py +++ b/src/vaner/setup/catalog_refresh.py @@ -27,6 +27,7 @@ import json import logging +import re from dataclasses import dataclass, field from datetime import UTC, datetime from importlib import resources @@ -35,6 +36,7 @@ logger = logging.getLogger(__name__) OLLAMA_REGISTRY_BASE = "https://registry.ollama.ai/v2/library/{name}/manifests/{tag}" +OLLAMA_LIBRARY_MODEL_URL = "https://ollama.com/library/{name}%3A{tag}" HF_API_BASE = "https://huggingface.co/api/models/{repo}" DEFAULT_HTTP_TIMEOUT = 8.0 OLLAMA_MODEL_LAYER_PREFIX = "application/vnd.ollama.image.model" @@ -54,6 +56,10 @@ class FamilySeed: recency_rank: int default_params_b: float default_download_size_gb: float + active_params_b: float + architecture: str + quantization: str + accelerator_tags: tuple[str, ...] parameters: dict[str, Any] = field(default_factory=dict) @@ -81,6 +87,10 @@ def families_from_seed(seed: dict[str, Any]) -> list[FamilySeed]: recency_rank=int(entry.get("recency_rank", 0)), default_params_b=float(entry.get("default_params_b", 0.0)), default_download_size_gb=float(entry.get("default_download_size_gb", 0.0)), + active_params_b=float(entry.get("active_params_b", 0.0) or 0.0), + architecture=str(entry.get("architecture", "dense") or "dense"), + quantization=str(entry.get("quantization", "") or ""), + accelerator_tags=tuple(str(t) for t in entry.get("accelerator_tags", []) if isinstance(t, str)), parameters=dict(entry.get("parameters", {})), ) ) @@ -95,15 +105,28 @@ def quantization_bytes_per_param(seed: dict[str, Any], quant: str) -> float: return float(profile.get("bytes_per_param", 0.55)) -def estimate_memory_budget(params_b: float, bytes_per_param: float, context_window: int) -> tuple[float, float]: +def estimate_memory_budget( + params_b: float, + bytes_per_param: float, + context_window: int, + *, + active_params_b: float = 0.0, + architecture: str = "dense", +) -> tuple[float, float]: """Return (min_effective_gb, recommended_effective_gb).""" weights_gb = params_b * bytes_per_param if weights_gb <= 0: return 0.0, 0.0 - context_overhead = max(0.5, weights_gb * (context_window / 32768) * 0.18) - min_gb = round(weights_gb * 1.12 + 0.5, 1) - rec_gb = round(weights_gb + context_overhead + 1.5, 1) + if architecture.lower() == "moe" and active_params_b > 0: + kv_reference_gb = max(active_params_b * bytes_per_param, weights_gb * 0.08) + context_overhead = max(2.0, kv_reference_gb * (context_window / 32768) * 0.18) + min_gb = round(weights_gb * 1.08 + 8.0, 1) + rec_gb = round(weights_gb + context_overhead + 8.0, 1) + else: + context_overhead = max(0.5, weights_gb * (context_window / 32768) * 0.18) + min_gb = round(weights_gb * 1.12 + 0.5, 1) + rec_gb = round(weights_gb + context_overhead + 1.5, 1) return min_gb, max(rec_gb, min_gb) @@ -155,6 +178,38 @@ def manifest_weights_bytes(manifest: dict[str, Any]) -> int: return total +def fetch_ollama_library_details(family: str, *, tag: str = "latest", timeout: float = DEFAULT_HTTP_TIMEOUT) -> dict[str, Any] | None: + """Best-effort model details from the public Ollama library page. + + Some currently published Ollama tags are visible and runnable through + the library UI before their registry manifests are readable through the + plain OCI endpoint. Treat the page as a weaker verifier: it must show an + `ollama run :` command and a concrete local size. Cloud-only + rows such as `:cloud` intentionally do not pass this check. + """ + + import urllib.error + import urllib.request + + url = OLLAMA_LIBRARY_MODEL_URL.format(name=family, tag=tag) + model_id = f"{family}:{tag}" + try: + req = urllib.request.Request(url, headers={"User-Agent": "vaner-catalog-refresh/1.0"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + html = resp.read().decode("utf-8", errors="replace") + except (urllib.error.URLError, TimeoutError, OSError) as exc: + logger.debug("ollama library page %s failed: %s", model_id, exc) + return None + + if f"ollama run {model_id}" not in html: + return None + size_match = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*GB", html) + if not size_match: + return None + size_gb = float(size_match.group(1)) + return {"download_size_gb": size_gb, "source": url} + + def fetch_hf_params_b(repo: str, *, timeout: float = DEFAULT_HTTP_TIMEOUT) -> float | None: """Best-effort lookup of HF parameter count, in billions.""" @@ -186,6 +241,7 @@ def build_registry_entry_for_family( quant: str, online: bool, manifest_fetcher=fetch_ollama_manifest, + library_fetcher=fetch_ollama_library_details, ) -> dict[str, Any] | None: """Translate one family into a single ``:`` registry row. @@ -193,30 +249,43 @@ def build_registry_entry_for_family( mode) — caller skips it. """ - bytes_per_param = quantization_bytes_per_param(seed, quant) + effective_quant = family.quantization or quant + bytes_per_param = quantization_bytes_per_param(seed, effective_quant) params_b: float = 0.0 download_gb: float = 0.0 - if online: + if online and family.runtime == "ollama": manifest = manifest_fetcher(family.ollama_family, tag=family.ollama_tag) - if manifest is None: - return None - weights_bytes = manifest_weights_bytes(manifest) - if weights_bytes <= 0: - logger.debug("ollama manifest for %s has no model layer", family.id) - return None - download_gb = round(weights_bytes / (1024**3), 1) - # Derive params from on-disk size + quant profile. This is more - # honest than guessing; the user pulls exactly these bytes. - params_b = round(weights_bytes / (1024**3) / bytes_per_param, 1) + if manifest is not None: + weights_bytes = manifest_weights_bytes(manifest) + if weights_bytes <= 0: + logger.debug("ollama manifest for %s has no model layer", family.id) + return None + download_gb = round(weights_bytes / (1024**3), 1) + # Derive params from on-disk size + quant profile. This is more + # honest than guessing; the user pulls exactly these bytes. + params_b = round(weights_bytes / (1024**3) / bytes_per_param, 1) + else: + details = library_fetcher(family.ollama_family, tag=family.ollama_tag) + if not details: + return None + download_gb = float(details["download_size_gb"]) + params_b = float(family.default_params_b or 0.0) else: - # Offline path: use seed sizing when available, otherwise emit a - # placeholder so the registry has a row per family. + # Offline and non-Ollama paths use seed sizing. Non-Ollama runtimes + # such as MLX are installed through their own package/model manager, + # so the Ollama manifest probe is not the source of truth. params_b = float(family.default_params_b or 0.0) download_gb = float(family.default_download_size_gb or 0.0) context_window = int(family.parameters.get("context_window", 8192)) - min_gb, rec_gb = estimate_memory_budget(params_b, bytes_per_param, context_window) + min_gb, rec_gb = estimate_memory_budget( + params_b, + bytes_per_param, + context_window, + active_params_b=family.active_params_b, + architecture=family.architecture, + ) parameters = dict(family.parameters) parameters.setdefault("num_ctx", context_window) @@ -237,7 +306,10 @@ def build_registry_entry_for_family( "parameters": parameters, "family_id": family.id, "params_b": params_b, - "quantization": quant, + "active_params_b": family.active_params_b, + "architecture": family.architecture, + "quantization": effective_quant, + "accelerator_tags": list(family.accelerator_tags), } @@ -246,6 +318,7 @@ def build_registry( online: bool = True, seed: dict[str, Any] | None = None, manifest_fetcher=fetch_ollama_manifest, + library_fetcher=fetch_ollama_library_details, ) -> dict[str, Any]: """Produce a full ``model_registry.json`` payload. @@ -266,6 +339,7 @@ def build_registry( quant=default_quant, online=online, manifest_fetcher=manifest_fetcher, + library_fetcher=library_fetcher, ) except Exception as exc: # pragma: no cover - defensive logger.warning("catalog: failed for %s (%s)", family.id, exc) diff --git a/src/vaner/setup/config_io.py b/src/vaner/setup/config_io.py index f69a233..0fae9fc 100644 --- a/src/vaner/setup/config_io.py +++ b/src/vaner/setup/config_io.py @@ -76,6 +76,7 @@ def persist_runtime_recommendation(repo_root: Path, recommendation: dict[str, An params = selected.get("params") if isinstance(selected.get("params"), dict) else {} capability = selected.get("capability") if isinstance(selected.get("capability"), dict) else {} runtime_params = selected.get("runtime_params") if isinstance(selected.get("runtime_params"), dict) else {} + sampling_params = selected.get("sampling_params") if isinstance(selected.get("sampling_params"), dict) else {} reasoning_mode = str(params.get("reasoning_mode") or "allowed") max_response_tokens = int(params.get("max_response_tokens") or 3072) reasoning_token_budget = int(params.get("reasoning_token_budget") or 4096) @@ -83,7 +84,7 @@ def persist_runtime_recommendation(repo_root: Path, recommendation: dict[str, An # Keep Vaner's own evidence package large enough to use long-context # models, while leaving most of the window for the user's prompt, # generated answer, reasoning budget, and runtime overhead. - max_context_tokens = min(65536, max(8192, context_window // 4)) + max_context_tokens = min(262144, max(16384, context_window // 3)) hardware = recommendation.get("hardware", {}) memory_source = hardware.get("memory_source") if isinstance(hardware, dict) else None accelerator_type = hardware.get("accelerator_type") if isinstance(hardware, dict) else None @@ -107,6 +108,8 @@ def persist_runtime_recommendation(repo_root: Path, recommendation: dict[str, An "reasoning_mode": reasoning_mode, "max_response_tokens": max_response_tokens, "reasoning_token_budget": reasoning_token_budget, + "runtime_options": runtime_params, + "sampling_options": sampling_params, }, ) text = update_toml_section( @@ -116,6 +119,8 @@ def persist_runtime_recommendation(repo_root: Path, recommendation: dict[str, An "endpoint": base_url.removesuffix("/v1") if runtime == "ollama" else base_url, "model": model_id, "backend": "ollama" if runtime == "ollama" else "openai", + "runtime_options": runtime_params, + "sampling_options": sampling_params, }, ) text = remove_toml_keys(text, "exploration", {"exploration_endpoint", "exploration_model", "exploration_backend"}) @@ -156,6 +161,12 @@ def toml_literal(value: object) -> str: else: items.append(toml_literal(item)) return "[" + ", ".join(items) + "]" + if isinstance(value, dict): + items = [] + for key, item in value.items(): + key_text = str(key).replace("\\", "\\\\").replace('"', '\\"') + items.append(f'"{key_text}" = {toml_literal(item)}') + return "{ " + ", ".join(items) + " }" escaped = str(value).replace("\\", "\\\\").replace('"', '\\"') return f'"{escaped}"' diff --git a/src/vaner/setup/hardware.py b/src/vaner/setup/hardware.py index 4af0b1f..5022355 100644 --- a/src/vaner/setup/hardware.py +++ b/src/vaner/setup/hardware.py @@ -42,6 +42,17 @@ MemoryKind = Literal["vram", "unified", "system", "unknown"] +_NVIDIA_UNIFIED_MEMORY_MARKERS = ( + "dgx spark", + "gb10", + "grace blackwell", +) + + +def _is_nvidia_unified_memory_name(name: str) -> bool: + normalized = name.lower() + return any(marker in normalized for marker in _NVIDIA_UNIFIED_MEMORY_MARKERS) + @dataclass(frozen=True, slots=True) class GPUDevice: @@ -95,6 +106,9 @@ class HardwareProfile: # identify discrete devices — consumers should fall back to the # ``gpu`` / ``gpu_vram_gb`` summary fields in that case. gpu_devices: tuple[GPUDevice, ...] = field(default_factory=tuple) + # Decimal GB free on the user's home volume. Model setup uses this + # to avoid recommending a large local download that cannot fit. + disk_free_gb: int = 0 # --------------------------------------------------------------------------- @@ -140,6 +154,17 @@ def _read_meminfo_gb() -> int | None: return None +def _probe_disk_free_gb(path: Path | None = None) -> int: + """Best-effort decimal GB free on the volume that will hold model caches.""" + + try: + usage = shutil.disk_usage(path or Path.home()) + return max(0, round(usage.free / 1_000_000_000)) + except Exception: + logger.debug("disk free probe failed", exc_info=True) + return 0 + + def _sysctl_memsize_gb() -> int | None: """Run ``sysctl hw.memsize`` (macOS) and convert to GB. Best-effort.""" try: @@ -288,7 +313,7 @@ def _probe_gpu_devices_nvidia_pynvml() -> tuple[GPUDevice, ...] | None: kind="nvidia", memory_total_bytes=total_bytes, memory_display_gb=vram_gb, - memory_kind="vram", + memory_kind="unified" if _is_nvidia_unified_memory_name(name) else "vram", ) ) except Exception: @@ -342,7 +367,7 @@ def _probe_gpu_devices_nvidia_smi() -> tuple[GPUDevice, ...] | None: kind="nvidia", memory_total_bytes=total_bytes, memory_display_gb=vram_gb, - memory_kind="vram", + memory_kind="unified" if _is_nvidia_unified_memory_name(name) else "vram", ) ) return tuple(devices) if devices else None @@ -741,6 +766,7 @@ def detect() -> HardwareProfile: thermal = _probe_thermal() runtimes = _probe_runtimes() models = _probe_models(runtimes) + disk_free_gb = _probe_disk_free_gb() # When the OS probe fails entirely we still need a literal value for the # frozen dataclass; fall back to "linux" but force the tier to "unknown" @@ -778,6 +804,7 @@ def detect() -> HardwareProfile: memory_is_unified=memory_is_unified, tier="unknown", gpu_devices=devices, + disk_free_gb=disk_free_gb, ) final_tier: HardwareTier = "unknown" if os_kind is None else tier_for(profile) return HardwareProfile( @@ -795,6 +822,7 @@ def detect() -> HardwareProfile: memory_is_unified=memory_is_unified, tier=final_tier, gpu_devices=devices, + disk_free_gb=disk_free_gb, ) diff --git a/src/vaner/setup/model_recommendation.py b/src/vaner/setup/model_recommendation.py index 3d8e600..04d41a5 100644 --- a/src/vaner/setup/model_recommendation.py +++ b/src/vaner/setup/model_recommendation.py @@ -29,6 +29,12 @@ class RecommendedModel: min_effective_memory_gb: float recommended_effective_memory_gb: float parameters: dict[str, Any] + family_id: str = "" + params_b: float = 0.0 + active_params_b: float = 0.0 + architecture: str = "dense" + quantization: str = "" + accelerator_tags: tuple[str, ...] = () @classmethod def from_raw(cls, raw: dict[str, Any]) -> RecommendedModel: @@ -44,6 +50,12 @@ def from_raw(cls, raw: dict[str, Any]) -> RecommendedModel: min_effective_memory_gb=float(raw.get("min_effective_memory_gb", 0)), recommended_effective_memory_gb=float(raw.get("recommended_effective_memory_gb", raw.get("min_effective_memory_gb", 0))), parameters=dict(raw.get("parameters", {})), + family_id=str(raw.get("family_id", raw.get("id", ""))), + params_b=float(raw.get("params_b", 0) or 0), + active_params_b=float(raw.get("active_params_b", 0) or 0), + architecture=str(raw.get("architecture", "dense") or "dense"), + quantization=str(raw.get("quantization", "") or ""), + accelerator_tags=tuple(str(v) for v in raw.get("accelerator_tags", []) if isinstance(v, str)), ) @@ -112,6 +124,10 @@ def _fallback_registry(warning: str) -> ModelRegistry: "max_response_tokens": 2048, "reasoning_token_budget": 2048, }, + family_id="qwen3", + params_b=4.0, + architecture="dense", + quantization="Q4_K_M", ), ), ) @@ -137,7 +153,19 @@ def recommend_local_model( for model in reg.models: fit = _fit_status(model, effective_memory_gb) installed_match = (model.runtime, model.id) in installed + disk_status = _disk_status(model, hw) + if disk_status["status"] == "insufficient" and not installed_match: + rejected.append( + { + "model_id": model.id, + "runtime": model.runtime, + "reason": "insufficient_disk", + **disk_status, + } + ) + continue runtime_available = model.runtime in available_runtimes + runtime_installable = _runtime_installable(model.runtime, hw) if fit == "too_large": rejected.append( { @@ -149,10 +177,20 @@ def recommend_local_model( } ) continue - if model.runtime != "ollama" and not runtime_available: + if not runtime_available and not runtime_installable: rejected.append({"model_id": model.id, "runtime": model.runtime, "reason": "runtime_unavailable"}) continue - score = _score_model(model, workload_tags, installed_match, runtime_available, fit) + score = _score_model( + model, + workload_tags, + installed_match, + runtime_available, + fit, + effective_memory_gb=effective_memory_gb, + answers=answers, + hardware=hw, + disk_status=disk_status["status"], + ) candidates.append( ( score, @@ -161,12 +199,14 @@ def recommend_local_model( "fit": fit, "already_installed": installed_match, "runtime_available": runtime_available, + "runtime_installable": runtime_installable, + "disk": disk_status, }, ) ) if not candidates: - fallback = min(reg.models, key=lambda m: m.min_effective_memory_gb) + fallback = min(reg.models, key=lambda m: (m.download_size_gb or 0.0, m.min_effective_memory_gb)) candidates.append( ( 0, @@ -175,6 +215,7 @@ def recommend_local_model( "fit": "fallback_cpu", "already_installed": (fallback.runtime, fallback.id) in installed, "runtime_available": fallback.runtime in available_runtimes, + "disk": _disk_status(fallback, hw), }, ) ) @@ -183,10 +224,10 @@ def recommend_local_model( score, selected, selected_diag = candidates[0] runtime_available = selected.runtime in available_runtimes already_installed = (selected.runtime, selected.id) in installed - needs_runtime_install = selected.runtime == "ollama" and not runtime_available + needs_runtime_install = not runtime_available needs_model_download = not already_installed user_explanation = _plain_explanation(hw, selected, effective_memory_gb, memory_source, already_installed) - install_plan = _install_plan(selected, needs_runtime_install, needs_model_download) + install_plan = _install_plan(selected, needs_runtime_install, needs_model_download, disk_status=selected_diag.get("disk")) runtime = _runtime_payload(selected.runtime) work_styles_tuple: tuple[str, ...] = tuple(answers.work_styles) if answers else () selected_payload = _selected_payload( @@ -222,6 +263,9 @@ def recommend_local_model( "accelerator_label": _accelerator_label(hw), "effective_gb_q4": effective_memory_gb, "memory_source": memory_source, + "disk_free_gb": hw.disk_free_gb, + "gpu_count": _gpu_count(hw), + "gpu_total_memory_gb": _gpu_total_memory_gb(hw), "notes": [], }, "selected": selected_payload, @@ -251,6 +295,7 @@ def recommend_local_model( "raw_hardware": { "memory_total_bytes": hw.memory_total_bytes, "memory_is_unified": hw.memory_is_unified, + "disk_free_gb": hw.disk_free_gb, "gpu_devices": [ { "name": d.name, @@ -266,8 +311,11 @@ def recommend_local_model( } -def _effective_memory_gb(hw: HardwareProfile) -> tuple[float, Literal["vram", "unified", "system", "cpu"]]: - if hw.memory_is_unified and hw.memory_display_gb: +def _effective_memory_gb(hw: HardwareProfile) -> tuple[float, Literal["vram", "unified", "inferred_gpu", "system", "cpu"]]: + if _looks_like_nvidia_unified_memory(hw) and hw.memory_display_gb: + reserve = 8 if hw.memory_display_gb >= 24 else 4 + return max(2.0, float(hw.memory_display_gb - reserve)), "unified" + if hw.memory_is_unified and hw.gpu == "apple_silicon" and hw.memory_display_gb: reserve = 8 if hw.memory_display_gb >= 24 else 4 return max(2.0, float(hw.memory_display_gb - reserve)), "unified" gpu_memories = [d.memory_display_gb for d in hw.gpu_devices if d.memory_kind == "vram" and d.memory_display_gb] @@ -276,17 +324,43 @@ def _effective_memory_gb(hw: HardwareProfile) -> tuple[float, Literal["vram", "u return max(2.0, float(max(gpu_memories) - 2)), "vram" if hw.gpu_vram_gb: return max(2.0, float(hw.gpu_vram_gb - 2)), "vram" + if hw.gpu == "nvidia" and hw.memory_display_gb >= 96: + # NVIDIA + large host memory + missing VRAM telemetry is common on + # new developer-class systems where NVML/nvidia-smi reporting may be + # incomplete or unified-memory platforms are not named clearly. Do + # not treat host RAM as fully usable GPU memory, but do avoid a tiny + # CPU-class default. + return 30.0, "inferred_gpu" if hw.gpu in {"nvidia", "amd"}: - # A discrete GPU without readable VRAM is not enough evidence for - # a large-model recommendation. Stay conservative until diagnostics - # can read the actual accelerator memory. - return min(8.0, max(2.0, float((hw.memory_display_gb or hw.ram_gb) - 8))), "system" + # A discrete GPU without readable VRAM is not enough evidence for a + # large local-model recommendation. System RAM is useful for the OS + # and caches, not for fast Vaner inference. + return 2.0, "system" if hw.memory_display_gb: - reserve = 6 if hw.memory_display_gb >= 16 else 3 - return max(2.0, float(hw.memory_display_gb - reserve)), "system" + return 2.0, "system" return 2.0, "cpu" +def _looks_like_nvidia_unified_memory(hw: HardwareProfile) -> bool: + if hw.gpu != "nvidia": + return False + if hw.memory_is_unified: + return True + names = " ".join(device.name.lower() for device in hw.gpu_devices) + return any(marker in names for marker in ("dgx spark", "gb10", "grace blackwell")) + + +def _gpu_count(hw: HardwareProfile) -> int: + return len([d for d in hw.gpu_devices if d.kind not in {"cpu", "integrated"}]) or (1 if hw.gpu in {"nvidia", "amd"} else 0) + + +def _gpu_total_memory_gb(hw: HardwareProfile) -> int: + values = [int(d.memory_display_gb or 0) for d in hw.gpu_devices if d.memory_display_gb and d.memory_kind in {"vram", "unified"}] + if values: + return sum(values) + return int(hw.gpu_vram_gb or 0) + + def _workload_tags(answers: SetupAnswers | None) -> set[str]: tags = {"general", "summarization"} if answers is None: @@ -313,23 +387,50 @@ def _fit_status(model: RecommendedModel, effective_memory_gb: float) -> Literal[ hardware that can clearly run the larger one. The strict "recommended" tier remains the upper bound. """ - # `min + 4` GB ≈ weights + a sensible 32K-ish KV-cache budget, which - # is what `compute_effective_context_window` actually picks at - # runtime on a card sized at the model's `min_effective_memory_gb`. - relaxed_recommended = model.min_effective_memory_gb + 4.0 + # The registry budgets are calculated against the model's architectural + # max context. Setup picks a runtime-effective context later, so fit + # should test the practical floor-context load too: weights + runtime + # reserve + one 32K KV slice. This is especially important for current + # MoE models where active params make context cheaper than total params + # imply, while weights still need to fit. + kv_reference_gb = _kv_reference_gb(model) or model.download_size_gb + practical_min = (model.download_size_gb or 0.0) + 3.0 + max(0.5, kv_reference_gb * 0.16) + relaxed_recommended = max(practical_min + 4.0, model.min_effective_memory_gb) if effective_memory_gb >= min(model.recommended_effective_memory_gb, relaxed_recommended): return "recommended" - if effective_memory_gb >= model.min_effective_memory_gb: + if effective_memory_gb >= min(model.min_effective_memory_gb, practical_min): return "fits" return "too_large" +def _disk_status(model: RecommendedModel, hw: HardwareProfile) -> dict[str, Any]: + free_gb = int(getattr(hw, "disk_free_gb", 0) or 0) + download_gb = max(0.0, float(model.download_size_gb or 0.0)) + # Keep room for the compressed download, expanded cache/metadata, and a + # little operational headroom. Installed models still report a need here; + # `already_installed` gets scored separately and the install plan can skip + # the download step. + required_gb = round(download_gb * 1.15 + 8.0, 1) if download_gb > 0 else 0.0 + if free_gb <= 0 or required_gb <= 0: + return {"status": "unknown", "free_gb": free_gb, "required_gb": required_gb} + if free_gb < required_gb: + return {"status": "insufficient", "free_gb": free_gb, "required_gb": required_gb} + if free_gb < required_gb + 25.0: + return {"status": "tight", "free_gb": free_gb, "required_gb": required_gb} + return {"status": "enough", "free_gb": free_gb, "required_gb": required_gb} + + def _score_model( model: RecommendedModel, workload_tags: set[str], installed_match: bool, runtime_available: bool, fit: str, + *, + effective_memory_gb: float, + answers: SetupAnswers | None, + hardware: HardwareProfile, + disk_status: str = "unknown", ) -> float: """Score a candidate against the user's hardware + workload tags. @@ -347,6 +448,12 @@ def _score_model( score += tag_overlap * 35.0 score += max(0.0, 30.0 - model.download_size_gb) * 0.5 score += model.recency_rank * 1.0 + score += _context_score(model) + score += _hardware_utilization_score(model, effective_memory_gb, answers) + score += _runtime_affinity_score(model, hardware, runtime_available) + score += _architecture_score(model, hardware) + if model.recency_rank < 80: + score -= (80 - model.recency_rank) * 8.0 if fit == "recommended": score += 75 elif fit == "fits": @@ -355,6 +462,111 @@ def _score_model( score += 120 elif runtime_available: score += 30 + if disk_status == "tight": + score -= 50 + if answers is not None: + if answers.priority in {"speed", "low_resource"} or answers.compute_posture == "light": + score -= max(0.0, model.download_size_gb - 30.0) * 2.1 + if answers.priority == "quality": + score += model.quality_rank * 0.35 + if answers.compute_posture == "available_power": + score += min(90.0, model.recommended_effective_memory_gb * 0.35) + if answers.background_posture == "deep_run_aggressive": + score += min(80.0, _max_context_window(model) / 131072.0 * 10.0) + return score + + +def _runtime_installable(runtime: Runtime, hw: HardwareProfile) -> bool: + if runtime == "ollama": + return True + if runtime == "mlx": + return hw.os == "darwin" and hw.gpu == "apple_silicon" + if runtime == "vllm": + return hw.os == "linux" and hw.gpu == "nvidia" + return False + + +def _max_context_window(model: RecommendedModel) -> int: + try: + return int(model.parameters.get("context_window", _CONTEXT_WINDOW_FLOOR)) + except (TypeError, ValueError): + return _CONTEXT_WINDOW_FLOOR + + +def _context_score(model: RecommendedModel) -> float: + # Reward native long-context models without letting context alone beat + # model quality. 32K => 0, 262K => about 45, 1M => about 75. + window = max(_CONTEXT_WINDOW_FLOOR, _max_context_window(model)) + multiples = max(1.0, window / _CONTEXT_WINDOW_FLOOR) + import math + + return min(90.0, math.log2(multiples) * 15.0) + + +def _hardware_utilization_score(model: RecommendedModel, effective_memory_gb: float, answers: SetupAnswers | None) -> float: + if effective_memory_gb <= 0 or model.recommended_effective_memory_gb <= 0: + return 0.0 + usage = min(1.0, model.recommended_effective_memory_gb / effective_memory_gb) + # Balanced users on very large boxes should not get a tiny-model default. + # Speed / low-resource explicitly opts back toward smaller models. + if answers and (answers.priority in {"speed", "low_resource"} or answers.compute_posture == "light"): + return -40.0 * usage + return min(160.0, 180.0 * (usage**0.5)) + + +def _runtime_affinity_score(model: RecommendedModel, hw: HardwareProfile, runtime_available: bool) -> float: + score = 0.0 + tags = set(model.accelerator_tags) + if hw.gpu == "apple_silicon": + if model.runtime == "mlx": + score += 125.0 + elif model.runtime == "ollama": + score += 20.0 + if "apple_silicon" in tags or "unified_memory" in tags: + score += 35.0 + if model.quantization.upper() in {"MXFP8", "MLX"} or "mlx" in tags: + score += 55.0 + if "blackwell" in tags or model.quantization.upper() == "NVFP4": + score -= 70.0 + elif hw.gpu == "nvidia": + if model.runtime == "vllm": + score += 90.0 + elif model.runtime == "ollama": + score += 35.0 + if "cuda" in tags or "nvidia" in tags: + score += 30.0 + if _looks_like_nvidia_unified_memory(hw): + if "dgx_spark" in tags or "unified_memory" in tags: + score += 90.0 + elif "dgx_spark" in tags: + score -= 55.0 + if "blackwell" in tags and _has_blackwell_gpu(hw): + score += 65.0 + if model.quantization.upper() in {"MXFP8", "MLX"}: + score -= 35.0 + elif model.runtime == "ollama": + score += 25.0 + if not runtime_available and model.runtime != "ollama": + score -= 25.0 + return score + + +def _has_blackwell_gpu(hw: HardwareProfile) -> bool: + names = " ".join(device.name.lower() for device in hw.gpu_devices) + return any(marker in names for marker in ("rtx 50", "5090", "5080", "5070", "5060", "blackwell", "pro 6000", "dgx spark", "gb10")) + + +def _architecture_score(model: RecommendedModel, hw: HardwareProfile) -> float: + if model.architecture.lower() != "moe": + return 0.0 + active = model.active_params_b or model.params_b + total = model.params_b or active + if active <= 0 or total <= 0: + return 25.0 + sparse_ratio = max(0.0, min(1.0, 1.0 - (active / total))) + score = 35.0 + sparse_ratio * 45.0 + if (hw.memory_is_unified or _looks_like_nvidia_unified_memory(hw)) and hw.memory_display_gb >= 128: + score += 30.0 return score @@ -367,6 +579,22 @@ def _runtime_payload(runtime: Runtime) -> dict[str, Any]: "native_endpoint": OLLAMA_NATIVE_ENDPOINT, "install_managed": True, } + if runtime == "mlx": + return { + "id": "mlx", + "label": "MLX", + "base_url": "http://127.0.0.1:8080/v1", + "native_endpoint": "http://127.0.0.1:8080", + "install_managed": True, + } + if runtime == "vllm": + return { + "id": "vllm", + "label": "vLLM", + "base_url": "http://127.0.0.1:8000/v1", + "native_endpoint": "http://127.0.0.1:8000", + "install_managed": True, + } return {"id": runtime, "label": runtime, "base_url": "", "install_managed": False} @@ -430,6 +658,7 @@ def compute_effective_context_window( effective_memory_gb: float, work_styles: tuple[str, ...] = (), runtime: str = "ollama", + kv_reference_gb: float | None = None, ) -> int: """Pick a runtime-effective context window. @@ -441,14 +670,14 @@ def compute_effective_context_window( from the hardware probe. - ``work_styles``: the wizard's archetype answers (coding, research, …) — long-context archetypes get a higher target. - - ``runtime``: today only ``ollama`` is wired; left as an input so - future runtimes (vLLM, llama.cpp w/ flash-attn) can override. + - ``runtime``: runtime family identifier; local runtimes can tune + context/KV behavior as support evolves. Returns the chosen window, clamped to [floor, max]. Heuristic — KV-cache scales roughly linearly with both context length and weight size; for Q4 + GQA models a 32K context costs - around 18% of weight memory. We turn that around: pick the largest + around 16% of weight memory. We turn that around: pick the largest multiple of 32K that fits in the headroom we have after weights and a small safety reserve. """ @@ -469,10 +698,11 @@ def compute_effective_context_window( # extra reserve keeps long-context defaults from pinning VRAM at the edge. runtime_reserve_gb = 3.0 headroom_gb = max(0.0, effective_memory_gb - weights_gb - runtime_reserve_gb) - # Cost of KV at the family's 32K reference. 0.18 is the rough Q4+GQA + # Cost of KV at the family's 32K reference. 0.16 is the rough Q4+GQA # constant; flash-attn / paged-attention runtimes get more headroom # implicitly because they pack the cache more tightly. - cost_per_32k = max(0.5, weights_gb * 0.18) + kv_gb = kv_reference_gb if kv_reference_gb and kv_reference_gb > 0 else weights_gb + cost_per_32k = max(0.5, kv_gb * 0.16) if cost_per_32k <= 0: return min(cap, max(floor, target)) @@ -483,11 +713,47 @@ def compute_effective_context_window( return min(cap, floor) affordable = floor * max(1, multiples) chosen = min(cap, max(target, affordable)) + chosen = min(chosen, _default_context_tier_cap(effective_memory_gb, weights_gb, runtime)) # Round down to the nearest multiple of 8K so num_ctx is friendly. chosen = (chosen // 8192) * 8192 return min(cap, max(floor, chosen)) +def _default_context_tier_cap(effective_memory_gb: float, weights_gb: float, runtime: str) -> int: + """Cap first-run context by memory tier. + + This is intentionally more conservative than "what might fit". Vaner's + default runner is Ollama, and onboarding must avoid accidental CPU + offload / KV pressure. Larger windows remain available through advanced + or custom profiles; this function chooses the safe first-run default. + """ + + if effective_memory_gb <= 0: + return _CONTEXT_WINDOW_FLOOR + if effective_memory_gb < 12: + return 32768 + if effective_memory_gb < 18: + return 65536 + if effective_memory_gb < 28: + return 131072 + if effective_memory_gb < 44: + return 65536 if weights_gb >= 20 else 131072 + if effective_memory_gb < 96: + return 262144 + if runtime == "ollama": + return 262144 + return 524288 + + +def _kv_reference_gb(model: RecommendedModel) -> float: + if model.architecture.lower() == "moe" and model.active_params_b > 0: + # MoE models load all weights, but the active expert set gives a + # better default proxy for KV/context scaling than total parameters. + active_weight_gb = model.active_params_b * 0.55 + return max(active_weight_gb, model.download_size_gb * 0.08) + return model.download_size_gb or 0.0 + + def _split_parameters(parameters: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: """Split a registry ``parameters`` block into capability / runtime / sampling. @@ -539,6 +805,7 @@ def _selected_payload( effective_memory_gb=float(effective_memory_gb or 0.0), work_styles=work_styles, runtime=model.runtime, + kv_reference_gb=_kv_reference_gb(model), ) capability["context_window"] = effective_ctx capability["max_context_window"] = max_ctx @@ -557,8 +824,15 @@ def _selected_payload( "download_size_gb": model.download_size_gb, "min_effective_memory_gb": model.min_effective_memory_gb, "recommended_effective_memory_gb": model.recommended_effective_memory_gb, + "family_id": model.family_id, + "params_b": model.params_b, + "active_params_b": model.active_params_b, + "architecture": model.architecture, + "quantization": model.quantization, + "accelerator_tags": list(model.accelerator_tags), "already_installed": bool(diag.get("already_installed")), "fit": diag.get("fit", "unknown"), + "disk": diag.get("disk", {"status": "unknown"}), "params": combined, "runtime_params": runtime_params, "model_params": combined, @@ -567,19 +841,59 @@ def _selected_payload( } -def _install_plan(model: RecommendedModel, needs_runtime_install: bool, needs_model_download: bool) -> list[dict[str, Any]]: +def _install_plan( + model: RecommendedModel, + needs_runtime_install: bool, + needs_model_download: bool, + *, + disk_status: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: steps = [{"id": "save_config", "label": "Save Vaner settings", "required": True}] + if disk_status and disk_status.get("status") in {"tight", "insufficient"}: + steps.append( + { + "id": "confirm_disk_space", + "label": "Free disk space" if disk_status.get("status") == "insufficient" else "Confirm disk space", + "required": True, + "free_gb": disk_status.get("free_gb"), + "required_gb": disk_status.get("required_gb"), + } + ) if needs_runtime_install: - steps.append({"id": "install_runtime", "label": "Install Ollama", "required": True}) + if model.runtime == "mlx": + steps.append( + { + "id": "install_runtime", + "label": "Install MLX", + "required": True, + "command": ["python", "-m", "pip", "install", "mlx-lm"], + } + ) + elif model.runtime == "vllm": + steps.append( + { + "id": "install_runtime", + "label": "Install vLLM", + "required": True, + "command": ["python", "-m", "pip", "install", "vllm"], + } + ) + else: + steps.append({"id": "install_runtime", "label": "Install Ollama", "required": True}) else: steps.append({"id": "check_runtime", "label": "Check local model runner", "required": True}) if needs_model_download: + command: list[str] = [] + if model.runtime == "ollama": + command = ["ollama", "pull", model.id] + elif model.runtime == "mlx": + command = ["mlx_lm.server", "--model", model.id, "--port", "8080"] steps.append( { "id": "download_model", "label": f"Download {model.display_name}", "required": True, - "command": ["ollama", "pull", model.id] if model.runtime == "ollama" else [], + "command": command, } ) else: @@ -632,7 +946,12 @@ def _plain_explanation( if memory_source == "vram": return f"Vaner found enough GPU memory for {model.display_name} with headroom for normal desktop use.{installed}" if memory_source == "unified": - return f"Vaner found unified memory and chose {model.display_name} with safe headroom for macOS and the model runner.{installed}" + return f"Vaner found unified accelerator memory and chose {model.display_name} with safe headroom for the desktop and model runner.{installed}" + if memory_source == "inferred_gpu": + return ( + f"Vaner found an NVIDIA GPU but could not read exact GPU memory, so it chose {model.display_name} " + f"as a cautious GPU-class default instead of a tiny CPU model.{installed}" + ) if hw.gpu == "none": return ( f"Vaner did not find a dedicated GPU, so it chose {model.display_name} as a safer local setup. " @@ -647,8 +966,11 @@ def _hardware_summary(hw: HardwareProfile, effective_memory_gb: float, memory_so "accelerator_type": hw.gpu, "effective_memory_gb": effective_memory_gb, "memory_source": memory_source, + "gpu_count": _gpu_count(hw), + "gpu_total_memory_gb": _gpu_total_memory_gb(hw), "system_memory_gb": hw.memory_display_gb or hw.ram_gb, - "is_unified_memory": hw.memory_is_unified, + "is_unified_memory": hw.memory_is_unified or _looks_like_nvidia_unified_memory(hw), + "disk_free_gb": hw.disk_free_gb, "tier": hw.tier, } diff --git a/src/vaner/setup/serializers.py b/src/vaner/setup/serializers.py index ad743bb..44163e7 100644 --- a/src/vaner/setup/serializers.py +++ b/src/vaner/setup/serializers.py @@ -104,6 +104,7 @@ def hardware_to_dict(hw: HardwareProfile) -> dict[str, Any]: "thermal_constrained": hw.thermal_constrained, "detected_runtimes": list(hw.detected_runtimes), "detected_models": [list(row) for row in hw.detected_models], + "disk_free_gb": hw.disk_free_gb, "tier": hw.tier, } diff --git a/src/vaner/store/artefacts.py b/src/vaner/store/artefacts.py index e9981e5..60a4a3e 100644 --- a/src/vaner/store/artefacts.py +++ b/src/vaner/store/artefacts.py @@ -787,26 +787,74 @@ async def list( cursor = await db.execute(query, tuple(params)) rows = await cursor.fetchall() - artefacts: list[Artefact] = [] + return [self._artefact_from_row(row) for row in rows] + + async def list_by_keys(self, keys: set[str] | list[str] | tuple[str, ...], *, limit: int = 200) -> list[Artefact]: + ordered = list(dict.fromkeys(str(key) for key in keys if str(key))) + if not ordered: + return [] + capped = ordered[: max(1, int(limit))] + placeholders = ",".join("?" for _ in capped) + query = ( + "SELECT key, kind, source_path, source_mtime, generated_at, model, content, " + "metadata_json, relevance_score, access_count, last_accessed, signal_id " + f"FROM artefacts WHERE key IN ({placeholders})" + ) + async with self._connect() as db: + cursor = await db.execute(query, tuple(capped)) + rows = await cursor.fetchall() + by_key = {row[0]: self._artefact_from_row(row) for row in rows} + return [by_key[key] for key in capped if key in by_key] + + async def list_by_source_paths(self, paths: set[str] | list[str] | tuple[str, ...], *, limit: int = 200) -> list[Artefact]: + ordered = list(dict.fromkeys(str(path) for path in paths if str(path))) + if not ordered: + return [] + capped = ordered[: max(1, int(limit))] + placeholders = ",".join("?" for _ in capped) + query = ( + "SELECT key, kind, source_path, source_mtime, generated_at, model, content, " + "metadata_json, relevance_score, access_count, last_accessed, signal_id " + f"FROM artefacts WHERE source_path IN ({placeholders})" + ) + async with self._connect() as db: + cursor = await db.execute(query, tuple(capped)) + rows = await cursor.fetchall() + by_path: dict[str, list[Artefact]] = {} for row in rows: - artefacts.append( - Artefact( - key=row[0], - kind=ArtefactKind(row[1]), - source_path=row[2], - source_mtime=row[3], - generated_at=row[4], - model=row[5], - content=row[6], - metadata=json.loads(row[7]), - relevance_score=row[8], - access_count=row[9], - last_accessed=row[10], - signal_id=row[11], - ) - ) + artefact = self._artefact_from_row(row) + by_path.setdefault(artefact.source_path, []).append(artefact) + artefacts: list[Artefact] = [] + for path in capped: + artefacts.extend(by_path.get(path, [])) return artefacts + async def list_source_paths(self, *, limit: int = 2000) -> list[str]: + async with self._connect() as db: + cursor = await db.execute( + "SELECT DISTINCT source_path FROM artefacts WHERE source_path != '' ORDER BY source_path LIMIT ?", + (max(1, int(limit)),), + ) + rows = await cursor.fetchall() + return [str(row[0]) for row in rows if row[0]] + + @staticmethod + def _artefact_from_row(row: tuple[object, ...]) -> Artefact: + return Artefact( + key=row[0], + kind=ArtefactKind(row[1]), + source_path=row[2], + source_mtime=row[3], + generated_at=row[4], + model=row[5], + content=row[6], + metadata=json.loads(row[7]), + relevance_score=row[8], + access_count=row[9], + last_accessed=row[10], + signal_id=row[11], + ) + async def mark_accessed(self, key: str) -> None: async with self._connect() as db: await db.execute( diff --git a/src/vaner/store/scenarios/sqlite.py b/src/vaner/store/scenarios/sqlite.py index 2358245..ce7c482 100644 --- a/src/vaner/store/scenarios/sqlite.py +++ b/src/vaner/store/scenarios/sqlite.py @@ -4,6 +4,7 @@ import re import time from collections import defaultdict +from dataclasses import dataclass from pathlib import Path from typing import cast @@ -25,6 +26,28 @@ ScenarioVisibility, ) +SCENARIO_SAMPLE_MIN_INTERVAL_SECONDS = 15.0 +SCENARIO_SAMPLE_KEEP_ROWS = 50_000 + + +@dataclass(frozen=True) +class ScenarioSample: + ts: float + scenario_id: str + relevance: float + readiness: str + confidence: float + freshness: str + visible_priority: float + visibility: str + lifecycle_motion: str + status: str + pinned: bool + active: bool + cycle_id: str | None = None + job_id: str | None = None + source_event_id: str | None = None + class ScenarioStore: def __init__(self, db_path: Path) -> None: @@ -77,11 +100,36 @@ async def initialize(self) -> None: ) """ ) + await db.execute( + """ + CREATE TABLE IF NOT EXISTS scenario_samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts REAL NOT NULL, + scenario_id TEXT NOT NULL, + relevance REAL NOT NULL, + readiness TEXT NOT NULL, + confidence REAL NOT NULL, + freshness TEXT NOT NULL, + visible_priority REAL NOT NULL, + visibility TEXT NOT NULL, + lifecycle_motion TEXT NOT NULL, + status TEXT NOT NULL, + pinned INTEGER NOT NULL DEFAULT 0, + active INTEGER NOT NULL DEFAULT 0, + cycle_id TEXT, + job_id TEXT, + source_event_id TEXT, + FOREIGN KEY (scenario_id) REFERENCES scenarios(id) ON DELETE CASCADE + ) + """ + ) await db.execute("CREATE INDEX IF NOT EXISTS idx_scenarios_kind ON scenarios(kind)") await db.execute("CREATE INDEX IF NOT EXISTS idx_scenarios_score ON scenarios(score DESC)") await db.execute("CREATE INDEX IF NOT EXISTS idx_scenarios_freshness ON scenarios(freshness)") await db.execute("CREATE INDEX IF NOT EXISTS idx_scenario_evidence_sid ON scenario_evidence(scenario_id)") await db.execute("CREATE INDEX IF NOT EXISTS idx_prompt_macro_clusters_centroid ON prompt_macro_clusters(centroid_label)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_scenario_samples_sid_ts ON scenario_samples(scenario_id, ts)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_scenario_samples_ts ON scenario_samples(ts)") await self._add_column_if_missing(db, "ALTER TABLE scenarios ADD COLUMN context_envelope_json TEXT NOT NULL DEFAULT '{}'") await self._add_column_if_missing(db, "ALTER TABLE scenarios ADD COLUMN memory_state TEXT NOT NULL DEFAULT 'candidate'") await self._add_column_if_missing(db, "ALTER TABLE scenarios ADD COLUMN memory_confidence REAL NOT NULL DEFAULT 0.0") @@ -259,6 +307,37 @@ async def list_top(self, *, kind: str | None = None, limit: int = 10, visibility evidence_map = await self._load_evidence_for_scenarios(db, [str(row["id"]) for row in rows]) return [self._row_to_scenario(row, evidence_map.get(str(row["id"]), [])) for row in rows] + async def list_samples( + self, + *, + scenario_ids: list[str] | None = None, + start_ts: float | None = None, + end_ts: float | None = None, + limit: int = 20_000, + ) -> list[ScenarioSample]: + query = "SELECT * FROM scenario_samples" + predicates: list[str] = [] + params: list[object] = [] + if scenario_ids: + placeholders = ", ".join("?" for _ in scenario_ids) + predicates.append(f"scenario_id IN ({placeholders})") + params.extend(scenario_ids) + if start_ts is not None: + predicates.append("ts >= ?") + params.append(float(start_ts)) + if end_ts is not None: + predicates.append("ts <= ?") + params.append(float(end_ts)) + if predicates: + query += " WHERE " + " AND ".join(predicates) + query += " ORDER BY ts ASC LIMIT ?" + params.append(max(1, min(100_000, int(limit)))) + async with aiosqlite.connect(self.db_path) as db: + db.row_factory = aiosqlite.Row + cur = await db.execute(query, params) + rows = await cur.fetchall() + return [self._sample_from_row(row) for row in rows] + async def get(self, scenario_id: str) -> Scenario | None: async with aiosqlite.connect(self.db_path) as db: db.row_factory = aiosqlite.Row @@ -658,6 +737,7 @@ def _row_to_scenario(self, row: aiosqlite.Row, evidence_rows: list[aiosqlite.Row async def _refresh_lifecycle_for_ids(self, db: aiosqlite.Connection, scenario_ids: list[str], *, now: float | None = None) -> None: if not scenario_ids: return + sample_ts = time.time() if now is None else now placeholders = ", ".join("?" for _ in scenario_ids) db.row_factory = aiosqlite.Row cur = await db.execute(f"SELECT * FROM scenarios WHERE id IN ({placeholders})", scenario_ids) @@ -686,6 +766,98 @@ async def _refresh_lifecycle_for_ids(self, db: aiosqlite.Connection, scenario_id refreshed.id, ), ) + await self._record_sample_if_changed(db, refreshed, ts=sample_ts) + + async def _record_sample_if_changed(self, db: aiosqlite.Connection, scenario: Scenario, *, ts: float) -> None: + cur = await db.execute( + """ + SELECT ts, relevance, readiness, confidence, freshness, visible_priority, + visibility, lifecycle_motion, status, pinned, active + FROM scenario_samples + WHERE scenario_id = ? + ORDER BY ts DESC + LIMIT 1 + """, + (scenario.id,), + ) + latest = await cur.fetchone() + status = _scenario_status(scenario) + active = 1 if status == "active" else 0 + pinned = 1 if int(scenario.pinned) else 0 + if latest is not None: + age = ts - float(latest[0]) + numeric_delta = max( + abs(float(latest[1]) - float(scenario.relevance)), + abs(float(latest[3]) - float(scenario.confidence)), + abs(float(latest[5]) - float(scenario.visible_priority)), + ) + categorical_same = ( + str(latest[2]) == scenario.readiness + and str(latest[4]) == scenario.freshness + and str(latest[6]) == scenario.visibility + and str(latest[7]) == scenario.lifecycle_motion + and str(latest[8]) == status + and int(latest[9]) == pinned + and int(latest[10]) == active + ) + if categorical_same and age < SCENARIO_SAMPLE_MIN_INTERVAL_SECONDS: + return + if categorical_same and numeric_delta < 0.002 and age < 300: + return + await db.execute( + """ + INSERT INTO scenario_samples ( + ts, scenario_id, relevance, readiness, confidence, freshness, + visible_priority, visibility, lifecycle_motion, status, pinned, + active, cycle_id, job_id, source_event_id + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL) + """, + ( + float(ts), + scenario.id, + float(scenario.relevance), + scenario.readiness, + float(scenario.confidence), + scenario.freshness, + float(scenario.visible_priority), + scenario.visibility, + scenario.lifecycle_motion, + status, + pinned, + active, + ), + ) + await db.execute( + """ + DELETE FROM scenario_samples + WHERE id IN ( + SELECT id FROM scenario_samples + ORDER BY ts DESC + LIMIT -1 OFFSET ? + ) + """, + (SCENARIO_SAMPLE_KEEP_ROWS,), + ) + + def _sample_from_row(self, row: aiosqlite.Row) -> ScenarioSample: + return ScenarioSample( + ts=float(row["ts"]), + scenario_id=str(row["scenario_id"]), + relevance=float(row["relevance"]), + readiness=str(row["readiness"]), + confidence=float(row["confidence"]), + freshness=str(row["freshness"]), + visible_priority=float(row["visible_priority"]), + visibility=str(row["visibility"]), + lifecycle_motion=str(row["lifecycle_motion"]), + status=str(row["status"]), + pinned=bool(row["pinned"]), + active=bool(row["active"]), + cycle_id=str(row["cycle_id"]) if row["cycle_id"] is not None else None, + job_id=str(row["job_id"]) if row["job_id"] is not None else None, + source_event_id=str(row["source_event_id"]) if row["source_event_id"] is not None else None, + ) async def _add_column_if_missing(self, db: aiosqlite.Connection, ddl: str) -> None: try: @@ -693,3 +865,17 @@ async def _add_column_if_missing(self, db: aiosqlite.Connection, ddl: str) -> No except aiosqlite.OperationalError as exc: if "duplicate column" not in str(exc).lower(): raise + + +def _scenario_status(scenario: Scenario) -> str: + if scenario.last_outcome == "useful": + return "completed" + if scenario.last_outcome in {"irrelevant", "wrong"}: + return "rejected" + if scenario.visibility == "archived" or scenario.freshness == "stale": + return "stale" + if scenario.readiness == "ready": + return "ready" + if scenario.readiness == "cooling": + return "cooling" + return "prep" diff --git a/tests/test_broker/test_answerable.py b/tests/test_broker/test_answerable.py index 575e955..89f8b14 100644 --- a/tests/test_broker/test_answerable.py +++ b/tests/test_broker/test_answerable.py @@ -147,3 +147,68 @@ def test_evidence_assembly_safe_labels_transport_limited_context(): assert assembly.items_transport_limited >= 1 assert any(decision.decision == "transport_limited" for decision in assembly.decisions) assert briefing.sections[0].items + + +def test_answerable_briefing_keeps_multiple_evidence_roles_from_one_file(): + content = "\n".join( + [ + "CACHE_HIT_THRESHOLD = 0.87", + "RAW_REWARD_WEIGHT = 0.15", + *[f"# filler {index}" for index in range(35)], + "def compute_reward(inputs):", + " raw_reward = inputs.cache_tier + inputs.quality_lift", + " return {'reward_total': raw_reward}", + *[f"# more filler {index}" for index in range(35)], + "def apply_cache_threshold(similarity):", + " return similarity >= CACHE_HIT_THRESHOLD", + ] + ) + + briefing = build_answerable_briefing( + "Explain cache thresholds and the raw reward flow.", + [_artefact("src/vaner/learning/reward.py", content)], + max_tokens=260, + assembly_mode="safe", + ) + + text = briefing.text + assert "CACHE_HIT_THRESHOLD" in text + assert "RAW_REWARD_WEIGHT" in text + assert "raw_reward" in text + assert "apply_cache_threshold" in text + assert "role=constant_or_default" in text + assert "L" in briefing.sections[0].items[0].excerpt + assert briefing.metadata.evidence_assembly.items_protected >= 1 + + +def test_answerable_briefing_preserves_llm_exploration_json_contract_and_parser(): + content = "\n".join( + [ + 'JSON_CONTRACT_POLICY = """Return JSON with ranked_files and follow_on."""', + *[f"# prompt filler {index}" for index in range(30)], + "def parse_exploration_response(text):", + " payload = json.loads(text)", + " ranked_files = payload['ranked_files']", + " follow_on = payload.get('follow_on', [])", + " return ranked_files, follow_on", + *[f"# behavior filler {index}" for index in range(30)], + "def enqueue_follow_on_scenarios(frontier, follow_on):", + " for scenario in follow_on:", + " frontier.add(scenario)", + ] + ) + + briefing = build_answerable_briefing( + "Explain the LLM exploration JSON contract, parser behavior, and follow-on behavior.", + [_artefact("src/vaner/policy/internal_llm.py", content)], + max_tokens=280, + assembly_mode="safe", + ) + + text = briefing.text + assert "JSON_CONTRACT_POLICY" in text + assert "json.loads" in text + assert "ranked_files" in text + assert "follow_on" in text + assert "enqueue_follow_on_scenarios" in text + assert briefing.metadata.evidence_assembly.items_protected >= 1 diff --git a/tests/test_broker/test_compressor.py b/tests/test_broker/test_compressor.py index 53ed591..85a6576 100644 --- a/tests/test_broker/test_compressor.py +++ b/tests/test_broker/test_compressor.py @@ -32,10 +32,24 @@ def test_compressor_empty_input(): def test_compressor_single_chunk_over_budget(): artefacts = [_artefact("a", "a.py", "word " * 400)] context, token_map, used, kept = compress_context(artefacts, max_tokens=10) - assert context == "" - assert token_map["a"] > 10 - assert used == 0 - assert kept == set() + assert "a.py" in context + assert "[trimmed]" in context + assert token_map["a"] <= 10 + assert used == token_map["a"] + assert kept == {"a"} + + +def test_compressor_trims_top_scored_chunk_instead_of_dropping_it(): + artefacts = [_artefact("low", "low.py", "tiny"), _artefact("high", "high.py", "alpha " * 500)] + context, token_map, used, kept = compress_context( + artefacts, + max_tokens=24, + score_by_key={"high": 10.0, "low": 1.0}, + ) + assert "high.py" in context + assert context.index("high.py") < context.find("low.py") if "low.py" in context else True + assert "high" in kept + assert used == sum(token_map[key] for key in kept) def test_compressor_counts_tokens_once(): @@ -56,3 +70,119 @@ def test_compressor_prefers_high_score_when_order_unsorted(): ) assert kept == {"high", "low"} assert context.index("high.py") < context.index("low.py") + + +def test_compressor_compacts_large_chunks_to_implementation_anchors(): + content = "\n".join( + [ + "intro " * 300, + "Schema: CREATE TABLE artefacts (key TEXT PRIMARY KEY, source_path TEXT NOT NULL)", + "Functions: list_by_keys(keys: list[str]) -> list[Artefact]", + "filler " * 300, + ] + ) + artefacts = [_artefact("store", "src/vaner/store/artefacts.py", content)] + context, _, used, kept = compress_context(artefacts, max_tokens=180) + + assert kept == {"store"} + assert used <= 180 + assert "CREATE TABLE artefacts" in context + assert "list_by_keys" in context + assert "compacted to important implementation anchors" in context + + +def test_query_aware_compressor_preserves_dispersed_reward_ingredients(): + content = "\n".join( + [ + "from dataclasses import dataclass", + "DEFAULT_REWARD_WEIGHTS = {'cache_tier': 0.25, 'quality_lift': 0.35, 'raw_reward': 0.10}", + *[f"# filler {index}" for index in range(40)], + "def compute_reward(inputs):", + " raw_reward = inputs.cache_tier + inputs.quality_lift", + " return {'reward_total': raw_reward, 'reward_components': DEFAULT_REWARD_WEIGHTS}", + *[f"# gap {index}" for index in range(40)], + "def update_replay_priority(outcome):", + " downstream_consumer = outcome.reward_total", + " return 1.0 + downstream_consumer", + ] + ) + artefacts = [_artefact("reward", "src/vaner/learning/reward.py", content)] + + context, token_map, used, kept = compress_context( + artefacts, + max_tokens=170, + score_by_key={"reward": 30.0}, + query="How does reward computation use default weights and downstream consumers?", + ) + + assert kept == {"reward"} + assert used <= 170 + assert token_map["reward"] == used + assert "DEFAULT_REWARD_WEIGHTS" in context + assert "reward_total" in context + assert "downstream_consumer" in context + assert "@@ lines" in context + + +def test_query_aware_compressor_keeps_non_redundant_bridging_file(): + direct = _artefact( + "direct", + "src/vaner/intent/scorer.py", + "\n".join( + [ + "def feature_vector_for_artefact(item):", + " return [item.path_score, item.content_score]", + *["# scorer filler" for _ in range(80)], + ] + ), + ) + bridge = _artefact( + "bridge", + "src/vaner/intent/trainer.py", + "def train_target_from_reward(payload):\n" + " target = (payload['reward_total'] + 1.0) * 0.5\n" + " return target\n", + ) + + context, _, used, kept = compress_context( + [direct, bridge], + max_tokens=150, + score_by_key={"direct": 50.0, "bridge": 5.0}, + query="How does the IntentScorer use GBDT feature groups, training target, and blending logic?", + ) + + assert used <= 150 + assert {"direct", "bridge"} <= kept + assert "feature_vector_for_artefact" in context + assert "train_target_from_reward" in context + assert "reward_total" in context + + +def test_query_aware_compressor_preserves_schema_and_methods_under_budget(): + content = "\n".join( + [ + "class ArtefactStore:", + " SCHEMA = '''CREATE TABLE context_packages (id TEXT PRIMARY KEY, injected_context TEXT)'''", + *[" # storage filler" for _ in range(50)], + " def persist_context_package(self, package):", + " self.conn.execute('INSERT INTO context_packages VALUES (?, ?)', (package.id, package.injected_context))", + *[" # retrieval filler" for _ in range(50)], + " def get_context_package(self, package_id):", + " return self.conn.execute('SELECT id, injected_context FROM context_packages WHERE id=?', (package_id,)).fetchone()", + ] + ) + artefacts = [_artefact("store", "src/vaner/store/artefacts.py", content)] + + context, _, used, kept = compress_context( + artefacts, + max_tokens=190, + score_by_key={"store": 40.0}, + query="How does the ArtefactStore persist and retrieve context packages? What database schema does it use?", + ) + + assert kept == {"store"} + assert used <= 190 + assert "CREATE TABLE context_packages" in context + assert "persist_context_package" in context + assert "get_context_package" in context + assert "SELECT id, injected_context" in context diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index d0c01ac..172f994 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -4,6 +4,7 @@ import time +from vaner.broker.context_preparation import infer_context_preparation_profile from vaner.broker.selector import select_artefacts from vaner.models.artefact import Artefact, ArtefactKind @@ -379,6 +380,210 @@ def test_select_artefacts_prioritizes_llm_exploration_flow_files(): ] +def test_select_artefacts_resolves_rollout_rehearsal_paraphrase(): + artefacts = [ + Artefact( + key="file_summary:src/runtime/release_notes.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/runtime/release_notes.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="Release notes formatter for changelog entries and deployment announcements.", + ), + Artefact( + key="file_summary:src/runtime/traffic_escrow.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/runtime/traffic_escrow.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content=( + "TrafficEscrow controller coordinates rehearse_proxy replay runs, " + "smoke policy checks, and staged promote gates." + ), + ), + ] + + selected = select_artefacts( + "What prevents a candidate release from getting full traffic until replay and smoke checks pass?", + artefacts, + top_n=1, + ) + + assert selected[0].source_path == "src/runtime/traffic_escrow.py" + + +def test_select_artefacts_resolves_vectorization_region_paraphrase(): + artefacts = [ + Artefact( + key="file_summary:src/incidents/general_latency.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/incidents/general_latency.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="Generic latency incident notes for customer-facing status updates.", + ), + Artefact( + key="file_summary:src/incidents/eu_apac_embedding_egress.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/incidents/eu_apac_embedding_egress.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content=( + "Embedding batch incident: eu-west residency stamp lag caused " + "ap-southeast edge fallback and cross-region egress." + ), + ), + ] + + selected = select_artefacts( + "Why did a Western Europe tenant get routed to a Southeast Asia edge during a vectorization spike?", + artefacts, + top_n=1, + ) + + assert selected[0].source_path == "src/incidents/eu_apac_embedding_egress.py" + + +def test_select_artefacts_resolves_low_precision_numeric_mode_paraphrase(): + artefacts = [ + Artefact( + key="file_summary:src/runtime/model_limits.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/runtime/model_limits.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="Model admission limits and request queue accounting.", + ), + Artefact( + key="file_summary:src/runtime/precision_annealing.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/runtime/precision_annealing.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="kernel_stability_threshold precision annealing pass rate checks before stepping from fp32 to int8.", + ), + ] + + selected = select_artefacts( + "What default pass rate is required before stepping down from the safest numeric mode in low bit inference?", + artefacts, + top_n=1, + ) + + assert selected[0].source_path == "src/runtime/precision_annealing.py" + + +def test_select_artefacts_prefers_reward_computation_source_over_generic_policy(): + artefacts = [ + Artefact( + key="file_summary:src/vaner/intent/scoring_policy.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/vaner/intent/scoring_policy.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="score weights priority scenario policy multiplicative nudges", + ), + Artefact( + key="file_summary:src/vaner/learning/reward.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/vaner/learning/reward.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="compute_reward RewardInput cache_tier similarity quality_lift host_outcome judge_score reward_total reward_components", + ), + ] + + selected = select_artefacts( + "How does the reward computation work? What signals does it combine to produce the final reward value?", + artefacts, + top_n=1, + ) + + assert selected[0].source_path == "src/vaner/learning/reward.py" + + +def test_select_artefacts_keeps_intent_scorer_and_features_together(): + artefacts = [ + Artefact( + key="file_summary:src/vaner/intent/scorer.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/vaner/intent/scorer.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="class IntentScorer HistGradientBoosting model score predict feature vector", + ), + Artefact( + key="file_summary:src/vaner/intent/features.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/vaner/intent/features.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content=( + "extract_hybrid_features feature_vector_for_artefact active signal flags replay priority " + "reward target component weights" + ), + ), + Artefact( + key="file_summary:src/vaner/router/proxy.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/vaner/router/proxy.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="proxy routing request response", + ), + ] + + selected = select_artefacts( + "How does the IntentScorer use GBDT models? What features does it extract and how are they combined?", + artefacts, + top_n=2, + ) + + assert {item.source_path for item in selected} == {"src/vaner/intent/scorer.py", "src/vaner/intent/features.py"} + + +def test_select_artefacts_prefers_artefact_store_schema_over_package_metadata(): + artefacts = [ + Artefact( + key="file_summary:src/vaner/store/artefacts.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="src/vaner/store/artefacts.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="class ArtefactStore CREATE TABLE artefacts SELECT key INSERT INTO artefacts context packages persist retrieve schema", + ), + Artefact( + key="file_summary:ui/cockpit/package.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="ui/cockpit/package.json", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="package scripts dependencies", + ), + ] + + selected = select_artefacts( + "How does the ArtefactStore persist and retrieve context packages? What database schema does it use?", + artefacts, + top_n=1, + ) + + assert selected[0].source_path == "src/vaner/store/artefacts.py" + + def test_select_artefacts_custom_scorer_changes_ranking(): artefacts = [ Artefact( @@ -535,3 +740,53 @@ def custom_scorer(_: str, artefact: Artefact) -> float: selected = select_artefacts("anything", artefacts, top_n=2, scorer=custom_scorer) assert [a.key for a in selected] == ["file_summary:top.py"] + + +def test_context_profile_infers_multi_source_synthesis_without_benchmark_labels(): + profile = infer_context_preparation_profile( + "List every customer escalation across Slack and Jira after March 2026 and summarize the common risk." + ) + + assert profile.need == "multi_source_synthesis" + assert profile.archetype in {"general", "operator"} + assert "slack" in profile.source_hints + assert "jira" in profile.source_hints + assert any(constraint.kind == "restrictive_language" and constraint.value == "after" for constraint in profile.constraints) + + +def test_select_artefacts_disables_global_gate_for_multi_source_context_need(): + artefacts = [ + Artefact( + key="file_summary:top.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="top.py", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="top result", + metadata={"corpus_id": "repo"}, + ), + Artefact( + key="file_summary:low_notes.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="low_notes.md", + source_mtime=time.time(), + generated_at=time.time(), + model="test", + content="low result", + metadata={"corpus_id": "notes"}, + ), + ] + + def custom_scorer(_: str, artefact: Artefact) -> float: + return 10.0 if artefact.key.endswith("top.py") else 2.0 + + selected = select_artefacts( + "summarize all related evidence", + artefacts, + top_n=2, + scorer=custom_scorer, + context_need="multi_source_synthesis", + ) + + assert [a.key for a in selected] == ["file_summary:top.py", "file_summary:low_notes.md"] diff --git a/tests/test_cli/test_config.py b/tests/test_cli/test_config.py index d7ea552..c2aa744 100644 --- a/tests/test_cli/test_config.py +++ b/tests/test_cli/test_config.py @@ -274,7 +274,7 @@ def test_load_config_warns_and_defaults_invalid_semantic_sections( assert config.policy.bundle_overrides == {} assert config.integrations.guidance_variant == "canonical" assert config.max_age_seconds == 3600 - assert config.max_context_tokens == 4096 + assert config.max_context_tokens == 8192 assert "Ignoring invalid Vaner config section [setup]" in caplog.text assert "Ignoring invalid Vaner config section [policy]" in caplog.text assert "Ignoring invalid Vaner config section [integrations]" in caplog.text diff --git a/tests/test_clients/test_structured_output.py b/tests/test_clients/test_structured_output.py index cfc91ec..bc541d2 100644 --- a/tests/test_clients/test_structured_output.py +++ b/tests/test_clients/test_structured_output.py @@ -141,6 +141,31 @@ def _handler(req: httpx.Request) -> httpx.Response: assert captured["body"]["options"]["num_predict"] == 128 +@pytest.mark.asyncio +async def test_ollama_extra_options_are_merged_with_num_predict(monkeypatch): + captured: dict[str, dict] = {} + + def _handler(req: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(req.content or b"{}") + return httpx.Response(200, json={"response": "{}"}) + + monkeypatch.setattr(httpx, "AsyncClient", _stub_async_client(_handler)) + + call = ollama_llm_structured( + model="qwen3.6:27b", + base_url="http://localhost:11434", + max_tokens=256, + extra_body={"options": {"num_ctx": 131072, "temperature": 0.7, "top_k": 20}}, + ) + await call("hi") + assert captured["body"]["options"] == { + "num_ctx": 131072, + "temperature": 0.7, + "top_k": 20, + "num_predict": 256, + } + + @pytest.mark.asyncio async def test_ollama_response_format_translates_to_format_json(monkeypatch): captured: dict[str, dict] = {} diff --git a/tests/test_daemon/test_generator.py b/tests/test_daemon/test_generator.py index 189883c..e517a1d 100644 --- a/tests/test_daemon/test_generator.py +++ b/tests/test_daemon/test_generator.py @@ -8,7 +8,14 @@ import httpx from vaner.daemon.engine import generator as generator_mod -from vaner.daemon.engine.generator import _llm_summarize, agenerate_file_summary, generate_artefact, generate_diff_summary +from vaner.daemon.engine.generator import ( + DIFF_SUMMARY_PROMPT, + FILE_SUMMARY_PROMPT, + _llm_summarize, + agenerate_file_summary, + generate_artefact, + generate_diff_summary, +) from vaner.models.config import BackendConfig, GenerationConfig, VanerConfig _real_async_client = httpx.AsyncClient @@ -30,10 +37,42 @@ def test_generate_artefact_for_normal_file(temp_repo): artefact = generate_artefact(source, temp_repo) assert artefact.source_path == "normal.py" assert "Functions:" in artefact.content + assert "f(limit: int)" in artefact.content assert "Constants:" in artefact.content assert "Limits:" in artefact.content +def test_generate_artefact_preserves_sql_schema_anchors(temp_repo): + source = temp_repo / "store.py" + source.write_text( + """ +import aiosqlite + +class Store: + async def initialize(self) -> None: + await self.db.execute(\"\"\" + CREATE TABLE artefacts ( + key TEXT PRIMARY KEY, + source_path TEXT NOT NULL, + metadata_json TEXT NOT NULL + ) + \"\"\") + await self.db.execute("CREATE INDEX idx_artefacts_source_path ON artefacts(source_path)") + + async def list_by_keys(self, keys: list[str]) -> list[str]: + return keys +""", + encoding="utf-8", + ) + + artefact = generate_artefact(source, temp_repo) + + assert "Schema:" in artefact.content + assert "CREATE TABLE artefacts" in artefact.content + assert "CREATE INDEX idx_artefacts_source_path" in artefact.content + assert "list_by_keys(self, keys: list[str]) -> list[str]" in artefact.content + + def test_generate_artefact_for_empty_file(temp_repo): source = temp_repo / "empty.py" source.write_text("\n", encoding="utf-8") @@ -59,6 +98,14 @@ def test_generate_diff_summary_redacts_patterns(temp_repo): assert "REDACTED" in artefact.content +def test_llm_summary_prompts_include_internal_evidence_policy(): + for prompt in (FILE_SUMMARY_PROMPT, DIFF_SUMMARY_PROMPT): + assert "Vaner internal LLM policy" in prompt + assert "Evidence summary policy" in prompt + assert "Preserve implementation anchors" in prompt + assert "Do not let predictions or likely intent become factual behavior" in prompt + + def test_agenerate_file_summary_uses_llm_when_enabled(temp_repo, monkeypatch): source = temp_repo / "llm.py" source.write_text("def x():\n return 1\n", encoding="utf-8") diff --git a/tests/test_daemon/test_http.py b/tests/test_daemon/test_http.py index 7c2403b..cb7cdd6 100644 --- a/tests/test_daemon/test_http.py +++ b/tests/test_daemon/test_http.py @@ -184,6 +184,83 @@ async def _seed() -> None: assert history.json()["scenarios"][0]["visibility"] == "archived" +def test_heatmap_replay_endpoint_returns_persisted_samples(temp_repo) -> None: + now = time.time() + + async def _seed() -> None: + store = ScenarioStore(temp_repo / ".vaner" / "scenarios.db") + await store.initialize() + await store.upsert( + Scenario( + id="scn_heatmap", + kind="debug", + score=0.9, + confidence=0.8, + entities=["src/main.py"], + prepared_context="ctx", + freshness="fresh", + created_at=now, + last_refreshed_at=now, + ) + ) + + asyncio.run(_seed()) + + config = VanerConfig( + repo_root=temp_repo, + store_path=temp_repo / ".vaner" / "store.db", + telemetry_path=temp_repo / ".vaner" / "telemetry.db", + ) + app = create_daemon_http_app(config) + with TestClient(app) as client: + response = client.get(f"/heatmap/replay?from_ts={now - 5}&to_ts={now + 5}&limit=10") + + assert response.status_code == 200 + payload = response.json() + assert payload["metadata"]["synthetic"] is False + assert [item["id"] for item in payload["scenarios"]] == ["scn_heatmap"] + assert payload["samples"] + assert payload["samples"][0]["scenario_id"] == "scn_heatmap" + + +def test_heatmap_replay_stream_emits_real_snapshot(temp_repo) -> None: + now = time.time() + + async def _seed() -> None: + store = ScenarioStore(temp_repo / ".vaner" / "scenarios.db") + await store.initialize() + await store.upsert( + Scenario( + id="scn_heatmap_stream", + kind="debug", + score=0.9, + confidence=0.8, + entities=["src/main.py"], + prepared_context="ctx", + freshness="fresh", + created_at=now, + last_refreshed_at=now, + ) + ) + + asyncio.run(_seed()) + + config = VanerConfig( + repo_root=temp_repo, + store_path=temp_repo / ".vaner" / "store.db", + telemetry_path=temp_repo / ".vaner" / "telemetry.db", + ) + app = create_daemon_http_app(config) + with TestClient(app) as client: + with client.stream("GET", "/heatmap/replay/stream?range_seconds=60&limit=1") as response: + text = "".join(response.iter_text()) + + assert response.status_code == 200 + assert "event: replay_snapshot" in text + assert '"synthetic": false' in text + assert "scn_heatmap_stream" in text + + def test_scenario_stream_route_not_shadowed_by_id_route(temp_repo) -> None: config = VanerConfig( repo_root=temp_repo, diff --git a/tests/test_engine/test_context_budget.py b/tests/test_engine/test_context_budget.py new file mode 100644 index 0000000..10f066f --- /dev/null +++ b/tests/test_engine/test_context_budget.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from vaner.engine import _adaptive_selection_top_n, _effective_context_budget +from vaner.models.config import BackendConfig, VanerConfig + + +def _config(tmp_path, *, max_context_tokens: int = 8192, num_ctx: int | None = None) -> VanerConfig: + runtime_options = {"num_ctx": num_ctx} if num_ctx is not None else {} + return VanerConfig( + repo_root=tmp_path, + store_path=tmp_path / ".vaner" / "store.db", + telemetry_path=tmp_path / ".vaner" / "telemetry.db", + max_context_tokens=max_context_tokens, + backend=BackendConfig(name="ollama", runtime_options=runtime_options), + ) + + +def test_effective_context_budget_uses_explicit_request(tmp_path): + assert _effective_context_budget(_config(tmp_path, num_ctx=131_072), requested=2048) == 2048 + + +def test_effective_context_budget_expands_for_large_local_context(tmp_path): + assert _effective_context_budget(_config(tmp_path, max_context_tokens=4096, num_ctx=131_072)) == 43_690 + + +def test_effective_context_budget_expands_for_million_token_runtime(tmp_path): + assert _effective_context_budget(_config(tmp_path, max_context_tokens=4096, num_ctx=1_048_576)) == 262_144 + + +def test_adaptive_selection_top_n_expands_for_multifacet_large_context(): + prompt = "How does the ArtefactStore persist packages? What schema does it use and how are rows retrieved?" + assert _adaptive_selection_top_n(prompt, requested=8, max_context_tokens=43_690) > 8 diff --git a/tests/test_engine/test_deep_drill.py b/tests/test_engine/test_deep_drill.py index f7e6dfc..254501e 100644 --- a/tests/test_engine/test_deep_drill.py +++ b/tests/test_engine/test_deep_drill.py @@ -70,6 +70,10 @@ async def _capturing_llm(prompt: str) -> str: assert ranked == [] assert len(follow_on) == 5, f"expected 5 follow-ons, got {len(follow_on)}" + assert "Vaner internal LLM policy" in captured["prompt"] + assert "JSON contract" in captured["prompt"] + assert "Prediction policy" in captured["prompt"] + assert "Return JSON only (no markdown fences, no extra keys)" in captured["prompt"] assert "HIGH-PRIORITY" in captured["prompt"] assert "0-5" in captured["prompt"] diff --git a/tests/test_engine/test_exploration_parallelism.py b/tests/test_engine/test_exploration_parallelism.py index c8a6f8f..0ac08ac 100644 --- a/tests/test_engine/test_exploration_parallelism.py +++ b/tests/test_engine/test_exploration_parallelism.py @@ -24,6 +24,7 @@ _merge_llm_ranked_with_seed_paths, ) from vaner.intent.adapter import CodeRepoAdapter +from vaner.intent.governor import PredictionGovernor from vaner.models.config import ComputeConfig, ExplorationConfig # --------------------------------------------------------------------------- @@ -114,6 +115,17 @@ def test_llm_rerank_preserves_deterministic_seed_paths() -> None: ] +def test_llm_rerank_reserves_space_for_seed_paths_when_llm_list_is_full() -> None: + merged = _merge_llm_ranked_with_seed_paths( + [f"src/vaner/noisy_{idx}.py" for idx in range(8)], + ["src/vaner/store/artefacts.py", "src/vaner/learning/reward.py"], + ) + + assert len(merged) == 8 + assert "src/vaner/store/artefacts.py" in merged + assert "src/vaner/learning/reward.py" in merged + + def test_core_group_matching_boosts_specific_mechanism_query() -> None: assert _core_group_matches_recent_query( "external LLM exploration flow ranked_files file ranking follow_on follow-on scenario proposal", @@ -249,3 +261,33 @@ async def flaky_llm(prompt: str) -> str: # all, the flaky first call must not crash the surrounding cycle. await engine.precompute_cycle() assert call_count >= 0 # tautologically true — proof is in the no-raise above + + +@pytest.mark.asyncio +async def test_normal_background_refills_empty_frontier_with_continuation_agenda(temp_repo): + """Normal precompute should keep preparing useful adjacent work after the first frontier drains.""" + + extra_files = { + "src/example/cache.py": "CACHE_LIMIT = 3\n", + "src/example/worker.py": "def run_worker():\n return 'ok'\n", + "tests/test_cache_behavior.py": "def test_cache_limit():\n assert True\n", + "docs/release-notes.md": "# Release notes\n", + } + for rel_path, content in extra_files.items(): + path = temp_repo / rel_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + adapter = CodeRepoAdapter(temp_repo) + engine = VanerEngine(adapter=adapter, llm=None) + engine.config.compute.idle_only = False + engine.config.compute.max_cycle_seconds = 20 + engine.config.compute.adaptive_cycle_budget = False + engine.config.exploration.llm_gate = "none" + await engine.prepare() + + await engine.precompute_cycle(governor=PredictionGovernor(mode=PredictionGovernor.Mode.BUDGET, budget_units=20)) + + assert engine._cycle_policy_state["continuation_rounds_last_cycle"] > 0 # noqa: SLF001 + explored = engine.get_explored_scenarios() + assert any("normal continuation" in scenario.reason for scenario in explored) diff --git a/tests/test_engine/test_structured_evidence_seeding.py b/tests/test_engine/test_structured_evidence_seeding.py index e35fc62..065144e 100644 --- a/tests/test_engine/test_structured_evidence_seeding.py +++ b/tests/test_engine/test_structured_evidence_seeding.py @@ -43,3 +43,83 @@ async def test_precompute_seeds_structured_direct_scenarios_before_broad_arc(tem assert explored[0].source == "structured_direct" assert "src/vaner/intent/frontier.py" in explored[0].unit_ids assert all(".mypy_cache" not in path for scenario in explored for path in scenario.unit_ids) + + +@pytest.mark.asyncio +async def test_precompute_latest_query_exact_anchor_beats_generic_signal_paths(temp_repo) -> None: + (temp_repo / "src" / "vaner" / "learning").mkdir(parents=True) + (temp_repo / "src" / "vaner" / "signals").mkdir(parents=True) + (temp_repo / "src" / "vaner" / "intent").mkdir(parents=True) + (temp_repo / "src" / "vaner" / "learning" / "reward.py").write_text( + "class RewardInput:\n pass\n\n" + "def compute_reward(inputs):\n" + " return {'reward_total': 0.0, 'reward_components': {}}\n", + encoding="utf-8", + ) + (temp_repo / "src" / "vaner" / "signals" / "schema.py").write_text( + "class SignalEvent:\n pass\n", + encoding="utf-8", + ) + (temp_repo / "src" / "vaner" / "intent" / "scoring_policy.py").write_text( + "class ScoringPolicy:\n pass\n", + encoding="utf-8", + ) + + engine = VanerEngine(adapter=CodeRepoAdapter(temp_repo)) + engine.config.compute.idle_only = False + engine.config.exploration.llm_gate = "none" + await engine.initialize() + query = "How does the reward computation work? What signals does it combine to produce the final reward value?" + engine._arc_model.observe(query) + await engine.store.insert_query_history( + session_id="s", + query_text=query, + selected_paths=[], + hit_precomputed=False, + token_used=0, + ) + + await engine.precompute_cycle() + + explored = engine.get_explored_scenarios() + assert explored + assert explored[0].source == "structured_direct" + assert "src/vaner/learning/reward.py" in explored[0].unit_ids + + +@pytest.mark.asyncio +async def test_precompute_latest_query_exact_anchor_keeps_store_schema_source(temp_repo) -> None: + (temp_repo / "src" / "vaner" / "store").mkdir(parents=True) + (temp_repo / "src" / "vaner" / "daemon").mkdir(parents=True) + (temp_repo / "src" / "vaner" / "store" / "artefacts.py").write_text( + "class ArtefactStore:\n" + " def initialize(self):\n" + " return 'CREATE TABLE artefacts key content metadata SELECT INSERT'\n", + encoding="utf-8", + ) + (temp_repo / "src" / "vaner" / "daemon" / "http.py").write_text( + "def _artefact_store():\n" + " return 'route factory for context package database schema'\n", + encoding="utf-8", + ) + + engine = VanerEngine(adapter=CodeRepoAdapter(temp_repo)) + engine.config.compute.idle_only = False + engine.config.exploration.llm_gate = "none" + await engine.initialize() + query = "How does the ArtefactStore persist and retrieve context packages? What database schema does it use?" + engine._arc_model.observe(query) + await engine.store.insert_query_history( + session_id="s", + query_text=query, + selected_paths=[], + hit_precomputed=False, + token_used=0, + ) + + await engine.precompute_cycle() + + explored = engine.get_explored_scenarios() + assert explored + assert explored[0].source == "structured_direct" + assert "src/vaner/store/artefacts.py" in explored[0].unit_ids diff --git a/tests/test_intent/test_drafter.py b/tests/test_intent/test_drafter.py index 94abea6..45db0f9 100644 --- a/tests/test_intent/test_drafter.py +++ b/tests/test_intent/test_drafter.py @@ -155,6 +155,10 @@ async def _llm(prompt: str) -> str: ) assert result is not None assert len(calls) == 2 # rewrite + draft + assert "Vaner internal LLM policy" in calls[0] + assert "Prediction policy" in calls[0] + assert "Draft policy" in calls[1] + assert "Use tentative wording when source evidence is incomplete" in calls[1] assert result.predicted_prompt == "canonicalised prompt" assert result.draft_answer == "draft body" diff --git a/tests/test_intent/test_evidence_resolver.py b/tests/test_intent/test_evidence_resolver.py index 4e1c42f..9a7523e 100644 --- a/tests/test_intent/test_evidence_resolver.py +++ b/tests/test_intent/test_evidence_resolver.py @@ -198,3 +198,59 @@ def test_weak_component_match_stays_evidence_ready_only() -> None: assert not ready assert reason == "weak_component_match" + + +def test_semantic_aliases_resolve_rollout_rehearsal_terms() -> None: + structured = structured_from_prediction_fields( + label="Explain release dry-run gate", + description="What prevents a candidate release from getting full traffic until replay and smoke checks pass?", + anchor="candidate release full traffic dry run", + confidence=0.8, + ) + + targets = resolve_evidence_targets( + structured, + available_paths=[ + "src/runtime/traffic_escrow.py", + "src/runtime/release_notes.py", + "docs/rollout.md", + ], + artefacts_by_key={ + "file_summary:src/runtime/traffic_escrow.py": _Artefact( + "TrafficEscrow controller coordinates rehearse_proxy replay runs, smoke policy checks, and staged promote gates." + ), + "file_summary:src/runtime/release_notes.py": _Artefact("Release notes formatter for changelog entries."), + "file_summary:docs/rollout.md": _Artefact("Rollout overview for staged canaries."), + }, + ) + + assert targets + assert targets[0].path == "src/runtime/traffic_escrow.py" + + +def test_semantic_aliases_resolve_vectorization_routing_terms() -> None: + structured = structured_from_prediction_fields( + label="Explain vectorization routing issue", + description="Why did a Western Europe tenant get routed to a Southeast Asia edge during a vectorization load spike?", + anchor="western europe vectorization routed southeast asia", + confidence=0.8, + ) + + targets = resolve_evidence_targets( + structured, + available_paths=[ + "src/incidents/eu_apac_embedding_egress.py", + "src/incidents/general_latency.py", + "docs/regions.md", + ], + artefacts_by_key={ + "file_summary:src/incidents/eu_apac_embedding_egress.py": _Artefact( + "Embedding batch incident: eu-west residency stamp lag caused ap-southeast edge fallback and cross-region egress." + ), + "file_summary:src/incidents/general_latency.py": _Artefact("Generic latency incident notes."), + "file_summary:docs/regions.md": _Artefact("Region naming conventions."), + }, + ) + + assert targets + assert targets[0].path == "src/incidents/eu_apac_embedding_egress.py" diff --git a/tests/test_intent/test_symbol_index.py b/tests/test_intent/test_symbol_index.py index 319edf1..2516d85 100644 --- a/tests/test_intent/test_symbol_index.py +++ b/tests/test_intent/test_symbol_index.py @@ -87,3 +87,50 @@ def test_generic_lowercase_methods_do_not_outrank_component_paths(tmp_path: Path ) assert paths[0] == "src/vaner/intent/cache.py" + + +def test_exact_path_ranking_prefers_reward_module_over_generic_signal_paths(tmp_path: Path) -> None: + (tmp_path / "src" / "vaner" / "learning").mkdir(parents=True) + (tmp_path / "src" / "vaner" / "signals").mkdir(parents=True) + (tmp_path / "src" / "vaner" / "learning" / "reward.py").write_text( + "class RewardInput:\n pass\n\n" + "def compute_reward(inputs):\n" + " return {'reward_total': 0.0, 'reward_components': {}}\n", + encoding="utf-8", + ) + (tmp_path / "src" / "vaner" / "signals" / "schema.py").write_text( + "class SignalEvent:\n pass\n", + encoding="utf-8", + ) + + paths = rank_exact_paths( + tmp_path, + "How does the reward computation work? What signals does it combine to produce the final reward value?", + max_paths=3, + ) + + assert paths[0] == "src/vaner/learning/reward.py" + + +def test_exact_path_ranking_prefers_public_store_definition_over_private_factory(tmp_path: Path) -> None: + (tmp_path / "src" / "vaner" / "store").mkdir(parents=True) + (tmp_path / "src" / "vaner" / "daemon").mkdir(parents=True) + (tmp_path / "src" / "vaner" / "store" / "artefacts.py").write_text( + "class ArtefactStore:\n" + " def persist(self):\n" + " return 'CREATE TABLE artefacts SELECT key INSERT INTO artefacts'\n", + encoding="utf-8", + ) + (tmp_path / "src" / "vaner" / "daemon" / "http.py").write_text( + "def _artefact_store():\n" + " return 'factory for context package database schema route'\n", + encoding="utf-8", + ) + + paths = rank_exact_paths( + tmp_path, + "How does the ArtefactStore persist and retrieve context packages? What database schema does it use?", + max_paths=3, + ) + + assert paths[0] == "src/vaner/store/artefacts.py" diff --git a/tests/test_policy/test_internal_llm.py b/tests/test_policy/test_internal_llm.py new file mode 100644 index 0000000..f129d1a --- /dev/null +++ b/tests/test_policy/test_internal_llm.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from vaner.policy.internal_llm import ( + CORE_POLICY, + DRAFT_POLICY, + EVIDENCE_SUMMARY_POLICY, + JSON_CONTRACT_POLICY, + PREDICTION_POLICY, + internal_llm_policy, +) + + +def test_core_policy_is_contract_preserving_not_persona() -> None: + assert "Preserve the requested output contract over style" in CORE_POLICY + assert "paths, symbols, constants" in CORE_POLICY + assert "assistant persona" not in CORE_POLICY.lower() + + +def test_json_policy_bans_common_parser_breakers() -> None: + policy = internal_llm_policy(JSON_CONTRACT_POLICY) + assert "valid JSON only" in policy + assert "no markdown fences" in policy + assert "no markdown fences, preamble, commentary, or trailing prose" in policy + assert "not chain-of-thought" in policy + + +def test_task_overlays_are_short_for_local_models() -> None: + for policy in [ + internal_llm_policy(EVIDENCE_SUMMARY_POLICY), + internal_llm_policy(JSON_CONTRACT_POLICY, PREDICTION_POLICY), + internal_llm_policy(DRAFT_POLICY), + ]: + assert len(policy.split()) <= 140 diff --git a/tests/test_setup/test_catalog_refresh.py b/tests/test_setup/test_catalog_refresh.py index 7b2ea15..5dc2479 100644 --- a/tests/test_setup/test_catalog_refresh.py +++ b/tests/test_setup/test_catalog_refresh.py @@ -14,6 +14,7 @@ build_registry_entry_for_family, estimate_memory_budget, families_from_seed, + fetch_ollama_library_details, load_catalog_seed, manifest_weights_bytes, quantization_bytes_per_param, @@ -37,6 +38,10 @@ def _fake_manifest(weight_bytes: int) -> dict[str, Any]: } +def _first_ollama_family(): + return next(f for f in families_from_seed(_seed()) if f.runtime == "ollama") + + def test_seed_loads_with_family_metadata() -> None: seed = _seed() assert "quantization_profiles" in seed @@ -44,7 +49,10 @@ def test_seed_loads_with_family_metadata() -> None: families = seed.get("families", []) assert families, "seed must list at least one family" for family in families: - assert family.get("ollama_family"), f"family {family['id']} missing ollama_family" + if family.get("runtime", "ollama") == "ollama": + assert family.get("ollama_family"), f"family {family['id']} missing ollama_family" + else: + assert family.get("default_download_size_gb", 0) > 0, f"family {family['id']} missing non-Ollama sizing" params = family["parameters"] assert "context_window" in params assert "temperature" in params, f"family {family['id']} missing sampling defaults" @@ -73,6 +81,9 @@ def test_estimate_memory_budget_recommended_exceeds_min() -> None: _, rec_short = estimate_memory_budget(7.0, 0.55, 8192) _, rec_long = estimate_memory_budget(7.0, 0.55, 131072) assert rec_long > rec_short + _, moe_rec = estimate_memory_budget(284.0, 0.55, 1048576, active_params_b=13.0, architecture="moe") + _, dense_rec = estimate_memory_budget(284.0, 0.55, 1048576) + assert moe_rec < dense_rec def test_manifest_weights_bytes_sums_model_layers() -> None: @@ -93,8 +104,7 @@ def test_manifest_weights_bytes_zero_when_no_model_layer() -> None: def test_build_entry_online_uses_manifest_size() -> None: seed = _seed() - families = families_from_seed(seed) - family = families[0] + family = _first_ollama_family() # ~24 GB on disk → at Q4_K_M (0.55 GB/B) ≈ 43.6 B params. fake_bytes = int(24 * 1024**3) @@ -119,17 +129,51 @@ def test_build_entry_online_uses_manifest_size() -> None: def test_build_entry_online_skips_when_manifest_missing() -> None: seed = _seed() - families = families_from_seed(seed) entry = build_registry_entry_for_family( seed, - families[0], + _first_ollama_family(), quant="Q4_K_M", online=True, manifest_fetcher=lambda _f, *, tag="latest": None, + library_fetcher=lambda _f, *, tag="latest": None, ) assert entry is None +def test_build_entry_online_can_use_ollama_library_page_when_manifest_missing() -> None: + seed = _seed() + family = _first_ollama_family() + entry = build_registry_entry_for_family( + seed, + family, + quant="Q4_K_M", + online=True, + manifest_fetcher=lambda _f, *, tag="latest": None, + library_fetcher=lambda _f, *, tag="latest": {"download_size_gb": 22.0, "source": "https://ollama.com/library/example"}, + ) + assert entry is not None + assert entry["id"] == f"{family.ollama_family}:{family.ollama_tag}" + assert entry["download_size_gb"] == 22.0 + assert entry["params_b"] == family.default_params_b + + +def test_ollama_library_details_require_run_command_and_size(monkeypatch) -> None: + class _Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self) -> bytes: + return b"ollama run qwen3.6:35b-a3b-coding-nvfp4 cd2692a833e6 22GB" + + monkeypatch.setattr("urllib.request.urlopen", lambda _req, timeout=0: _Response()) + details = fetch_ollama_library_details("qwen3.6", tag="35b-a3b-coding-nvfp4") + assert details is not None + assert details["download_size_gb"] == 22.0 + + def test_build_registry_offline_emits_one_per_family() -> None: payload = build_registry(online=False) seed_families = len(_seed()["families"]) @@ -203,7 +247,7 @@ def fake_fetcher(family_name: str, *, tag: str = "latest"): return _fake_manifest(int(8 * 1024**3)) return None - payload = build_registry(online=True, seed=seed, manifest_fetcher=fake_fetcher) + payload = build_registry(online=True, seed=seed, manifest_fetcher=fake_fetcher, library_fetcher=lambda _f, *, tag="latest": None) assert len(payload["models"]) == 1 assert payload["models"][0]["family_id"] == families[0].id skipped = payload.get("skipped", []) diff --git a/tests/test_setup/test_hardware.py b/tests/test_setup/test_hardware.py index 82f8a04..ca443f3 100644 --- a/tests/test_setup/test_hardware.py +++ b/tests/test_setup/test_hardware.py @@ -196,6 +196,14 @@ def test_models_lmstudio_parses_ids() -> None: ] +def test_disk_free_probe_reports_decimal_gb() -> None: + class Usage: + free = 123_400_000_000 + + with patch.object(hw.shutil, "disk_usage", return_value=Usage()): + assert hw._probe_disk_free_gb() == 123 + + def test_thermal_probe_returns_false_off_linux() -> None: with patch.object(hw.sys, "platform", "darwin"): assert hw._probe_thermal() is False @@ -224,6 +232,39 @@ def test_gpu_probe_handles_subprocess_failure() -> None: assert vram is None +def test_nvidia_smi_marks_dgx_spark_memory_as_unified() -> None: + class Result: + returncode = 0 + stdout = "NVIDIA DGX Spark GB10, 131072\n" + + with ( + patch.object(hw.shutil, "which", return_value="/usr/bin/nvidia-smi"), + patch.object(hw.subprocess, "run", return_value=Result()), + ): + devices = hw._probe_gpu_devices_nvidia_smi() + + assert devices is not None + assert devices[0].name == "NVIDIA DGX Spark GB10" + assert devices[0].memory_display_gb == 128 + assert devices[0].memory_kind == "unified" + + +def test_nvidia_smi_keeps_regular_rtx_memory_as_vram() -> None: + class Result: + returncode = 0 + stdout = "NVIDIA GeForce RTX 5090, 32768\n" + + with ( + patch.object(hw.shutil, "which", return_value="/usr/bin/nvidia-smi"), + patch.object(hw.subprocess, "run", return_value=Result()), + ): + devices = hw._probe_gpu_devices_nvidia_smi() + + assert devices is not None + assert devices[0].memory_display_gb == 32 + assert devices[0].memory_kind == "vram" + + # --------------------------------------------------------------------------- # detect() composition # --------------------------------------------------------------------------- diff --git a/tests/test_setup/test_model_recommendation.py b/tests/test_setup/test_model_recommendation.py index 6d81b93..13f6c1f 100644 --- a/tests/test_setup/test_model_recommendation.py +++ b/tests/test_setup/test_model_recommendation.py @@ -76,16 +76,105 @@ def test_recommendation_user_layer_hides_diagnostics() -> None: assert "diagnostics" not in payload["user"] +def test_system_ram_does_not_expand_default_local_model() -> None: + payload = recommend_local_model( + answers=_answers("mixed"), + hardware=_profile( + ram_gb=64, + memory_total_bytes=64 * 1024**3, + memory_display_gb=64, + memory_is_unified=False, + gpu="none", + gpu_vram_gb=None, + gpu_devices=(), + ), + ) + assert payload["hardware"]["memory_source"] == "system" + assert payload["hardware"]["effective_memory_gb"] == 2.0 + assert payload["selected"]["model_id"] == "gemma4:e2b" + assert payload["selected"]["capability"]["context_window"] == 32768 + + +def test_unknown_high_memory_nvidia_uses_cautious_gpu_class_recommendation() -> None: + payload = recommend_local_model( + answers=_answers("mixed"), + hardware=_profile( + ram_gb=128, + memory_total_bytes=128 * 1024**3, + memory_display_gb=128, + memory_is_unified=False, + gpu="nvidia", + gpu_vram_gb=None, + gpu_devices=(), + ), + ) + assert payload["hardware"]["memory_source"] == "inferred_gpu" + assert payload["hardware"]["effective_memory_gb"] == 30.0 + assert payload["selected"]["model_id"] != "gemma4:e2b" + assert payload["selected"]["download_size_gb"] >= 16 + + +def test_unknown_low_memory_nvidia_stays_conservative_without_vram_signal() -> None: + payload = recommend_local_model( + answers=_answers("mixed"), + hardware=_profile( + ram_gb=32, + memory_total_bytes=32 * 1024**3, + memory_display_gb=32, + memory_is_unified=False, + gpu="nvidia", + gpu_vram_gb=None, + gpu_devices=(), + ), + ) + assert payload["hardware"]["memory_source"] == "system" + assert payload["selected"]["model_id"] == "gemma4:e2b" + + +def test_dgx_spark_uses_unified_memory_and_gpt_oss_120b() -> None: + payload = recommend_local_model( + answers=_answers("coding", "research"), + hardware=_profile( + ram_gb=128, + memory_total_bytes=128 * 1024**3, + memory_display_gb=128, + memory_is_unified=False, + disk_free_gb=500, + gpu="nvidia", + gpu_vram_gb=None, + gpu_devices=( + GPUDevice( + name="NVIDIA DGX Spark GB10", + vendor="NVIDIA", + kind="nvidia", + memory_total_bytes=None, + memory_display_gb=None, + memory_kind="unknown", + ), + ), + ), + ) + selected = payload["selected"] + assert payload["hardware"]["memory_source"] == "unified" + assert payload["hardware"]["effective_memory_gb"] == 120.0 + assert payload["hardware"]["is_unified_memory"] is True + assert selected["model_id"] == "gpt-oss:120b" + assert selected["quantization"] == "MXFP4" + assert selected["params_b"] >= 100 + assert selected["active_params_b"] == 5.1 + assert selected["capability"]["context_window"] == 131072 + + def test_recommendation_prefers_installed_compatible_model() -> None: # Pick whichever bundled model has a compatible installed match. payload = recommend_local_model( - hardware=_profile(detected_models=(("ollama", "qwen3.5:latest", "10GB"),)), + hardware=_profile(detected_models=(("ollama", "gemma4:e4b", "9GB"),)), ) - assert payload["selected"]["model_id"] == "qwen3.5:latest" + assert payload["selected"]["model_id"] == "gemma4:e4b" assert payload["user"]["needs_model_download"] is False -def test_recommendation_prefers_qwen36_27b_on_32gb_vram_over_installed_35b() -> None: +def test_recommendation_prefers_current_fit_on_32gb_vram_over_legacy_installed_model() -> None: payload = recommend_local_model( answers=_answers("coding", "research"), hardware=_profile( @@ -104,12 +193,112 @@ def test_recommendation_prefers_qwen36_27b_on_32gb_vram_over_installed_35b() -> ), ) selected = payload["selected"] - assert selected["model_id"] == "qwen3.6:27b" - assert selected["capability"]["context_window"] == 131072 - assert selected["runtime_params"]["num_ctx"] == 131072 + assert selected["model_id"] == "qwen3.6:35b-a3b-coding-nvfp4" + assert selected["capability"]["context_window"] == 65536 + assert selected["runtime_params"]["num_ctx"] == 65536 assert payload["user"]["needs_model_download"] is True +def test_recommendation_uses_current_ollama_moe_on_huge_apple_unified_memory() -> None: + payload = recommend_local_model( + answers=_answers("mixed"), + hardware=_profile( + os="darwin", + ram_gb=512, + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_is_unified=True, + disk_free_gb=300, + gpu="apple_silicon", + gpu_vram_gb=None, + gpu_devices=( + GPUDevice( + name="Apple M3 Ultra", + vendor="Apple", + kind="apple_silicon", + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_kind="unified", + ), + ), + detected_runtimes=("ollama",), + ), + ) + selected = payload["selected"] + assert selected["model_id"] == "qwen3.6:35b-a3b-coding-mxfp8" + assert selected["runtime"] == "ollama" + assert selected["architecture"] == "moe" + assert selected["quantization"] == "MXFP8" + assert selected["capability"]["context_window"] == 262144 + assert selected["runtime_params"]["num_ctx"] == 262144 + assert payload["user"]["needs_runtime_install"] is False + + +def test_speed_posture_avoids_oversized_stale_context_model_on_huge_apple() -> None: + payload = recommend_local_model( + answers=SetupAnswers( + work_styles=("mixed",), + priority="speed", + compute_posture="balanced", + cloud_posture="ask_first", + background_posture="normal", + ), + hardware=_profile( + os="darwin", + ram_gb=512, + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_is_unified=True, + disk_free_gb=300, + gpu="apple_silicon", + gpu_vram_gb=None, + gpu_devices=( + GPUDevice( + name="Apple M3 Ultra", + vendor="Apple", + kind="apple_silicon", + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_kind="unified", + ), + ), + detected_runtimes=("ollama",), + ), + ) + assert payload["selected"]["model_id"] != "llama4:16x17b" + + +def test_disk_space_filters_large_downloads() -> None: + payload = recommend_local_model( + answers=_answers("mixed"), + hardware=_profile( + os="darwin", + ram_gb=512, + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_is_unified=True, + disk_free_gb=25, + gpu="apple_silicon", + gpu_vram_gb=None, + gpu_devices=( + GPUDevice( + name="Apple M3 Ultra", + vendor="Apple", + kind="apple_silicon", + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_kind="unified", + ), + ), + detected_runtimes=("ollama",), + ), + ) + assert payload["selected"]["model_id"] == "gemma4:e4b" + assert payload["selected"]["disk"]["status"] in {"enough", "tight"} + rejected = payload["diagnostics"]["rejected_models"] + assert any(row["model_id"] == "qwen3.6:35b-a3b-coding-nvfp4" and row["reason"] == "insufficient_disk" for row in rejected) + + def test_qwen36_27b_context_keeps_headroom_on_32gb_vram() -> None: chosen = compute_effective_context_window( max_context_window=262144, @@ -121,6 +310,30 @@ def test_qwen36_27b_context_keeps_headroom_on_32gb_vram() -> None: assert chosen == 131072 +def test_heavy_qwen_context_caps_at_64k_on_32gb_vram() -> None: + chosen = compute_effective_context_window( + max_context_window=262144, + weights_gb=22.0, + effective_memory_gb=30.0, + work_styles=("coding", "research"), + runtime="ollama", + kv_reference_gb=1.8, + ) + assert chosen == 65536 + + +def test_heavy_qwen_context_uses_256k_on_48gb_plus_vram() -> None: + chosen = compute_effective_context_window( + max_context_window=262144, + weights_gb=22.0, + effective_memory_gb=46.0, + work_styles=("coding", "research"), + runtime="ollama", + kv_reference_gb=1.8, + ) + assert chosen == 262144 + + def test_persist_runtime_recommendation_writes_backend(tmp_path: Path) -> None: repo = tmp_path / "repo" repo.mkdir() @@ -131,10 +344,53 @@ def test_persist_runtime_recommendation_writes_backend(tmp_path: Path) -> None: parsed = tomllib.loads((repo / ".vaner" / "config.toml").read_text(encoding="utf-8")) assert parsed["backend"]["name"] == "ollama" assert parsed["backend"]["model"] == payload["selected"]["model_id"] + assert parsed["backend"]["runtime_options"]["num_ctx"] == payload["selected"]["runtime_params"]["num_ctx"] + assert parsed["backend"]["sampling_options"] == payload["selected"]["sampling_params"] assert parsed["exploration"]["model"] == payload["selected"]["model_id"] + assert parsed["exploration"]["runtime_options"]["num_ctx"] == payload["selected"]["runtime_params"]["num_ctx"] + assert parsed["exploration"]["sampling_options"] == payload["selected"]["sampling_params"] assert "exploration_model" not in parsed["exploration"] assert parsed["compute"]["device"] == "cuda" - assert parsed["limits"]["max_context_tokens"] == payload["selected"]["capability"]["context_window"] // 4 + assert parsed["limits"]["max_context_tokens"] == payload["selected"]["capability"]["context_window"] // 3 + + +def test_persist_huge_apple_recommendation_keeps_ollama_backend(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + answers = _answers("mixed") + persist_setup_and_policy(repo, answers, "local_balanced") + payload = recommend_local_model( + answers=answers, + hardware=_profile( + os="darwin", + ram_gb=512, + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_is_unified=True, + disk_free_gb=300, + gpu="apple_silicon", + gpu_vram_gb=None, + gpu_devices=( + GPUDevice( + name="Apple M3 Ultra", + vendor="Apple", + kind="apple_silicon", + memory_total_bytes=512 * 1000**3, + memory_display_gb=512, + memory_kind="unified", + ), + ), + detected_runtimes=("ollama",), + ), + ) + persist_runtime_recommendation(repo, payload) + parsed = tomllib.loads((repo / ".vaner" / "config.toml").read_text(encoding="utf-8")) + assert parsed["backend"]["name"] == "ollama" + assert parsed["backend"]["base_url"] == "http://127.0.0.1:11434/v1" + assert parsed["exploration"]["backend"] == "ollama" + assert parsed["exploration"]["runtime_options"]["num_ctx"] == payload["selected"]["runtime_params"]["num_ctx"] + assert parsed["compute"]["device"] == "mps" + assert parsed["limits"]["max_context_tokens"] == payload["selected"]["capability"]["context_window"] // 3 def test_models_recommended_cli(monkeypatch) -> None: diff --git a/tests/test_store/test_artefacts.py b/tests/test_store/test_artefacts.py index 92a5405..7c1892b 100644 --- a/tests/test_store/test_artefacts.py +++ b/tests/test_store/test_artefacts.py @@ -30,6 +30,43 @@ async def test_store_upsert_and_list(tmp_path): assert rows[0].key == artefact.key +@pytest.mark.asyncio +async def test_store_lists_artefacts_by_keys_and_source_paths(tmp_path): + store = ArtefactStore(tmp_path / "store.db") + await store.initialize() + now = time.time() + artefacts = [ + Artefact( + key="file_summary:a.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="a.py", + source_mtime=now, + generated_at=now, + model="test", + content="alpha", + ), + Artefact( + key="file_summary:b.py", + kind=ArtefactKind.FILE_SUMMARY, + source_path="b.py", + source_mtime=now, + generated_at=now, + model="test", + content="beta", + ), + ] + for artefact in artefacts: + await store.upsert(artefact) + + by_key = await store.list_by_keys(["file_summary:b.py", "missing", "file_summary:a.py"]) + by_path = await store.list_by_source_paths(["b.py", "a.py"]) + paths = await store.list_source_paths() + + assert [row.key for row in by_key] == ["file_summary:b.py", "file_summary:a.py"] + assert [row.source_path for row in by_path] == ["b.py", "a.py"] + assert paths == ["a.py", "b.py"] + + @pytest.mark.asyncio async def test_store_initialize_migrates_synthetic_v6_database(tmp_path): db_path = tmp_path / "store.db" diff --git a/tests/test_store/test_store_staleness.py b/tests/test_store/test_store_staleness.py index f4aae2f..fb56775 100644 --- a/tests/test_store/test_store_staleness.py +++ b/tests/test_store/test_store_staleness.py @@ -8,6 +8,7 @@ import pytest from vaner.models.artefact import Artefact, ArtefactKind +from vaner.models.scenario import Scenario from vaner.store.artefacts import ArtefactStore from vaner.store.scenarios.sqlite import ScenarioStore @@ -110,3 +111,52 @@ async def test_scenario_store_initialize_migrates_legacy_scenarios_table(tmp_pat assert "memory_confidence" in columns assert "memory_evidence_hashes_json" in columns assert "prior_successes" in columns + + +@pytest.mark.asyncio +async def test_scenario_store_records_real_heatmap_samples(tmp_path): + store = ScenarioStore(tmp_path / "scenarios.db") + await store.initialize() + await store.upsert( + Scenario( + id="scn_sample", + kind="change", + score=0.8, + confidence=0.7, + entities=["src/app.py"], + prepared_context="ready", + freshness="fresh", + created_at=100.0, + last_refreshed_at=100.0, + ) + ) + + samples = await store.list_samples(scenario_ids=["scn_sample"], start_ts=0.0, end_ts=time.time()) + + assert len(samples) == 1 + assert samples[0].scenario_id == "scn_sample" + assert samples[0].readiness == "ready" + assert samples[0].status == "ready" + + +@pytest.mark.asyncio +async def test_scenario_store_samples_change_on_feedback(tmp_path): + store = ScenarioStore(tmp_path / "scenarios.db") + await store.initialize() + await store.upsert( + Scenario( + id="scn_feedback", + kind="debug", + score=0.6, + confidence=0.6, + prepared_context="ready", + freshness="recent", + ) + ) + await store.record_outcome("scn_feedback", "useful") + + samples = await store.list_samples(scenario_ids=["scn_feedback"], start_ts=0.0, end_ts=time.time()) + + assert samples + assert samples[-1].status == "completed" + assert samples[-1].freshness == "fresh" diff --git a/ui/cockpit/src/App.tsx b/ui/cockpit/src/App.tsx index 3cb33c4..cd19452 100644 --- a/ui/cockpit/src/App.tsx +++ b/ui/cockpit/src/App.tsx @@ -27,7 +27,7 @@ import { useActiveWork } from './api/useActiveWork' import { useEvents } from './api/useEvents' import { useFocusRuntime } from './api/useFocusRuntime' import { useLiveWork, type LiveWorkSelection } from './api/useLiveWork' -import { usePipelineEvents } from './api/usePipelineEvents' +import { usePipelineEvents, type PipelineEvent } from './api/usePipelineEvents' import { usePreparedWork } from './api/usePreparedWork' import { useScenarios } from './api/useScenarios' import { @@ -49,6 +49,7 @@ import { type CommandItem, } from './components/chrome' import { EventStreamPanel } from './components/EventStreamPanel' +import { HeatmapReplayView } from './components/HeatmapReplayView' import { Inspector } from './components/Inspector' import { LearningPanel } from './components/LearningPanel' import { LiveWorkInspector } from './components/LiveWorkInspector' @@ -65,6 +66,7 @@ import type { ImpactSummary, LatestInvalidationSignal, LimitSettings, + LiveWorkEvent, MCPSettings, PredictionsByState, PredictionSummary, @@ -86,6 +88,7 @@ const COCKPIT_BUILD_SHA = (import.meta as unknown as { env?: { VITE_COCKPIT_SHA? type ScenarioScope = 'live' | 'session' | 'focus' | 'history' const PREDICTION_STATE_ORDER = ['ready', 'drafting', 'evidence_gathering', 'grounding', 'queued', 'stale'] +const LIVE_STALE_GRACE_SECONDS = 30 * 60 function predictionReadiness(prediction: PredictionSummary): string { return String(prediction.readiness ?? prediction.run?.readiness ?? 'queued') @@ -185,6 +188,8 @@ function scenarioFromPrediction(prediction: PredictionSummary): UIScenario { readiness: ready ? 'ready' : readiness === 'stale' ? 'cooling' : 'warming', visibility: ready ? 'prominent' : readiness === 'stale' ? 'cooling' : 'warming', lifecycleMotion: ready ? 'rising' : readiness === 'stale' ? 'falling' : 'stable', + createdAt: prediction.created_at ?? null, + lastRefreshedAt: prediction.updated_at ?? prediction.created_at ?? null, lastReinforcedAt: prediction.updated_at ?? prediction.created_at ?? null, archivedAt: null, visibilityReason: ready ? 'prepared prediction is ready' : 'background prediction is warming', @@ -203,6 +208,165 @@ function isPredictionScenarioId(id: string): boolean { return id.startsWith('prediction:') } +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function liveWorkEventsFromPipeline(events: PipelineEvent[]): LiveWorkEvent[] { + const rows: LiveWorkEvent[] = [] + for (const event of events) { + if (event.kind !== 'work.snapshot') { + continue + } + const items = Array.isArray(event.payload.items) ? event.payload.items : [] + for (const item of items) { + if (!isRecord(item)) { + continue + } + const entityType = String(item.entity_type ?? '') + const entityId = String(item.entity_id ?? '') + const eventId = String(item.event_id ?? '') + if (!entityType || !entityId || !eventId) { + continue + } + rows.push(item as unknown as LiveWorkEvent) + } + } + return rows.sort((a, b) => Number(b.ts ?? 0) - Number(a.ts ?? 0)) +} + +function isActiveLiveWorkEvent(event: LiveWorkEvent): boolean { + return ( + ['queued', 'running', 'grounding', 'evidence_gathering', 'drafting'].includes(event.status) || + ['model', 'progress', 'prediction_precompute', 'queued'].includes(event.stage) + ) +} + +function scenarioCandidatesForLiveWork(event: LiveWorkEvent): string[] { + const candidates: string[] = [] + if (event.entity_type === 'prediction' && event.entity_id) { + candidates.push(`prediction:${event.entity_id}`) + } + if (event.entity_type === 'scenario' && event.entity_id) { + candidates.push(event.entity_id) + } + if (event.scenario_id) { + candidates.push(event.scenario_id) + } + return candidates +} + +function predictionScenarioIdFromLiveWork(event: LiveWorkEvent): string | null { + return event.entity_type === 'prediction' && event.entity_id ? `prediction:${event.entity_id}` : null +} + +function activeScenarioIdFromLiveWork(events: PipelineEvent[], scenarios: UIScenario[]): string | null { + const scenarioIds = new Set(scenarios.map((scenario) => scenario.id)) + const workEvents = liveWorkEventsFromPipeline(events) + const activeEvents = workEvents.filter(isActiveLiveWorkEvent) + for (const event of activeEvents.length ? activeEvents : workEvents) { + const candidates = event.scenario_id ? [event.scenario_id, ...scenarioCandidatesForLiveWork(event)] : scenarioCandidatesForLiveWork(event) + for (const candidate of candidates) { + if (scenarioIds.has(candidate)) { + return candidate + } + } + } + return null +} + +function activeScenarioCandidateFromPipelineEvent(event: PipelineEvent | undefined): string | null { + if (!event) { + return null + } + if (event.scn) { + return event.scn + } + const activeEvent = liveWorkEventsFromPipeline([event]).find(isActiveLiveWorkEvent) + return activeEvent ? activeEvent.scenario_id ?? scenarioCandidatesForLiveWork(activeEvent)[0] ?? null : null +} + +function parentScenarioFromLiveWorkEvent(event: LiveWorkEvent): UIScenario | null { + const id = predictionScenarioIdFromLiveWork(event) ?? (event.entity_type === 'scenario' ? event.entity_id : null) + if (!id) return null + const targets = Array.isArray(event.targets) ? event.targets.filter((target): target is string => typeof target === 'string' && target.length > 0) : [] + const active = isActiveLiveWorkEvent(event) + const summary = event.summary || `${event.entity_type} ${event.entity_id}` + return { + id, + kind: event.stage === 'model' ? 'research' : 'change', + title: active ? `Working: ${summary}` : summary, + score: active ? 0.98 : 0.78, + relevance: active ? 0.98 : 0.76, + confidence: active ? 0.92 : 0.7, + visiblePriority: active ? 0.99 : 0.76, + freshness: active ? 'fresh' : 'recent', + readiness: active ? 'warming' : 'ready', + visibility: active ? 'prominent' : 'warming', + lifecycleMotion: active ? 'rising' : 'stable', + createdAt: Number(event.ts ?? Date.now() / 1000), + lastRefreshedAt: Number(event.ts ?? Date.now() / 1000), + lastReinforcedAt: Number(event.ts ?? Date.now() / 1000), + archivedAt: null, + visibilityReason: active ? 'active background work' : 'recent background work', + depth: 0, + parent: null, + path: targets[0] ?? event.scenario_id ?? event.entity_id, + skill: null, + decisionState: active ? 'active' : 'pending', + reason: summary, + entities: targets.slice(0, 10), + pinned: active, + } +} + +function childScenarioFromLiveWorkEvent(event: LiveWorkEvent): UIScenario | null { + if (!event.scenario_id || event.scenario_id === event.entity_id) { + return null + } + const parent = predictionScenarioIdFromLiveWork(event) + const targets = Array.isArray(event.targets) ? event.targets.filter((target): target is string => typeof target === 'string' && target.length > 0) : [] + const active = isActiveLiveWorkEvent(event) + return { + id: event.scenario_id, + kind: event.stage === 'model' ? 'research' : 'change', + title: targets[0] ? `Exploring ${targets[0]}` : event.summary || `Scenario ${event.scenario_id.slice(0, 8)}`, + score: active ? 0.94 : 0.7, + relevance: active ? 0.92 : 0.68, + confidence: active ? 0.82 : 0.65, + visiblePriority: active ? 0.92 : 0.68, + freshness: active ? 'fresh' : 'recent', + readiness: active ? 'warming' : 'ready', + visibility: active ? 'prominent' : 'warming', + lifecycleMotion: active ? 'rising' : 'stable', + createdAt: Number(event.ts ?? Date.now() / 1000), + lastRefreshedAt: Number(event.ts ?? Date.now() / 1000), + lastReinforcedAt: Number(event.ts ?? Date.now() / 1000), + archivedAt: null, + visibilityReason: active ? 'active scenario under prepared work' : 'recent scenario under prepared work', + depth: parent ? 1 : 0, + parent, + path: targets[0] ?? event.scenario_id, + skill: null, + decisionState: active ? 'active' : 'pending', + reason: event.summary, + entities: targets.slice(0, 10), + pinned: false, + } +} + +function scenariosFromLiveWorkEvent(event: LiveWorkEvent): UIScenario[] { + return [parentScenarioFromLiveWorkEvent(event), childScenarioFromLiveWorkEvent(event)].filter((scenario): scenario is UIScenario => Boolean(scenario)) +} + +function scenarioBelongsOnActiveMap(scenario: UIScenario, nowSeconds = Date.now() / 1000): boolean { + if (scenario.pinned || scenario.visibility === 'prominent' || scenario.freshness !== 'stale') { + return true + } + const reinforcedAt = Number(scenario.lastReinforcedAt ?? 0) + return reinforcedAt > 0 && nowSeconds - reinforcedAt <= LIVE_STALE_GRACE_SECONDS +} + function formatDecisionTime(assembledAt: number): string { return new Date(assembledAt * 1000).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' }) } @@ -258,6 +422,7 @@ function App() { const [pinnedFacts, setPinnedFacts] = useState([]) const [selectedId, setSelectedId] = useState(null) const [liveSelection, setLiveSelection] = useState(null) + const [scenarioAutoFocus, setScenarioAutoFocus] = useState(false) const [selectedDecisionId, setSelectedDecisionId] = useState(null) const [paletteOpen, setPaletteOpen] = useState(false) const [drawerOpen, setDrawerOpen] = useState(false) @@ -303,6 +468,8 @@ function App() { readiness: 'ready', visibility: 'prominent', lifecycleMotion: 'rising', + createdAt: draft.created_at ?? null, + lastRefreshedAt: draft.updated_at ?? draft.created_at ?? null, lastReinforcedAt: Date.now() / 1000, archivedAt: null, visibilityReason: 'active draft plan', @@ -328,12 +495,28 @@ function App() { [predictions], ) + const liveWorkScenarios = useMemo(() => { + const byId = new Map() + for (const event of liveWorkEventsFromPipeline(pipeline.events).filter(isActiveLiveWorkEvent).slice(0, 8)) { + for (const scenario of scenariosFromLiveWorkEvent(event)) { + if (!byId.has(scenario.id)) { + byId.set(scenario.id, scenario) + } + } + } + return [...byId.values()] + }, [pipeline.events]) + const scenarios = useMemo(() => { const withLive = (rows: UIScenario[]) => { if (scenarioScope === 'history') { return rows } - const combined = [...predictionScenarios, ...rows.filter((row) => !predictionScenarios.some((prediction) => prediction.id === row.id))] + const activeRows = scenarioScope === 'live' || scenarioScope === 'focus' + ? rows.filter((row) => scenarioBelongsOnActiveMap(row)) + : rows + const seeded = [...predictionScenarios, ...liveWorkScenarios.filter((live) => !predictionScenarios.some((prediction) => prediction.id === live.id))] + const combined = [...seeded, ...activeRows.filter((row) => !seeded.some((seed) => seed.id === row.id))] return livePlanScenario ? [livePlanScenario, ...combined.filter((row) => row.id !== livePlanScenario.id)] : combined } if (scenarioScope === 'focus' && !focusMatches) { @@ -349,7 +532,7 @@ function App() { const refreshedAt = scenarioMap[scenario.id]?.last_refreshed_at ?? scenarioMap[scenario.id]?.created_at return typeof refreshedAt === 'number' && refreshedAt >= bootstrapPayload.daemon_started_at! })) - }, [bootstrapPayload?.daemon_started_at, focusMatches, livePlanScenario, predictionScenarios, scenarioMap, scenarioResult.scenarios, scenarioScope]) + }, [bootstrapPayload?.daemon_started_at, focusMatches, livePlanScenario, liveWorkScenarios, predictionScenarios, scenarioMap, scenarioResult.scenarios, scenarioScope]) const preparedCards = useMemo(() => { const freshnessRank = (label: string) => { @@ -409,7 +592,7 @@ function App() { }, [cockpit.accent]) useEffect(() => { - const pulseTarget = pipeline.events[0]?.scn + const pulseTarget = activeScenarioCandidateFromPipelineEvent(pipeline.events[0]) if (!pulseTarget) { return } @@ -433,6 +616,19 @@ function App() { } }, [scenarios, selectedId]) + const scenarioAutoFocusTargetId = useMemo( + () => activeScenarioIdFromLiveWork(pipeline.events, scenarios), + [pipeline.events, scenarios], + ) + const scenarioAutoFocusTarget = scenarios.find((scenario) => scenario.id === scenarioAutoFocusTargetId) ?? null + + useEffect(() => { + if (!scenarioAutoFocus || view !== 'scenario-map' || !scenarioAutoFocusTargetId || selectedId === scenarioAutoFocusTargetId) { + return + } + selectScenario(scenarioAutoFocusTargetId) + }, [scenarioAutoFocus, scenarioAutoFocusTargetId, selectedId, view]) + useEffect(() => { if (!preparedCards.length) { return @@ -552,6 +748,13 @@ function App() { setSelectedId(id) if (id && isPredictionScenarioId(id)) { setLiveSelection({ entityType: 'prediction', entityId: id.replace(/^prediction:/, '') }) + } else if (id) { + const scenario = scenarios.find((item) => item.id === id) + if (scenario?.parent && isPredictionScenarioId(scenario.parent)) { + setLiveSelection({ entityType: 'prediction', entityId: scenario.parent.replace(/^prediction:/, '') }) + return + } + setLiveSelection(null) } else { setLiveSelection(null) } @@ -711,9 +914,10 @@ function App() { '2': 'prepared-work', '3': 'now', '4': 'scenario-map', - '5': 'timeline', - '6': 'board', - '7': 'evidence', + '5': 'heatmap', + '6': 'timeline', + '7': 'board', + '8': 'evidence', } const nextView = shortcutView[event.key] if (nextView) { @@ -826,9 +1030,10 @@ function App() { { id: 'view-prepared-work', kind: 'view', label: 'View Prepared Work', hint: '2', run: () => setView('prepared-work') }, { id: 'view-now', kind: 'view', label: 'View Now', hint: '3', run: () => setView('now') }, { id: 'view-scenario-map', kind: 'view', label: 'View Scenario Map', hint: '4', run: () => setView('scenario-map') }, - { id: 'view-timeline', kind: 'view', label: 'View Timeline', hint: '5', run: () => setView('timeline') }, - { id: 'view-board', kind: 'view', label: 'View Board', hint: '6', run: () => setView('board') }, - { id: 'view-evidence', kind: 'view', label: 'View Evidence', hint: '7', run: () => setView('evidence') }, + { id: 'view-heatmap', kind: 'view', label: 'View Heatmap Replay', hint: '5', run: () => setView('heatmap') }, + { id: 'view-timeline', kind: 'view', label: 'View Timeline', hint: '6', run: () => setView('timeline') }, + { id: 'view-board', kind: 'view', label: 'View Board', hint: '7', run: () => setView('board') }, + { id: 'view-evidence', kind: 'view', label: 'View Evidence', hint: '8', run: () => setView('evidence') }, { id: 'clear-events', kind: 'action', @@ -898,9 +1103,10 @@ function App() { : scenarioScope === 'history' ? 'No archived suggestions yet. Cooling scenarios will move here instead of staying on the live map.' : 'No suggestions have been created in this cockpit session yet. Switch to History to inspect older suggestions.' + const embeddedInspector = view === 'heatmap' && mode !== 'proxy' return ( -
+
scenario.pinned).map((scenario) => scenario.id))} signals={pipeline.signals} @@ -1014,6 +1224,18 @@ function App() { onSelectPrediction={selectPrediction} /> ) : null} + {view === 'heatmap' ? ( + void handleFeedback(id, result)} + onPinScenario={(id) => void handleTogglePin(id)} + onOpenScenarioMap={() => setView('scenario-map')} + onOpenEvidence={() => setView('evidence')} + /> + ) : null} {view === 'timeline' ? ( ) : null} + {embeddedInspector ? null : (
+ )} setPaletteOpen(false)} commands={commands} /> { return request('/work/active') } +export function getHeatmapReplay(params: { fromTs: number; toTs: number; limit?: number }): Promise { + const query = new URLSearchParams() + query.set('from_ts', String(params.fromTs / 1000)) + query.set('to_ts', String(params.toTs / 1000)) + if (params.limit) query.set('limit', String(params.limit)) + return request(`/heatmap/replay?${query.toString()}`) +} + +export function streamHeatmapReplay(params: { rangeMs: number; limit?: number }): EventSource { + const query = new URLSearchParams() + query.set('range_seconds', String(params.rangeMs / 1000)) + if (params.limit) query.set('limit', String(params.limit)) + return openEventSource(`/heatmap/replay/stream?${query.toString()}`) +} + export function getLiveWork(entityType: string, entityId: string, limit = 80): Promise { const query = new URLSearchParams() query.set('entity_type', entityType) diff --git a/ui/cockpit/src/components/CockpitViews.tsx b/ui/cockpit/src/components/CockpitViews.tsx index 899c687..857b3f4 100644 --- a/ui/cockpit/src/components/CockpitViews.tsx +++ b/ui/cockpit/src/components/CockpitViews.tsx @@ -89,6 +89,13 @@ export function FocusView({ const proactiveTone = typeof focusState?.proactive_allowed === 'boolean' ? (focusState.proactive_allowed ? 'ok' : 'warn') : 'muted' const worker = jobs?.worker const queue = jobs?.queue + const profile = jobs?.profile ?? {} + const continuationRounds = numberFromProfile(profile.continuation_rounds) + const continuationAdmitted = numberFromProfile(profile.continuation_admitted) + const produced = numberFromProfile(profile.produced) + const precomputeMs = numberFromProfile(profile.precompute_ms) + const totalMs = numberFromProfile(profile.total_ms) + const noScenarioReason = stringFromProfile(profile.no_scenario_reason) const codexCapability = capabilities?.composer_adapters?.find((adapter) => adapter.host_app === 'codex-cli') const globalPlans = sources?.sources?.global_client_plans const directionPredictions = predictions @@ -171,6 +178,16 @@ export function FocusView({ {worker.state ?? 'unknown'}
{[worker.phase, worker.defer_reason, queue ? `queue ${queue.size ?? 0}/${queue.max_size ?? 0}` : null].filter(Boolean).join(' · ')} + {Object.keys(profile).length ? ( +
+ 0 ? 'ok' : 'muted'} /> + 0 ? 'ok' : 'muted'} /> + + + + {noScenarioReason ? : null} +
+ ) : null} ) : null} {focusJobs.length ? ( @@ -832,6 +849,33 @@ function relativeAge(epoch: number): string { return `${Math.round(delta / 86400)}d ago` } +function numberFromProfile(value: number | string | undefined): number { + if (typeof value === 'number' && Number.isFinite(value)) return value + if (typeof value === 'string') { + const parsed = Number(value) + return Number.isFinite(parsed) ? parsed : 0 + } + return 0 +} + +function stringFromProfile(value: number | string | undefined): string { + return typeof value === 'string' ? value : '' +} + +function formatCount(value: number): string { + if (!Number.isFinite(value)) return '0' + return `${Math.round(value)}` +} + +function formatDurationMs(value: number): string { + if (!Number.isFinite(value) || value <= 0) return '0s' + if (value < 1000) return `${Math.round(value)}ms` + if (value < 60_000) return `${(value / 1000).toFixed(1)}s` + const minutes = Math.floor(value / 60_000) + const seconds = Math.round((value % 60_000) / 1000) + return `${minutes}m ${seconds}s` +} + const viewScrollStyle: React.CSSProperties = { height: '100%', overflow: 'auto', @@ -879,6 +923,14 @@ const focusJobStyle: React.CSSProperties = { minWidth: 0, } +const focusProfileGridStyle: React.CSSProperties = { + display: 'grid', + gap: 6, + marginTop: 10, + paddingTop: 10, + borderTop: '1px solid var(--line-hair)', +} + const signalStyle: React.CSSProperties = { border: '1px solid var(--line-hair)', borderRadius: 'var(--r-1)', diff --git a/ui/cockpit/src/components/HeatmapReplayView.tsx b/ui/cockpit/src/components/HeatmapReplayView.tsx new file mode 100644 index 0000000..e5c7995 --- /dev/null +++ b/ui/cockpit/src/components/HeatmapReplayView.tsx @@ -0,0 +1,831 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' + +import { adaptScenario } from '../api/adapt' +import { getHeatmapReplay, streamHeatmapReplay } from '../api/client' +import type { PipelineEvent } from '../api/usePipelineEvents' +import { + buildScenarioHeatmapData, + filterEventsByType, + normalizeLiveWorkHeatmapEvents, + scenarioStateAtCursor, + type HeatmapEvent, + type HeatmapEventType, + type HeatmapGroupBy, + type HeatmapRow, + type HeatmapSortBy, +} from '../lib/heatmap' +import type { UIScenario } from '../types' + +type TimeRange = '5m' | '15m' | '1h' | 'session' +type ReplaySpeed = 0.5 | 1 | 2 | 4 + +const TIME_RANGES: Array<{ id: TimeRange; label: string; ms: number }> = [ + { id: '5m', label: '5m', ms: 5 * 60_000 }, + { id: '15m', label: '15m', ms: 15 * 60_000 }, + { id: '1h', label: '1h', ms: 60 * 60_000 }, + { id: 'session', label: 'session', ms: 4 * 60 * 60_000 }, +] +const EVENT_TYPES: HeatmapEventType[] = ['work', 'model', 'tool', 'file', 'test', 'error', 'risk', 'context', 'user', 'suggestion'] +const LEFT_WIDTH = 300 +const TOP_HEIGHT = 44 +const ROW_HEIGHT = 32 +const BOTTOM_HEIGHT = 46 + +interface HeatmapReplayViewProps { + scenarios: UIScenario[] + events: PipelineEvent[] + selectedScenarioId: string | null + onSelectScenario: (id: string) => void + onScenarioFeedback: (id: string, result: 'useful' | 'partial' | 'irrelevant') => void + onPinScenario: (id: string) => void + onOpenScenarioMap: () => void + onOpenEvidence: () => void +} + +type HoverState = + | { kind: 'cell'; x: number; y: number; row: HeatmapRow; sampleIndex: number; timeMs: number; event?: HeatmapEvent } + | { kind: 'event'; x: number; y: number; row: HeatmapRow; event: HeatmapEvent } + | null + +export function HeatmapReplayView({ + scenarios, + events, + selectedScenarioId, + onSelectScenario, + onScenarioFeedback, + onPinScenario, + onOpenScenarioMap, + onOpenEvidence, +}: HeatmapReplayViewProps) { + const canvasRef = useRef(null) + const wrapRef = useRef(null) + const [size, setSize] = useState({ w: 1000, h: 620 }) + const [range, setRange] = useState('15m') + const [groupBy, setGroupBy] = useState('cluster') + const [sortBy, setSortBy] = useState('active') + const [showStale, setShowStale] = useState(false) + const [showLowConfidence, setShowLowConfidence] = useState(true) + const [onlyPinned, setOnlyPinned] = useState(false) + const [onlyWithEvents, setOnlyWithEvents] = useState(false) + const [enabledEvents, setEnabledEvents] = useState>(() => new Set(EVENT_TYPES)) + const [mode, setMode] = useState<'live' | 'replay'>('live') + const [playing, setPlaying] = useState(false) + const [speed, setSpeed] = useState(1) + const [now, setNow] = useState(() => Date.now()) + const [cursor, setCursor] = useState(() => Date.now()) + const [hover, setHover] = useState(null) + const [selectedEvent, setSelectedEvent] = useState(null) + const [payload, setPayload] = useState> | null>(null) + const [error, setError] = useState(null) + + useEffect(() => { + const element = wrapRef.current + if (!element) return + const observer = new ResizeObserver((entries) => { + for (const entry of entries) { + setSize({ w: Math.max(680, entry.contentRect.width), h: Math.max(420, entry.contentRect.height) }) + } + }) + observer.observe(element) + return () => observer.disconnect() + }, []) + + useEffect(() => { + const timer = window.setInterval(() => { + const next = Date.now() + setNow(next) + if (mode === 'live') { + setCursor(next) + } + }, 1000) + return () => window.clearInterval(timer) + }, [mode]) + + useEffect(() => { + let cancelled = false + const load = async () => { + try { + const toTs = Date.now() + const fromTs = toTs - rangeMsFor(range) + const next = await getHeatmapReplay({ fromTs, toTs, limit: 140 }) + if (cancelled) return + setPayload(next) + setError(null) + } catch (err) { + if (!cancelled) setError(err instanceof Error ? err.message : String(err)) + } + } + if (mode !== 'replay') return + void load() + return () => { + cancelled = true + } + }, [mode, range]) + + useEffect(() => { + if (mode !== 'live') return + const source = streamHeatmapReplay({ rangeMs: rangeMsFor(range), limit: 140 }) + const handleMessage = (event: MessageEvent) => { + try { + setPayload(JSON.parse(event.data)) + setError(null) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } + } + source.addEventListener('replay_snapshot', handleMessage as EventListener) + source.onmessage = handleMessage + source.onerror = () => { + setError('Heatmap stream disconnected; retrying...') + } + return () => { + source.removeEventListener('replay_snapshot', handleMessage as EventListener) + source.close() + } + }, [mode, range]) + + useEffect(() => { + if (!playing || mode !== 'replay') return + const timer = window.setInterval(() => { + setCursor((current) => { + const rangeMs = rangeMsFor(range) + const end = now + const next = current + 1000 * speed + if (next >= end) { + setPlaying(false) + setMode('live') + return end + } + return Math.max(end - rangeMs, next) + }) + }, 1000) + return () => window.clearInterval(timer) + }, [mode, now, playing, range, speed]) + + const replayScenarios = useMemo(() => (payload?.scenarios ?? []).map(adaptScenario), [payload?.scenarios]) + const replayEvents = useMemo(() => normalizeLiveWorkHeatmapEvents(payload?.events ?? []), [payload?.events]) + + const rows = useMemo( + () => + buildScenarioHeatmapData(replayScenarios, events, { + now, + rangeMs: rangeMsFor(range), + activeScenarioId: selectedScenarioId, + showStale, + showLowConfidence, + onlyPinned, + onlyWithEvents, + eventTypes: enabledEvents, + groupBy, + sortBy, + buckets: Math.min(180, Math.max(72, Math.floor((size.w - LEFT_WIDTH) / 8))), + samples: payload?.samples ?? [], + events: replayEvents, + }), + [enabledEvents, events, groupBy, now, onlyPinned, onlyWithEvents, payload?.samples, range, replayEvents, replayScenarios, selectedScenarioId, showLowConfidence, showStale, size.w, sortBy], + ) + + const visibleRows = useMemo(() => rows.slice(0, Math.max(1, Math.floor((size.h - TOP_HEIGHT - BOTTOM_HEIGHT) / ROW_HEIGHT))), [rows, size.h]) + const selectedRow = selectedScenarioId ? rows.find((row) => row.scenario.id === selectedScenarioId) ?? null : null + const selectedSample = selectedRow ? scenarioStateAtCursor(selectedRow, cursor) : null + const filteredEvents = useMemo(() => filterEventsByType(rows.flatMap((row) => row.events), enabledEvents), [enabledEvents, rows]) + + const hitTest = useCallback( + (clientX: number, clientY: number): HoverState => { + const rect = canvasRef.current?.getBoundingClientRect() + if (!rect) return null + const x = clientX - rect.left + const y = clientY - rect.top + const plot = plotRect(size) + if (x < 0 || y < 0 || x > size.w || y > size.h) return null + const rowIndex = Math.floor((y - TOP_HEIGHT) / ROW_HEIGHT) + const row = visibleRows[rowIndex] + if (!row) return null + const rangeMs = rangeMsFor(range) + const start = now - rangeMs + for (const event of row.events) { + if (!enabledEvents.has(event.type)) continue + const ex = LEFT_WIDTH + ((event.timestamp - start) / rangeMs) * plot.w + const ey = TOP_HEIGHT + rowIndex * ROW_HEIGHT + ROW_HEIGHT / 2 + const radius = 4 + event.magnitude * 7 + if (Math.hypot(x - ex, y - ey) <= radius + 3) { + return { kind: 'event', x, y, row, event } + } + } + if (x >= LEFT_WIDTH && x <= LEFT_WIDTH + plot.w) { + const timeMs = start + ((x - LEFT_WIDTH) / plot.w) * rangeMs + return { kind: 'cell', x, y, row, sampleIndex: nearestSampleIndex(row, timeMs), timeMs } + } + return { kind: 'cell', x, y, row, sampleIndex: Math.max(0, row.samples.length - 1), timeMs: now } + }, + [enabledEvents, now, range, size, visibleRows], + ) + + useEffect(() => { + const canvas = canvasRef.current + const ctx = canvas?.getContext('2d') + if (!canvas || !ctx) return + const dpr = window.devicePixelRatio || 1 + canvas.width = Math.floor(size.w * dpr) + canvas.height = Math.floor(size.h * dpr) + canvas.style.width = `${size.w}px` + canvas.style.height = `${size.h}px` + ctx.setTransform(dpr, 0, 0, dpr, 0, 0) + drawHeatmap(ctx, { + width: size.w, + height: size.h, + rows: visibleRows, + now, + rangeMs: rangeMsFor(range), + cursor, + mode, + selectedScenarioId, + eventTypes: enabledEvents, + colors: canvasColors(canvas), + }) + }, [cursor, enabledEvents, mode, now, range, selectedScenarioId, size, visibleRows]) + + const jumpLive = () => { + setMode('live') + setPlaying(false) + setCursor(Date.now()) + } + + return ( +
+
+
+
HEATMAP REPLAY
+
+ Scenario intensity over time +
+
+ { + setMode('replay') + setPlaying((value) => !value) + }} + onJumpLive={jumpLive} + onSpeed={setSpeed} + onRange={(next) => { + setRange(next) + setCursor(Date.now() - rangeMsFor(next) * 0.25) + setMode('replay') + }} + /> +
+ setSortBy(value as HeatmapSortBy)} options={['active', 'readiness', 'relevance', 'confidence', 'recent']} /> + + setShowLowConfidence(!value)} label="hide low conf" /> + + +
+
+ +
+ {EVENT_TYPES.map((type) => ( + + ))} +
+ +
+
+ {error ? ( +
+
Could not load heatmap replay
+
{error}
+
+ ) : !rows.length || !payload?.samples?.length ? ( +
+
No recorded heatmap samples yet
+
Heatmap data will appear as real scenario score samples are recorded.
+
+ ) : null} + setHover(hitTest(event.clientX, event.clientY))} + onPointerLeave={() => setHover(null)} + onClick={(event) => { + const hit = hitTest(event.clientX, event.clientY) + if (!hit) return + onSelectScenario(hit.row.scenario.id) + if (hit.kind === 'event') { + setSelectedEvent(hit.event) + } else { + setSelectedEvent(null) + } + }} + onDoubleClick={jumpLive} + onPointerDown={(event) => { + const rect = canvasRef.current?.getBoundingClientRect() + if (!rect) return + const x = event.clientX - rect.left + if (x < LEFT_WIDTH) return + const plot = plotRect(size) + const nextCursor = now - rangeMsFor(range) + ((x - LEFT_WIDTH) / plot.w) * rangeMsFor(range) + setCursor(Math.max(now - rangeMsFor(range), Math.min(now, nextCursor))) + setMode('replay') + setPlaying(false) + }} + style={{ display: 'block', cursor: 'crosshair' }} + /> + {hover ? : null} +
+ +
+
+ ) +} + +function ReplayControls({ + mode, + playing, + speed, + range, + onPlay, + onJumpLive, + onSpeed, + onRange, +}: { + mode: 'live' | 'replay' + playing: boolean + speed: ReplaySpeed + range: TimeRange + onPlay: () => void + onJumpLive: () => void + onSpeed: (speed: ReplaySpeed) => void + onRange: (range: TimeRange) => void +}) { + return ( +
+ + + {([0.5, 1, 2, 4] as ReplaySpeed[]).map((item) => ( + + ))} +
+ {TIME_RANGES.map((item) => ( + + ))} +
+ ) +} + +function HeatmapTooltip({ hover, now, rangeMs }: { hover: NonNullable; now: number; rangeMs: number }) { + const sample = hover.kind === 'cell' + ? hover.row.samples[hover.sampleIndex] ?? scenarioStateAtCursor(hover.row, hover.timeMs) + : scenarioStateAtCursor(hover.row, hover.event.timestamp) + const event = hover.kind === 'event' ? hover.event : undefined + const nearby = hover.row.events.filter((item) => Math.abs(item.timestamp - sample.timestamp) < rangeMs / 40).slice(0, 3) + return ( +
+
{formatClock(event?.timestamp ?? sample.timestamp, now)}
+
{hover.row.scenario.title}
+
+ intensity {Math.round(sample.intensity * 100)} · rel {Math.round(sample.relevance * 100)} · ready {Math.round(sample.readiness * 100)} · conf {Math.round(sample.confidence * 100)} +
+ {event ?
{event.title}
: null} + {!event && nearby.length ?
{nearby.map((item) => item.title).join(' · ')}
: null} +
+ ) +} + +function HeatmapInspector({ + row, + sample, + event, + events, + cursor, + onOpenScenarioMap, + onOpenEvidence, + onScenarioFeedback, + onPinScenario, +}: { + row: HeatmapRow | null + sample: ReturnType | null + event: HeatmapEvent | null + events: HeatmapEvent[] + cursor: number + onOpenScenarioMap: () => void + onOpenEvidence: () => void + onScenarioFeedback: (id: string, result: 'useful' | 'partial' | 'irrelevant') => void + onPinScenario: (id: string) => void +}) { + if (!row) { + return ( + + ) + } + const latest = row.events.filter((item) => item.timestamp <= cursor).slice(-6).reverse() + return ( + + ) +} + +function drawHeatmap( + ctx: CanvasRenderingContext2D, + args: { + width: number + height: number + rows: HeatmapRow[] + now: number + rangeMs: number + cursor: number + mode: 'live' | 'replay' + selectedScenarioId: string | null + eventTypes: Set + colors: Record + }, +) { + ctx.clearRect(0, 0, args.width, args.height) + ctx.fillStyle = args.colors.bg0 + ctx.fillRect(0, 0, args.width, args.height) + const plot = plotRect({ w: args.width, h: args.height }) + const start = args.now - args.rangeMs + + ctx.strokeStyle = args.colors.grid + ctx.lineWidth = 1 + for (let i = 0; i <= 6; i += 1) { + const x = LEFT_WIDTH + (plot.w / 6) * i + ctx.beginPath() + ctx.moveTo(x, TOP_HEIGHT) + ctx.lineTo(x, TOP_HEIGHT + args.rows.length * ROW_HEIGHT) + ctx.stroke() + ctx.fillStyle = args.colors.fg4 + ctx.font = '10px JetBrains Mono, monospace' + ctx.fillText(formatClock(start + (args.rangeMs / 6) * i, args.now), x + 4, 24) + } + + args.rows.forEach((row, rowIndex) => { + const y = TOP_HEIGHT + rowIndex * ROW_HEIGHT + const selected = row.scenario.id === args.selectedScenarioId + ctx.fillStyle = selected ? args.colors.selectedRow : rowIndex % 2 ? args.colors.rowOdd : args.colors.rowEven + ctx.fillRect(0, y, args.width, ROW_HEIGHT) + if (row.active) { + ctx.fillStyle = args.colors.ok + ctx.fillRect(0, y + 4, 3, ROW_HEIGHT - 8) + } + if (row.scenario.pinned) { + ctx.fillStyle = args.colors.amber + ctx.beginPath() + ctx.arc(14, y + ROW_HEIGHT / 2, 3, 0, Math.PI * 2) + ctx.fill() + } + ctx.fillStyle = args.colors.fg2 + ctx.font = '11px Space Grotesk, sans-serif' + ctx.fillText(truncate(row.scenario.title, 34), 26, y + 15) + ctx.fillStyle = args.colors.fg4 + ctx.font = '10px JetBrains Mono, monospace' + ctx.fillText(`${row.status} · ${Math.round(row.currentIntensity * 100)}`, 26, y + 27) + + const heatW = Math.max(4, Math.min(18, plot.w / 120)) + row.samples.forEach((sample) => { + const x = LEFT_WIDTH + ((sample.timestamp - start) / args.rangeMs) * plot.w + if (x < LEFT_WIDTH - heatW || x > LEFT_WIDTH + plot.w + heatW) return + ctx.fillStyle = heatColor(sample.intensity, args.colors[`kind-${row.scenario.kind}`] || args.colors.accent, args.colors) + ctx.fillRect(Math.max(LEFT_WIDTH, x - heatW / 2), y + 4, heatW, ROW_HEIGHT - 8) + }) + ctx.strokeStyle = args.colors.grid + ctx.beginPath() + ctx.moveTo(0, y + ROW_HEIGHT) + ctx.lineTo(args.width, y + ROW_HEIGHT) + ctx.stroke() + }) + + args.rows.forEach((row, rowIndex) => { + const y = TOP_HEIGHT + rowIndex * ROW_HEIGHT + ROW_HEIGHT / 2 + row.events.forEach((event) => { + if (!args.eventTypes.has(event.type)) return + const x = LEFT_WIDTH + ((event.timestamp - start) / args.rangeMs) * plot.w + if (x < LEFT_WIDTH || x > LEFT_WIDTH + plot.w) return + const future = args.mode === 'replay' && event.timestamp > args.cursor + const radius = 4 + event.magnitude * 7 + ctx.globalAlpha = future ? 0.18 : 0.9 + ctx.fillStyle = eventColor(event.type) + ctx.strokeStyle = event.severity === 'error' ? args.colors.err : args.colors.bg0 + ctx.lineWidth = event.severity === 'error' ? 2 : 1 + ctx.beginPath() + ctx.arc(x, y, radius, 0, Math.PI * 2) + ctx.fill() + ctx.stroke() + ctx.globalAlpha = 1 + }) + }) + + const cursorX = LEFT_WIDTH + ((args.cursor - start) / args.rangeMs) * plot.w + ctx.strokeStyle = args.mode === 'live' ? args.colors.ok : args.colors.accent + ctx.lineWidth = 2 + ctx.beginPath() + ctx.moveTo(cursorX, TOP_HEIGHT - 8) + ctx.lineTo(cursorX, TOP_HEIGHT + args.rows.length * ROW_HEIGHT + 8) + ctx.stroke() + ctx.fillStyle = args.mode === 'live' ? args.colors.ok : args.colors.accent + ctx.font = '10px JetBrains Mono, monospace' + ctx.fillText(args.mode === 'live' ? 'LIVE' : 'REPLAY', Math.min(cursorX + 6, args.width - 64), TOP_HEIGHT - 13) +} + +function rangeMsFor(range: TimeRange): number { + return TIME_RANGES.find((item) => item.id === range)?.ms ?? 15 * 60_000 +} + +function plotRect(size: { w: number; h: number }) { + return { x: LEFT_WIDTH, y: TOP_HEIGHT, w: Math.max(100, size.w - LEFT_WIDTH - 24), h: Math.max(100, size.h - TOP_HEIGHT - BOTTOM_HEIGHT) } +} + +function nearestSampleIndex(row: HeatmapRow, timeMs: number): number { + if (!row.samples.length) return 0 + let bestIndex = 0 + let bestDistance = Math.abs(row.samples[0].timestamp - timeMs) + for (let index = 1; index < row.samples.length; index += 1) { + const distance = Math.abs(row.samples[index].timestamp - timeMs) + if (distance < bestDistance) { + bestDistance = distance + bestIndex = index + } + } + return bestIndex +} + +function canvasColors(canvas: HTMLCanvasElement): Record { + const styles = getComputedStyle(canvas) + return { + bg0: styles.getPropertyValue('--bg-0').trim() || '#111', + bg1: styles.getPropertyValue('--bg-1').trim() || '#18181d', + fg2: styles.getPropertyValue('--fg-2').trim() || '#ccc', + fg4: styles.getPropertyValue('--fg-4').trim() || '#777', + grid: styles.getPropertyValue('--line-hair').trim() || '#333', + ok: styles.getPropertyValue('--ok').trim() || '#6cc76c', + amber: styles.getPropertyValue('--amber').trim() || '#e6b656', + err: styles.getPropertyValue('--err').trim() || '#d66', + accent: styles.getPropertyValue('--accent').trim() || '#8cc', + 'kind-research': styles.getPropertyValue('--kind-research').trim() || '#ba86f0', + 'kind-explain': styles.getPropertyValue('--kind-explain').trim() || '#e6b656', + 'kind-change': styles.getPropertyValue('--kind-change').trim() || '#6ab7ff', + 'kind-debug': styles.getPropertyValue('--kind-debug').trim() || '#d76b5a', + 'kind-refactor': styles.getPropertyValue('--kind-refactor').trim() || '#74c6a2', + selectedRow: 'rgba(120, 190, 180, 0.10)', + rowOdd: 'rgba(255,255,255,0.018)', + rowEven: 'rgba(255,255,255,0.006)', + } +} + +function heatColor(intensity: number, color: string, colors: Record): string { + const alpha = 0.08 + intensity * 0.74 + if (intensity > 0.82) return `color-mix(in oklch, ${colors.amber} ${Math.round(intensity * 72)}%, transparent)` + return `color-mix(in oklch, ${color} ${Math.round(alpha * 100)}%, transparent)` +} + +function eventColor(type: HeatmapEventType): string { + return { + work: '#6ab7ff', + model: '#e6b656', + tool: '#a36cc7', + file: '#74c6a2', + test: '#6cc76c', + error: '#d76b5a', + risk: '#e58b58', + context: '#8db5ff', + suggestion: '#ba86f0', + user: '#d6d6d6', + }[type] +} + +function formatClock(timestamp: number, now: number): string { + const delta = Math.round((timestamp - now) / 1000) + if (Math.abs(delta) < 60) return `${delta}s` + if (Math.abs(delta) < 3600) return `${Math.round(delta / 60)}m` + return new Date(timestamp).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) +} + +function truncate(value: string, limit: number): string { + return value.length > limit ? `${value.slice(0, limit - 1)}…` : value +} + +function Select({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (value: string) => void }) { + return ( + + ) +} + +function Toggle({ checked, onChange, label }: { checked: boolean; onChange: (checked: boolean) => void; label: string }) { + return ( + + ) +} + +function Panel({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+
{title.toUpperCase()}
+
{children}
+
+ ) +} + +function ScoreGrid({ values }: { values: Array<[string, number]> }) { + return ( +
+ {values.map(([label, value]) => ( +
+
{label}
+
{Math.round(value * 100)}
+
+ ))} +
+ ) +} + +function MetaLine({ label, value }: { label: string; value: string }) { + return
{label ? `${label}: ` : ''}{value}
+} + +const viewStyle: React.CSSProperties = { + height: '100%', + minHeight: 0, + display: 'grid', + gridTemplateRows: 'auto auto minmax(0, 1fr)', + background: 'var(--bg-0)', +} +const toolbarStyle: React.CSSProperties = { + display: 'grid', + gridTemplateColumns: 'minmax(220px, 1fr) auto minmax(280px, 1fr)', + gap: 14, + alignItems: 'center', + padding: '14px 18px 10px', + borderBottom: '1px solid var(--line-1)', + background: 'linear-gradient(180deg, var(--bg-1), var(--bg-0))', +} +const eventToggleStyle: React.CSSProperties = { + display: 'flex', + gap: 6, + padding: '8px 18px', + borderBottom: '1px solid var(--line-hair)', + background: 'var(--bg-0)', + overflowX: 'auto', +} +const mainStyle: React.CSSProperties = { minHeight: 0, display: 'grid', gridTemplateColumns: 'minmax(0, 1fr) 310px' } +const canvasWrapStyle: React.CSSProperties = { position: 'relative', minHeight: 0, overflow: 'hidden' } +const inspectorStyle: React.CSSProperties = { + borderLeft: '1px solid var(--line-1)', + background: 'var(--bg-1)', + padding: 16, + overflow: 'auto', +} +const emptyStyle: React.CSSProperties = { + position: 'absolute', + inset: 0, + display: 'grid', + placeContent: 'center', + textAlign: 'center', + color: 'var(--fg-4)', + fontSize: 12, + zIndex: 2, + pointerEvents: 'none', +} +const tooltipStyle: React.CSSProperties = { + position: 'absolute', + zIndex: 4, + width: 280, + padding: 10, + border: '1px solid var(--line-1)', + borderRadius: 'var(--r-2)', + background: 'color-mix(in oklch, var(--bg-1) 96%, transparent)', + boxShadow: 'var(--shadow-md)', + pointerEvents: 'none', +} +const selectStyle: React.CSSProperties = { + background: 'var(--bg-2)', + border: '1px solid var(--line-1)', + borderRadius: 'var(--r-1)', + color: 'var(--fg-2)', + padding: '4px 6px', + fontSize: 10, +} +const eyebrow: React.CSSProperties = { color: 'var(--fg-4)', fontSize: 10, letterSpacing: 1.1, marginBottom: 7 } +const scoreCardStyle: React.CSSProperties = { + border: '1px solid var(--line-hair)', + background: 'var(--bg-inset)', + borderRadius: 'var(--r-2)', + padding: 9, +} +const eventLineStyle: React.CSSProperties = { + display: 'grid', + gridTemplateColumns: '54px minmax(0, 1fr)', + gap: 8, + padding: '6px 0', + borderBottom: '1px solid var(--line-hair)', + fontSize: 11.5, +} + +function controlButtonStyle(active: boolean): React.CSSProperties { + return { + border: '1px solid var(--line-1)', + background: active ? 'var(--accent-bg)' : 'var(--bg-2)', + color: active ? 'var(--fg-1)' : 'var(--fg-3)', + borderRadius: 'var(--r-1)', + padding: '5px 8px', + fontFamily: 'var(--font-mono)', + fontSize: 10, + cursor: 'pointer', + } +} + +function eventTypeButtonStyle(active: boolean, type: HeatmapEventType): React.CSSProperties { + return { + ...controlButtonStyle(active), + borderColor: active ? eventColor(type) : 'var(--line-1)', + color: active ? eventColor(type) : 'var(--fg-4)', + } +} diff --git a/ui/cockpit/src/components/Inspector.tsx b/ui/cockpit/src/components/Inspector.tsx index a919853..a9ac8c6 100644 --- a/ui/cockpit/src/components/Inspector.tsx +++ b/ui/cockpit/src/components/Inspector.tsx @@ -101,6 +101,22 @@ export function Inspector({ )}
{scenario.reason || 'No reason recorded.'}
+
+
LIFECYCLE
+
+ + + + + + +
+ {scenario.visibilityReason ? ( +
+ {scenario.visibilityReason} +
+ ) : null} +
{scenario.path || 'No path recorded.'}
@@ -160,6 +176,15 @@ function Section({ title, children }: { title: string; children: React.ReactNode ) } +function LifecycleField({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ) +} + const sectionTitle: React.CSSProperties = { fontSize: 10, letterSpacing: 1.1, color: 'var(--fg-4)', marginBottom: 8 } const buttonStyle: React.CSSProperties = { border: '1px solid var(--line-1)', @@ -178,6 +203,11 @@ const cardStyle: React.CSSProperties = { } const preStyle: React.CSSProperties = { ...cardStyle, color: 'var(--fg-2)', whiteSpace: 'pre-wrap', overflow: 'auto', fontSize: 11 } const emptyStyle: React.CSSProperties = { color: 'var(--fg-4)', fontSize: 12 } +const lifecycleGridStyle: React.CSSProperties = { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(128px, 1fr))', + gap: 8, +} function formatTimestamp(epoch: number): string { if (!epoch) return 'unknown' @@ -188,6 +218,17 @@ function formatTimestamp(epoch: number): string { return `${Math.round(dt / 86400)}d ago` } +function formatLifecycleTimestamp(epoch: number | null): string { + if (!epoch) return 'unknown' + const date = new Date(epoch * 1000) + return `${formatTimestamp(epoch)} · ${date.toLocaleString([], { + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + })}` +} + const FACTOR_PALETTE = [ '#5eb2ff', '#e6b656', diff --git a/ui/cockpit/src/components/LiveWorkInspector.tsx b/ui/cockpit/src/components/LiveWorkInspector.tsx index e4bf4e7..edcd620 100644 --- a/ui/cockpit/src/components/LiveWorkInspector.tsx +++ b/ui/cockpit/src/components/LiveWorkInspector.tsx @@ -61,7 +61,7 @@ export function LiveWorkInspector({ snapshot, live, error, title, onClose }: Liv {event.stage} · {event.status} {relativeAge(event.ts)}
-
{event.summary}
+
{liveWorkEventSummary(event)}
{event.model || event.latency_ms || event.artifact_kind ? (
{[event.model, event.latency_ms ? `${Math.round(event.latency_ms)}ms` : null, event.artifact_kind].filter(Boolean).join(' · ')} @@ -104,6 +104,27 @@ function relativeAge(epoch: number): string { return `${Math.round(delta / 86400)}d ago` } +function liveWorkEventSummary(event: LiveWorkSnapshot['events'][number]): string { + const payload = event.metadata?.payload + if (!isRecord(payload)) { + return event.summary + } + const tokensUsed = Number(payload.tokens_used) + const tokenBudget = Number(payload.token_budget) + if (!Number.isFinite(tokensUsed) || !Number.isFinite(tokenBudget) || tokenBudget <= 0) { + return event.summary + } + const complete = Number(payload.scenarios_complete) + const over = Math.max(0, Math.round(tokensUsed - tokenBudget)) + const base = `Progress: ${Math.round(tokensUsed)} tokens used / ${Math.round(tokenBudget)} token target` + const suffix = Number.isFinite(complete) ? `, ${Math.round(complete)} scenarios complete` : '' + return over > 0 ? `${base} (${over} over target)${suffix}.` : `${base}${suffix}.` +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + const eyebrowStyle: React.CSSProperties = { fontSize: 10, letterSpacing: 1.2, color: 'var(--fg-4)', marginBottom: 8 } const sectionTitle: React.CSSProperties = { fontSize: 10, letterSpacing: 1.1, color: 'var(--fg-4)', marginBottom: 8 } const buttonStyle: React.CSSProperties = { diff --git a/ui/cockpit/src/components/PipelineCanvas.test.tsx b/ui/cockpit/src/components/PipelineCanvas.test.tsx index 68e37ea..411e672 100644 --- a/ui/cockpit/src/components/PipelineCanvas.test.tsx +++ b/ui/cockpit/src/components/PipelineCanvas.test.tsx @@ -1,5 +1,5 @@ -import { render, screen } from '@testing-library/react' -import { describe, expect, it } from 'vitest' +import { fireEvent, render, screen } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' import { PipelineCanvas } from './PipelineCanvas' import type { @@ -113,6 +113,32 @@ describe('PipelineCanvas', () => { expect(screen.getByText(/No scenarios are available from the daemon yet/)).toBeInTheDocument() }) + it('renders the scenario auto-focus toggle', () => { + const onAutoFocusChange = vi.fn() + render( + undefined} + autoFocusEnabled={false} + autoFocusTargetLabel="Working prediction" + onAutoFocusChange={onAutoFocusChange} + activePulses={new Set()} + pinnedIds={new Set()} + signals={[]} + targets={[]} + artefacts={[]} + decisions={[]} + model={model()} + cycle={cycle()} + events={EVENTS} + />, + ) + + fireEvent.click(screen.getByRole('switch', { name: /auto-focus/i })) + expect(onAutoFocusChange).toHaveBeenCalledWith(true) + }) + it('flashes the model lane with a spinner while LLM requests are pending', () => { const pending = new Map([ ['ev-1', { path: 'src/a.py', model: 'qwen', startedAt: Date.now() / 1000 }], diff --git a/ui/cockpit/src/components/PipelineCanvas.tsx b/ui/cockpit/src/components/PipelineCanvas.tsx index e1dc446..6f6a2ac 100644 --- a/ui/cockpit/src/components/PipelineCanvas.tsx +++ b/ui/cockpit/src/components/PipelineCanvas.tsx @@ -35,6 +35,10 @@ export interface PipelineCanvasProps { emptyHint?: string selectedId: string | null onSelect: (id: string) => void + autoFocusEnabled?: boolean + autoFocusTargetId?: string | null + autoFocusTargetLabel?: string | null + onAutoFocusChange?: (enabled: boolean) => void activePulses: Set pinnedIds: Set signals: SignalRow[] @@ -66,6 +70,10 @@ export function PipelineCanvas({ emptyHint = 'No scenarios are available from the daemon yet. They will appear here after Vaner observes useful workspace activity.', selectedId, onSelect, + autoFocusEnabled = false, + autoFocusTargetId = null, + autoFocusTargetLabel = null, + onAutoFocusChange, activePulses, pinnedIds, signals, @@ -109,6 +117,42 @@ export function PipelineCanvas({ Higher bubbles are relevant now. Size = readiness, opacity = freshness, border = confidence.
+
+ + + {autoFocusTargetId ? autoFocusTargetLabel ?? autoFocusTargetId : 'waiting for active node'} + +
{activePlan ? (
ACTIVE PLAN
@@ -134,6 +178,22 @@ export function PipelineCanvas({ ) } +const autoFocusControlStyle: React.CSSProperties = { + position: 'absolute', + right: 18, + bottom: 58, + zIndex: 4, + display: 'flex', + alignItems: 'center', + gap: 10, + maxWidth: 'calc(100% - 36px)', + padding: '7px 10px', + border: '1px solid var(--line-1)', + borderRadius: 'var(--r-2)', + background: 'color-mix(in oklch, var(--bg-1) 94%, transparent)', + boxShadow: '0 10px 28px rgba(0,0,0,.2)', +} + const activePlanStyle: React.CSSProperties = { position: 'absolute', right: 18, diff --git a/ui/cockpit/src/components/ScenarioCluster.test.ts b/ui/cockpit/src/components/ScenarioCluster.test.ts index dbfbb89..960f86c 100644 --- a/ui/cockpit/src/components/ScenarioCluster.test.ts +++ b/ui/cockpit/src/components/ScenarioCluster.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { computeEdges, initialLayout, jaccard, scenarioPaths, scenarioScoreScale, tickForces } from './ScenarioCluster' +import { computeEdges, fadingScenario, initialLayout, jaccard, scenarioPaths, scenarioScoreScale, tickForces } from './ScenarioCluster' import type { UIScenario } from '../types' function scenario(overrides: Partial): UIScenario { @@ -16,6 +16,8 @@ function scenario(overrides: Partial): UIScenario { readiness: 'warming', visibility: 'warming', lifecycleMotion: 'stable', + createdAt: null, + lastRefreshedAt: null, lastReinforcedAt: null, archivedAt: null, visibilityReason: '', @@ -67,6 +69,18 @@ describe('scenarioScoreScale', () => { }) }) +describe('fadingScenario', () => { + it('marks removed nodes as archived fading nodes', () => { + const faded = fadingScenario(scenario({ id: 'old', relevance: 0.8, visiblePriority: 0.75 }), 1000) + expect(faded.id).toBe('old') + expect(faded.visibility).toBe('archived') + expect(faded.lifecycleMotion).toBe('fading') + expect(faded.freshness).toBe('stale') + expect(faded.relevance).toBeLessThan(0.3) + expect(faded.archivedAt).toBe(1) + }) +}) + describe('computeEdges', () => { it('includes explicit parent edges regardless of path overlap', () => { const edges = computeEdges([ diff --git a/ui/cockpit/src/components/ScenarioCluster.tsx b/ui/cockpit/src/components/ScenarioCluster.tsx index b9d5765..c0a49d6 100644 --- a/ui/cockpit/src/components/ScenarioCluster.tsx +++ b/ui/cockpit/src/components/ScenarioCluster.tsx @@ -4,6 +4,8 @@ import { KIND_COLOR } from '../lib/constants' import type { UIScenario, UIScenarioKind } from '../types' export type ClusterPosition = { x: number; y: number; vx?: number; vy?: number; _manual?: boolean } +const EXIT_RETENTION_MS = 2600 +const LEGEND_KINDS: UIScenarioKind[] = ['research', 'explain', 'change', 'debug', 'refactor'] export interface ScenarioEdge { from: string @@ -63,6 +65,41 @@ function scenarioNodeBadge(scenario: UIScenario): string { return `${Math.round(scenario.relevance * 100)}%` } +export function fadingScenario(scenario: UIScenario, nowMs = Date.now()): UIScenario { + return { + ...scenario, + relevance: Math.min(scenario.relevance, 0.28), + visiblePriority: Math.min(scenario.visiblePriority, 0.28), + freshness: 'stale', + readiness: 'cooling', + visibility: 'archived', + lifecycleMotion: 'fading', + archivedAt: nowMs / 1000, + visibilityReason: scenario.visibilityReason || 'no longer in the active daemon frontier', + } +} + +function scenarioMotionMarkerStyle(color: string, motion: UIScenario['lifecycleMotion']): React.CSSProperties | null { + if (motion === 'stable') { + return null + } + const pointsUp = motion === 'rising' + return { + position: 'absolute', + right: -13, + top: '50%', + transform: 'translateY(-50%)', + width: 0, + height: 0, + borderLeft: '4px solid transparent', + borderRight: '4px solid transparent', + borderBottom: pointsUp ? `7px solid ${color}` : undefined, + borderTop: pointsUp ? undefined : `7px solid ${color}`, + opacity: motion === 'fading' ? 0.45 : 0.8, + filter: 'drop-shadow(0 0 4px rgba(0,0,0,.28))', + } +} + /** * Jaccard similarity between two sets of path strings. Used to derive * implicit edges between scenarios that touch the same files even when they @@ -323,6 +360,8 @@ export function ScenarioCluster({ const [, forceTick] = useState(0) const [view, setView] = useState({ x: 0, y: 0, scale: 1 }) const dragRef = useRef<{ id: string; moved: boolean } | null>(null) + const previousScenariosRef = useRef>(new Map()) + const [exitingScenarios, setExitingScenarios] = useState>({}) useEffect(() => { const element = wrapRef.current @@ -339,9 +378,53 @@ export function ScenarioCluster({ }, []) useEffect(() => { - edgesRef.current = computeEdges(scenarios) + const now = Date.now() + const currentIds = new Set(scenarios.map((scenario) => scenario.id)) + setExitingScenarios((previous) => { + const next: Record = {} + for (const [id, item] of Object.entries(previous)) { + if (item.expiresAt > now && !currentIds.has(id)) { + next[id] = item + } + } + for (const [id, scenario] of previousScenariosRef.current.entries()) { + if (!currentIds.has(id)) { + next[id] = { scenario: fadingScenario(scenario, now), expiresAt: now + EXIT_RETENTION_MS } + } + } + return next + }) + previousScenariosRef.current = new Map(scenarios.map((scenario) => [scenario.id, scenario])) + }, [scenarios]) + + useEffect(() => { + if (!Object.keys(exitingScenarios).length) { + return + } + const timer = window.setTimeout(() => { + const now = Date.now() + setExitingScenarios((previous) => { + const next = Object.fromEntries(Object.entries(previous).filter(([, item]) => item.expiresAt > now)) + return next + }) + }, EXIT_RETENTION_MS + 80) + return () => window.clearTimeout(timer) + }, [exitingScenarios]) + + const displayScenarios = useMemo(() => { + const currentIds = new Set(scenarios.map((scenario) => scenario.id)) + return [ + ...scenarios, + ...Object.values(exitingScenarios) + .map((item) => item.scenario) + .filter((scenario) => !currentIds.has(scenario.id)), + ] + }, [exitingScenarios, scenarios]) + + useEffect(() => { + edgesRef.current = computeEdges(displayScenarios) const previous = posRef.current - const fresh = initialLayout(scenarios, size.w || 800, size.h || 520) + const fresh = initialLayout(displayScenarios, size.w || 800, size.h || 520) for (const id of Object.keys(fresh)) { if (previous[id]) { fresh[id] = { ...fresh[id], ...previous[id] } @@ -349,11 +432,11 @@ export function ScenarioCluster({ } posRef.current = fresh forceTick((value) => value + 1) - }, [scenarios, size.h, size.w]) + }, [displayScenarios, size.h, size.w]) const gravityTargets = useMemo( - () => Object.fromEntries(scenarios.map((scenario) => [scenario.id, scenarioTargetY(scenario, size.h || 520)])), - [scenarios, size.h], + () => Object.fromEntries(displayScenarios.map((scenario) => [scenario.id, scenarioTargetY(scenario, size.h || 520)])), + [displayScenarios, size.h], ) useEffect(() => { @@ -479,7 +562,7 @@ export function ScenarioCluster({ // to recompute when it changes even though the linter doesn't see the // indirect dependency. // eslint-disable-next-line react-hooks/exhaustive-deps - }, [selectedId, scenarios]) + }, [selectedId, displayScenarios]) useEffect(() => { if (!selectedId) { @@ -498,7 +581,7 @@ export function ScenarioCluster({ const positions = posRef.current - if (!scenarios.length) { + if (!displayScenarios.length) { return (
- {scenarios.map((scenario) => { + {displayScenarios.map((scenario) => { const position = positions[scenario.id] if (!position) { return null @@ -585,6 +668,8 @@ export function ScenarioCluster({ const freshnessOpacity = scenarioFreshnessOpacity(scenario) const borderWidth = plan || prediction ? 2 : 1 + Math.max(0.4, scenario.confidence) * 1.4 const stalePinned = pinned && (scenario.freshness === 'stale' || scenario.relevance < 0.65) + const exiting = scenario.visibility === 'archived' || scenario.lifecycleMotion === 'fading' + const motionStyle = scenarioMotionMarkerStyle(color, scenario.lifecycleMotion) return (
onNodePointerDown(event, scenario.id)} + onPointerDown={(event) => { + if (exiting) { + event.stopPropagation() + return + } + onNodePointerDown(event, scenario.id) + }} style={{ position: 'absolute', left: position.x - radius, @@ -600,10 +691,10 @@ export function ScenarioCluster({ width: radius * 2, height: radius * 2, borderRadius: plan ? '28%' : '50%', - cursor: 'pointer', + cursor: exiting ? 'default' : 'pointer', background: rejected ? 'transparent' : `color-mix(in oklch, ${color} ${plan || prediction || scenario.readiness === 'ready' ? 30 : 18}%, transparent)`, border: `${borderWidth}px solid ${color}`, - opacity: rejected ? 0.25 : dim ? 0.42 : freshnessOpacity, + opacity: rejected ? 0.25 : exiting ? 0.24 : dim ? 0.42 : freshnessOpacity, boxShadow: selected ? `0 0 0 3px var(--bg-0), 0 0 0 5px ${color}, 0 0 26px ${color}80` : pinned @@ -613,7 +704,7 @@ export function ScenarioCluster({ : chosen ? `0 0 18px ${color}80` : 'none', - transition: 'box-shadow .2s, opacity .2s', + transition: 'box-shadow .2s, opacity .65s', animation: pulse ? 'dc-pulse 1.2s infinite' : 'none', }} > @@ -674,48 +765,13 @@ export function ScenarioCluster({ > {scenarioNodeBadge(scenario)}
-
+ {motionStyle ?
: null}
) })}
-
- - - Top = relevant now · size = readiness · opacity = freshness · border = confidence - -
+
- + @@ -748,6 +804,72 @@ export function ScenarioCluster({ ) } +function ScenarioLegend() { + return ( +
+
+ LEGEND +
+
+ {LEGEND_KINDS.map((kind) => ( +
+ + {kind} +
+ ))} +
+
+ + plan + + prepared/live work +
+
+ ) +} + +const legendStyle: React.CSSProperties = { + position: 'absolute', + bottom: 12, + left: 12, + zIndex: 3, + display: 'grid', + gap: 8, + width: 390, + maxWidth: 'calc(100% - 36px)', + background: 'color-mix(in oklch, var(--bg-1) 94%, transparent)', + border: '1px solid var(--line-1)', + borderRadius: 'var(--r-2)', + padding: '10px 11px', + boxShadow: '0 10px 28px rgba(0,0,0,.2)', + color: 'var(--fg-3)', + fontSize: 10.5, +} + +const legendGridStyle: React.CSSProperties = { + display: 'grid', + gridTemplateColumns: 'repeat(5, minmax(0, 1fr))', + gap: 7, +} + +const legendItemStyle: React.CSSProperties = { + display: 'inline-flex', + alignItems: 'center', + gap: 5, + minWidth: 0, + fontFamily: 'var(--font-mono)', + whiteSpace: 'nowrap', +} + +const legendBubbleStyle: React.CSSProperties = { + display: 'inline-flex', + width: 10, + height: 10, + flex: '0 0 auto', + borderRadius: '50%', + border: '1.5px solid var(--accent)', +} + const clusterBtn: React.CSSProperties = { background: 'var(--bg-2)', border: 'none', diff --git a/ui/cockpit/src/components/chrome.tsx b/ui/cockpit/src/components/chrome.tsx index 4511cbe..bfe1b15 100644 --- a/ui/cockpit/src/components/chrome.tsx +++ b/ui/cockpit/src/components/chrome.tsx @@ -27,16 +27,17 @@ export interface CommandItem { run: () => void } -export type CockpitView = 'focus' | 'prepared-work' | 'now' | 'scenario-map' | 'timeline' | 'board' | 'evidence' +export type CockpitView = 'focus' | 'prepared-work' | 'now' | 'scenario-map' | 'heatmap' | 'timeline' | 'board' | 'evidence' const COCKPIT_VIEW_OPTIONS: Array<{ id: CockpitView; label: string; shortcut: string }> = [ { id: 'focus', label: 'Focus', shortcut: '1' }, { id: 'prepared-work', label: 'Prepared Work', shortcut: '2' }, { id: 'now', label: 'Now', shortcut: '3' }, { id: 'scenario-map', label: 'Scenario Map', shortcut: '4' }, - { id: 'timeline', label: 'Timeline', shortcut: '5' }, - { id: 'board', label: 'Board', shortcut: '6' }, - { id: 'evidence', label: 'Evidence', shortcut: '7' }, + { id: 'heatmap', label: 'Heatmap', shortcut: '5' }, + { id: 'timeline', label: 'Timeline', shortcut: '6' }, + { id: 'board', label: 'Board', shortcut: '7' }, + { id: 'evidence', label: 'Evidence', shortcut: '8' }, ] interface TopBarProps { diff --git a/ui/cockpit/src/lib/heatmap.test.ts b/ui/cockpit/src/lib/heatmap.test.ts new file mode 100644 index 0000000..743f9a5 --- /dev/null +++ b/ui/cockpit/src/lib/heatmap.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, it } from 'vitest' + +import { + buildScenarioHeatmapData, + filterEventsByType, + normalizeHeatmapEvents, + scenarioIntensity, + scenarioStateAtCursor, + sortHeatmapRows, + type HeatmapEvent, +} from './heatmap' +import type { PipelineEvent } from '../api/usePipelineEvents' +import type { UIScenario } from '../types' + +function scenario(overrides: Partial): UIScenario { + return { + id: 's1', + kind: 'change', + title: 'Scenario', + score: 0.5, + relevance: 0.5, + confidence: 0.5, + visiblePriority: 0.5, + freshness: 'recent', + readiness: 'warming', + visibility: 'warming', + lifecycleMotion: 'stable', + createdAt: 100, + lastRefreshedAt: 100, + lastReinforcedAt: 100, + archivedAt: null, + visibilityReason: '', + depth: 0, + parent: null, + path: 'src/app.ts', + skill: null, + decisionState: 'pending', + reason: '', + entities: [], + pinned: false, + ...overrides, + } +} + +function event(overrides: Partial): PipelineEvent { + return { + id: 'e1', + t: '00:00', + tag: 'work', + color: 'var(--accent)', + msg: 'Model request completed', + scn: 's1', + stage: 'model', + kind: 'llm.response', + ts: 100, + path: 'src/app.ts', + cycleId: null, + payload: { latency_ms: 1200 }, + ...overrides, + } +} + +describe('scenarioIntensity', () => { + it('boosts active scenarios without saturating them', () => { + const base = scenarioIntensity(scenario({ readiness: 'ready', relevance: 0.8, confidence: 0.7, freshness: 'fresh' })) + const active = scenarioIntensity(scenario({ readiness: 'ready', relevance: 0.8, confidence: 0.7, freshness: 'fresh' }), 's1') + expect(active).toBeGreaterThan(base) + expect(active).toBeLessThan(1) + }) + + it('decays stale rejected scenarios visibly', () => { + const hot = scenarioIntensity(scenario({ readiness: 'ready', relevance: 0.8, confidence: 0.8, freshness: 'fresh' })) + const stale = scenarioIntensity(scenario({ readiness: 'ready', relevance: 0.8, confidence: 0.8, freshness: 'stale', decisionState: 'rejected' })) + expect(stale).toBeLessThan(hot * 0.5) + }) +}) + +describe('normalizeHeatmapEvents', () => { + it('turns work snapshots into scenario-linked heatmap events', () => { + const rows = normalizeHeatmapEvents([ + event({ + id: 'snapshot', + kind: 'work.snapshot', + stage: 'work', + payload: { + items: [ + { + event_id: 'lw-1', + ts: 101, + entity_type: 'prediction', + entity_id: 'p1', + scenario_id: 's2', + stage: 'model', + status: 'running', + summary: 'Model request started', + targets: ['src/a.ts'], + }, + ], + }, + }), + ]) + expect(rows[0]).toMatchObject({ id: 'lw-1', scenarioId: 's2', type: 'model' }) + }) +}) + +describe('buildScenarioHeatmapData', () => { + it('generates rows with changing samples and linked events', () => { + const rows = buildScenarioHeatmapData( + [scenario({ id: 's1', lastReinforcedAt: 95 })], + [event({ ts: 100 })], + { + now: 120_000, + rangeMs: 60_000, + buckets: 32, + activeScenarioId: 's1', + samples: [ + { ts: 70, scenario_id: 's1', relevance: 0.3, readiness: 'warming', confidence: 0.5, freshness: 'recent', visible_priority: 0.3, visibility: 'warming', lifecycle_motion: 'stable', status: 'prep', pinned: false, active: false }, + { ts: 110, scenario_id: 's1', relevance: 0.8, readiness: 'ready', confidence: 0.7, freshness: 'fresh', visible_priority: 0.8, visibility: 'prominent', lifecycle_motion: 'rising', status: 'active', pinned: false, active: true }, + ], + }, + ) + expect(rows).toHaveLength(1) + expect(rows[0].samples).toHaveLength(2) + expect(rows[0].events).toHaveLength(1) + expect(Math.max(...rows[0].samples.map((sample) => sample.intensity))).toBeGreaterThan( + Math.min(...rows[0].samples.map((sample) => sample.intensity)), + ) + }) + + it('can filter stale and eventless rows', () => { + const rows = buildScenarioHeatmapData( + [ + scenario({ id: 'hot', freshness: 'fresh' }), + scenario({ id: 'old', freshness: 'stale' }), + ], + [], + { now: 120_000, rangeMs: 60_000, showStale: false, onlyWithEvents: true }, + ) + expect(rows).toHaveLength(0) + }) +}) + +describe('scenarioStateAtCursor', () => { + it('returns the latest sample at or before the cursor', () => { + const row = buildScenarioHeatmapData( + [scenario({ id: 's1' })], + [], + { + now: 120_000, + rangeMs: 60_000, + samples: [ + { ts: 70, scenario_id: 's1', relevance: 0.2, readiness: 'warming', confidence: 0.4, freshness: 'recent', visible_priority: 0.2, visibility: 'warming', lifecycle_motion: 'stable', status: 'prep', pinned: false, active: false }, + { ts: 90, scenario_id: 's1', relevance: 0.7, readiness: 'ready', confidence: 0.8, freshness: 'fresh', visible_priority: 0.7, visibility: 'prominent', lifecycle_motion: 'rising', status: 'ready', pinned: false, active: false }, + ], + }, + )[0] + const sample = scenarioStateAtCursor(row, 91_000) + expect(sample).toBe(row.samples[1]) + }) +}) + +describe('filtering and sorting', () => { + it('filters events by enabled type', () => { + const events: HeatmapEvent[] = [ + { id: 'a', timestamp: 1, scenarioId: 's1', type: 'model', title: '', description: '', severity: 'info', magnitude: 0.5, paths: [] }, + { id: 'b', timestamp: 2, scenarioId: 's1', type: 'error', title: '', description: '', severity: 'error', magnitude: 0.8, paths: [] }, + ] + expect(filterEventsByType(events, new Set(['error']))).toHaveLength(1) + }) + + it('sorts active rows first', () => { + const rows = buildScenarioHeatmapData( + [scenario({ id: 'a', relevance: 0.2 }), scenario({ id: 'b', relevance: 0.9 })], + [], + { now: 120_000, rangeMs: 60_000, activeScenarioId: 'a' }, + ) + expect(sortHeatmapRows(rows, 'active')[0].scenario.id).toBe('a') + }) +}) diff --git a/ui/cockpit/src/lib/heatmap.ts b/ui/cockpit/src/lib/heatmap.ts new file mode 100644 index 0000000..9e0f6dd --- /dev/null +++ b/ui/cockpit/src/lib/heatmap.ts @@ -0,0 +1,394 @@ +import type { PipelineEvent } from '../api/usePipelineEvents' +import type { LiveWorkEvent, ScenarioHeatmapSample, UIScenario } from '../types' + +export type HeatmapEventType = 'work' | 'model' | 'tool' | 'file' | 'test' | 'error' | 'risk' | 'context' | 'suggestion' | 'user' +export type HeatmapGroupBy = 'cluster' | 'status' | 'file-area' | 'type' +export type HeatmapSortBy = 'active' | 'readiness' | 'relevance' | 'confidence' | 'recent' + +export interface HeatmapEvent { + id: string + timestamp: number + scenarioId: string | null + type: HeatmapEventType + title: string + description: string + severity: 'info' | 'warn' | 'error' + magnitude: number + paths: string[] +} + +export interface HeatmapSample { + timestamp: number + intensity: number + readiness: number + relevance: number + confidence: number + freshness: number +} + +export interface HeatmapRow { + scenario: UIScenario + status: string + cluster: string + active: boolean + currentIntensity: number + samples: HeatmapSample[] + events: HeatmapEvent[] +} + +export interface HeatmapBuildOptions { + now: number + rangeMs: number + buckets?: number + activeScenarioId?: string | null + showCompleted?: boolean + showStale?: boolean + showLowConfidence?: boolean + onlyPinned?: boolean + onlyWithEvents?: boolean + eventTypes?: Set + groupBy?: HeatmapGroupBy + sortBy?: HeatmapSortBy + samples?: ScenarioHeatmapSample[] + events?: HeatmapEvent[] +} + +export function clamp01(value: number): number { + return Math.max(0, Math.min(1, value)) +} + +export function readinessValue(readiness: UIScenario['readiness']): number { + return ({ ready: 1, warming: 0.62, cooling: 0.3, unprepared: 0.16 } as Record)[readiness] ?? 0.35 +} + +export function freshnessValue(freshness: UIScenario['freshness']): number { + return ({ fresh: 1, recent: 0.68, stale: 0.16 } as Record)[freshness] ?? 0.45 +} + +export function scenarioStatus(scenario: UIScenario, activeScenarioId?: string | null): string { + if (scenario.id === activeScenarioId || scenario.decisionState === 'active') return 'active' + if (scenario.decisionState === 'rejected') return 'rejected' + if (scenario.visibility === 'archived' || scenario.freshness === 'stale') return 'stale' + if (scenario.readiness === 'ready') return 'ready' + if (scenario.readiness === 'warming') return 'prep' + if (scenario.readiness === 'cooling') return 'cooling' + return 'prep' +} + +export function scenarioIntensity(scenario: UIScenario, activeScenarioId?: string | null): number { + const relevance = clamp01(Number(scenario.relevance) || 0) + const readiness = readinessValue(scenario.readiness) + const confidence = clamp01(Number(scenario.confidence) || 0) + const freshness = freshnessValue(scenario.freshness) + const status = scenarioStatus(scenario, activeScenarioId) + const risk = status === 'rejected' ? 0 : status === 'stale' ? 0.38 : 0.86 + let intensity = relevance * 0.35 + readiness * 0.25 + confidence * 0.2 + freshness * 0.1 + risk * 0.1 + if (scenario.id === activeScenarioId) intensity += 0.08 + if (scenario.pinned) intensity = Math.max(intensity, 0.34) + if (status === 'stale') intensity *= 0.58 + if (status === 'rejected') intensity *= 0.34 + return clamp01(intensity) +} + +export function normalizeHeatmapEvents(events: PipelineEvent[]): HeatmapEvent[] { + const rows: HeatmapEvent[] = [] + for (const event of events) { + if (event.kind === 'work.snapshot') { + const items = Array.isArray(event.payload.items) ? event.payload.items : [] + for (const item of items) { + if (!isRecord(item)) continue + const live = item as unknown as LiveWorkEvent + rows.push({ + id: String(live.event_id || `${event.id}-${rows.length}`), + timestamp: Number(live.ts || event.ts) * 1000, + scenarioId: live.scenario_id || (live.entity_type === 'prediction' ? `prediction:${live.entity_id}` : live.entity_type === 'scenario' ? live.entity_id : null), + type: heatmapEventType(String(live.stage || ''), String(live.status || ''), String(live.summary || '')), + title: live.summary || `${live.stage} ${live.status}`, + description: live.safe_preview || live.summary || '', + severity: live.status === 'error' ? 'error' : live.status === 'blocked' ? 'warn' : 'info', + magnitude: heatmapMagnitude(String(live.stage || ''), Number(live.latency_ms || 0), live.token_usage), + paths: Array.isArray(live.targets) ? live.targets.filter((target): target is string => typeof target === 'string') : [], + }) + } + continue + } + + rows.push({ + id: event.id, + timestamp: event.ts * 1000, + scenarioId: event.scn, + type: heatmapEventType(event.stage, event.kind, event.msg), + title: event.msg || event.kind, + description: event.path || event.kind, + severity: event.kind.includes('error') ? 'error' : event.kind.includes('risk') ? 'warn' : 'info', + magnitude: heatmapMagnitude(event.stage, Number(event.payload.latency_ms || 0), event.payload), + paths: event.path ? [event.path] : event.payload.paths && Array.isArray(event.payload.paths) ? event.payload.paths.filter((path): path is string => typeof path === 'string') : [], + }) + } + return dedupeEvents(rows).sort((a, b) => a.timestamp - b.timestamp) +} + +export function normalizeLiveWorkHeatmapEvents(events: LiveWorkEvent[]): HeatmapEvent[] { + return dedupeEvents(events.map((live, index) => ({ + id: String(live.event_id || `live-${index}`), + timestamp: Number(live.ts || 0) * 1000, + scenarioId: live.scenario_id || (live.entity_type === 'prediction' ? `prediction:${live.entity_id}` : live.entity_type === 'scenario' ? live.entity_id : null), + type: heatmapEventType(String(live.stage || ''), String(live.status || ''), String(live.summary || '')), + title: live.summary || `${live.stage} ${live.status}`, + description: live.safe_preview || live.summary || '', + severity: live.status === 'error' ? 'error' : live.status === 'blocked' ? 'warn' : 'info', + magnitude: heatmapMagnitude(String(live.stage || ''), Number(live.latency_ms || 0), live.token_usage), + paths: Array.isArray(live.targets) ? live.targets.filter((target): target is string => typeof target === 'string') : [], + }))).sort((a, b) => a.timestamp - b.timestamp) +} + +export function buildScenarioHeatmapData(scenarios: UIScenario[], pipelineEvents: PipelineEvent[], options: HeatmapBuildOptions): HeatmapRow[] { + const start = options.now - options.rangeMs + const events = (options.events ?? normalizeHeatmapEvents(pipelineEvents)).filter((event) => event.timestamp >= start && event.timestamp <= options.now) + const eventTypes = options.eventTypes + const filteredEvents = eventTypes ? events.filter((event) => eventTypes.has(event.type)) : events + const samplesByScenario = samplesByScenarioId(options.samples ?? [], start, options.now) + const byScenario = new Map() + for (const event of filteredEvents) { + if (!event.scenarioId) continue + const current = byScenario.get(event.scenarioId) ?? [] + current.push(event) + byScenario.set(event.scenarioId, current) + } + addEventBackedRows(scenarios, byScenario, samplesByScenario) + + const rows = scenarios + .filter((scenario) => filterScenario(scenario, byScenario.get(scenario.id) ?? [], options)) + .map((scenario): HeatmapRow => { + const scenarioEvents = byScenario.get(scenario.id) ?? [] + const samples = samplesByScenario.get(scenario.id) ?? [] + const currentIntensity = samples.length ? samples[samples.length - 1].intensity : scenarioIntensity(scenario, options.activeScenarioId) + return { + scenario, + status: scenarioStatus(scenario, options.activeScenarioId), + cluster: scenarioCluster(scenario, options.groupBy ?? 'cluster'), + active: scenario.id === options.activeScenarioId || scenario.decisionState === 'active', + currentIntensity, + samples, + events: scenarioEvents, + } + }) + return sortHeatmapRows(rows, options.sortBy ?? 'active') +} + +function addEventBackedRows( + scenarios: UIScenario[], + byScenario: Map, + samplesByScenario: Map, +): void { + const known = new Set(scenarios.map((scenario) => scenario.id)) + for (const [scenarioId, scenarioEvents] of byScenario.entries()) { + if (known.has(scenarioId) || !scenarioEvents.length) continue + const latest = scenarioEvents[scenarioEvents.length - 1] + const first = scenarioEvents[0] + const confidence = clamp01(Math.max(0.35, ...scenarioEvents.map((event) => event.magnitude))) + const active = scenarioEvents.some((event) => Date.now() - event.timestamp < 20_000 && event.severity !== 'error') + scenarios.push({ + id: scenarioId, + kind: 'research', + title: scenarioId.startsWith('prediction:') ? `Prediction ${scenarioId.replace('prediction:', '').slice(0, 8)}` : latest.title, + score: confidence, + relevance: confidence, + confidence, + visiblePriority: confidence, + freshness: active ? 'fresh' : 'recent', + readiness: active ? 'warming' : 'ready', + visibility: active ? 'prominent' : 'warming', + lifecycleMotion: active ? 'rising' : 'stable', + createdAt: first.timestamp / 1000, + lastRefreshedAt: latest.timestamp / 1000, + lastReinforcedAt: latest.timestamp / 1000, + archivedAt: null, + visibilityReason: 'Real live-work events without a stored scenario id.', + depth: 1, + parent: scenarioId.startsWith('prediction:') ? 'live-work' : null, + path: latest.paths[0] ?? '', + skill: null, + decisionState: active ? 'active' : 'idle', + reason: latest.description || latest.title, + entities: latest.paths, + pinned: false, + }) + samplesByScenario.set( + scenarioId, + scenarioEvents.map((event) => ({ + timestamp: event.timestamp, + intensity: clamp01(0.28 + event.magnitude * 0.64), + readiness: active ? 0.62 : 1, + relevance: clamp01(0.35 + event.magnitude * 0.55), + confidence, + freshness: active ? 1 : 0.68, + })), + ) + known.add(scenarioId) + } +} + +export function scenarioStateAtCursor(row: HeatmapRow, cursorMs: number): HeatmapSample { + if (!row.samples.length) { + return { + timestamp: cursorMs, + intensity: row.currentIntensity, + readiness: readinessValue(row.scenario.readiness), + relevance: row.scenario.relevance, + confidence: row.scenario.confidence, + freshness: freshnessValue(row.scenario.freshness), + } + } + let best = row.samples[0] + for (const sample of row.samples) { + if (sample.timestamp <= cursorMs) { + best = sample + } else { + break + } + } + return best ?? row.samples[0] +} + +export function filterEventsByType(events: HeatmapEvent[], enabled: Set): HeatmapEvent[] { + return events.filter((event) => enabled.has(event.type)) +} + +export function sortHeatmapRows(rows: HeatmapRow[], sortBy: HeatmapSortBy): HeatmapRow[] { + return [...rows].sort((a, b) => { + if (sortBy === 'active') { + const activeDelta = Number(b.active) - Number(a.active) + if (activeDelta) return activeDelta + const eventDelta = latestEventTs(b) - latestEventTs(a) + if (eventDelta) return eventDelta + return b.currentIntensity - a.currentIntensity + } + if (sortBy === 'readiness') return readinessValue(b.scenario.readiness) - readinessValue(a.scenario.readiness) + if (sortBy === 'relevance') return b.scenario.relevance - a.scenario.relevance + if (sortBy === 'confidence') return b.scenario.confidence - a.scenario.confidence + return latestEventTs(b) - latestEventTs(a) + }) +} + +export function scenarioCluster(scenario: UIScenario, groupBy: HeatmapGroupBy): string { + if (groupBy === 'status') return scenarioStatus(scenario) + if (groupBy === 'type') return scenario.kind + if (groupBy === 'file-area') return scenario.path.split('/').slice(0, 2).join('/') || 'workspace' + return scenario.parent ?? scenario.kind +} + +function sampleAt( + scenario: UIScenario, + events: HeatmapEvent[], + timestamp: number, + start: number, + now: number, + bucketMs: number, + currentIntensity: number, +): HeatmapSample { + const reinforcedAt = Number(scenario.lastReinforcedAt || scenario.lastRefreshedAt || scenario.createdAt || now) * 1000 + const rangeProgress = clamp01((timestamp - start) / Math.max(1, now - start)) + const warmup = timestamp < reinforcedAt ? 0.28 + rangeProgress * 0.48 : 0.82 + rangeProgress * 0.18 + let intensity = currentIntensity * warmup + for (const event of events) { + const distance = Math.abs(event.timestamp - timestamp) + if (distance > bucketMs * 3) continue + intensity += event.magnitude * 0.18 * Math.exp(-distance / Math.max(1, bucketMs * 1.2)) + } + return { + timestamp, + intensity: clamp01(intensity), + readiness: readinessValue(scenario.readiness), + relevance: scenario.relevance, + confidence: scenario.confidence, + freshness: freshnessValue(scenario.freshness), + } +} + +function samplesByScenarioId(samples: ScenarioHeatmapSample[], start: number, end: number): Map { + const grouped = new Map() + for (const sample of samples) { + const timestamp = Number(sample.ts || 0) * 1000 + if (timestamp < start || timestamp > end) continue + const row: HeatmapSample = { + timestamp, + intensity: sampleIntensity(sample), + readiness: readinessValue(sample.readiness as UIScenario['readiness']), + relevance: clamp01(Number(sample.relevance) || 0), + confidence: clamp01(Number(sample.confidence) || 0), + freshness: freshnessValue(sample.freshness as UIScenario['freshness']), + } + const current = grouped.get(sample.scenario_id) ?? [] + current.push(row) + grouped.set(sample.scenario_id, current) + } + for (const rows of grouped.values()) { + rows.sort((a, b) => a.timestamp - b.timestamp) + } + return grouped +} + +function sampleIntensity(sample: ScenarioHeatmapSample): number { + const readiness = readinessValue(sample.readiness as UIScenario['readiness']) + const freshness = freshnessValue(sample.freshness as UIScenario['freshness']) + const relevance = clamp01(Number(sample.relevance) || 0) + const confidence = clamp01(Number(sample.confidence) || 0) + const risk = sample.status === 'rejected' ? 0 : sample.status === 'stale' ? 0.38 : 0.86 + let intensity = relevance * 0.35 + readiness * 0.25 + confidence * 0.2 + freshness * 0.1 + risk * 0.1 + if (sample.active) intensity += 0.08 + if (sample.pinned) intensity = Math.max(intensity, 0.34) + if (sample.status === 'stale') intensity *= 0.58 + if (sample.status === 'rejected') intensity *= 0.34 + return clamp01(intensity) +} + +function filterScenario(scenario: UIScenario, events: HeatmapEvent[], options: HeatmapBuildOptions): boolean { + const status = scenarioStatus(scenario, options.activeScenarioId) + if (options.onlyPinned && !scenario.pinned) return false + if (options.onlyWithEvents && !events.length) return false + if (options.showCompleted === false && status === 'completed') return false + if (options.showStale === false && status === 'stale') return false + if (options.showLowConfidence === false && scenario.confidence < 0.45) return false + return true +} + +function heatmapEventType(stage: string, kind: string, text = ''): HeatmapEventType { + const value = `${stage} ${kind} ${text}`.toLowerCase() + if (value.includes('error') || value.includes('failed')) return 'error' + if (value.includes('risk') || value.includes('blocked')) return 'risk' + if (value.includes('test')) return 'test' + if (value.includes('file') || value.includes('artefact')) return 'file' + if (value.includes('context') || value.includes('briefing')) return 'context' + if (value.includes('suggest') || value.includes('prediction')) return 'suggestion' + if (value.includes('signal') || value.includes('user') || value.includes('codex')) return 'user' + if (value.includes('model') || value.includes('llm')) return 'model' + if (value.includes('tool')) return 'tool' + return 'work' +} + +function heatmapMagnitude(stage: string, latencyMs: number, usage: unknown): number { + const tokenUsage = isRecord(usage) && isRecord(usage.token_usage) ? usage.token_usage : usage + const totalTokens = isRecord(tokenUsage) ? Number(tokenUsage.total_tokens || 0) : 0 + const modelBoost = stage.includes('model') ? 0.18 : 0 + return clamp01(0.32 + modelBoost + Math.min(0.26, latencyMs / 12000) + Math.min(0.24, totalTokens / 10000)) +} + +function dedupeEvents(events: HeatmapEvent[]): HeatmapEvent[] { + const seen = new Set() + const out: HeatmapEvent[] = [] + for (const event of events) { + if (seen.has(event.id)) continue + seen.add(event.id) + out.push(event) + } + return out +} + +function latestEventTs(row: HeatmapRow): number { + return row.events.reduce((latest, event) => Math.max(latest, event.timestamp), 0) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/ui/cockpit/src/styles/tokens.css b/ui/cockpit/src/styles/tokens.css index 58eb6ea..0c2de25 100644 --- a/ui/cockpit/src/styles/tokens.css +++ b/ui/cockpit/src/styles/tokens.css @@ -219,10 +219,21 @@ textarea { color: var(--fg-1); } +.cockpit-root--wide-graph { + grid-template-areas: + "top top" + "rail graph"; + grid-template-columns: 260px minmax(0, 1fr); +} + @media (max-width: 1120px) { .cockpit-root { grid-template-columns: 240px minmax(0, 1fr) 320px; } + + .cockpit-root--wide-graph { + grid-template-columns: 240px minmax(0, 1fr); + } } @media (max-width: 880px) { diff --git a/ui/cockpit/src/types.ts b/ui/cockpit/src/types.ts index 643028d..2b0bd54 100644 --- a/ui/cockpit/src/types.ts +++ b/ui/cockpit/src/types.ts @@ -19,6 +19,8 @@ export interface UIScenario { readiness: UIReadiness visibility: UIVisibility lifecycleMotion: UILifecycleMotion + createdAt: number | null + lastRefreshedAt: number | null lastReinforcedAt: number | null archivedAt: number | null visibilityReason: string @@ -372,7 +374,7 @@ export interface JobsStatusPayload { dropped?: number coalesced?: number } - profile?: Record + profile?: Record } export interface SourcesPermissionsPayload { @@ -530,6 +532,41 @@ export interface ScenarioApiPayload { lifecycle_components?: Array<{ label: string; value: number; description?: string }> } +export interface ScenarioHeatmapSample { + ts: number + scenario_id: string + relevance: number + readiness: string + confidence: number + freshness: string + visible_priority: number + visibility: string + lifecycle_motion: string + status: string + pinned: boolean + active: boolean + cycle_id?: string | null + job_id?: string | null + source_event_id?: string | null +} + +export interface HeatmapReplayPayload { + from_ts: number + to_ts: number + scenarios: ScenarioApiPayload[] + samples: ScenarioHeatmapSample[] + events: LiveWorkEvent[] + metadata?: { + sample_source?: string + event_source?: string + synthetic?: boolean + sample_count?: number + event_count?: number + scenario_count?: number + complete?: boolean + } +} + // --------------------------------------------------------------------------- // Cockpit refresh: goals, artefacts, predictions-by-state, work-product // self-eval + lifecycle, learning summary. These mirror the JSON shapes the From f43d92a07aeb64de981ef60e632926600c98f849 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Wed, 6 May 2026 23:29:55 +0200 Subject: [PATCH 03/22] Preserve upstream source rank in context selection --- src/vaner/broker/selector.py | 36 ++++++++++++- tests/test_broker/test_selector.py | 86 ++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index bfe82c6..bdd10d9 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -29,6 +29,19 @@ def _recency_bonus(artefact: Artefact, decay_half_life_seconds: int = 1800) -> f return 0.1 + (0.9 * decay) +def _source_rank_prior(artefact: Artefact) -> float: + """Bounded prior from an upstream context source ranking, when present.""" + + raw_rank = artefact.metadata.get("retrieval_rank") or artefact.metadata.get("source_rank") + try: + rank = int(raw_rank) + except (TypeError, ValueError): + return 0.0 + if rank <= 0: + return 0.0 + return min(15.0, 15.0 / math.sqrt(rank)) + + _COMMON_WORDS = frozenset( "the and are for but not that with this from have been will can its also more" " use used using used using using into they their there when than then what" @@ -885,6 +898,16 @@ def select_artefacts( ) ) score += constraint_bonus + source_prior = _source_rank_prior(artefact) + if source_prior: + factors.append( + ScoreFactor( + name="source_rank_prior", + contribution=source_prior, + detail="trusted upstream context source ranked this artefact highly", + ) + ) + score += source_prior if capture_factors is not None: capture_factors[artefact.key] = factors scored_rows.append((score, artefact)) @@ -895,6 +918,8 @@ def select_artefacts( threshold_multiplier = competitive_threshold_multiplier(context_need) min_competitive_score = ranked[0][0] * threshold_multiplier if ranked else 0.0 selected_buckets: set[str] = set() + deferred_for_diversity: list[Artefact] = [] + diversity_floor = max(1, top_n // 2) for score, artefact in ranked: if score < min_competitive_score: if capture_drop_reasons is not None: @@ -905,8 +930,9 @@ def select_artefacts( if ( context_need in {"multi_source_synthesis", "conflict_resolution", "research_mapping", "decision_support"} and bucket in selected_buckets - and len(selected) < max(1, top_n // 2) + and len(selected) < diversity_floor ): + deferred_for_diversity.append(artefact) if capture_drop_reasons is not None: capture_drop_reasons[artefact.key] = "deferred_for_context_diversity" continue @@ -918,6 +944,14 @@ def select_artefacts( selected.append(artefact) seen_corpora.add(corpus_id) selected_buckets.add(bucket) + if len(selected) >= diversity_floor: + while deferred_for_diversity and len(selected) < top_n: + deferred = deferred_for_diversity.pop(0) + if deferred in selected: + continue + selected.append(deferred) + seen_corpora.add(str(deferred.metadata.get("corpus_id", "default"))) + selected_buckets.add(_diversity_bucket(deferred, context_need)) if len(selected) >= top_n: break if capture_drop_reasons is not None: diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 172f994..0bd321f 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -65,6 +65,92 @@ def test_select_artefacts_prefers_git_and_working_set_matches(): assert selected[0].key == "file_summary:b.py" +def test_select_artefacts_uses_bounded_source_rank_prior(): + now = time.time() + artefacts = [ + Artefact( + key="file_summary:late.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="late.md", + source_mtime=now, + generated_at=now, + model="test", + content="tenant cutover latency threshold escalation sustained", + metadata={"retrieval_rank": 12}, + ), + Artefact( + key="file_summary:early.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="early.md", + source_mtime=now, + generated_at=now, + model="test", + content="tenant cutover latency threshold escalation sustained", + metadata={"retrieval_rank": 1}, + ), + ] + factors: dict[str, list] = {} + + selected = select_artefacts( + "tenant cutover latency threshold escalation sustained", + artefacts, + top_n=1, + capture_factors=factors, + ) + + assert selected[0].key == "file_summary:early.md" + assert any(factor.name == "source_rank_prior" for factor in factors["file_summary:early.md"]) + + +def test_select_artefacts_backfills_deferred_diversity_candidates(): + now = time.time() + artefacts = [ + Artefact( + key=f"file_summary:strong-{index}.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path=f"docs/shared/strong-{index}.md", + source_mtime=now, + generated_at=now, + model="test", + content="rollback rollback rollback conflict current updated baseline score", + ) + for index in range(4) + ] + artefacts.extend( + [ + Artefact( + key="file_summary:coverage-a.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/coverage-a/source.md", + source_mtime=now, + generated_at=now, + model="test", + content="baseline score", + ), + Artefact( + key="file_summary:coverage-b.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/coverage-b/source.md", + source_mtime=now, + generated_at=now, + model="test", + content="updated score", + ), + ] + ) + + selected = select_artefacts( + "compare current updated rollback conflict baseline score", + artefacts, + top_n=4, + context_need="conflict_resolution", + ) + + selected_keys = {artefact.key for artefact in selected} + assert "file_summary:strong-1.md" in selected_keys + assert "file_summary:strong-2.md" in selected_keys + + def test_select_artefacts_origin_rerank_prefers_definition_files(): artefacts = [ Artefact( From 20ce3f8fb5840bdb0070a3135b0f05246e8c62e0 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 07:33:39 +0200 Subject: [PATCH 04/22] Improve coverage-aware context selection --- src/vaner/broker/context_preparation.py | 67 +++++++++++--- src/vaner/broker/selector.py | 113 +++++++++++++++++++++++- src/vaner/engine.py | 3 + src/vaner/models/context_preparation.py | 2 +- tests/test_broker/test_selector.py | 95 +++++++++++++++++++- 5 files changed, 266 insertions(+), 14 deletions(-) diff --git a/src/vaner/broker/context_preparation.py b/src/vaner/broker/context_preparation.py index becb5b9..55d81cd 100644 --- a/src/vaner/broker/context_preparation.py +++ b/src/vaner/broker/context_preparation.py @@ -14,6 +14,7 @@ ContextSourceStats, PreparedContextDiagnostics, ) +from vaner.semantic_aliases import engineering_semantic_aliases _PATH_RE = re.compile(r"\b(?:[\w.-]+/)+[\w.-]+\b") _QUOTED_RE = re.compile(r"[`'\"]([^`'\"]{3,100})[`'\"]") @@ -27,6 +28,11 @@ r"\b(?:only|except|before|after|latest|current|superseded|newest|oldest|no later than|at least|at most)\b", re.IGNORECASE, ) +_QUERY_STOPWORDS = frozenset( + "about across after again also and any are because before between can could did does during every for from had has have" + " how into its list most not only our over should since than that the their them then there these this those through what" + " when where which while who why with would".split() +) def infer_context_preparation_profile(prompt: str) -> ContextPreparationProfile: @@ -41,7 +47,7 @@ def infer_context_preparation_profile(prompt: str) -> ContextPreparationProfile: notes: list[str] = [] if any(constraint.kind == "restrictive_language" for constraint in constraints): notes.append("hard_constraints_detected") - if need in {"multi_source_synthesis", "conflict_resolution", "research_mapping"}: + if need in {"multi_source_synthesis", "source_evidence", "conflict_resolution", "research_mapping"}: notes.append("coverage_sensitive") return ContextPreparationProfile( need=need, @@ -57,20 +63,28 @@ def infer_context_preparation_profile(prompt: str) -> ContextPreparationProfile: def query_variants(prompt: str, profile: ContextPreparationProfile, *, max_variants: int = 6) -> list[str]: variants = [prompt] + keywords = extract_query_keywords(prompt) facet_terms = [facet.value for facet in profile.facets if len(facet.value) > 2] constraint_terms = [constraint.value for constraint in profile.constraints if constraint.kind != "restrictive_language"] + alias_terms = sorted(engineering_semantic_aliases(prompt, [*keywords, *facet_terms], stopwords=_QUERY_STOPWORDS)) + if keywords: + variants.append(" ".join(keywords[:14])) if facet_terms: variants.append(" ".join(facet_terms[:8])) if constraint_terms: variants.append(" ".join([*constraint_terms[:4], *facet_terms[:6]])) if profile.need in {"multi_source_synthesis", "research_mapping", "decision_support"}: - variants.append(" ".join([*facet_terms[:10], "summary status decision evidence"])) + variants.append(" ".join([*facet_terms[:10], *keywords[:8], "summary status decision evidence"])) + if profile.need == "source_evidence": + variants.append(" ".join([*facet_terms[:8], *keywords[:8], "source evidence claim citation provenance"])) if profile.need == "conflict_resolution": - variants.append(" ".join([*facet_terms[:8], "current superseded conflict updated latest"])) + variants.append(" ".join([*facet_terms[:8], *keywords[:8], "current superseded conflict updated latest"])) if profile.need == "implementation_support": - variants.append(" ".join([*facet_terms[:10], "implementation caller dependency test"])) + variants.append(" ".join([*facet_terms[:10], *keywords[:8], "implementation caller dependency test"])) if profile.need == "absence_check": - variants.append(" ".join([*facet_terms[:8], "source truth reference policy"])) + variants.append(" ".join([*facet_terms[:8], *keywords[:8], "source truth reference policy"])) + if alias_terms: + variants.append(" ".join([*keywords[:8], *alias_terms[:10]])) deduped = [] for variant in variants: normalized = " ".join(variant.split()) @@ -81,6 +95,27 @@ def query_variants(prompt: str, profile: ContextPreparationProfile, *, max_varia return deduped +def extract_query_keywords(prompt: str, *, limit: int = 24) -> list[str]: + """Extract FTS-friendly terms from a conversational prompt.""" + + keywords: list[str] = [] + seen: set[str] = set() + for token in re.findall(r"\b[A-Za-z0-9][A-Za-z0-9_-]{2,}\b", prompt): + normalized = token.strip("_-").lower() + if len(normalized) < 3 or normalized in _QUERY_STOPWORDS: + continue + if normalized not in seen: + seen.add(normalized) + keywords.append(normalized) + for part in re.split(r"[_-]+", normalized): + if len(part) >= 3 and part not in _QUERY_STOPWORDS and part not in seen: + seen.add(part) + keywords.append(part) + if len(keywords) >= limit: + break + return keywords[:limit] + + def exact_reference_candidates(prompt: str, available_paths: list[str], *, limit: int = 32) -> list[str]: references = set(_PATH_RE.findall(prompt)) references.update(match.group(1).strip() for match in _QUOTED_RE.finditer(prompt)) @@ -118,6 +153,7 @@ def hard_constraints_satisfied(prompt: str, artefact: Artefact, profile: Context text = f"{artefact.source_path}\n{artefact.content}".lower() satisfied: list[str] = [] missing: list[str] = [] + blocking_missing: list[str] = [] for constraint in profile.constraints: value = constraint.value.lower() if constraint.kind == "restrictive_language": @@ -127,13 +163,17 @@ def hard_constraints_satisfied(prompt: str, artefact: Artefact, profile: Context satisfied.append(constraint.value) elif constraint.required: missing.append(constraint.value) - if _constraint_terms(prompt) and not satisfied and profile.need in {"direct_reference", "conflict_resolution", "absence_check"}: + if constraint.kind in {"path", "quoted_reference"}: + blocking_missing.append(constraint.value) + if blocking_missing: + return False, satisfied, blocking_missing + if _constraint_terms(prompt) and not satisfied and profile.need in {"absence_check"}: return False, satisfied, missing - return not missing, satisfied, missing + return True, satisfied, missing def competitive_threshold_multiplier(need: str | None) -> float: - if need in {"multi_source_synthesis", "conflict_resolution", "research_mapping", "decision_support"}: + if need in {"multi_source_synthesis", "source_evidence", "conflict_resolution", "research_mapping", "decision_support"}: return 0.0 if need in {"implementation_support", "absence_check", "working_set_extension", "creative_grounding"}: return 0.20 @@ -243,13 +283,18 @@ def _infer_need(lowered: str, archetype: str): return "conflict_resolution" if re.search(r"\b(not found|unavailable|missing|is there|do we have|absence|not available|cannot find)\b", lowered): return "absence_check" + if re.search(r"\b(according to|source|evidence|citation|cite|provenance|claim|where did .* come from)\b", lowered): + return "source_evidence" if archetype == "developer" and re.search(r"\b(implement|fix|debug|change|affected|callers|tests?|dependency|regression)\b", lowered): return "implementation_support" if archetype == "writer": return "creative_grounding" if archetype == "researcher": return "research_mapping" - if re.search(r"\b(all|every|list|compare|across|summarize|synthesize|count|which .* and|what .* and)\b", lowered): + if re.search( + r"\b(all|every|list|compare|across|summarize|synthesize|count|which .* and|what .* and|which .* most|highest number)\b", + lowered, + ): return "multi_source_synthesis" if re.search(r"\b(decide|decision|recommend|tradeoff|risk|should we|plan)\b", lowered): return "decision_support" @@ -278,7 +323,7 @@ def _extract_facets(prompt: str, constraints: list[ContextConstraint]) -> list[C facets: list[ContextFacet] = [] for token in re.findall(r"\b[A-Za-z][A-Za-z0-9_-]{3,}\b", prompt): lowered = token.lower() - if lowered in constrained_values or lowered in {"what", "where", "when", "which", "does", "with", "from", "that", "this", "should"}: + if lowered in constrained_values or lowered in _QUERY_STOPWORDS: continue required = any(char.isupper() for char in token[1:]) or "_" in token or "-" in token facets.append(ContextFacet(name="term", value=token, required=required)) @@ -296,6 +341,8 @@ def _extract_source_hints(lowered: str) -> list[str]: def _expected_evidence_count(lowered: str, need: str) -> int: if need in {"multi_source_synthesis", "research_mapping"}: return 4 + if need == "source_evidence": + return 2 if need == "conflict_resolution": return 2 if re.search(r"\b(all|every|complete|completeness|list)\b", lowered): diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index bdd10d9..6a69540 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -535,6 +535,7 @@ async def select_artefacts_fts( coverage_floor_enabled: bool = True, max_expansion_passes: int = 1, capture_prepared_context_diagnostics: list[PreparedContextDiagnostics] | None = None, + semantic_memory_enabled: bool = False, ) -> list[Artefact]: """Multi-source context candidate retrieval, then scorer re-rank. @@ -565,6 +566,12 @@ async def select_artefacts_fts( except Exception: keys = [] _record_source_ranking(source_rankings, source_by_key, source, keys) + if semantic_memory_enabled and hasattr(store, "select_artefacts_semantic"): + try: + semantic_keys = list(await store.select_artefacts_semantic(prompt, limit=retrieval_limit)) # type: ignore[attr-defined] + except Exception: + semantic_keys = [] + _record_source_ranking(source_rankings, source_by_key, "semantic_memory", semantic_keys) if context_enabled: try: available_paths = await store.list_source_paths(limit=max(5000, retrieval_limit * 10)) # type: ignore[union-attr] @@ -599,7 +606,7 @@ async def select_artefacts_fts( if artefact.key in seen: continue seen.add(artefact.key) - candidates.append(artefact) + candidates.append(_with_context_source_metadata(artefact, source_rankings, source_by_key)) selected = select_artefacts( prompt, candidates, @@ -635,7 +642,9 @@ async def select_artefacts_fts( if new_keys: _record_source_ranking(source_rankings, source_by_key, "coverage_floor", new_keys) recovered = await store.list_by_keys(set(new_keys), limit=max(retrieval_limit, len(new_keys))) # type: ignore[union-attr] - candidates.extend(recovered) + candidates.extend( + _with_context_source_metadata(artefact, source_rankings, source_by_key) for artefact in recovered + ) selected = select_artefacts( prompt, candidates, @@ -738,6 +747,32 @@ def _fuse_source_rankings(source_rankings: dict[str, list[str]], *, limit: int) ] +def _with_context_source_metadata( + artefact: Artefact, + source_rankings: dict[str, list[str]], + source_by_key: dict[str, set[str]], +) -> Artefact: + sources = sorted(source_by_key.get(artefact.key, set())) + if not sources: + return artefact + best_rank: int | None = None + best_source: str | None = None + for source in sources: + try: + rank = source_rankings.get(source, []).index(artefact.key) + 1 + except ValueError: + continue + if best_rank is None or rank < best_rank: + best_rank = rank + best_source = source + metadata = dict(artefact.metadata) + metadata.setdefault("context_sources", sources) + if best_rank is not None: + metadata.setdefault("retrieval_rank", best_rank) + metadata.setdefault("retrieval_source", best_source) + return artefact.model_copy(update={"metadata": metadata}) + + def _append_source_factors( capture_factors: dict[str, list[ScoreFactor]] | None, source_by_key: dict[str, set[str]], @@ -783,6 +818,70 @@ def _coverage_recovery_terms(profile: ContextPreparationProfile, diagnostics: Pr return " ".join(term for term in terms if term) +_COVERAGE_SENSITIVE_NEEDS = { + "multi_source_synthesis", + "source_evidence", + "conflict_resolution", + "research_mapping", + "decision_support", +} + + +def _artefact_context_text(artefact: Artefact) -> str: + return f"{artefact.source_path}\n{artefact.content}".lower() + + +def _matches_facet(artefact: Artefact, facet_value: str) -> bool: + value = facet_value.lower() + text = _artefact_context_text(artefact) + if value in text: + return True + parts = [part for part in re.split(r"[_\-\s]+", value) if len(part) > 2] + return bool(parts) and all(part in text for part in parts) + + +def _coverage_seed_artefacts( + ranked: list[tuple[float, Artefact]], + profile: ContextPreparationProfile | None, + *, + top_n: int, + min_score: float, +) -> list[Artefact]: + if profile is None or profile.need not in _COVERAGE_SENSITIVE_NEEDS or top_n <= 1: + return [] + selected: list[Artefact] = [] + selected_keys: set[str] = set() + + def add_best_matching(predicate: Callable[[Artefact], bool]) -> None: + if len(selected) >= top_n: + return + for score, artefact in ranked: + if score < min_score or artefact.key in selected_keys: + continue + if predicate(artefact): + selected.append(artefact) + selected_keys.add(artefact.key) + return + + if profile.need == "conflict_resolution": + add_best_matching( + lambda artefact: any(term in _artefact_context_text(artefact) for term in ("current", "latest", "updated", "newer", "now")) + ) + add_best_matching( + lambda artefact: any( + term in _artefact_context_text(artefact) for term in ("superseded", "old", "older", "deprecated", "previous") + ) + ) + + facet_limit = min(top_n, max(1, profile.expected_evidence_count), 6) + facets = sorted(profile.facets, key=lambda facet: (not facet.required, len(facet.value))) + for facet in facets[: max(facet_limit * 2, facet_limit)]: + if len(selected) >= facet_limit: + break + add_best_matching(lambda artefact, value=facet.value: _matches_facet(artefact, value)) + return selected + + def select_artefacts( prompt: str, artefacts: list[Artefact], @@ -920,7 +1019,17 @@ def select_artefacts( selected_buckets: set[str] = set() deferred_for_diversity: list[Artefact] = [] diversity_floor = max(1, top_n // 2) + + for artefact in _coverage_seed_artefacts(ranked, context_profile, top_n=top_n, min_score=min_competitive_score): + if artefact in selected: + continue + selected.append(artefact) + seen_corpora.add(str(artefact.metadata.get("corpus_id", "default"))) + selected_buckets.add(_diversity_bucket(artefact, context_need)) + for score, artefact in ranked: + if artefact in selected: + continue if score < min_competitive_score: if capture_drop_reasons is not None: capture_drop_reasons[artefact.key] = "below_competitive_threshold" diff --git a/src/vaner/engine.py b/src/vaner/engine.py index fa9d9cb..bb58ee8 100644 --- a/src/vaner/engine.py +++ b/src/vaner/engine.py @@ -1321,6 +1321,7 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int max_candidate_keys=self.config.context_preparation.max_candidate_keys, coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, max_expansion_passes=self.config.context_preparation.max_expansion_passes, + semantic_memory_enabled=self.config.context_preparation.semantic_memory_enabled, capture_prepared_context_diagnostics=prepared_context_diagnostics, ) seen = {a.key for a in selected} @@ -6078,6 +6079,7 @@ def _score_with_model(question: str, artefact) -> float: max_candidate_keys=self.config.context_preparation.max_candidate_keys, coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, max_expansion_passes=self.config.context_preparation.max_expansion_passes, + semantic_memory_enabled=self.config.context_preparation.semantic_memory_enabled, capture_prepared_context_diagnostics=prepared_context_diagnostics, ) if source_key is None and selected: @@ -6104,6 +6106,7 @@ def _score_with_source(question: str, artefact) -> float: max_candidate_keys=self.config.context_preparation.max_candidate_keys, coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, max_expansion_passes=self.config.context_preparation.max_expansion_passes, + semantic_memory_enabled=self.config.context_preparation.semantic_memory_enabled, capture_prepared_context_diagnostics=prepared_context_diagnostics, ) score_map = {artefact.key: self._intent_scorer.score(prompt, artefact, features=features) for artefact in selected} diff --git a/src/vaner/models/context_preparation.py b/src/vaner/models/context_preparation.py index ba42543..ead633c 100644 --- a/src/vaner/models/context_preparation.py +++ b/src/vaner/models/context_preparation.py @@ -11,6 +11,7 @@ "task_continuation", "evidence_gathering", "multi_source_synthesis", + "source_evidence", "decision_support", "conflict_resolution", "creative_grounding", @@ -81,4 +82,3 @@ class PreparedContextDiagnostics(BaseModel): latency_ms: float = 0.0 compactness_score: float = 1.0 provenance_coverage: float = 0.0 - diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 0bd321f..8f3b1c4 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -4,7 +4,7 @@ import time -from vaner.broker.context_preparation import infer_context_preparation_profile +from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants from vaner.broker.selector import select_artefacts from vaner.models.artefact import Artefact, ArtefactKind @@ -102,6 +102,46 @@ def test_select_artefacts_uses_bounded_source_rank_prior(): assert any(factor.name == "source_rank_prior" for factor in factors["file_summary:early.md"]) +def test_query_variants_add_local_keyword_and_alias_recall_terms(): + prompt = "What prevents a candidate release from getting full traffic until replay and smoke checks pass?" + profile = infer_context_preparation_profile(prompt) + + variants = " ".join(query_variants(prompt, profile)) + + assert "candidate release" in variants + assert "escrow" in variants + assert "promote" in variants + + +def test_source_evidence_profile_uses_evidence_mode(): + prompt = "What source evidence supports the rollout claim?" + profile = infer_context_preparation_profile(prompt) + + assert profile.need == "source_evidence" + assert any("source evidence claim citation provenance" in variant for variant in query_variants(prompt, profile)) + + +def test_date_constraints_report_but_do_not_drop_relevant_evidence_per_document(): + now = time.time() + prompt = "In March 2026, which deployment offering had the most request timeouts?" + profile = infer_context_preparation_profile(prompt) + artefacts = [ + Artefact( + key="file_summary:timeout.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="support/timeout.md", + source_mtime=now, + generated_at=now, + model="test", + content="Hosted API request timeout escalation with customer impact.", + ) + ] + + selected = select_artefacts(prompt, artefacts, top_n=1, context_need=profile.need, context_profile=profile) + + assert selected[0].key == "file_summary:timeout.md" + + def test_select_artefacts_backfills_deferred_diversity_candidates(): now = time.time() artefacts = [ @@ -151,6 +191,59 @@ def test_select_artefacts_backfills_deferred_diversity_candidates(): assert "file_summary:strong-2.md" in selected_keys +def test_select_artefacts_seeds_coverage_for_multi_source_synthesis(): + now = time.time() + artefacts = [ + Artefact( + key="file_summary:hosted.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/hosted.md", + source_mtime=now, + generated_at=now, + model="test", + content="Hosted API support escalations auth incident customer escalation escalation escalation escalation", + metadata={"retrieval_rank": 1}, + ), + Artefact( + key="file_summary:dedicated.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/dedicated.md", + source_mtime=now, + generated_at=now, + model="test", + content="Dedicated deployment support escalations auth customers", + metadata={"retrieval_rank": 40}, + ), + Artefact( + key="file_summary:private.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/private.md", + source_mtime=now, + generated_at=now, + model="test", + content="Private deployment support escalations auth customers", + metadata={"retrieval_rank": 41}, + ), + Artefact( + key="file_summary:generic.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/generic.md", + source_mtime=now, + generated_at=now, + model="test", + content="support escalations auth customers escalation escalation escalation escalation escalation", + metadata={"retrieval_rank": 2}, + ), + ] + prompt = "Across Hosted API, Dedicated, and Private deployment offerings, which had the most auth-related support escalations?" + profile = infer_context_preparation_profile(prompt) + + selected = select_artefacts(prompt, artefacts, top_n=3, context_need=profile.need, context_profile=profile) + + selected_paths = {artefact.source_path for artefact in selected} + assert {"docs/hosted.md", "docs/dedicated.md", "docs/private.md"} <= selected_paths + + def test_select_artefacts_origin_rerank_prefers_definition_files(): artefacts = [ Artefact( From 0ba35eca6c3832c7c2dc288c0ef4a929d57c20c9 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 09:05:08 +0200 Subject: [PATCH 05/22] Add semantic artefact memory source --- src/vaner/broker/selector.py | 7 +- src/vaner/engine.py | 3 + src/vaner/store/artefacts.py | 229 +++++++++++++++++++++++++++++ tests/test_broker/test_selector.py | 55 ++++++- tests/test_store/test_artefacts.py | 55 ++++++- 5 files changed, 344 insertions(+), 5 deletions(-) diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index 6a69540..61dd9d8 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -5,7 +5,7 @@ import math import re import time -from collections.abc import Callable +from collections.abc import Awaitable, Callable from fnmatch import fnmatch from vaner.broker.context_preparation import ( @@ -536,6 +536,7 @@ async def select_artefacts_fts( max_expansion_passes: int = 1, capture_prepared_context_diagnostics: list[PreparedContextDiagnostics] | None = None, semantic_memory_enabled: bool = False, + semantic_embed: Callable[[list[str]], Awaitable[list[list[float]]]] | None = None, ) -> list[Artefact]: """Multi-source context candidate retrieval, then scorer re-rank. @@ -566,9 +567,9 @@ async def select_artefacts_fts( except Exception: keys = [] _record_source_ranking(source_rankings, source_by_key, source, keys) - if semantic_memory_enabled and hasattr(store, "select_artefacts_semantic"): + if semantic_memory_enabled and semantic_embed is not None and hasattr(store, "select_artefacts_semantic"): try: - semantic_keys = list(await store.select_artefacts_semantic(prompt, limit=retrieval_limit)) # type: ignore[attr-defined] + semantic_keys = list(await store.select_artefacts_semantic(prompt, limit=retrieval_limit, embed=semantic_embed)) # type: ignore[attr-defined] except Exception: semantic_keys = [] _record_source_ranking(source_rankings, source_by_key, "semantic_memory", semantic_keys) diff --git a/src/vaner/engine.py b/src/vaner/engine.py index bb58ee8..76fddc1 100644 --- a/src/vaner/engine.py +++ b/src/vaner/engine.py @@ -1322,6 +1322,7 @@ async def query(self, prompt: str, *, max_tokens: int | None = None, top_n: int coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, max_expansion_passes=self.config.context_preparation.max_expansion_passes, semantic_memory_enabled=self.config.context_preparation.semantic_memory_enabled, + semantic_embed=self.embed, capture_prepared_context_diagnostics=prepared_context_diagnostics, ) seen = {a.key for a in selected} @@ -6080,6 +6081,7 @@ def _score_with_model(question: str, artefact) -> float: coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, max_expansion_passes=self.config.context_preparation.max_expansion_passes, semantic_memory_enabled=self.config.context_preparation.semantic_memory_enabled, + semantic_embed=self.embed, capture_prepared_context_diagnostics=prepared_context_diagnostics, ) if source_key is None and selected: @@ -6107,6 +6109,7 @@ def _score_with_source(question: str, artefact) -> float: coverage_floor_enabled=self.config.context_preparation.coverage_floor_enabled, max_expansion_passes=self.config.context_preparation.max_expansion_passes, semantic_memory_enabled=self.config.context_preparation.semantic_memory_enabled, + semantic_embed=self.embed, capture_prepared_context_diagnostics=prepared_context_diagnostics, ) score_map = {artefact.key: self._intent_scorer.score(prompt, artefact, features=features) for artefact in selected} diff --git a/src/vaner/store/artefacts.py b/src/vaner/store/artefacts.py index 60a4a3e..70ce835 100644 --- a/src/vaner/store/artefacts.py +++ b/src/vaner/store/artefacts.py @@ -5,9 +5,11 @@ import asyncio import hashlib import json +import math import re as _re import time import uuid +from collections.abc import Awaitable, Callable from pathlib import Path import aiosqlite @@ -26,6 +28,45 @@ ) from vaner.policy.privacy import sanitize_no_absolute_paths +SemanticEmbedder = Callable[[list[str]], Awaitable[list[list[float]]]] + + +def _cosine_similarity(a: list[float], b: list[float]) -> float: + if not a or not b or len(a) != len(b): + return 0.0 + dot = sum(left * right for left, right in zip(a, b, strict=False)) + mag_a = math.sqrt(sum(value * value for value in a)) + mag_b = math.sqrt(sum(value * value for value in b)) + if mag_a == 0.0 or mag_b == 0.0: + return 0.0 + return dot / (mag_a * mag_b) + + +def _semantic_chunks_for_artefact(artefact: Artefact, *, max_chunks: int = 24, max_chars: int = 1600) -> list[tuple[int, str]]: + text = f"{artefact.source_path}\n{artefact.content}".strip() + if not text: + return [] + paragraphs = [part.strip() for part in _re.split(r"\n\s*\n+", text) if part.strip()] + chunks: list[str] = [] + current = "" + for paragraph in paragraphs: + if len(paragraph) > max_chars: + for start in range(0, len(paragraph), max_chars): + part = paragraph[start : start + max_chars].strip() + if part: + chunks.append(part) + continue + if current and len(current) + len(paragraph) + 2 > max_chars: + chunks.append(current) + current = paragraph + else: + current = f"{current}\n\n{paragraph}".strip() if current else paragraph + if current: + chunks.append(current) + if not chunks: + chunks = [text[:max_chars]] + return [(index, chunk) for index, chunk in enumerate(chunks[: max(1, int(max_chunks))])] + class ArtefactStore: _PINNED_FACT_SCOPES = frozenset({"user", "project", "workflow"}) @@ -507,6 +548,23 @@ async def _initialize_schema(self, db: aiosqlite.Connection) -> None: await db.execute("CREATE INDEX IF NOT EXISTS idx_work_products_type ON work_products(type)") await db.execute("CREATE INDEX IF NOT EXISTS idx_work_products_target_key ON work_products(target_key)") await db.execute("CREATE INDEX IF NOT EXISTS idx_work_products_updated_at ON work_products(updated_at DESC)") + await db.execute( + """ + CREATE TABLE IF NOT EXISTS artefact_semantic_chunks ( + chunk_id TEXT PRIMARY KEY, + artefact_key TEXT NOT NULL, + source_path TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + text TEXT NOT NULL, + content_hash TEXT NOT NULL, + embedding_json TEXT NOT NULL, + embedding_model TEXT NOT NULL DEFAULT '', + updated_at REAL NOT NULL + ) + """ + ) + await db.execute("CREATE INDEX IF NOT EXISTS idx_semantic_chunks_key ON artefact_semantic_chunks(artefact_key)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_semantic_chunks_model ON artefact_semantic_chunks(embedding_model)") await db.execute( """ CREATE TABLE IF NOT EXISTS work_product_events ( @@ -703,6 +761,28 @@ async def _initialize_schema(self, db: aiosqlite.Connection) -> None: await db.execute("ALTER TABLE workspace_goals ADD COLUMN pc_unfinished_item_state TEXT NOT NULL DEFAULT 'none'") await db.execute("CREATE INDEX IF NOT EXISTS idx_workspace_goals_subgoal_of ON workspace_goals(subgoal_of)") await db.execute("INSERT OR IGNORE INTO schema_version(version) VALUES (8)") + current_schema_version = 8 + + if current_schema_version < 9: + await db.execute( + """ + CREATE TABLE IF NOT EXISTS artefact_semantic_chunks ( + chunk_id TEXT PRIMARY KEY, + artefact_key TEXT NOT NULL, + source_path TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + text TEXT NOT NULL, + content_hash TEXT NOT NULL, + embedding_json TEXT NOT NULL, + embedding_model TEXT NOT NULL DEFAULT '', + updated_at REAL NOT NULL + ) + """ + ) + await db.execute("CREATE INDEX IF NOT EXISTS idx_semantic_chunks_key ON artefact_semantic_chunks(artefact_key)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_semantic_chunks_model ON artefact_semantic_chunks(embedding_model)") + await db.execute("INSERT OR IGNORE INTO schema_version(version) VALUES (9)") + current_schema_version = 9 # FTS5 index on artefact source_path + content for sub-millisecond # candidate retrieval; the full scorer then re-ranks the top-N hits. @@ -755,6 +835,7 @@ async def upsert(self, artefact: Artefact) -> None: "INSERT INTO artefacts_fts(key, source_path, content) VALUES (?, ?, ?)", (artefact.key, artefact.source_path, artefact.content[:8192]), ) + await db.execute("DELETE FROM artefact_semantic_chunks WHERE artefact_key = ?", (artefact.key,)) await db.commit() async def get(self, key: str) -> Artefact | None: @@ -1706,6 +1787,154 @@ async def select_artefacts_fts(self, query: str, limit: int = 50) -> list[str]: except Exception: return [] + async def index_artefact_semantic( + self, + artefact: Artefact, + *, + embed: SemanticEmbedder, + embedding_model: str = "", + max_chunks: int = 24, + ) -> int: + """Persist dense semantic chunks for one artefact. + + This is a context-source index, not a prediction cache. It lets Vaner + retrieve semantically related raw evidence and then run the normal + context-preparation selection, coverage, provenance, and gap logic. + """ + + chunks = _semantic_chunks_for_artefact(artefact, max_chunks=max_chunks) + if not chunks: + await self.replace_artefact_semantic_chunks(artefact, [], [], embedding_model=embedding_model) + return 0 + vectors = await embed([chunk for _, chunk in chunks]) + await self.replace_artefact_semantic_chunks(artefact, chunks, vectors, embedding_model=embedding_model) + return len(chunks) + + async def replace_artefact_semantic_chunks( + self, + artefact: Artefact, + chunks: list[tuple[int, str]], + vectors: list[list[float]], + *, + embedding_model: str = "", + ) -> None: + if len(chunks) != len(vectors): + raise ValueError("semantic chunk/vector count mismatch") + now = time.time() + content_hash = hashlib.sha1(f"{artefact.source_path}\n{artefact.content}".encode()).hexdigest() # noqa: S324 + rows = [] + for chunk_index, text in chunks: + chunk_id = hashlib.sha1(f"{artefact.key}:{chunk_index}:{content_hash}".encode()).hexdigest() # noqa: S324 + vector = [float(value) for value in vectors[chunk_index]] + rows.append( + ( + chunk_id, + artefact.key, + artefact.source_path, + chunk_index, + text, + content_hash, + json.dumps(vector), + embedding_model, + now, + ) + ) + async with self._write_lock: + async with self._connect() as db: + await db.execute("DELETE FROM artefact_semantic_chunks WHERE artefact_key = ?", (artefact.key,)) + if rows: + await db.executemany( + """ + INSERT INTO artefact_semantic_chunks( + chunk_id, artefact_key, source_path, chunk_index, text, + content_hash, embedding_json, embedding_model, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + rows, + ) + await db.commit() + + async def rebuild_semantic_index( + self, + *, + embed: SemanticEmbedder, + embedding_model: str = "", + limit: int = 2000, + max_chunks_per_artefact: int = 24, + ) -> int: + artefacts = await self.list(limit=limit) + indexed = 0 + for artefact in artefacts: + indexed += await self.index_artefact_semantic( + artefact, + embed=embed, + embedding_model=embedding_model, + max_chunks=max_chunks_per_artefact, + ) + return indexed + + async def select_artefacts_semantic( + self, + query: str, + *, + embed: SemanticEmbedder | None = None, + limit: int = 50, + embedding_model: str = "", + chunk_limit: int = 2000, + ) -> list[str]: + """Return artefact keys ranked by semantic chunk similarity.""" + + if embed is None or not query.strip(): + return [] + try: + query_vectors = await embed([query]) + except Exception: + return [] + if not query_vectors: + return [] + query_vector = [float(value) for value in query_vectors[0]] + params: list[object] = [] + where = "" + if embedding_model: + where = "WHERE embedding_model = ?" + params.append(embedding_model) + params.append(max(1, int(chunk_limit))) + async with self._connect() as db: + cursor = await db.execute( + f""" + SELECT artefact_key, embedding_json + FROM artefact_semantic_chunks + {where} + ORDER BY updated_at DESC + LIMIT ? + """, + tuple(params), + ) + rows = await cursor.fetchall() + scores: dict[str, float] = {} + counts: dict[str, int] = {} + for key, embedding_json in rows: + try: + vector = [float(value) for value in json.loads(str(embedding_json))] + except Exception: + continue + score = _cosine_similarity(query_vector, vector) + if score <= 0.0: + continue + key_str = str(key) + counts[key_str] = counts.get(key_str, 0) + 1 + scores[key_str] = max(scores.get(key_str, 0.0), score) + min(0.05, counts[key_str] * 0.005) + return [key for key, _ in sorted(scores.items(), key=lambda item: (-item[1], item[0]))[: max(1, int(limit))]] + + async def semantic_index_snapshot(self) -> dict[str, int]: + async with self._connect() as db: + chunks_row = await (await db.execute("SELECT COUNT(*) FROM artefact_semantic_chunks")).fetchone() + artefacts_row = await (await db.execute("SELECT COUNT(DISTINCT artefact_key) FROM artefact_semantic_chunks")).fetchone() + return { + "chunks": int(chunks_row[0] or 0), + "artefacts": int(artefacts_row[0] or 0), + } + async def replace_quality_issues(self, issues: list[dict[str, object]]) -> None: async with self._connect() as db: await db.execute("DELETE FROM quality_issues") diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 8f3b1c4..c453f0b 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -4,9 +4,22 @@ import time +import pytest + from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants -from vaner.broker.selector import select_artefacts +from vaner.broker.selector import select_artefacts, select_artefacts_fts from vaner.models.artefact import Artefact, ArtefactKind +from vaner.store.artefacts import ArtefactStore + + +async def _fake_embed(texts: list[str]) -> list[list[float]]: + vectors: list[list[float]] = [] + for text in texts: + lowered = text.lower() + rollout = 1.0 if any(term in lowered for term in ("candidate", "release", "traffic", "replay", "smoke", "escrow")) else 0.0 + auth = 1.0 if any(term in lowered for term in ("auth", "credential", "token")) else 0.0 + vectors.append([rollout, auth]) + return vectors def test_select_artefacts_prefers_prompt_matches(): @@ -121,6 +134,46 @@ def test_source_evidence_profile_uses_evidence_mode(): assert any("source evidence claim citation provenance" in variant for variant in query_variants(prompt, profile)) +@pytest.mark.asyncio +async def test_select_artefacts_fts_uses_semantic_memory_source(tmp_path): + store = ArtefactStore(tmp_path / "store.db") + await store.initialize() + now = time.time() + rollout = Artefact( + key="file_summary:rollout.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/rollout.md", + source_mtime=now, + generated_at=now, + model="test", + content="Traffic escrow rehearses replay and smoke policy checks before promote.", + ) + unrelated = Artefact( + key="file_summary:auth.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/auth.md", + source_mtime=now, + generated_at=now, + model="test", + content="Credential rotation and token validation notes.", + ) + await store.upsert(rollout) + await store.upsert(unrelated) + await store.index_artefact_semantic(rollout, embed=_fake_embed, embedding_model="fake") + await store.index_artefact_semantic(unrelated, embed=_fake_embed, embedding_model="fake") + + selected = await select_artefacts_fts( + "candidate release gets full traffic after checks", + store, + top_n=1, + semantic_memory_enabled=True, + semantic_embed=_fake_embed, + ) + + assert selected[0].key == "file_summary:rollout.md" + assert "semantic_memory" in selected[0].metadata["context_sources"] + + def test_date_constraints_report_but_do_not_drop_relevant_evidence_per_document(): now = time.time() prompt = "In March 2026, which deployment offering had the most request timeouts?" diff --git a/tests/test_store/test_artefacts.py b/tests/test_store/test_artefacts.py index 7c1892b..a8f3dde 100644 --- a/tests/test_store/test_artefacts.py +++ b/tests/test_store/test_artefacts.py @@ -11,6 +11,17 @@ from vaner.store.artefacts import ArtefactStore +async def _fake_embed(texts: list[str]) -> list[list[float]]: + vectors: list[list[float]] = [] + for text in texts: + lowered = text.lower() + rollout = 1.0 if any(term in lowered for term in ("candidate", "release", "traffic", "replay", "smoke", "escrow")) else 0.0 + timeout = 1.0 if any(term in lowered for term in ("timeout", "deadline", "504")) else 0.0 + auth = 1.0 if any(term in lowered for term in ("auth", "credential", "token")) else 0.0 + vectors.append([rollout, timeout, auth]) + return vectors + + @pytest.mark.asyncio async def test_store_upsert_and_list(tmp_path): store = ArtefactStore(tmp_path / "store.db") @@ -110,9 +121,51 @@ async def test_store_initialize_migrates_synthetic_v6_database(tmp_path): version_row = await (await db.execute("SELECT MAX(version) FROM schema_version")).fetchone() prediction_columns = [row[1] for row in await (await db.execute("PRAGMA table_info(prediction_cache)")).fetchall()] goal_columns = [row[1] for row in await (await db.execute("PRAGMA table_info(workspace_goals)")).fetchall()] + semantic_columns = [row[1] for row in await (await db.execute("PRAGMA table_info(artefact_semantic_chunks)")).fetchall()] - assert version_row[0] == 8 + assert version_row[0] == 9 assert "last_accessed_at" in prediction_columns assert "subgoal_of" in goal_columns assert "pc_reconciliation_state" in goal_columns assert "pc_unfinished_item_state" in goal_columns + assert "embedding_json" in semantic_columns + + +@pytest.mark.asyncio +async def test_store_semantic_index_selects_by_embedding_similarity(tmp_path): + store = ArtefactStore(tmp_path / "store.db") + await store.initialize() + now = time.time() + rollout = Artefact( + key="file_summary:rollout.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/rollout.md", + source_mtime=now, + generated_at=now, + model="test", + content="Traffic escrow rehearses replay and smoke policy checks before promote.", + ) + unrelated = Artefact( + key="file_summary:auth.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/auth.md", + source_mtime=now, + generated_at=now, + model="test", + content="Credential rotation and token validation notes.", + ) + await store.upsert(rollout) + await store.upsert(unrelated) + await store.index_artefact_semantic(rollout, embed=_fake_embed, embedding_model="fake") + await store.index_artefact_semantic(unrelated, embed=_fake_embed, embedding_model="fake") + + selected = await store.select_artefacts_semantic( + "candidate release gets full traffic after checks", + embed=_fake_embed, + embedding_model="fake", + limit=1, + ) + snapshot = await store.semantic_index_snapshot() + + assert selected == ["file_summary:rollout.md"] + assert snapshot["artefacts"] == 2 From c468688c346b4ba0237bea4a7b9d7ed14d3ae46f Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 09:24:30 +0200 Subject: [PATCH 06/22] Improve source-class semantic retrieval --- src/vaner/broker/context_preparation.py | 49 ++++++++++ src/vaner/broker/selector.py | 110 +++++++++++++++++++-- src/vaner/semantic_aliases.py | 6 ++ tests/test_broker/test_selector.py | 121 +++++++++++++++++++++++- 4 files changed, 275 insertions(+), 11 deletions(-) diff --git a/src/vaner/broker/context_preparation.py b/src/vaner/broker/context_preparation.py index 55d81cd..9af703b 100644 --- a/src/vaner/broker/context_preparation.py +++ b/src/vaner/broker/context_preparation.py @@ -75,6 +75,9 @@ def query_variants(prompt: str, profile: ContextPreparationProfile, *, max_varia variants.append(" ".join([*constraint_terms[:4], *facet_terms[:6]])) if profile.need in {"multi_source_synthesis", "research_mapping", "decision_support"}: variants.append(" ".join([*facet_terms[:10], *keywords[:8], "summary status decision evidence"])) + if _looks_like_scheduling_query(prompt): + variants.append(" ".join([*keywords[:10], "schedule calendar invite booking event meeting time window"])) + variants.append(" ".join([*keywords[:8], "technical deep dive architecture review private hosting isolated network vpc"])) if profile.need == "source_evidence": variants.append(" ".join([*facet_terms[:8], *keywords[:8], "source evidence claim citation provenance"])) if profile.need == "conflict_resolution": @@ -95,6 +98,44 @@ def query_variants(prompt: str, profile: ContextPreparationProfile, *, max_varia return deduped +def source_path_hint_candidates(prompt: str, available_paths: list[str], *, limit: int = 64) -> list[str]: + """Return paths whose source class/path shape is directly implied by the prompt.""" + + lowered = prompt.lower() + hint_terms: set[str] = set() + if re.search(r"\b(postmortem|postmortems|retro|retrospective|rca|incident review)\b", lowered): + hint_terms.update({"postmortem", "postmortems", "rca"}) + if re.search(r"\b(runbook|playbook|procedure|operating guide)\b", lowered): + hint_terms.update({"runbook", "runbooks", "playbook", "playbooks"}) + if _looks_like_scheduling_query(prompt): + hint_terms.update({"calendar", "invite", "booking", "schedule", "meeting", "gmail"}) + if re.search(r"\b(architecture|technical|deep dive|review|workshop)\b", lowered): + hint_terms.update({"architecture", "review", "deepdive", "deep-dive", "technical", "workshop"}) + if re.search(r"\b(healthcare|health care|clinical|medical|hospital)\b", lowered): + hint_terms.update({"health", "healthcare", "clinical", "medical"}) + if re.search(r"\b(isolated network|private|vpc|on-prem|own network)\b", lowered): + hint_terms.update({"private", "privhost", "vpc", "network", "hosting"}) + if re.search(r"\b(issue|ticket|bug|support escalation|incident)\b", lowered): + hint_terms.update({"jira", "linear", "support", "incidents"}) + if not hint_terms: + return [] + ranked: list[tuple[int, str]] = [] + for path in available_paths: + path_lower = path.lower() + score = 0 + for term in hint_terms: + if term in path_lower: + score += 3 if f"/{term}" in path_lower or f"{term}/" in path_lower else 1 + if "postmortem" in hint_terms and "/postmortems/" in path_lower: + score += 8 + if "gmail" in hint_terms and path_lower.startswith("gmail/"): + score += 4 + if score: + ranked.append((score, path)) + ranked.sort(key=lambda item: (-item[0], item[1])) + return [path for _, path in ranked[:limit]] + + def extract_query_keywords(prompt: str, *, limit: int = 24) -> list[str]: """Extract FTS-friendly terms from a conversational prompt.""" @@ -116,6 +157,14 @@ def extract_query_keywords(prompt: str, *, limit: int = 24) -> list[str]: return keywords[:limit] +def _looks_like_scheduling_query(prompt: str) -> bool: + lowered = prompt.lower() + return bool( + re.search(r"\b(when|scheduled|schedule|calendar|invite|meeting|deep dive|time window|booking|booked)\b", lowered) + and re.search(r"\b(client|customer|call|review|demo|technical|architecture|workshop)\b", lowered) + ) + + def exact_reference_candidates(prompt: str, available_paths: list[str], *, limit: int = 32) -> list[str]: references = set(_PATH_RE.findall(prompt)) references.update(match.group(1).strip() for match in _QUOTED_RE.finditer(prompt)) diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index 61dd9d8..b0f8858 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -15,6 +15,7 @@ hard_constraints_satisfied, infer_context_preparation_profile, query_variants, + source_path_hint_candidates, ) from vaner.models.artefact import Artefact from vaner.models.context_preparation import ContextPreparationProfile, PreparedContextDiagnostics @@ -64,6 +65,7 @@ def _prompt_terms(prompt: str) -> list[str]: terms.extend(part.lower() for part in raw.split("_") if len(part) > 2) if lowered not in {"fastapi", "openapi"}: terms.extend(part.lower() for part in _camel_parts(raw) if len(part) > 2) + terms.extend(raw for raw in re.findall(r"\b\d{2,4}\b", prompt)) terms.extend(engineering_semantic_aliases(prompt, terms, stopwords=_COMMON_WORDS)) return list(dict.fromkeys(term for term in terms if len(term) > 2 and term not in _COMMON_WORDS)) @@ -567,22 +569,29 @@ async def select_artefacts_fts( except Exception: keys = [] _record_source_ranking(source_rankings, source_by_key, source, keys) - if semantic_memory_enabled and semantic_embed is not None and hasattr(store, "select_artefacts_semantic"): - try: - semantic_keys = list(await store.select_artefacts_semantic(prompt, limit=retrieval_limit, embed=semantic_embed)) # type: ignore[attr-defined] - except Exception: - semantic_keys = [] - _record_source_ranking(source_rankings, source_by_key, "semantic_memory", semantic_keys) if context_enabled: try: available_paths = await store.list_source_paths(limit=max(5000, retrieval_limit * 10)) # type: ignore[union-attr] except Exception: available_paths = [] exact_paths = set(exact_reference_candidates(prompt, available_paths, limit=min(32, max(8, top_n * 3)))) + source_hint_paths = set(source_path_hint_candidates(prompt, available_paths, limit=min(96, max(16, top_n * 8)))) else: + available_paths = [] exact_paths = set() + source_hint_paths = set() + if semantic_memory_enabled and semantic_embed is not None and hasattr(store, "select_artefacts_semantic"): + semantic_variants = variants if context_enabled else [prompt] + for index, variant in enumerate(semantic_variants): + try: + semantic_keys = list(await store.select_artefacts_semantic(variant, limit=retrieval_limit, embed=semantic_embed)) # type: ignore[attr-defined] + except Exception: + semantic_keys = [] + source = "semantic_memory" if index == 0 else "semantic_query_variants" + _record_source_ranking(source_rankings, source_by_key, source, semantic_keys) else: exact_paths = set() + source_hint_paths = set() preferred = preferred_keys or set() preferred_paths_set = preferred_paths or set() @@ -596,11 +605,21 @@ async def select_artefacts_fts( loaded: list[Artefact] = [] load_keys = set(fused_keys) | preferred loaded.extend(await store.list_by_keys(load_keys, limit=max(retrieval_limit, len(load_keys)))) # type: ignore[union-attr] - path_loads = set(preferred_paths_set) | exact_paths + path_loads = set(preferred_paths_set) | exact_paths | source_hint_paths if path_loads: - path_loaded = await store.list_by_source_paths(path_loads, limit=max(retrieval_limit, len(path_loads))) # type: ignore[union-attr] - loaded.extend(path_loaded) - _record_source_ranking(source_rankings, source_by_key, "exact_reference", [artefact.key for artefact in path_loaded]) + if exact_paths: + exact_loaded = await store.list_by_source_paths(exact_paths, limit=max(retrieval_limit, len(exact_paths))) # type: ignore[union-attr] + loaded.extend(exact_loaded) + _record_source_ranking(source_rankings, source_by_key, "exact_reference", [artefact.key for artefact in exact_loaded]) + if source_hint_paths: + hint_loaded = await store.list_by_source_paths(source_hint_paths, limit=max(retrieval_limit, len(source_hint_paths))) # type: ignore[union-attr] + loaded.extend(hint_loaded) + _record_source_ranking(source_rankings, source_by_key, "source_metadata", [artefact.key for artefact in hint_loaded]) + preferred_path_loads = preferred_paths_set - exact_paths - source_hint_paths + if preferred_path_loads: + loaded.extend( + await store.list_by_source_paths(preferred_path_loads, limit=max(retrieval_limit, len(preferred_path_loads))) # type: ignore[union-attr] + ) seen: set[str] = set() candidates = [] for artefact in loaded: @@ -732,6 +751,8 @@ def _fuse_source_rankings(source_rankings: dict[str, list[str]], *, limit: int) "generated_query_variants": 0.72, "relationship_graph": 0.9, "semantic_memory": 0.75, + "semantic_query_variants": 0.68, + "source_metadata": 0.82, "coverage_floor": 0.85, } scores: dict[str, float] = {} @@ -883,6 +904,55 @@ def add_best_matching(predicate: Callable[[Artefact], bool]) -> None: return selected +def _canonical_source_class_bonus(prompt: str, artefact: Artefact, profile: ContextPreparationProfile | None) -> float: + if profile is None or profile.need != "multi_source_synthesis": + return 0.0 + lowered = prompt.lower() + path = artefact.source_path.lower() + content_head = (artefact.content or "").lower()[:600] + if not re.search(r"\b(across|all|every|which .*most|highest number|count)\b", lowered): + return 0.0 + bonus = 0.0 + if re.search(r"\b(postmortem|postmortems|rca|incident review)\b", lowered): + if "/postmortems/" in path and path.startswith(("confluence/", "docs/", "google_drive/")): + bonus += 16.0 + elif path.startswith(("slack/", "gmail/")) and "postmortem" in path: + bonus -= 6.0 + if "template" in path or "template" in content_head: + bonus -= 14.0 + return bonus + + +def _scheduling_evidence_bonus(prompt: str, artefact: Artefact, profile: ContextPreparationProfile | None) -> float: + lowered = prompt.lower() + if profile is None or not re.search(r"\b(when|scheduled|schedule|calendar|invite|meeting|time window|booking|booked)\b", lowered): + return 0.0 + if not re.search(r"\b(client|customer|call|review|demo|technical|architecture|workshop)\b", lowered): + return 0.0 + path = artefact.source_path.lower() + text = _artefact_context_text(artefact)[:9000] + bonus = 0.0 + if path.startswith(("gmail/", "calendar/")): + bonus += 4.0 + elif path.startswith(("hubspot/", "fireflies/")) and re.search(r"\b(when|scheduled|time window|booking|booked)\b", lowered): + bonus -= 2.0 + if any(term in text for term in ("calendar", "invite", ".ics", "event/", "scheduled", "confirming", "works for our team")): + bonus += 5.0 + if "technical deep dive" in lowered and "technical deep dive" in text and "architecture review" in text: + bonus += 4.0 + if "isolated network" in lowered and "private" in text and any( + term in text for term in ("vpc", "on-prem", "isolated", "private hosting") + ): + bonus += 5.0 + numeric_terms = set(re.findall(r"\b\d{2,4}\b", lowered)) + matched_numeric = sum(1 for term in numeric_terms if term in text) + if matched_numeric: + bonus += min(4.0, matched_numeric * 2.0) + if re.search(r"\b(pacific|pt|time window)\b", lowered) and re.search(r"\b(?:pt|pst|pdt|-0[78]00|\d{1,2}:\d{2})\b", text): + bonus += 3.0 + return bonus + + def select_artefacts( prompt: str, artefacts: list[Artefact], @@ -998,6 +1068,26 @@ def select_artefacts( ) ) score += constraint_bonus + canonical_bonus = _canonical_source_class_bonus(prompt, artefact, context_profile) + if canonical_bonus: + factors.append( + ScoreFactor( + name="canonical_source_class", + contribution=canonical_bonus, + detail="aggregation over a source class prefers canonical durable sources over conversational chatter", + ) + ) + score += canonical_bonus + scheduling_bonus = _scheduling_evidence_bonus(prompt, artefact, context_profile) + if scheduling_bonus: + factors.append( + ScoreFactor( + name="scheduling_evidence", + contribution=scheduling_bonus, + detail="scheduling queries prefer confirmed invites, time windows, and deployment-specific meeting context", + ) + ) + score += scheduling_bonus source_prior = _source_rank_prior(artefact) if source_prior: factors.append( diff --git a/src/vaner/semantic_aliases.py b/src/vaner/semantic_aliases.py index 18a1cca..aef6401 100644 --- a/src/vaner/semantic_aliases.py +++ b/src/vaner/semantic_aliases.py @@ -52,6 +52,12 @@ def has_any(*phrases: str) -> bool: if has_any("egress", "residency", "cross region", "edge fallback", "routing anomaly", "routed"): aliases.update({"route", "routing", "egress", "residency", "fallback", "failover", "edge", "control", "plane"}) + if has_any("healthcare", "health care", "clinical", "medical", "hospital"): + aliases.update({"health", "healthcare", "clinical", "medical", "patient", "hipaa"}) + + if has_any("isolated network", "own network", "private network", "private hosting", "private deploy", "privnet", "vpc"): + aliases.update({"private", "privnet", "vpc", "vnet", "isolated", "hosting", "deploy", "deployment"}) + if has_any("makes things up", "made things up", "hallucination", "hallucinate", "joke", "meme"): aliases.update({"hallucination", "hallucinate", "meme", "memes", "satire", "tagging", "joke", "jokes"}) diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index c453f0b..394f8ab 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -6,7 +6,7 @@ import pytest -from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants +from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants, source_path_hint_candidates from vaner.broker.selector import select_artefacts, select_artefacts_fts from vaner.models.artefact import Artefact, ArtefactKind from vaner.store.artefacts import ArtefactStore @@ -126,6 +126,50 @@ def test_query_variants_add_local_keyword_and_alias_recall_terms(): assert "promote" in variants +def test_query_variants_add_scheduling_and_private_hosting_aliases(): + prompt = ( + "When is the 60 to 90 minute technical deep dive scheduled with the healthcare client " + "about running model serving inside their own isolated network?" + ) + profile = infer_context_preparation_profile(prompt) + + variants = " ".join(query_variants(prompt, profile, max_variants=8)) + + assert "calendar invite booking" in variants + assert "architecture review private hosting isolated network" in variants + + +def test_source_path_hints_surface_source_class_candidates(): + paths = [ + "slack/incidents/random-thread.json", + "confluence/oncall-and-incident-response/postmortems/streaming-stalls.json", + "confluence/oncall-and-incident-response/postmortems/rate-limit-misconfig.json", + ] + + selected = source_path_hint_candidates("Across all incident postmortems, which team owned action items?", paths) + + assert selected[:2] == [ + "confluence/oncall-and-incident-response/postmortems/rate-limit-misconfig.json", + "confluence/oncall-and-incident-response/postmortems/streaming-stalls.json", + ] + + +def test_source_path_hints_rank_specific_scheduling_threads(): + paths = [ + "gmail/team/generic-deepdive-scheduler.json", + "gmail/soojin/architecture-review-booking-next-steps-cytohealth.json", + "hubspot/company-health-network.json", + ] + prompt = ( + "When is the technical deep dive scheduled with the healthcare client " + "about running model serving inside their own isolated network?" + ) + + selected = source_path_hint_candidates(prompt, paths) + + assert selected[0] == "gmail/soojin/architecture-review-booking-next-steps-cytohealth.json" + + def test_source_evidence_profile_uses_evidence_mode(): prompt = "What source evidence supports the rollout claim?" profile = infer_context_preparation_profile(prompt) @@ -297,6 +341,81 @@ def test_select_artefacts_seeds_coverage_for_multi_source_synthesis(): assert {"docs/hosted.md", "docs/dedicated.md", "docs/private.md"} <= selected_paths +def test_multi_source_aggregation_prefers_canonical_postmortem_docs(): + now = time.time() + artefacts = [ + Artefact( + key="file_summary:slack-postmortem.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="slack/postmortems/action-items-thread.json", + source_mtime=now, + generated_at=now, + model="test", + content="postmortem action items action items assigned team action items", + ), + Artefact( + key="file_summary:template.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/oncall/postmortems/postmortem-template.json", + source_mtime=now, + generated_at=now, + model="test", + content="Postmortem template for assigned team action items", + ), + Artefact( + key="file_summary:canonical.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/oncall/postmortems/streaming-stalls-2026-01-12.json", + source_mtime=now, + generated_at=now, + model="test", + content="Postmortem: Streaming stalls. Follow-up action items assigned to runtime and SRE.", + ), + ] + prompt = "Across all incident postmortems, which team was assigned the most follow-up action items?" + profile = infer_context_preparation_profile(prompt) + + selected = select_artefacts(prompt, artefacts, top_n=1, context_need=profile.need, context_profile=profile) + + assert selected[0].key == "file_summary:canonical.json" + + +def test_scheduling_queries_prefer_confirmed_invite_context(): + now = time.time() + artefacts = [ + Artefact( + key="file_summary:generic-deepdive.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="gmail/team/generic-deepdive.json", + source_mtime=now, + generated_at=now, + model="test", + content="Technical deep dive coordination for hosted integration scheduler.", + ), + Artefact( + key="file_summary:confirmed-private-review.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="gmail/solutions/architecture-review-booking.json", + source_mtime=now, + generated_at=now, + model="test", + content=( + "Schedule private hosting architecture review. Confirming Tue Oct 24, 10:00-11:15 PT. " + "Invite attached. 60-90 minute architecture review / technical deep dive for private VPC deployment." + ), + ), + ] + prompt = ( + "When is the 60 to 90 minute technical deep dive scheduled with the healthcare client " + "about running model serving inside their own isolated network, and what is the time window in Pacific time?" + ) + profile = infer_context_preparation_profile(prompt) + + selected = select_artefacts(prompt, artefacts, top_n=1, context_need=profile.need, context_profile=profile) + + assert selected[0].key == "file_summary:confirmed-private-review.json" + + def test_select_artefacts_origin_rerank_prefers_definition_files(): artefacts = [ Artefact( From 9038dfabb26e9013de39812232914a736c09d9ab Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 09:44:00 +0200 Subject: [PATCH 07/22] Add explicit context preparation tools --- src/vaner/broker/aggregation.py | 252 ++++++++++++++++++++++++ src/vaner/broker/assembler.py | 2 + src/vaner/broker/context_preparation.py | 16 +- src/vaner/broker/preparation_policy.py | 85 ++++++++ src/vaner/broker/selector.py | 89 ++++++++- src/vaner/models/context_preparation.py | 42 ++++ tests/test_broker/test_selector.py | 124 ++++++++++++ 7 files changed, 608 insertions(+), 2 deletions(-) create mode 100644 src/vaner/broker/aggregation.py create mode 100644 src/vaner/broker/preparation_policy.py diff --git a/src/vaner/broker/aggregation.py b/src/vaner/broker/aggregation.py new file mode 100644 index 0000000..b35be8b --- /dev/null +++ b/src/vaner/broker/aggregation.py @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import re +import time +from collections import Counter, defaultdict + +from vaner.broker.context_preparation import extract_query_keywords +from vaner.models.artefact import Artefact, ArtefactKind +from vaner.models.context_preparation import ContextPreparationProfile, ContextToolTrace + + +def build_source_aggregation( + prompt: str, + candidates: list[Artefact], + profile: ContextPreparationProfile, + *, + source_by_key: dict[str, set[str]] | None = None, + max_sources: int = 40, + provenance_budget: int = 24, +) -> tuple[Artefact | None, ContextToolTrace]: + """Compress many relevant sources into a provenance-preserving evidence artefact.""" + + trace = ContextToolTrace(tool="aggregate_sources", input_count=len(candidates)) + if profile.need != "multi_source_synthesis" or len(candidates) < 3: + trace.notes.append("skipped:not_multi_source_or_too_few_candidates") + return None, trace + + source_candidates = _dedupe_candidates(candidates)[:max_sources] + if len(source_candidates) < 3: + trace.notes.append("skipped:dedupe_left_too_few_candidates") + return None, trace + + grouped = _group_findings(prompt, source_candidates) + facets = _facet_coverage(profile, source_candidates) + source_classes = Counter(_source_class(artefact.source_path) for artefact in source_candidates) + source_refs = _source_refs(source_candidates, source_by_key or {}, provenance_budget=provenance_budget) + gaps = _coverage_gaps(prompt, profile, source_candidates, grouped) + + sections = [ + "Prepared source aggregation", + f"Question: {prompt.strip()}", + "", + "Coverage:", + f"- sources_considered: {len(source_candidates)}", + "- source_classes: " + _format_counter(source_classes, limit=8), + "- facets_covered: " + (", ".join(facets) if facets else "none detected"), + "- gaps: " + ("; ".join(gaps) if gaps else "none detected"), + "", + "Grouped findings:", + *_finding_lines(grouped), + "", + "Representative provenance:", + *source_refs, + ] + content = "\n".join(sections).strip() + digest = hashlib.sha256((prompt + "\n" + "\n".join(a.key for a in source_candidates)).encode("utf-8")).hexdigest()[:16] + metadata = { + "context_sources": ["aggregate_sources"], + "aggregation_source_count": len(source_candidates), + "aggregation_source_keys": [artefact.key for artefact in source_candidates[:provenance_budget]], + "aggregation_source_paths": [artefact.source_path for artefact in source_candidates[:provenance_budget]], + "aggregation_groups": [{"value": value, "count": count} for value, count, _ in grouped[:12]], + "provenance": "prepared_context_aggregation", + } + trace.output_count = 1 + trace.notes.append(f"sources_considered:{len(source_candidates)}") + trace.notes.append(f"groups:{len(grouped)}") + return ( + Artefact( + key=f"prepared_context:source_aggregation:{digest}", + kind=ArtefactKind.FILE_SUMMARY, + source_path=f"prepared_context/source_aggregation/{digest}.md", + source_mtime=time.time(), + generated_at=time.time(), + model="vaner-context-tools", + content=content, + metadata=metadata, + ), + trace, + ) + + +def _dedupe_candidates(candidates: list[Artefact]) -> list[Artefact]: + seen: set[str] = set() + deduped: list[Artefact] = [] + for artefact in candidates: + if artefact.key in seen: + continue + seen.add(artefact.key) + deduped.append(artefact) + return deduped + + +def _source_class(path: str) -> str: + parts = [part for part in path.split("/") if part] + if len(parts) >= 3 and parts[2].lower() in {"postmortems", "runbooks", "incidents", "tickets"}: + return "/".join(parts[:3]) + if len(parts) >= 2: + return "/".join(parts[:2]) + return parts[0] if parts else "unknown" + + +def _facet_coverage(profile: ContextPreparationProfile, candidates: list[Artefact]) -> list[str]: + text = "\n".join(f"{artefact.source_path}\n{artefact.content}" for artefact in candidates).lower() + covered = [facet.value for facet in profile.facets if facet.value.lower() in text] + return list(dict.fromkeys(covered))[:16] + + +def _coverage_gaps( + prompt: str, + profile: ContextPreparationProfile, + candidates: list[Artefact], + grouped: list[tuple[str, int, list[Artefact]]], +) -> list[str]: + text = "\n".join(f"{artefact.source_path}\n{artefact.content}" for artefact in candidates).lower() + gaps: list[str] = [] + for constraint in profile.constraints: + if constraint.kind != "restrictive_language" and constraint.required and constraint.value.lower() not in text: + gaps.append(f"missing required constraint: {constraint.value}") + if _asks_for_grouped_counts(prompt) and not grouped: + gaps.append("no stable grouping/count signal found") + if len(candidates) >= 30: + gaps.append("aggregation truncated to source budget") + return gaps[:8] + + +def _group_findings(prompt: str, candidates: list[Artefact]) -> list[tuple[str, int, list[Artefact]]]: + prompt_keywords = set(extract_query_keywords(prompt, limit=16)) + group_to_sources: dict[str, list[Artefact]] = defaultdict(list) + for artefact in candidates: + values = _group_values(artefact.content) + if not values and _asks_for_theme_summary(prompt): + values = _theme_values(artefact.content, prompt_keywords) + for value in sorted(values): + if artefact not in group_to_sources[value]: + group_to_sources[value].append(artefact) + grouped = [(value, len(sources), sources) for value, sources in group_to_sources.items()] + return sorted(grouped, key=lambda item: (-item[1], item[0].lower())) + + +def _group_values(content: str) -> set[str]: + values: set[str] = set() + patterns = ( + r"\b(?:owner|owned by|assigned to|responsible team|team|teams|dri|dris)\s*[:=-]?\s*(?:the\s+)?([A-Za-z][A-Za-z0-9&/_ -]{1,48})", + r"\b([A-Za-z][A-Za-z0-9&/_-]{1,32})\s+(?:team|squad|group)\b", + ) + for pattern in patterns: + for match in re.finditer(pattern, content, flags=re.IGNORECASE): + values.update(_split_group_value(match.group(1))) + return {value for value in values if _valid_group_value(value)} + + +def _split_group_value(raw: str) -> set[str]: + cleaned = re.split(r"[.;()\n]", raw.strip(), maxsplit=1)[0] + parts = re.split(r"\s*(?:,|/|\band\b|&)\s*", cleaned, flags=re.IGNORECASE) + return {_normalize_group_value(part) for part in parts if part.strip()} + + +def _normalize_group_value(value: str) -> str: + tokens = [token for token in re.findall(r"[A-Za-z0-9_-]+", value) if token.lower() not in _GROUP_STOPWORDS] + if not tokens: + return "" + return " ".join(token.upper() if token.isupper() else token[:1].upper() + token[1:].lower() for token in tokens[:4]) + + +def _valid_group_value(value: str) -> bool: + lowered = value.lower() + return bool(value) and len(value) <= 48 and lowered not in _GROUP_STOPWORDS and not lowered.isdigit() + + +def _theme_values(content: str, prompt_keywords: set[str]) -> set[str]: + tokens = [ + token.lower() + for token in re.findall(r"\b[A-Za-z][A-Za-z0-9_-]{3,}\b", content) + if token.lower() not in prompt_keywords and token.lower() not in _THEME_STOPWORDS + ] + counts = Counter(tokens) + return {token for token, count in counts.most_common(5) if count >= 2} + + +def _asks_for_grouped_counts(prompt: str) -> bool: + return bool(re.search(r"\b(count|how many|most|highest number|by team|by owner|which team|which owner)\b", prompt, re.IGNORECASE)) + + +def _asks_for_theme_summary(prompt: str) -> bool: + return bool(re.search(r"\b(patterns?|themes?|summarize|synthesize|compare|across all)\b", prompt, re.IGNORECASE)) + + +def _finding_lines(grouped: list[tuple[str, int, list[Artefact]]]) -> list[str]: + if not grouped: + return ["- no grouped finding could be inferred from candidate sources"] + lines: list[str] = [] + for value, count, sources in grouped[:12]: + paths = ", ".join(artefact.source_path for artefact in sources[:3]) + if len(sources) > 3: + paths += f", +{len(sources) - 3} more" + lines.append(f"- {value}: {count} source(s); examples: {paths}") + return lines + + +def _source_refs(candidates: list[Artefact], source_by_key: dict[str, set[str]], *, provenance_budget: int) -> list[str]: + refs: list[str] = [] + for index, artefact in enumerate(candidates[:provenance_budget], start=1): + source_names = ",".join(sorted(source_by_key.get(artefact.key, set()))) or "candidate" + snippet = " ".join(artefact.content.split())[:220] + refs.append(f"- [{index}] {artefact.source_path} ({source_names}): {snippet}") + return refs + + +def _format_counter(counter: Counter[str], *, limit: int) -> str: + if not counter: + return "none" + return ", ".join(f"{key}={value}" for key, value in counter.most_common(limit)) + + +_GROUP_STOPWORDS = { + "and", + "assigned", + "item", + "items", + "team", + "teams", + "follow", + "follow-up", + "action", + "actions", + "owner", + "owned", + "responsible", + "the", + "to", + "for", +} + +_THEME_STOPWORDS = { + "that", + "this", + "with", + "from", + "have", + "were", + "been", + "their", + "there", + "source", + "sources", + "summary", + "evidence", +} diff --git a/src/vaner/broker/assembler.py b/src/vaner/broker/assembler.py index 43d6b96..2260be8 100644 --- a/src/vaner/broker/assembler.py +++ b/src/vaner/broker/assembler.py @@ -157,6 +157,8 @@ def _prepared_context_mode(diagnostics: PreparedContextDiagnostics | None, answe return "writing" if need == "decision_support": return "decision" + if need == "scheduling": + return "planning" if need == "conflict_resolution": return "conflict_resolution" if need == "absence_check": diff --git a/src/vaner/broker/context_preparation.py b/src/vaner/broker/context_preparation.py index 9af703b..a6932c7 100644 --- a/src/vaner/broker/context_preparation.py +++ b/src/vaner/broker/context_preparation.py @@ -10,8 +10,10 @@ ContextConstraint, ContextCoverageReport, ContextFacet, + ContextPreparationPlan, ContextPreparationProfile, ContextSourceStats, + ContextToolTrace, PreparedContextDiagnostics, ) from vaner.semantic_aliases import engineering_semantic_aliases @@ -224,7 +226,7 @@ def hard_constraints_satisfied(prompt: str, artefact: Artefact, profile: Context def competitive_threshold_multiplier(need: str | None) -> float: if need in {"multi_source_synthesis", "source_evidence", "conflict_resolution", "research_mapping", "decision_support"}: return 0.0 - if need in {"implementation_support", "absence_check", "working_set_extension", "creative_grounding"}: + if need in {"implementation_support", "absence_check", "working_set_extension", "creative_grounding", "scheduling"}: return 0.20 if need == "evidence_gathering": return 0.30 @@ -284,6 +286,8 @@ def build_prepared_context_diagnostics( selected: list[Artefact], fused_candidate_count: int, latency_ms: float, + preparation_plan: ContextPreparationPlan | None = None, + tool_traces: list[ContextToolTrace] | None = None, ) -> PreparedContextDiagnostics: selected_keys = {artefact.key for artefact in selected} source_counter: Counter[str] = Counter() @@ -297,12 +301,17 @@ def build_prepared_context_diagnostics( coverage = build_coverage_report(profile, selected, source_by_key) return PreparedContextDiagnostics( profile=profile, + preparation_plan=preparation_plan, + tool_traces=tool_traces or [], source_counts=[ ContextSourceStats(source=source, candidate_count=count, selected_count=selected_counter.get(source, 0)) for source, count in sorted(source_counter.items()) ], fused_candidate_count=fused_candidate_count, selected_count=len(selected), + aggregation_count=sum( + 1 for artefact in selected if str(artefact.metadata.get("provenance")) == "prepared_context_aggregation" + ), hard_constraints_extracted=constraints, hard_constraints_satisfied=[value for value in constraints if value not in coverage.missing_constraints], hard_constraints_missing=coverage.missing_constraints, @@ -334,6 +343,11 @@ def _infer_need(lowered: str, archetype: str): return "absence_check" if re.search(r"\b(according to|source|evidence|citation|cite|provenance|claim|where did .* come from)\b", lowered): return "source_evidence" + if re.search(r"\b(when|scheduled|schedule|calendar|invite|meeting|time window|booking|booked)\b", lowered) and re.search( + r"\b(client|customer|call|review|demo|technical|architecture|workshop)\b", + lowered, + ): + return "scheduling" if archetype == "developer" and re.search(r"\b(implement|fix|debug|change|affected|callers|tests?|dependency|regression)\b", lowered): return "implementation_support" if archetype == "writer": diff --git a/src/vaner/broker/preparation_policy.py b/src/vaner/broker/preparation_policy.py new file mode 100644 index 0000000..62877b3 --- /dev/null +++ b/src/vaner/broker/preparation_policy.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from vaner.models.context_preparation import ContextPreparationPlan, ContextPreparationProfile, ContextToolStep + + +def choose_preparation_plan(profile: ContextPreparationProfile, prompt: str) -> ContextPreparationPlan: + """Choose a bounded, deterministic context-preparation plan. + + The policy names general preparation capabilities, not benchmark or dataset + shortcuts. Tool execution remains bounded by selector/config budgets. + """ + + need = profile.need + if need == "multi_source_synthesis": + return ContextPreparationPlan( + name="multi_source_synthesis", + need=need, + steps=[ + ContextToolStep(tool="source_class_search", mode="canonical", budget=96), + ContextToolStep(tool="semantic_search", mode="query_variants", budget=96), + ContextToolStep(tool="aggregate_sources", mode="structured_provenance", budget=40), + ContextToolStep(tool="coverage_check", mode="facets_constraints", budget=0), + ], + ) + if need == "scheduling": + return ContextPreparationPlan( + name="scheduling_evidence", + need=need, + steps=[ + ContextToolStep(tool="time_entity_extraction", mode="strict", budget=0), + ContextToolStep(tool="source_class_search", mode="calendar_invite_preferred", budget=64), + ContextToolStep(tool="semantic_search", mode="query_variants", budget=64), + ContextToolStep(tool="conflict_scan", mode="time_evidence", budget=16), + ContextToolStep(tool="coverage_check", mode="facets_constraints", budget=0), + ], + ) + if need == "implementation_support": + return ContextPreparationPlan( + name="implementation_support", + need=need, + steps=[ + ContextToolStep(tool="exact_reference_lookup", mode="paths_symbols", budget=32), + ContextToolStep(tool="relationship_expand", mode="dependencies", budget=48), + ContextToolStep(tool="working_set_lookup", mode="recent_activity", budget=32), + ContextToolStep(tool="test_lookup", mode="adjacent_tests", budget=24), + ContextToolStep(tool="risk_check", mode="regression", budget=0), + ], + ) + if need == "source_evidence": + return ContextPreparationPlan( + name="source_evidence", + need=need, + steps=[ + ContextToolStep(tool="exact_reference_lookup", mode="quoted_refs", budget=32), + ContextToolStep(tool="semantic_search", mode="evidence_spans", budget=64), + ContextToolStep(tool="source_class_search", mode="canonical", budget=48), + ContextToolStep(tool="coverage_check", mode="provenance", budget=0), + ], + ) + if need == "conflict_resolution": + return ContextPreparationPlan( + name="conflict_resolution", + need=need, + steps=[ + ContextToolStep(tool="semantic_search", mode="current_and_superseded", budget=80), + ContextToolStep(tool="source_class_search", mode="canonical", budget=64), + ContextToolStep(tool="conflict_scan", mode="preserve_pairs", budget=24), + ContextToolStep(tool="coverage_check", mode="conflict_pairs", budget=0), + ], + ) + return ContextPreparationPlan( + name="evidence_gathering", + need=need, + steps=[ + ContextToolStep(tool="lexical_search", mode="balanced", budget=64), + ContextToolStep(tool="semantic_search", mode="optional", budget=64), + ContextToolStep(tool="coverage_check", mode="facets_constraints", budget=0), + ], + ) + + +def plan_uses_tool(plan: ContextPreparationPlan, tool_name: str) -> bool: + return any(step.tool == tool_name for step in plan.steps) diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index b0f8858..37717dd 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -8,6 +8,7 @@ from collections.abc import Awaitable, Callable from fnmatch import fnmatch +from vaner.broker.aggregation import build_source_aggregation from vaner.broker.context_preparation import ( build_prepared_context_diagnostics, competitive_threshold_multiplier, @@ -17,8 +18,9 @@ query_variants, source_path_hint_candidates, ) +from vaner.broker.preparation_policy import choose_preparation_plan, plan_uses_tool from vaner.models.artefact import Artefact -from vaner.models.context_preparation import ContextPreparationProfile, PreparedContextDiagnostics +from vaner.models.context_preparation import ContextPreparationProfile, ContextToolTrace, PreparedContextDiagnostics from vaner.models.decision import ScoreFactor from vaner.semantic_aliases import engineering_semantic_aliases @@ -551,6 +553,8 @@ async def select_artefacts_fts( started = time.monotonic() profile = infer_context_preparation_profile(prompt) context_enabled = context_preparation_mode != "legacy" + preparation_plan = choose_preparation_plan(profile, prompt) if context_enabled else None + tool_traces: list[ContextToolTrace] = [] source_by_key: dict[str, set[str]] = {} source_rankings: dict[str, list[str]] = {} @@ -576,10 +580,19 @@ async def select_artefacts_fts( available_paths = [] exact_paths = set(exact_reference_candidates(prompt, available_paths, limit=min(32, max(8, top_n * 3)))) source_hint_paths = set(source_path_hint_candidates(prompt, available_paths, limit=min(96, max(16, top_n * 8)))) + if exact_paths: + tool_traces.append( + ContextToolTrace(tool="exact_reference_lookup", input_count=len(available_paths), output_count=len(exact_paths)) + ) + if source_hint_paths: + tool_traces.append( + ContextToolTrace(tool="source_class_search", input_count=len(available_paths), output_count=len(source_hint_paths)) + ) else: available_paths = [] exact_paths = set() source_hint_paths = set() + semantic_trace = ContextToolTrace(tool="semantic_search") if semantic_memory_enabled and semantic_embed is not None and hasattr(store, "select_artefacts_semantic"): semantic_variants = variants if context_enabled else [prompt] for index, variant in enumerate(semantic_variants): @@ -589,6 +602,9 @@ async def select_artefacts_fts( semantic_keys = [] source = "semantic_memory" if index == 0 else "semantic_query_variants" _record_source_ranking(source_rankings, source_by_key, source, semantic_keys) + semantic_trace.input_count += 1 + semantic_trace.output_count += len(semantic_keys) + tool_traces.append(semantic_trace) else: exact_paths = set() source_hint_paths = set() @@ -649,6 +665,8 @@ async def select_artefacts_fts( selected=selected, fused_candidate_count=len(fused_keys), latency_ms=(time.monotonic() - started) * 1000.0, + preparation_plan=preparation_plan, + tool_traces=tool_traces, ) if diagnostics.coverage.gap_flags: recovery_terms = _coverage_recovery_terms(profile, diagnostics) @@ -680,6 +698,30 @@ async def select_artefacts_fts( context_need=profile.need, context_profile=profile, ) + if context_enabled and preparation_plan is not None and plan_uses_tool(preparation_plan, "aggregate_sources"): + aggregate_budget = _plan_budget(preparation_plan, "aggregate_sources", default=40) + aggregate_pool = _aggregation_candidate_pool( + prompt, + candidates, + selected, + profile, + exclude_private=exclude_private, + path_excludes=path_excludes or [], + source_by_key=source_by_key, + limit=aggregate_budget, + ) + aggregate, trace = build_source_aggregation( + prompt, + aggregate_pool, + profile, + source_by_key=source_by_key, + max_sources=aggregate_budget, + ) + tool_traces.append(trace) + if aggregate is not None: + _record_source_ranking(source_rankings, source_by_key, "aggregate_sources", [aggregate.key]) + aggregate = _with_context_source_metadata(aggregate, source_rankings, source_by_key) + selected = [aggregate, *[artefact for artefact in selected if artefact.key != aggregate.key]][:top_n] _append_source_factors(capture_factors, source_by_key, selected) if capture_prepared_context_diagnostics is not None: capture_prepared_context_diagnostics.append( @@ -689,6 +731,8 @@ async def select_artefacts_fts( selected=selected, fused_candidate_count=len(fused_keys), latency_ms=(time.monotonic() - started) * 1000.0, + preparation_plan=preparation_plan, + tool_traces=tool_traces, ) ) return selected @@ -719,6 +763,8 @@ async def select_artefacts_fts( selected=selected, fused_candidate_count=len(all_artefacts), latency_ms=(time.monotonic() - started) * 1000.0, + preparation_plan=preparation_plan, + tool_traces=tool_traces, ) ) return selected @@ -815,6 +861,47 @@ def _append_source_factors( ) +def _plan_budget(plan: object, tool_name: str, *, default: int) -> int: + steps = getattr(plan, "steps", []) + for step in steps: + if getattr(step, "tool", None) == tool_name and getattr(step, "budget", 0): + return int(step.budget) + return default + + +def _aggregation_candidate_pool( + prompt: str, + candidates: list[Artefact], + selected: list[Artefact], + profile: ContextPreparationProfile, + *, + exclude_private: bool, + path_excludes: list[str], + source_by_key: dict[str, set[str]], + limit: int, +) -> list[Artefact]: + selected_keys = {artefact.key for artefact in selected} + rows: list[tuple[float, Artefact]] = [] + for artefact in candidates: + if exclude_private and str(artefact.metadata.get("privacy_zone", "")).lower() == "private_local": + continue + if any(fnmatch(artefact.source_path, pattern) for pattern in path_excludes): + continue + constraints_ok, _, _ = hard_constraints_satisfied(prompt, artefact, profile) + if not constraints_ok: + continue + sources = source_by_key.get(artefact.key, set()) + source_signal = 0.7 if sources else 0.0 + if sources & {"source_metadata", "semantic_memory", "semantic_query_variants", "coverage_floor"}: + source_signal += 1.0 + score = score_artefact(prompt, artefact) + _canonical_source_class_bonus(prompt, artefact, profile) + source_signal + if artefact.key in selected_keys: + score += 2.0 + rows.append((score, artefact)) + rows.sort(key=lambda item: (-item[0], item[1].source_path)) + return [artefact for _, artefact in rows[: max(1, limit)]] + + def _diversity_bucket(artefact: Artefact, context_need: str | None) -> str: path = artefact.source_path if context_need == "implementation_support": diff --git a/src/vaner/models/context_preparation.py b/src/vaner/models/context_preparation.py index ead633c..0006720 100644 --- a/src/vaner/models/context_preparation.py +++ b/src/vaner/models/context_preparation.py @@ -12,6 +12,7 @@ "evidence_gathering", "multi_source_synthesis", "source_evidence", + "scheduling", "decision_support", "conflict_resolution", "creative_grounding", @@ -23,6 +24,22 @@ ContextArchetype = Literal["developer", "writer", "researcher", "operator", "general"] +ContextToolName = Literal[ + "lexical_search", + "semantic_search", + "source_class_search", + "exact_reference_lookup", + "relationship_expand", + "working_set_lookup", + "prepared_memory_lookup", + "aggregate_sources", + "coverage_check", + "conflict_scan", + "test_lookup", + "risk_check", + "time_entity_extraction", +] + class ContextFacet(BaseModel): name: str @@ -47,6 +64,28 @@ class ContextPreparationProfile(BaseModel): notes: list[str] = Field(default_factory=list) +class ContextToolStep(BaseModel): + tool: ContextToolName + mode: str = "balanced" + budget: int = 0 + stop_condition: str = "" + + +class ContextPreparationPlan(BaseModel): + name: str + need: ContextNeed = "evidence_gathering" + steps: list[ContextToolStep] = Field(default_factory=list) + max_recovery_passes: int = 1 + deterministic: bool = True + + +class ContextToolTrace(BaseModel): + tool: ContextToolName + input_count: int = 0 + output_count: int = 0 + notes: list[str] = Field(default_factory=list) + + class ContextCoverageReport(BaseModel): covered_facets: list[str] = Field(default_factory=list) missing_constraints: list[str] = Field(default_factory=list) @@ -68,9 +107,12 @@ class ContextSourceStats(BaseModel): class PreparedContextDiagnostics(BaseModel): profile: ContextPreparationProfile = Field(default_factory=ContextPreparationProfile) + preparation_plan: ContextPreparationPlan | None = None + tool_traces: list[ContextToolTrace] = Field(default_factory=list) source_counts: list[ContextSourceStats] = Field(default_factory=list) fused_candidate_count: int = 0 selected_count: int = 0 + aggregation_count: int = 0 hard_constraints_extracted: list[str] = Field(default_factory=list) hard_constraints_satisfied: list[str] = Field(default_factory=list) hard_constraints_missing: list[str] = Field(default_factory=list) diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 394f8ab..586c2c9 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -6,7 +6,9 @@ import pytest +from vaner.broker.aggregation import build_source_aggregation from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants, source_path_hint_candidates +from vaner.broker.preparation_policy import choose_preparation_plan from vaner.broker.selector import select_artefacts, select_artefacts_fts from vaner.models.artefact import Artefact, ArtefactKind from vaner.store.artefacts import ArtefactStore @@ -178,6 +180,67 @@ def test_source_evidence_profile_uses_evidence_mode(): assert any("source evidence claim citation provenance" in variant for variant in query_variants(prompt, profile)) +def test_preparation_policy_names_bounded_context_tools(): + profile = infer_context_preparation_profile( + "Across all incident reviews, which owner had the highest number of follow-up tasks?" + ) + + plan = choose_preparation_plan(profile, "Across all incident reviews, which owner had the highest number of follow-up tasks?") + + assert plan.name == "multi_source_synthesis" + assert plan.deterministic is True + assert [step.tool for step in plan.steps] == [ + "source_class_search", + "semantic_search", + "aggregate_sources", + "coverage_check", + ] + + +def test_source_aggregation_preserves_group_counts_and_provenance(): + now = time.time() + prompt = "Across all incident reviews, which team owned the most follow-up action items?" + profile = infer_context_preparation_profile(prompt) + artefacts = [ + Artefact( + key="file_summary:runtime.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/incidents/postmortems/runtime-latency.json", + source_mtime=now, + generated_at=now, + model="test", + content="Post-incident review. Follow-up action items assigned to Runtime and SRE.", + ), + Artefact( + key="file_summary:runtime-2.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/incidents/postmortems/rate-limit.json", + source_mtime=now, + generated_at=now, + model="test", + content="Review notes. Owner: Runtime. Action items track throttling defaults.", + ), + Artefact( + key="file_summary:data.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/incidents/postmortems/replay-gap.json", + source_mtime=now, + generated_at=now, + model="test", + content="Incident review. Assigned to Data Platform for replay coverage.", + ), + ] + + aggregate, trace = build_source_aggregation(prompt, artefacts, profile) + + assert aggregate is not None + assert trace.tool == "aggregate_sources" + assert trace.output_count == 1 + assert "Runtime: 2 source(s)" in aggregate.content + assert "Representative provenance:" in aggregate.content + assert aggregate.metadata["aggregation_source_count"] == 3 + + @pytest.mark.asyncio async def test_select_artefacts_fts_uses_semantic_memory_source(tmp_path): store = ArtefactStore(tmp_path / "store.db") @@ -218,6 +281,67 @@ async def test_select_artefacts_fts_uses_semantic_memory_source(tmp_path): assert "semantic_memory" in selected[0].metadata["context_sources"] +@pytest.mark.asyncio +async def test_select_artefacts_fts_injects_prepared_aggregation_for_multi_source_synthesis(tmp_path): + store = ArtefactStore(tmp_path / "store.db") + await store.initialize() + now = time.time() + for artefact in [ + Artefact( + key="file_summary:runtime.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/oncall/postmortems/runtime-latency.json", + source_mtime=now, + generated_at=now, + model="test", + content="Postmortem. Follow-up action items assigned to Runtime and SRE after incident review.", + ), + Artefact( + key="file_summary:runtime-2.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/oncall/postmortems/rate-limit.json", + source_mtime=now, + generated_at=now, + model="test", + content="Postmortem. Owner: Runtime. Action items cover throttling and monitoring.", + ), + Artefact( + key="file_summary:data.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/oncall/postmortems/replay-gap.json", + source_mtime=now, + generated_at=now, + model="test", + content="Postmortem. Assigned to Data Platform for replay and audit coverage.", + ), + Artefact( + key="file_summary:template.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/oncall/postmortems/template.json", + source_mtime=now, + generated_at=now, + model="test", + content="Postmortem template. Assigned team placeholder.", + ), + ]: + await store.upsert(artefact) + diagnostics = [] + + selected = await select_artefacts_fts( + "Across all incident postmortems, which team was assigned the most follow-up action items?", + store, + top_n=4, + capture_prepared_context_diagnostics=diagnostics, + ) + + assert selected[0].key.startswith("prepared_context:source_aggregation:") + assert "Runtime: 2 source(s)" in selected[0].content + assert diagnostics[-1].preparation_plan is not None + assert diagnostics[-1].preparation_plan.name == "multi_source_synthesis" + assert diagnostics[-1].aggregation_count == 1 + assert any(trace.tool == "aggregate_sources" and trace.output_count == 1 for trace in diagnostics[-1].tool_traces) + + def test_date_constraints_report_but_do_not_drop_relevant_evidence_per_document(): now = time.time() prompt = "In March 2026, which deployment offering had the most request timeouts?" From 77338c7ac2f17c191d9baca073c16b3be3eeedcd Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 10:06:18 +0200 Subject: [PATCH 08/22] Strengthen context aggregation provenance --- src/vaner/broker/aggregation.py | 195 ++++++++++++++++++++++-- src/vaner/broker/context_preparation.py | 4 + src/vaner/broker/preparation_policy.py | 6 + src/vaner/broker/selector.py | 33 +++- tests/test_broker/test_selector.py | 80 +++++++++- 5 files changed, 289 insertions(+), 29 deletions(-) diff --git a/src/vaner/broker/aggregation.py b/src/vaner/broker/aggregation.py index b35be8b..59b99d8 100644 --- a/src/vaner/broker/aggregation.py +++ b/src/vaner/broker/aggregation.py @@ -6,6 +6,7 @@ import re import time from collections import Counter, defaultdict +from dataclasses import dataclass from vaner.broker.context_preparation import extract_query_keywords from vaner.models.artefact import Artefact, ArtefactKind @@ -19,7 +20,7 @@ def build_source_aggregation( *, source_by_key: dict[str, set[str]] | None = None, max_sources: int = 40, - provenance_budget: int = 24, + provenance_budget: int | None = None, ) -> tuple[Artefact | None, ContextToolTrace]: """Compress many relevant sources into a provenance-preserving evidence artefact.""" @@ -33,11 +34,15 @@ def build_source_aggregation( trace.notes.append("skipped:dedupe_left_too_few_candidates") return None, trace + resolved_provenance_budget = max_sources if provenance_budget is None else provenance_budget grouped = _group_findings(prompt, source_candidates) facets = _facet_coverage(profile, source_candidates) source_classes = Counter(_source_class(artefact.source_path) for artefact in source_candidates) - source_refs = _source_refs(source_candidates, source_by_key or {}, provenance_budget=provenance_budget) + source_refs = _source_refs(source_candidates, source_by_key or {}, provenance_budget=resolved_provenance_budget) gaps = _coverage_gaps(prompt, profile, source_candidates, grouped) + provenance_truncated = len(source_candidates) > resolved_provenance_budget + if provenance_truncated: + gaps.append("provenance truncated to budget") sections = [ "Prepared source aggregation", @@ -47,6 +52,7 @@ def build_source_aggregation( f"- sources_considered: {len(source_candidates)}", "- source_classes: " + _format_counter(source_classes, limit=8), "- facets_covered: " + (", ".join(facets) if facets else "none detected"), + f"- provenance_coverage: {min(len(source_candidates), resolved_provenance_budget)}/{len(source_candidates)}", "- gaps: " + ("; ".join(gaps) if gaps else "none detected"), "", "Grouped findings:", @@ -60,9 +66,14 @@ def build_source_aggregation( metadata = { "context_sources": ["aggregate_sources"], "aggregation_source_count": len(source_candidates), - "aggregation_source_keys": [artefact.key for artefact in source_candidates[:provenance_budget]], - "aggregation_source_paths": [artefact.source_path for artefact in source_candidates[:provenance_budget]], - "aggregation_groups": [{"value": value, "count": count} for value, count, _ in grouped[:12]], + "aggregation_source_keys": [artefact.key for artefact in source_candidates[:resolved_provenance_budget]], + "aggregation_source_paths": [artefact.source_path for artefact in source_candidates[:resolved_provenance_budget]], + "aggregation_groups": [ + _group_metadata(group, evidence_budget=min(12, resolved_provenance_budget)) for group in grouped[:12] + ], + "aggregation_mode": _aggregation_mode(prompt), + "provenance_coverage_count": min(len(source_candidates), resolved_provenance_budget), + "provenance_truncated": provenance_truncated, "provenance": "prepared_context_aggregation", } trace.output_count = 1 @@ -109,11 +120,28 @@ def _facet_coverage(profile: ContextPreparationProfile, candidates: list[Artefac return list(dict.fromkeys(covered))[:16] +@dataclass(frozen=True) +class AggregationEvidence: + source_key: str + path: str + snippet: str + confidence: float = 0.7 + + +@dataclass(frozen=True) +class AggregationGroup: + value: str + count: int + sources: list[Artefact] + evidence: list[AggregationEvidence] + count_basis: str = "source_occurrence" + + def _coverage_gaps( prompt: str, profile: ContextPreparationProfile, candidates: list[Artefact], - grouped: list[tuple[str, int, list[Artefact]]], + grouped: list[AggregationGroup], ) -> list[str]: text = "\n".join(f"{artefact.source_path}\n{artefact.content}" for artefact in candidates).lower() gaps: list[str] = [] @@ -127,9 +155,14 @@ def _coverage_gaps( return gaps[:8] -def _group_findings(prompt: str, candidates: list[Artefact]) -> list[tuple[str, int, list[Artefact]]]: +def _group_findings(prompt: str, candidates: list[Artefact]) -> list[AggregationGroup]: + if _aggregation_mode(prompt) == "count_extracted_items": + item_groups = _group_extracted_items(candidates) + if item_groups: + return item_groups prompt_keywords = set(extract_query_keywords(prompt, limit=16)) group_to_sources: dict[str, list[Artefact]] = defaultdict(list) + group_to_evidence: dict[str, list[AggregationEvidence]] = defaultdict(list) for artefact in candidates: values = _group_values(artefact.content) if not values and _asks_for_theme_summary(prompt): @@ -137,8 +170,95 @@ def _group_findings(prompt: str, candidates: list[Artefact]) -> list[tuple[str, for value in sorted(values): if artefact not in group_to_sources[value]: group_to_sources[value].append(artefact) - grouped = [(value, len(sources), sources) for value, sources in group_to_sources.items()] - return sorted(grouped, key=lambda item: (-item[1], item[0].lower())) + group_to_evidence[value].append( + AggregationEvidence( + source_key=artefact.key, + path=artefact.source_path, + snippet=_best_snippet(artefact.content, value), + confidence=0.65, + ) + ) + grouped = [ + AggregationGroup( + value=value, + count=len(sources), + sources=sources, + evidence=group_to_evidence[value], + ) + for value, sources in group_to_sources.items() + ] + return sorted(grouped, key=lambda item: (-item.count, item.value.lower())) + + +def _group_extracted_items(candidates: list[Artefact]) -> list[AggregationGroup]: + group_to_sources: dict[str, list[Artefact]] = defaultdict(list) + group_to_evidence: dict[str, list[AggregationEvidence]] = defaultdict(list) + group_counts: Counter[str] = Counter() + for artefact in candidates: + for value, item_text in _extract_owned_items(artefact.content): + group_counts[value] += 1 + if artefact not in group_to_sources[value]: + group_to_sources[value].append(artefact) + group_to_evidence[value].append( + AggregationEvidence( + source_key=artefact.key, + path=artefact.source_path, + snippet=item_text[:260], + confidence=0.82, + ) + ) + grouped = [ + AggregationGroup( + value=value, + count=count, + sources=group_to_sources[value], + evidence=group_to_evidence[value], + count_basis="extracted_item", + ) + for value, count in group_counts.items() + ] + return sorted(grouped, key=lambda item: (-item.count, item.value.lower())) + + +def _extract_owned_items(content: str) -> list[tuple[str, str]]: + items: list[tuple[str, str]] = [] + in_followup_section = False + for raw_line in content.splitlines(): + line = raw_line.strip() + if not line: + continue + if re.match(r"^#{1,4}\s+follow[- ]?up action items\b", line, re.IGNORECASE): + in_followup_section = True + continue + if in_followup_section and line.startswith("#"): + in_followup_section = False + if ( + not in_followup_section + and not re.match(r"^(?:[-*]|\d+[.)])\s+", line) + and not re.search(r"\b(action items?|follow[- ]?up tasks?|tasks?|open items?)\b", line, re.IGNORECASE) + ): + continue + if not re.search(r"\b(owner|assigned|team|dri|responsible)\b", line, re.IGNORECASE): + continue + owners = _owners_from_line(line) + for owner in owners: + items.append((owner, line)) + return items + + +def _owners_from_line(line: str) -> set[str]: + owners: set[str] = set() + patterns = ( + r"\bowner team\s*:\s*([^.;\n]+)", + r"\bowner\s*:\s*([^.;\n]+)", + r"\bassigned to\s+([^.;\n]+)", + r"\bresponsible team\s*:\s*([^.;\n]+)", + r"\bdri\s*:\s*([^.;\n]+)", + ) + for pattern in patterns: + for match in re.finditer(pattern, line, re.IGNORECASE): + owners.update(_split_group_value(match.group(1))) + return {owner for owner in owners if _valid_group_value(owner)} def _group_values(content: str) -> set[str]: @@ -189,18 +309,40 @@ def _asks_for_theme_summary(prompt: str) -> bool: return bool(re.search(r"\b(patterns?|themes?|summarize|synthesize|compare|across all)\b", prompt, re.IGNORECASE)) -def _finding_lines(grouped: list[tuple[str, int, list[Artefact]]]) -> list[str]: +def _finding_lines(grouped: list[AggregationGroup]) -> list[str]: if not grouped: return ["- no grouped finding could be inferred from candidate sources"] lines: list[str] = [] - for value, count, sources in grouped[:12]: - paths = ", ".join(artefact.source_path for artefact in sources[:3]) - if len(sources) > 3: - paths += f", +{len(sources) - 3} more" - lines.append(f"- {value}: {count} source(s); examples: {paths}") + for group in grouped[:12]: + unit = "item(s)" if group.count_basis == "extracted_item" else "source(s)" + paths = ", ".join(artefact.source_path for artefact in group.sources[:3]) + if len(group.sources) > 3: + paths += f", +{len(group.sources) - 3} more" + lines.append(f"- {group.value}: {group.count} {unit}; examples: {paths}") return lines +def _group_metadata(group: AggregationGroup, *, evidence_budget: int) -> dict[str, object]: + evidence = group.evidence[:evidence_budget] + return { + "value": group.value, + "count": group.count, + "count_basis": group.count_basis, + "source_keys": [artefact.key for artefact in group.sources], + "source_paths": [artefact.source_path for artefact in group.sources], + "evidence": [ + { + "source_key": item.source_key, + "path": item.path, + "snippet": item.snippet, + "confidence": item.confidence, + } + for item in evidence + ], + "evidence_truncated": len(group.evidence) > len(evidence), + } + + def _source_refs(candidates: list[Artefact], source_by_key: dict[str, set[str]], *, provenance_budget: int) -> list[str]: refs: list[str] = [] for index, artefact in enumerate(candidates[:provenance_budget], start=1): @@ -216,6 +358,29 @@ def _format_counter(counter: Counter[str], *, limit: int) -> str: return ", ".join(f"{key}={value}" for key, value in counter.most_common(limit)) +def _aggregation_mode(prompt: str) -> str: + if re.search(r"\b(action items?|follow[- ]?up tasks?|tasks?|tickets?|open items?)\b", prompt, re.IGNORECASE) and re.search( + r"\b(count|how many|most|highest number|by team|by owner|which team|which owner|assigned)\b", + prompt, + re.IGNORECASE, + ): + return "count_extracted_items" + if _asks_for_grouped_counts(prompt): + return "count_mentions" + if _asks_for_theme_summary(prompt): + return "summarize_themes" + return "group_by_entity" + + +def _best_snippet(content: str, value: str) -> str: + value_lower = value.lower() + for line in content.splitlines(): + cleaned = " ".join(line.split()) + if value_lower in cleaned.lower(): + return cleaned[:260] + return " ".join(content.split())[:260] + + _GROUP_STOPWORDS = { "and", "assigned", diff --git a/src/vaner/broker/context_preparation.py b/src/vaner/broker/context_preparation.py index a6932c7..0b7a232 100644 --- a/src/vaner/broker/context_preparation.py +++ b/src/vaner/broker/context_preparation.py @@ -130,6 +130,10 @@ def source_path_hint_candidates(prompt: str, available_paths: list[str], *, limi score += 3 if f"/{term}" in path_lower or f"{term}/" in path_lower else 1 if "postmortem" in hint_terms and "/postmortems/" in path_lower: score += 8 + if path_lower.startswith(("confluence/", "docs/", "google_drive/")): + score += 6 + elif path_lower.startswith(("slack/", "gmail/")): + score -= 3 if "gmail" in hint_terms and path_lower.startswith("gmail/"): score += 4 if score: diff --git a/src/vaner/broker/preparation_policy.py b/src/vaner/broker/preparation_policy.py index 62877b3..aba5957 100644 --- a/src/vaner/broker/preparation_policy.py +++ b/src/vaner/broker/preparation_policy.py @@ -83,3 +83,9 @@ def choose_preparation_plan(profile: ContextPreparationProfile, prompt: str) -> def plan_uses_tool(plan: ContextPreparationPlan, tool_name: str) -> bool: return any(step.tool == tool_name for step in plan.steps) + + +def plan_tool_names(plan: ContextPreparationPlan | None) -> set[str]: + if plan is None: + return set() + return {str(step.tool) for step in plan.steps} diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index 37717dd..528a5d8 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -18,7 +18,7 @@ query_variants, source_path_hint_candidates, ) -from vaner.broker.preparation_policy import choose_preparation_plan, plan_uses_tool +from vaner.broker.preparation_policy import choose_preparation_plan, plan_tool_names, plan_uses_tool from vaner.models.artefact import Artefact from vaner.models.context_preparation import ContextPreparationProfile, ContextToolTrace, PreparedContextDiagnostics from vaner.models.decision import ScoreFactor @@ -554,6 +554,7 @@ async def select_artefacts_fts( profile = infer_context_preparation_profile(prompt) context_enabled = context_preparation_mode != "legacy" preparation_plan = choose_preparation_plan(profile, prompt) if context_enabled else None + enabled_context_tools = plan_tool_names(preparation_plan) tool_traces: list[ContextToolTrace] = [] source_by_key: dict[str, set[str]] = {} @@ -657,6 +658,7 @@ async def select_artefacts_fts( capture_drop_reasons=capture_drop_reasons, context_need=profile.need if context_enabled else None, context_profile=profile if context_enabled else None, + enabled_context_tools=enabled_context_tools, ) if context_enabled and coverage_floor_enabled and max_expansion_passes > 0: diagnostics = build_prepared_context_diagnostics( @@ -697,6 +699,7 @@ async def select_artefacts_fts( capture_drop_reasons=capture_drop_reasons, context_need=profile.need, context_profile=profile, + enabled_context_tools=enabled_context_tools, ) if context_enabled and preparation_plan is not None and plan_uses_tool(preparation_plan, "aggregate_sources"): aggregate_budget = _plan_budget(preparation_plan, "aggregate_sources", default=40) @@ -708,6 +711,7 @@ async def select_artefacts_fts( exclude_private=exclude_private, path_excludes=path_excludes or [], source_by_key=source_by_key, + canonical_source_class_enabled="source_class_search" in enabled_context_tools, limit=aggregate_budget, ) aggregate, trace = build_source_aggregation( @@ -751,10 +755,11 @@ async def select_artefacts_fts( path_bonuses=path_bonuses, path_excludes=path_excludes, capture_factors=capture_factors, - capture_drop_reasons=capture_drop_reasons, - context_need=profile.need if context_enabled else None, - context_profile=profile if context_enabled else None, - ) + capture_drop_reasons=capture_drop_reasons, + context_need=profile.need if context_enabled else None, + context_profile=profile if context_enabled else None, + enabled_context_tools=enabled_context_tools, + ) if capture_prepared_context_diagnostics is not None: capture_prepared_context_diagnostics.append( build_prepared_context_diagnostics( @@ -878,6 +883,7 @@ def _aggregation_candidate_pool( exclude_private: bool, path_excludes: list[str], source_by_key: dict[str, set[str]], + canonical_source_class_enabled: bool, limit: int, ) -> list[Artefact]: selected_keys = {artefact.key for artefact in selected} @@ -894,7 +900,8 @@ def _aggregation_candidate_pool( source_signal = 0.7 if sources else 0.0 if sources & {"source_metadata", "semantic_memory", "semantic_query_variants", "coverage_floor"}: source_signal += 1.0 - score = score_artefact(prompt, artefact) + _canonical_source_class_bonus(prompt, artefact, profile) + source_signal + canonical_bonus = _canonical_source_class_bonus(prompt, artefact, profile) if canonical_source_class_enabled else 0.0 + score = score_artefact(prompt, artefact) + canonical_bonus + source_signal if artefact.key in selected_keys: score += 2.0 rows.append((score, artefact)) @@ -1054,12 +1061,14 @@ def select_artefacts( capture_drop_reasons: dict[str, str] | None = None, context_need: str | None = None, context_profile: ContextPreparationProfile | None = None, + enabled_context_tools: set[str] | None = None, ) -> list[Artefact]: preferred_paths = preferred_paths or set() preferred_keys = preferred_keys or set() path_bonuses = path_bonuses or [] path_excludes = path_excludes or [] apply_origin_rerank = _is_origin_question(prompt) + enabled_context_tools = enabled_context_tools or set() scored_rows: list[tuple[float, Artefact]] = [] for artefact in artefacts: @@ -1155,7 +1164,11 @@ def select_artefacts( ) ) score += constraint_bonus - canonical_bonus = _canonical_source_class_bonus(prompt, artefact, context_profile) + canonical_bonus = ( + _canonical_source_class_bonus(prompt, artefact, context_profile) + if "source_class_search" in enabled_context_tools + else 0.0 + ) if canonical_bonus: factors.append( ScoreFactor( @@ -1165,7 +1178,11 @@ def select_artefacts( ) ) score += canonical_bonus - scheduling_bonus = _scheduling_evidence_bonus(prompt, artefact, context_profile) + scheduling_bonus = ( + _scheduling_evidence_bonus(prompt, artefact, context_profile) + if {"time_entity_extraction", "conflict_scan"} & enabled_context_tools + else 0.0 + ) if scheduling_bonus: factors.append( ScoreFactor( diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 586c2c9..aa9ed12 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -8,7 +8,7 @@ from vaner.broker.aggregation import build_source_aggregation from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants, source_path_hint_candidates -from vaner.broker.preparation_policy import choose_preparation_plan +from vaner.broker.preparation_policy import choose_preparation_plan, plan_tool_names from vaner.broker.selector import select_artefacts, select_artefacts_fts from vaner.models.artefact import Artefact, ArtefactKind from vaner.store.artefacts import ArtefactStore @@ -236,9 +236,18 @@ def test_source_aggregation_preserves_group_counts_and_provenance(): assert aggregate is not None assert trace.tool == "aggregate_sources" assert trace.output_count == 1 - assert "Runtime: 2 source(s)" in aggregate.content + assert "Runtime: 2 item(s)" in aggregate.content assert "Representative provenance:" in aggregate.content assert aggregate.metadata["aggregation_source_count"] == 3 + runtime_group = next(group for group in aggregate.metadata["aggregation_groups"] if group["value"] == "Runtime") + assert runtime_group["count_basis"] == "extracted_item" + assert runtime_group["source_keys"] == ["file_summary:runtime.json", "file_summary:runtime-2.json"] + assert {item["source_key"] for item in runtime_group["evidence"]} == { + "file_summary:runtime.json", + "file_summary:runtime-2.json", + } + assert aggregate.metadata["provenance_coverage_count"] == 3 + assert aggregate.metadata["provenance_truncated"] is False @pytest.mark.asyncio @@ -335,7 +344,7 @@ async def test_select_artefacts_fts_injects_prepared_aggregation_for_multi_sourc ) assert selected[0].key.startswith("prepared_context:source_aggregation:") - assert "Runtime: 2 source(s)" in selected[0].content + assert "Runtime: 2 item(s)" in selected[0].content assert diagnostics[-1].preparation_plan is not None assert diagnostics[-1].preparation_plan.name == "multi_source_synthesis" assert diagnostics[-1].aggregation_count == 1 @@ -358,7 +367,14 @@ def test_date_constraints_report_but_do_not_drop_relevant_evidence_per_document( ) ] - selected = select_artefacts(prompt, artefacts, top_n=1, context_need=profile.need, context_profile=profile) + selected = select_artefacts( + prompt, + artefacts, + top_n=1, + context_need=profile.need, + context_profile=profile, + enabled_context_tools=plan_tool_names(choose_preparation_plan(profile, prompt)), + ) assert selected[0].key == "file_summary:timeout.md" @@ -499,7 +515,14 @@ def test_multi_source_aggregation_prefers_canonical_postmortem_docs(): prompt = "Across all incident postmortems, which team was assigned the most follow-up action items?" profile = infer_context_preparation_profile(prompt) - selected = select_artefacts(prompt, artefacts, top_n=1, context_need=profile.need, context_profile=profile) + selected = select_artefacts( + prompt, + artefacts, + top_n=1, + context_need=profile.need, + context_profile=profile, + enabled_context_tools=plan_tool_names(choose_preparation_plan(profile, prompt)), + ) assert selected[0].key == "file_summary:canonical.json" @@ -535,11 +558,56 @@ def test_scheduling_queries_prefer_confirmed_invite_context(): ) profile = infer_context_preparation_profile(prompt) - selected = select_artefacts(prompt, artefacts, top_n=1, context_need=profile.need, context_profile=profile) + selected = select_artefacts( + prompt, + artefacts, + top_n=1, + context_need=profile.need, + context_profile=profile, + enabled_context_tools=plan_tool_names(choose_preparation_plan(profile, prompt)), + ) assert selected[0].key == "file_summary:confirmed-private-review.json" +def test_context_specific_rank_boosts_require_enabled_policy_tools(): + now = time.time() + prompt = "When is the technical deep dive scheduled with the customer?" + profile = infer_context_preparation_profile(prompt) + artefact = Artefact( + key="file_summary:confirmed.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="gmail/solutions/architecture-review-booking.json", + source_mtime=now, + generated_at=now, + model="test", + content="Confirming Tue Oct 24, 10:00-11:15 PT. Invite attached.", + ) + factors_without_policy: dict[str, list] = {} + factors_with_policy: dict[str, list] = {} + + select_artefacts( + prompt, + [artefact], + top_n=1, + context_need=profile.need, + context_profile=profile, + capture_factors=factors_without_policy, + ) + select_artefacts( + prompt, + [artefact], + top_n=1, + context_need=profile.need, + context_profile=profile, + capture_factors=factors_with_policy, + enabled_context_tools=plan_tool_names(choose_preparation_plan(profile, prompt)), + ) + + assert not any(factor.name == "scheduling_evidence" for factor in factors_without_policy[artefact.key]) + assert any(factor.name == "scheduling_evidence" for factor in factors_with_policy[artefact.key]) + + def test_select_artefacts_origin_rerank_prefers_definition_files(): artefacts = [ Artefact( From 9c4d23110b5819cfa440c8c9bfae1b6d7e8adf61 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 12:33:24 +0200 Subject: [PATCH 09/22] Add scheduling evidence preparation --- src/vaner/broker/preparation_policy.py | 1 + src/vaner/broker/scheduling.py | 216 ++++++++++++++++++++++++ src/vaner/broker/selector.py | 42 ++--- src/vaner/models/context_preparation.py | 1 + tests/test_broker/test_selector.py | 57 +++++++ 5 files changed, 290 insertions(+), 27 deletions(-) create mode 100644 src/vaner/broker/scheduling.py diff --git a/src/vaner/broker/preparation_policy.py b/src/vaner/broker/preparation_policy.py index aba5957..ebc9126 100644 --- a/src/vaner/broker/preparation_policy.py +++ b/src/vaner/broker/preparation_policy.py @@ -33,6 +33,7 @@ def choose_preparation_plan(profile: ContextPreparationProfile, prompt: str) -> ContextToolStep(tool="source_class_search", mode="calendar_invite_preferred", budget=64), ContextToolStep(tool="semantic_search", mode="query_variants", budget=64), ContextToolStep(tool="conflict_scan", mode="time_evidence", budget=16), + ContextToolStep(tool="prepare_scheduling_evidence", mode="confirmed_time_source", budget=24), ContextToolStep(tool="coverage_check", mode="facets_constraints", budget=0), ], ) diff --git a/src/vaner/broker/scheduling.py b/src/vaner/broker/scheduling.py new file mode 100644 index 0000000..19b3f20 --- /dev/null +++ b/src/vaner/broker/scheduling.py @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import re +import time +from dataclasses import dataclass + +from vaner.models.artefact import Artefact, ArtefactKind +from vaner.models.context_preparation import ContextPreparationProfile, ContextToolTrace + + +@dataclass(frozen=True) +class SchedulingEvidence: + source_key: str + path: str + snippet: str + score: float + time_windows: list[str] + confidence: float + status: str = "candidate" + + +def build_scheduling_evidence( + prompt: str, + candidates: list[Artefact], + profile: ContextPreparationProfile, + *, + max_sources: int = 24, +) -> tuple[Artefact | None, ContextToolTrace]: + """Prepare compact meeting/time evidence from scheduling candidates.""" + + trace = ContextToolTrace(tool="prepare_scheduling_evidence", input_count=len(candidates)) + if profile.need != "scheduling": + trace.notes.append("skipped:not_scheduling") + return None, trace + evidence = [ + item + for item in ( + _scheduling_evidence_for(prompt, artefact, profile) + for artefact in _dedupe_candidates(candidates) + ) + if item is not None + ] + evidence.sort(key=lambda item: (-item.score, item.path)) + evidence = evidence[:max_sources] + if not evidence: + trace.notes.append("skipped:no_scheduling_evidence") + return None, trace + + sections = [ + "Prepared scheduling evidence", + f"Question: {prompt.strip()}", + "", + "Candidate time evidence:", + *_evidence_lines(evidence), + "", + "Coverage:", + f"- sources_considered: {len(candidates)}", + f"- evidence_sources: {len(evidence)}", + "- gaps: " + ("none detected" if any(item.time_windows for item in evidence) else "no explicit time window found"), + ] + digest = hashlib.sha256((prompt + "\n" + "\n".join(item.source_key for item in evidence)).encode("utf-8")).hexdigest()[:16] + metadata = { + "context_sources": ["prepare_scheduling_evidence"], + "scheduling_evidence_source_keys": [item.source_key for item in evidence], + "scheduling_evidence_source_paths": [item.path for item in evidence], + "scheduling_evidence": [ + { + "source_key": item.source_key, + "path": item.path, + "snippet": item.snippet, + "score": item.score, + "time_windows": item.time_windows, + "confidence": item.confidence, + "status": item.status, + } + for item in evidence + ], + "provenance": "prepared_scheduling_evidence", + } + trace.output_count = 1 + trace.notes.append(f"evidence_sources:{len(evidence)}") + trace.notes.append(f"top_score:{evidence[0].score:.1f}") + return ( + Artefact( + key=f"prepared_context:scheduling_evidence:{digest}", + kind=ArtefactKind.FILE_SUMMARY, + source_path=f"prepared_context/scheduling_evidence/{digest}.md", + source_mtime=time.time(), + generated_at=time.time(), + model="vaner-context-tools", + content="\n".join(sections).strip(), + metadata=metadata, + ), + trace, + ) + + +def score_scheduling_evidence(prompt: str, artefact: Artefact, profile: ContextPreparationProfile | None = None) -> float: + item = _scheduling_evidence_for(prompt, artefact, profile) + return item.score if item is not None else 0.0 + + +def _scheduling_evidence_for( + prompt: str, + artefact: Artefact, + profile: ContextPreparationProfile | None = None, +) -> SchedulingEvidence | None: + lowered = prompt.lower() + if profile is not None and profile.need != "scheduling": + return None + if not re.search(r"\b(when|scheduled|schedule|calendar|invite|meeting|time window|booking|booked)\b", lowered): + return None + path = artefact.source_path.lower() + text = f"{artefact.source_path}\n{artefact.content}" + text_lower = text.lower() + score = 0.0 + if path.startswith(("gmail/", "calendar/")): + score += 8.0 + elif path.startswith(("hubspot/", "fireflies/")): + score -= 2.0 + + time_windows = _time_windows(text) + if time_windows: + score += min(18.0, 8.0 + 3.0 * len(time_windows)) + if re.search(r"\b(?:pt|pst|pdt|pacific|-0[78]00)\b", text_lower): + score += 6.0 + if any(term in text_lower for term in ("calendar", "invite", ".ics", "calendar.redwood", "event/")): + score += 9.0 + status = "candidate" + if any(term in text_lower for term in ("works for our team", "confirmed", "confirming", "accepted", "locked in")): + score += 12.0 + status = "confirmed" + elif any(term in text_lower for term in ("proposed", "tentative", "hold", "requesting")): + score += 2.0 + status = "tentative" + + if "technical deep dive" in lowered and "technical deep dive" in text_lower: + score += 6.0 + if "architecture" in lowered and "architecture review" in text_lower: + score += 5.0 + if re.search(r"\b(isolated network|private|vpc|on-prem|own network)\b", lowered) and any( + term in text_lower for term in ("private", "vpc", "on-prem", "isolated", "network diagram", "private hosting") + ): + score += 8.0 + if re.search(r"\b(healthcare|health care|clinical|medical|hospital)\b", lowered) and any( + term in text_lower for term in ("health", "clinical", "medical", "patient", "phi", "hipaa") + ): + score += 10.0 + + numeric_terms = set(re.findall(r"\b\d{2,4}\b", lowered)) + if numeric_terms: + score += min(6.0, 2.0 * sum(1 for term in numeric_terms if term in text_lower)) + if not time_windows and score < 18.0: + return None + confidence = min(0.95, 0.35 + score / 80.0) + return SchedulingEvidence( + source_key=artefact.key, + path=artefact.source_path, + snippet=_best_scheduling_snippet(artefact.content), + score=score, + time_windows=time_windows, + confidence=confidence, + status=status, + ) + + +def _time_windows(text: str) -> list[str]: + patterns = ( + r"\b(?:mon|tue|wed|thu|fri|sat|sun)[a-z]*\s+[a-z]{3,9}\s+\d{1,2}\s*[—-]\s*\d{1,2}:\d{2}\s*[–-]\s*\d{1,2}:\d{2}(?:\s*[ap]m)?(?:\s*(?:pt|pst|pdt|pacific))?", + r"\b(?:mon|tue|wed|thu|fri|sat|sun)[a-z]*\s+[a-z]{3,9}\s+\d{1,2}\s+\d{1,2}:\d{2}(?:\s*[ap]m)?(?:\s*(?:pt|pst|pdt|pacific))?", + r"\b\d{1,2}:\d{2}\s*(?:am|pm)?\s*[–-]\s*\d{1,2}:\d{2}\s*(?:am|pm)?(?:\s*(?:pt|pst|pdt|pacific))?", + ) + windows: list[str] = [] + for pattern in patterns: + for match in re.finditer(pattern, text, flags=re.IGNORECASE): + value = " ".join(match.group(0).split()) + if value not in windows: + windows.append(value) + return windows[:8] + + +def _best_scheduling_snippet(content: str) -> str: + lines = [" ".join(line.split()) for line in content.splitlines() if line.strip()] + preferred = [ + line + for line in lines + if re.search( + r"\b(works for our team|confirmed|confirming|invite|calendar|scheduled|proposed|pt|pst|pdt|\d{1,2}:\d{2})\b", + line, + re.IGNORECASE, + ) + ] + return "\n".join(preferred[:5])[:700] if preferred else " ".join(content.split())[:700] + + +def _evidence_lines(evidence: list[SchedulingEvidence]) -> list[str]: + lines = [] + for index, item in enumerate(evidence[:8], start=1): + windows = "; ".join(item.time_windows) if item.time_windows else "time not explicit" + lines.append(f"- [{index}] {item.status} score={item.score:.1f} time={windows} source={item.path}") + lines.append(f" evidence: {item.snippet[:320]}") + return lines + + +def _dedupe_candidates(candidates: list[Artefact]) -> list[Artefact]: + seen: set[str] = set() + deduped: list[Artefact] = [] + for artefact in candidates: + if artefact.key in seen: + continue + seen.add(artefact.key) + deduped.append(artefact) + return deduped diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index 528a5d8..83f1d82 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -19,6 +19,7 @@ source_path_hint_candidates, ) from vaner.broker.preparation_policy import choose_preparation_plan, plan_tool_names, plan_uses_tool +from vaner.broker.scheduling import build_scheduling_evidence, score_scheduling_evidence from vaner.models.artefact import Artefact from vaner.models.context_preparation import ContextPreparationProfile, ContextToolTrace, PreparedContextDiagnostics from vaner.models.decision import ScoreFactor @@ -726,6 +727,19 @@ async def select_artefacts_fts( _record_source_ranking(source_rankings, source_by_key, "aggregate_sources", [aggregate.key]) aggregate = _with_context_source_metadata(aggregate, source_rankings, source_by_key) selected = [aggregate, *[artefact for artefact in selected if artefact.key != aggregate.key]][:top_n] + if context_enabled and preparation_plan is not None and plan_uses_tool(preparation_plan, "prepare_scheduling_evidence"): + schedule_budget = _plan_budget(preparation_plan, "prepare_scheduling_evidence", default=24) + scheduling, trace = build_scheduling_evidence( + prompt, + candidates, + profile, + max_sources=schedule_budget, + ) + tool_traces.append(trace) + if scheduling is not None: + _record_source_ranking(source_rankings, source_by_key, "prepare_scheduling_evidence", [scheduling.key]) + scheduling = _with_context_source_metadata(scheduling, source_rankings, source_by_key) + selected = [scheduling, *[artefact for artefact in selected if artefact.key != scheduling.key]][:top_n] _append_source_factors(capture_factors, source_by_key, selected) if capture_prepared_context_diagnostics is not None: capture_prepared_context_diagnostics.append( @@ -1018,33 +1032,7 @@ def _canonical_source_class_bonus(prompt: str, artefact: Artefact, profile: Cont def _scheduling_evidence_bonus(prompt: str, artefact: Artefact, profile: ContextPreparationProfile | None) -> float: - lowered = prompt.lower() - if profile is None or not re.search(r"\b(when|scheduled|schedule|calendar|invite|meeting|time window|booking|booked)\b", lowered): - return 0.0 - if not re.search(r"\b(client|customer|call|review|demo|technical|architecture|workshop)\b", lowered): - return 0.0 - path = artefact.source_path.lower() - text = _artefact_context_text(artefact)[:9000] - bonus = 0.0 - if path.startswith(("gmail/", "calendar/")): - bonus += 4.0 - elif path.startswith(("hubspot/", "fireflies/")) and re.search(r"\b(when|scheduled|time window|booking|booked)\b", lowered): - bonus -= 2.0 - if any(term in text for term in ("calendar", "invite", ".ics", "event/", "scheduled", "confirming", "works for our team")): - bonus += 5.0 - if "technical deep dive" in lowered and "technical deep dive" in text and "architecture review" in text: - bonus += 4.0 - if "isolated network" in lowered and "private" in text and any( - term in text for term in ("vpc", "on-prem", "isolated", "private hosting") - ): - bonus += 5.0 - numeric_terms = set(re.findall(r"\b\d{2,4}\b", lowered)) - matched_numeric = sum(1 for term in numeric_terms if term in text) - if matched_numeric: - bonus += min(4.0, matched_numeric * 2.0) - if re.search(r"\b(pacific|pt|time window)\b", lowered) and re.search(r"\b(?:pt|pst|pdt|-0[78]00|\d{1,2}:\d{2})\b", text): - bonus += 3.0 - return bonus + return score_scheduling_evidence(prompt, artefact, profile) def select_artefacts( diff --git a/src/vaner/models/context_preparation.py b/src/vaner/models/context_preparation.py index 0006720..9c1cd86 100644 --- a/src/vaner/models/context_preparation.py +++ b/src/vaner/models/context_preparation.py @@ -33,6 +33,7 @@ "working_set_lookup", "prepared_memory_lookup", "aggregate_sources", + "prepare_scheduling_evidence", "coverage_check", "conflict_scan", "test_lookup", diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index aa9ed12..33866c1 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -9,6 +9,7 @@ from vaner.broker.aggregation import build_source_aggregation from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants, source_path_hint_candidates from vaner.broker.preparation_policy import choose_preparation_plan, plan_tool_names +from vaner.broker.scheduling import build_scheduling_evidence from vaner.broker.selector import select_artefacts, select_artefacts_fts from vaner.models.artefact import Artefact, ArtefactKind from vaner.store.artefacts import ArtefactStore @@ -197,6 +198,22 @@ def test_preparation_policy_names_bounded_context_tools(): ] +def test_preparation_policy_names_scheduling_evidence_tool(): + prompt = "When is the technical deep dive scheduled with the healthcare client?" + profile = infer_context_preparation_profile(prompt) + plan = choose_preparation_plan(profile, prompt) + + assert profile.need == "scheduling" + assert [step.tool for step in plan.steps] == [ + "time_entity_extraction", + "source_class_search", + "semantic_search", + "conflict_scan", + "prepare_scheduling_evidence", + "coverage_check", + ] + + def test_source_aggregation_preserves_group_counts_and_provenance(): now = time.time() prompt = "Across all incident reviews, which team owned the most follow-up action items?" @@ -250,6 +267,46 @@ def test_source_aggregation_preserves_group_counts_and_provenance(): assert aggregate.metadata["provenance_truncated"] is False +def test_scheduling_evidence_prepares_confirmed_time_source(): + now = time.time() + prompt = ( + "When is the 60 to 90 minute technical deep dive scheduled with the healthcare client " + "about running model serving inside their own isolated network, and what is the time window in Pacific time?" + ) + profile = infer_context_preparation_profile(prompt) + artefacts = [ + Artefact( + key="file_summary:generic.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="gmail/sales/generic-private-review.json", + source_mtime=now, + generated_at=now, + model="test", + content="Proposed private hosting review slots next month. No confirmed attendees yet.", + ), + Artefact( + key="file_summary:confirmed.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="gmail/solutions/architecture-review-booking-next-steps-cytohealth.json", + source_mtime=now, + generated_at=now, + model="test", + content=( + "CytoHealth private VPC architecture review. Tue Oct 24 10:00-11:15 PT works for our team. " + "Invite attached invite-20281024.ics. PHI-sensitive workloads and isolated network deployment." + ), + ), + ] + + scheduling, trace = build_scheduling_evidence(prompt, artefacts, profile) + + assert scheduling is not None + assert trace.output_count == 1 + assert scheduling.metadata["scheduling_evidence_source_keys"][0] == "file_summary:confirmed.json" + assert scheduling.metadata["scheduling_evidence"][0]["status"] == "confirmed" + assert "10:00-11:15 PT" in scheduling.content + + @pytest.mark.asyncio async def test_select_artefacts_fts_uses_semantic_memory_source(tmp_path): store = ArtefactStore(tmp_path / "store.db") From 5a615b30ca199f7ef3c1537539d0a19d773eafa0 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 12:37:49 +0200 Subject: [PATCH 10/22] Document benchmark development guardrails --- ARCHITECTURE.md | 9 ++ docs/benchmarks/README.md | 6 ++ docs/benchmarks/public-methodology.md | 7 ++ docs/context-preparation.md | 124 ++++++++++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 docs/context-preparation.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c21a68f..3202b43 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -42,6 +42,15 @@ Vaner is a local-first predictive context engine for coding assistants. - `vaner why [decision-id] [--list|--verbose|--json]` - `vaner query --explain [--verbose|--json]` +## Context preparation + +Vaner prepares context through explicit needs, bounded preparation policies, +general context tools, traces, and provenance-preserving packages. Benchmark +work must improve these reusable product capabilities instead of teaching the +engine dataset-specific routes or answer patterns. See +[`docs/context-preparation.md`](docs/context-preparation.md) for the current +architecture notes and benchmark-driven development guardrails. + ## Claude Code plugin packaging Vaner ships a Claude Code plugin under `plugins/vaner/`, catalogued by a repo-root marketplace at `.claude-plugin/marketplace.json`. The plugin is a thin wrapper — it registers the `vaner` MCP server, the `vaner-feedback` skill, and a SessionStart hook that checks whether the `vaner` CLI is on PATH. The actual code still ships via PyPI and `scripts/install.sh`; the plugin only wires Claude Code to the installed CLI. Version parity between `pyproject.toml`, `plugins/vaner/.claude-plugin/plugin.json`, and the marketplace entry is enforced by `scripts/bump-plugin-version.sh` (pre-commit and CI). diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index ac8947d..e709e91 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -39,6 +39,12 @@ but doesn't measure whether the briefing actually helps the backend LLM produce a better answer. File-recall is still reported as a secondary diagnostic but is not the ship gate. +Benchmark misses should be treated as diagnostic evidence for general +context-preparation failures, not as invitations to encode benchmark-specific +routes or answer patterns. See +[`docs/context-preparation.md`](../context-preparation.md) for the current +benchmark-driven development guardrails. + ## Methodology See the evaluation harness for the session replay runner. diff --git a/docs/benchmarks/public-methodology.md b/docs/benchmarks/public-methodology.md index 44365ba..c872e8f 100644 --- a/docs/benchmarks/public-methodology.md +++ b/docs/benchmarks/public-methodology.md @@ -13,5 +13,12 @@ path coverage, latency, estimated or provider-reported cost, and archetype-level strengths and weaknesses. Public reports may name public corpora, public repos, models, and aggregate metrics. +Benchmark-driven product work must follow the context-preparation guardrails in +[`docs/context-preparation.md`](../context-preparation.md): benchmarks are +diagnostic lenses, and fixes should name the general failure mode, improve a +reusable Vaner capability, report product-shaped metrics, avoid dataset-only +product logic, and run context-specific behavior through explicit policies or +tools. + Private training data, learned priors, raw prompts, judge transcripts, policy weights, and internal failure traces remain outside the public repository. diff --git a/docs/context-preparation.md b/docs/context-preparation.md new file mode 100644 index 0000000..2c3a1a8 --- /dev/null +++ b/docs/context-preparation.md @@ -0,0 +1,124 @@ +# Context Preparation Architecture + +Vaner is a context prediction and preparation engine. Retrieval is one +mechanism inside that engine, not the product abstraction. + +The intended preparation spine is: + +```text +ContextNeed +-> PreparationPolicy +-> ContextTools +-> ContextToolTrace +-> ContextPackage +``` + +The core should infer what kind of context the next step needs, choose a +bounded preparation policy, execute general context tools, preserve provenance, +record coverage and gaps, and assemble the smallest sufficient context package. + +## Benchmark-Driven Development Guardrails + +EnterpriseRAG and similar benchmarks should be used as diagnostic lenses, not +as product targets. A benchmark miss is useful when it reveals a general Vaner +context-preparation failure. Fixes should improve Vaner's reusable preparation +engine, not encode dataset-specific shortcuts. + +For any benchmark-driven change, the PR should answer the questions below. + +### 1. What Is The General Failure Mode? + +Name the underlying product failure, not only the benchmark case. + +Good examples: + +- multi-source synthesis exceeded the raw context budget; +- scheduling evidence was discovered but not selected; +- source-class expansion missed canonical documents; +- aggregation lacked row-level provenance; +- conflict or absence evidence was not represented in the final package. + +Bad examples: + +- `qst_0437` needs these documents; +- `qst_0184` expects this Gmail path; +- EnterpriseRAG asks this exact question shape. + +### 2. Why Is The Capability Non-Benchmark-Specific? + +The change should apply to normal Vaner use cases outside the benchmark. + +Examples: + +- `aggregate_sources` should work for postmortems, customer feedback, research + notes, incidents, meeting notes, and similar broad synthesis tasks. +- `prepare_scheduling_evidence` should work for invites, calendars, email + threads, meeting notes, time windows, and timezone evidence generally. +- source-class hints should be a general candidate expansion mechanism, not a + hidden shortcut to benchmark documents. + +### 3. Which Product-Shaped Metrics Improved? + +Do not report only raw benchmark recall if the product path is richer than raw +top-k selection. + +Prefer metrics such as: + +- raw selected document recall; +- aggregation input recall; +- aggregation provenance recall; +- scheduling evidence recall; +- final package support coverage; +- coverage and gap detection quality; +- tool trace correctness; +- provenance completeness. + +For multi-source synthesis, raw selected documents may not be the main success +path. Aggregation input and provenance coverage may be the more relevant +measures. + +### 4. Is Product Code Free Of Dataset-Only Language? + +Product code should not contain: + +- benchmark question IDs; +- expected benchmark document IDs; +- dataset-specific source names; +- hardcoded answer shapes for one benchmark; +- routing keyed to EnterpriseRAG-specific phrasing. + +Dataset-specific logic belongs in tests, fixtures, harness code, or +diagnostics, not in Vaner's product engine. + +### 5. Is Context-Specific Behavior Behind Explicit Policies Or Tools? + +Context-specific behavior should be invoked by the selected preparation policy, +not hidden as always-on selector behavior. + +Examples: + +- scheduling ranking belongs behind the scheduling policy or tool path; +- aggregation belongs behind `multi_source_synthesis`; +- source-class and canonical ranking belong behind source-class or + canonical-rank tools; +- conflict logic belongs behind `conflict_resolution`; +- absence logic belongs behind `absence_check`. + +The selector should become less magical over time. The preparation trace should +explain why a behavior ran. + +## Review Standard + +A benchmark-driven PR is acceptable if it can clearly show: + +```text +benchmark miss +-> general failure mode +-> reusable Vaner capability +-> product-shaped metric improvement +-> no dataset-specific product logic +-> explicit policy/tool path +``` + +A benchmark-driven PR is not acceptable if it mainly teaches Vaner a +benchmark-specific route, source, or answer pattern. From e69857f78cc178ef7bc4ab744c353efa0151efb2 Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 13:49:57 +0200 Subject: [PATCH 11/22] Normalize aggregation owner team labels --- src/vaner/broker/aggregation.py | 8 +++++ tests/test_broker/test_selector.py | 55 ++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/src/vaner/broker/aggregation.py b/src/vaner/broker/aggregation.py index 59b99d8..e0db677 100644 --- a/src/vaner/broker/aggregation.py +++ b/src/vaner/broker/aggregation.py @@ -249,6 +249,7 @@ def _extract_owned_items(content: str) -> list[tuple[str, str]]: def _owners_from_line(line: str) -> set[str]: owners: set[str] = set() patterns = ( + r"\bowning team\s*:\s*([^.;\n]+)", r"\bowner team\s*:\s*([^.;\n]+)", r"\bowner\s*:\s*([^.;\n]+)", r"\bassigned to\s+([^.;\n]+)", @@ -281,6 +282,8 @@ def _split_group_value(raw: str) -> set[str]: def _normalize_group_value(value: str) -> str: tokens = [token for token in re.findall(r"[A-Za-z0-9_-]+", value) if token.lower() not in _GROUP_STOPWORDS] + while len(tokens) > 1 and tokens[0].lower() in _GROUP_PREFIX_STOPWORDS: + tokens.pop(0) if not tokens: return "" return " ".join(token.upper() if token.isupper() else token[:1].upper() + token[1:].lower() for token in tokens[:4]) @@ -400,6 +403,11 @@ def _best_snippet(content: str, value: str) -> str: "for", } +_GROUP_PREFIX_STOPWORDS = { + "eng", + "engineering", +} + _THEME_STOPWORDS = { "that", "this", diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 33866c1..ad1c9ce 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -267,6 +267,61 @@ def test_source_aggregation_preserves_group_counts_and_provenance(): assert aggregate.metadata["provenance_truncated"] is False +def test_source_aggregation_normalizes_owning_team_prefixes(): + now = time.time() + prompt = "Across all incident reviews, which team owned the most follow-up action items?" + profile = infer_context_preparation_profile(prompt) + artefacts = [ + Artefact( + key="file_summary:control-plane.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/incidents/postmortems/control-plane.json", + source_mtime=now, + generated_at=now, + model="test", + content=( + "## Follow-up action items\n" + "1) Add schema validation. Owning team: Eng Platform.\n" + "2) Add rollback smoke test. Owning team: Eng Platform.\n" + ), + ), + Artefact( + key="file_summary:streaming.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/incidents/postmortems/streaming.json", + source_mtime=now, + generated_at=now, + model="test", + content=( + "## Follow-up action items\n" + "1) Add stream stall alert. Owning team: Eng SRE.\n" + "2) Document gateway ownership. Owner team: Platform.\n" + ), + ), + Artefact( + key="file_summary:rbac.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="confluence/incidents/postmortems/rbac.json", + source_mtime=now, + generated_at=now, + model="test", + content=( + "## Follow-up action items\n" + "1) Add policy diff approval. Owner team: Security.\n" + ), + ), + ] + + aggregate, _trace = build_source_aggregation(prompt, artefacts, profile) + + assert aggregate is not None + groups = {group["value"]: group for group in aggregate.metadata["aggregation_groups"]} + assert groups["Platform"]["count"] == 3 + assert groups["SRE"]["count"] == 1 + assert "Eng Platform" not in groups + assert "Eng SRE" not in groups + + def test_scheduling_evidence_prepares_confirmed_time_source(): now = time.time() prompt = ( From 287d2430da8c996d237c041a31813a9678ff982f Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 14:02:14 +0200 Subject: [PATCH 12/22] Add structured aggregation extraction rows --- src/vaner/broker/aggregation.py | 372 +++++++++++++++++++++++++---- tests/test_broker/test_selector.py | 60 ++++- 2 files changed, 391 insertions(+), 41 deletions(-) diff --git a/src/vaner/broker/aggregation.py b/src/vaner/broker/aggregation.py index e0db677..bb31857 100644 --- a/src/vaner/broker/aggregation.py +++ b/src/vaner/broker/aggregation.py @@ -6,7 +6,7 @@ import re import time from collections import Counter, defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field from vaner.broker.context_preparation import extract_query_keywords from vaner.models.artefact import Artefact, ArtefactKind @@ -35,7 +35,8 @@ def build_source_aggregation( return None, trace resolved_provenance_budget = max_sources if provenance_budget is None else provenance_budget - grouped = _group_findings(prompt, source_candidates) + extraction_spec = _infer_extraction_spec(prompt) + grouped, extracted_rows = _group_findings(prompt, source_candidates, extraction_spec) facets = _facet_coverage(profile, source_candidates) source_classes = Counter(_source_class(artefact.source_path) for artefact in source_candidates) source_refs = _source_refs(source_candidates, source_by_key or {}, provenance_budget=resolved_provenance_budget) @@ -55,6 +56,12 @@ def build_source_aggregation( f"- provenance_coverage: {min(len(source_candidates), resolved_provenance_budget)}/{len(source_candidates)}", "- gaps: " + ("; ".join(gaps) if gaps else "none detected"), "", + "Extraction:", + f"- item_type: {extraction_spec.item_type}", + f"- group_by: {extraction_spec.group_by}", + f"- count_basis: {extraction_spec.count_basis}", + f"- extracted_row_count: {len(extracted_rows)}", + "", "Grouped findings:", *_finding_lines(grouped), "", @@ -72,6 +79,11 @@ def build_source_aggregation( _group_metadata(group, evidence_budget=min(12, resolved_provenance_budget)) for group in grouped[:12] ], "aggregation_mode": _aggregation_mode(prompt), + "aggregation_extraction_spec": _extraction_spec_metadata(extraction_spec), + "count_basis": extraction_spec.count_basis, + "extracted_row_count": len(extracted_rows), + "extracted_rows": [_extracted_row_metadata(row) for row in extracted_rows[:resolved_provenance_budget]], + "extraction_confidence": _mean_confidence(extracted_rows), "provenance_coverage_count": min(len(source_candidates), resolved_provenance_budget), "provenance_truncated": provenance_truncated, "provenance": "prepared_context_aggregation", @@ -79,6 +91,9 @@ def build_source_aggregation( trace.output_count = 1 trace.notes.append(f"sources_considered:{len(source_candidates)}") trace.notes.append(f"groups:{len(grouped)}") + trace.notes.append(f"count_basis:{extraction_spec.count_basis}") + trace.notes.append(f"item_type:{extraction_spec.item_type}") + trace.notes.append(f"extracted_rows:{len(extracted_rows)}") return ( Artefact( key=f"prepared_context:source_aggregation:{digest}", @@ -120,6 +135,24 @@ def _facet_coverage(profile: ContextPreparationProfile, candidates: list[Artefac return list(dict.fromkeys(covered))[:16] +@dataclass(frozen=True) +class AggregationExtractionSpec: + item_type: str = "generic_theme" + scope_hints: list[str] = field(default_factory=list) + group_by: str = "theme" + count_basis: str = "source_count" + evidence_required: list[str] = field( + default_factory=lambda: [ + "source_key", + "source_path", + "snippet", + "extracted_text", + "normalized_group", + "confidence", + ] + ) + + @dataclass(frozen=True) class AggregationEvidence: source_key: str @@ -128,13 +161,29 @@ class AggregationEvidence: confidence: float = 0.7 +@dataclass(frozen=True) +class ExtractedAggregationRow: + item_type: str + extracted_text: str + source_key: str + source_path: str + snippet: str + group_type: str + group_raw_value: str + group_normalized_value: str + status: str | None = None + date: str | None = None + confidence: float = 0.7 + + @dataclass(frozen=True) class AggregationGroup: value: str count: int sources: list[Artefact] evidence: list[AggregationEvidence] - count_basis: str = "source_occurrence" + count_basis: str = "source_count" + rows: list[ExtractedAggregationRow] = field(default_factory=list) def _coverage_gaps( @@ -155,11 +204,16 @@ def _coverage_gaps( return gaps[:8] -def _group_findings(prompt: str, candidates: list[Artefact]) -> list[AggregationGroup]: - if _aggregation_mode(prompt) == "count_extracted_items": - item_groups = _group_extracted_items(candidates) +def _group_findings( + prompt: str, + candidates: list[Artefact], + extraction_spec: AggregationExtractionSpec, +) -> tuple[list[AggregationGroup], list[ExtractedAggregationRow]]: + if extraction_spec.count_basis == "extracted_item_count": + rows = _extract_rows(candidates, extraction_spec) + item_groups = _group_extracted_rows(candidates, rows, extraction_spec) if item_groups: - return item_groups + return item_groups, rows prompt_keywords = set(extract_query_keywords(prompt, limit=16)) group_to_sources: dict[str, list[Artefact]] = defaultdict(list) group_to_evidence: dict[str, list[AggregationEvidence]] = defaultdict(list) @@ -184,44 +238,89 @@ def _group_findings(prompt: str, candidates: list[Artefact]) -> list[Aggregation count=len(sources), sources=sources, evidence=group_to_evidence[value], + count_basis=extraction_spec.count_basis, ) for value, sources in group_to_sources.items() ] - return sorted(grouped, key=lambda item: (-item.count, item.value.lower())) + return sorted(grouped, key=lambda item: (-item.count, item.value.lower())), [] -def _group_extracted_items(candidates: list[Artefact]) -> list[AggregationGroup]: +def _group_extracted_rows( + candidates: list[Artefact], + rows: list[ExtractedAggregationRow], + extraction_spec: AggregationExtractionSpec, +) -> list[AggregationGroup]: + source_lookup = {artefact.key: artefact for artefact in candidates} group_to_sources: dict[str, list[Artefact]] = defaultdict(list) group_to_evidence: dict[str, list[AggregationEvidence]] = defaultdict(list) + group_to_rows: dict[str, list[ExtractedAggregationRow]] = defaultdict(list) group_counts: Counter[str] = Counter() - for artefact in candidates: - for value, item_text in _extract_owned_items(artefact.content): - group_counts[value] += 1 - if artefact not in group_to_sources[value]: - group_to_sources[value].append(artefact) - group_to_evidence[value].append( - AggregationEvidence( - source_key=artefact.key, - path=artefact.source_path, - snippet=item_text[:260], - confidence=0.82, - ) + for row in rows: + value = row.group_normalized_value + artefact = source_lookup.get(row.source_key) + group_counts[value] += 1 + if artefact is not None and artefact not in group_to_sources[value]: + group_to_sources[value].append(artefact) + group_to_rows[value].append(row) + group_to_evidence[value].append( + AggregationEvidence( + source_key=row.source_key, + path=row.source_path, + snippet=row.snippet, + confidence=row.confidence, ) + ) grouped = [ AggregationGroup( value=value, count=count, sources=group_to_sources[value], evidence=group_to_evidence[value], - count_basis="extracted_item", + count_basis=extraction_spec.count_basis, + rows=group_to_rows[value], ) for value, count in group_counts.items() ] return sorted(grouped, key=lambda item: (-item.count, item.value.lower())) -def _extract_owned_items(content: str) -> list[tuple[str, str]]: - items: list[tuple[str, str]] = [] +def _extract_rows(candidates: list[Artefact], extraction_spec: AggregationExtractionSpec) -> list[ExtractedAggregationRow]: + scoped_section_available = _has_matching_section(candidates, extraction_spec.scope_hints) + rows: list[ExtractedAggregationRow] = [] + for artefact in candidates: + for line in _candidate_item_lines(artefact.content, extraction_spec, scoped_section_available=scoped_section_available): + for raw_group, normalized_group in _owner_values_from_line(line): + rows.append( + ExtractedAggregationRow( + item_type=extraction_spec.item_type, + extracted_text=line, + source_key=artefact.key, + source_path=artefact.source_path, + snippet=line[:260], + group_type=extraction_spec.group_by, + group_raw_value=raw_group, + group_normalized_value=normalized_group, + status=_status_from_line(line), + date=_date_from_line(line), + confidence=0.84 if scoped_section_available else 0.76, + ) + ) + return rows + + +def _candidate_item_lines( + content: str, + extraction_spec: AggregationExtractionSpec, + *, + scoped_section_available: bool, +) -> list[str]: + lines: list[str] = [] + if scoped_section_available and extraction_spec.scope_hints: + for section_line in _section_lines_matching(content, extraction_spec.scope_hints): + if _line_has_grouping(section_line): + lines.append(section_line) + return lines + in_followup_section = False for raw_line in content.splitlines(): line = raw_line.strip() @@ -235,19 +334,64 @@ def _extract_owned_items(content: str) -> list[tuple[str, str]]: if ( not in_followup_section and not re.match(r"^(?:[-*]|\d+[.)])\s+", line) - and not re.search(r"\b(action items?|follow[- ]?up tasks?|tasks?|open items?)\b", line, re.IGNORECASE) + and not _line_matches_item_type(line, extraction_spec) ): continue - if not re.search(r"\b(owner|assigned|team|dri|responsible)\b", line, re.IGNORECASE): + if not _line_has_grouping(line): + continue + lines.append(line) + return lines + + +def _has_matching_section(candidates: list[Artefact], hints: list[str]) -> bool: + if not hints: + return False + return any(_section_lines_matching(artefact.content, hints) for artefact in candidates) + + +def _section_lines_matching(content: str, hints: list[str]) -> list[str]: + matched_lines: list[str] = [] + in_matching_section = False + for raw_line in content.splitlines(): + line = raw_line.strip() + if not line: + continue + heading = _heading_text(line) + if heading is not None: + in_matching_section = any(hint.lower() in heading.lower() for hint in hints) continue - owners = _owners_from_line(line) - for owner in owners: - items.append((owner, line)) - return items + if in_matching_section: + matched_lines.append(line) + return matched_lines + + +def _heading_text(line: str) -> str | None: + match = re.match(r"^#{1,6}\s+(.+)$", line) + if match: + return match.group(1).strip() + match = re.match(r"^h[1-6]\.\s+(.+)$", line, re.IGNORECASE) + if match: + return match.group(1).strip() + return None + + +def _line_matches_item_type(line: str, extraction_spec: AggregationExtractionSpec) -> bool: + pattern = _ITEM_TYPE_PATTERNS.get(extraction_spec.item_type) + if pattern is None: + return True + return bool(re.search(pattern, line, re.IGNORECASE)) + + +def _line_has_grouping(line: str) -> bool: + return bool(re.search(r"\b(owner|owning|assigned|team|dri|responsible)\b", line, re.IGNORECASE)) def _owners_from_line(line: str) -> set[str]: - owners: set[str] = set() + return {normalized for _raw, normalized in _owner_values_from_line(line)} + + +def _owner_values_from_line(line: str) -> list[tuple[str, str]]: + owners: set[tuple[str, str]] = set() patterns = ( r"\bowning team\s*:\s*([^.;\n]+)", r"\bowner team\s*:\s*([^.;\n]+)", @@ -258,8 +402,8 @@ def _owners_from_line(line: str) -> set[str]: ) for pattern in patterns: for match in re.finditer(pattern, line, re.IGNORECASE): - owners.update(_split_group_value(match.group(1))) - return {owner for owner in owners if _valid_group_value(owner)} + owners.update(_split_group_value_pairs(match.group(1))) + return sorted(owners, key=lambda item: item[1].lower()) def _group_values(content: str) -> set[str]: @@ -275,15 +419,30 @@ def _group_values(content: str) -> set[str]: def _split_group_value(raw: str) -> set[str]: + return {normalized for _raw, normalized in _split_group_value_pairs(raw)} + + +def _split_group_value_pairs(raw: str) -> set[tuple[str, str]]: cleaned = re.split(r"[.;()\n]", raw.strip(), maxsplit=1)[0] + cleaned = re.split(r"\s+\b(?:after|for|due|priority|target)\b", cleaned, maxsplit=1, flags=re.IGNORECASE)[0] parts = re.split(r"\s*(?:,|/|\band\b|&)\s*", cleaned, flags=re.IGNORECASE) - return {_normalize_group_value(part) for part in parts if part.strip()} + values: set[tuple[str, str]] = set() + for part in parts: + stripped = part.strip() + if not stripped: + continue + normalized = _normalize_group_value(stripped) + if _valid_group_value(normalized): + values.add((stripped, normalized)) + return values def _normalize_group_value(value: str) -> str: tokens = [token for token in re.findall(r"[A-Za-z0-9_-]+", value) if token.lower() not in _GROUP_STOPWORDS] while len(tokens) > 1 and tokens[0].lower() in _GROUP_PREFIX_STOPWORDS: tokens.pop(0) + while len(tokens) > 1 and tokens[-1].lower() in _GROUP_PREFIX_STOPWORDS: + tokens.pop() if not tokens: return "" return " ".join(token.upper() if token.isupper() else token[:1].upper() + token[1:].lower() for token in tokens[:4]) @@ -317,7 +476,7 @@ def _finding_lines(grouped: list[AggregationGroup]) -> list[str]: return ["- no grouped finding could be inferred from candidate sources"] lines: list[str] = [] for group in grouped[:12]: - unit = "item(s)" if group.count_basis == "extracted_item" else "source(s)" + unit = "item(s)" if group.count_basis == "extracted_item_count" else "source(s)" paths = ", ".join(artefact.source_path for artefact in group.sources[:3]) if len(group.sources) > 3: paths += f", +{len(group.sources) - 3} more" @@ -342,6 +501,7 @@ def _group_metadata(group: AggregationGroup, *, evidence_budget: int) -> dict[st } for item in evidence ], + "extracted_rows": [_extracted_row_metadata(row) for row in group.rows[:evidence_budget]], "evidence_truncated": len(group.evidence) > len(evidence), } @@ -362,11 +522,8 @@ def _format_counter(counter: Counter[str], *, limit: int) -> str: def _aggregation_mode(prompt: str) -> str: - if re.search(r"\b(action items?|follow[- ]?up tasks?|tasks?|tickets?|open items?)\b", prompt, re.IGNORECASE) and re.search( - r"\b(count|how many|most|highest number|by team|by owner|which team|which owner|assigned)\b", - prompt, - re.IGNORECASE, - ): + spec = _infer_extraction_spec(prompt) + if spec.count_basis == "extracted_item_count": return "count_extracted_items" if _asks_for_grouped_counts(prompt): return "count_mentions" @@ -375,6 +532,123 @@ def _aggregation_mode(prompt: str) -> str: return "group_by_entity" +def _infer_extraction_spec(prompt: str) -> AggregationExtractionSpec: + lower = prompt.lower() + item_type = "generic_theme" + item_patterns = { + "action_item": r"\b(action items?|follow[- ]?up tasks?|tasks?|tickets?|open items?)\b", + "decision": r"\b(decisions?|decided|approved|adopted)\b", + "risk": r"\b(risks?|blockers?|concerns?)\b", + "customer_request": r"\b(customer requests?|feature requests?|requests?)\b", + "open_question": r"\b(open questions?|questions?)\b", + "meeting_time": r"\b(meeting time|scheduled|calendar|invite|time window)\b", + } + for candidate_type, pattern in item_patterns.items(): + if re.search(pattern, lower): + item_type = candidate_type + break + + scope_hints: list[str] = [] + scope_terms = { + "follow-up": r"\bfollow[- ]?up\b", + "risk": r"\brisks?\b", + "owner": r"\bowners?\b", + "deadline": r"\b(deadlines?|due dates?)\b", + } + for value, pattern in scope_terms.items(): + if re.search(pattern, lower): + scope_hints.append(value) + if not scope_hints and item_type != "generic_theme": + scope_hints.extend(_ITEM_TYPE_SECTION_HINTS.get(item_type, [])) + + if re.search(r"\b(owner|dri|assignee)\b", lower): + group_by = "owner" + elif re.search(r"\b(team|teams)\b", lower): + group_by = "team" + elif "status" in lower: + group_by = "status" + elif re.search(r"\b(date|when|deadline|due)\b", lower): + group_by = "date" + elif "source" in lower: + group_by = "source" + else: + group_by = "theme" + + asks_for_item_rollup = item_type != "generic_theme" and ( + _asks_for_grouped_counts(prompt) or re.search(r"\b(list|show|summarize|across all|by)\b", lower) + ) + if asks_for_item_rollup: + count_basis = "extracted_item_count" + elif _asks_for_grouped_counts(prompt): + count_basis = "mention_count" + else: + count_basis = "source_count" + + return AggregationExtractionSpec( + item_type=item_type, + scope_hints=list(dict.fromkeys(scope_hints)), + group_by=group_by, + count_basis=count_basis, + ) + + +def _extraction_spec_metadata(spec: AggregationExtractionSpec) -> dict[str, object]: + return { + "item_type": spec.item_type, + "scope_hints": spec.scope_hints, + "group_by": spec.group_by, + "count_basis": spec.count_basis, + "evidence_required": spec.evidence_required, + } + + +def _extracted_row_metadata(row: ExtractedAggregationRow) -> dict[str, object]: + return { + "item_type": row.item_type, + "extracted_text": row.extracted_text, + "source_key": row.source_key, + "source_path": row.source_path, + "snippet": row.snippet, + "group": { + "type": row.group_type, + "raw_value": row.group_raw_value, + "normalized_value": row.group_normalized_value, + }, + "status": row.status, + "date": row.date, + "confidence": row.confidence, + } + + +def _status_from_line(line: str) -> str | None: + lowered = line.lower() + if "done" in lowered or "completed" in lowered: + return "completed" + if "blocked" in lowered: + return "blocked" + if "open" in lowered or "todo" in lowered or "pending" in lowered: + return "open" + return None + + +def _date_from_line(line: str) -> str | None: + match = re.search( + r"\b(?:due|target(?: date)?)\s*:\s*([0-9]{4}-[0-9]{2}-[0-9]{2}|[A-Za-z]{3,9}\s+\d{1,2}(?:,\s*\d{4})?)", + line, + re.IGNORECASE, + ) + if match: + return match.group(1) + match = re.search(r"\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b", line) + return match.group(0) if match else None + + +def _mean_confidence(rows: list[ExtractedAggregationRow]) -> float | None: + if not rows: + return None + return round(sum(row.confidence for row in rows) / len(rows), 3) + + def _best_snippet(content: str, value: str) -> str: value_lower = value.lower() for line in content.splitlines(): @@ -408,6 +682,24 @@ def _best_snippet(content: str, value: str) -> str: "engineering", } +_ITEM_TYPE_PATTERNS = { + "action_item": r"\b(action items?|follow[- ]?up tasks?|tasks?|tickets?|open items?|owner|owning|assigned|responsible|dri)\b", + "decision": r"\b(decisions?|decided|approved|adopted|accepted|rejected|owner|dri)\b", + "risk": r"\b(risks?|blockers?|concerns?|mitigation|owner|dri)\b", + "customer_request": r"\b(customer requests?|feature requests?|requests?|asks?|owner|dri|product area)\b", + "open_question": r"\b(open questions?|questions?|owner|dri)\b", + "meeting_time": r"\b(meeting time|scheduled|calendar|invite|time window|owner|dri)\b", +} + +_ITEM_TYPE_SECTION_HINTS = { + "action_item": ["action item", "tasks", "open items"], + "decision": ["decision"], + "risk": ["risk"], + "customer_request": ["request"], + "open_question": ["open question"], + "meeting_time": ["meeting time", "scheduled", "calendar", "invite"], +} + _THEME_STOPWORDS = { "that", "this", diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index ad1c9ce..90e7226 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -257,12 +257,16 @@ def test_source_aggregation_preserves_group_counts_and_provenance(): assert "Representative provenance:" in aggregate.content assert aggregate.metadata["aggregation_source_count"] == 3 runtime_group = next(group for group in aggregate.metadata["aggregation_groups"] if group["value"] == "Runtime") - assert runtime_group["count_basis"] == "extracted_item" + assert runtime_group["count_basis"] == "extracted_item_count" assert runtime_group["source_keys"] == ["file_summary:runtime.json", "file_summary:runtime-2.json"] assert {item["source_key"] for item in runtime_group["evidence"]} == { "file_summary:runtime.json", "file_summary:runtime-2.json", } + assert aggregate.metadata["count_basis"] == "extracted_item_count" + assert aggregate.metadata["aggregation_extraction_spec"]["item_type"] == "action_item" + assert aggregate.metadata["extracted_row_count"] == 4 + assert runtime_group["extracted_rows"][0]["group"]["normalized_value"] == "Runtime" assert aggregate.metadata["provenance_coverage_count"] == 3 assert aggregate.metadata["provenance_truncated"] is False @@ -322,6 +326,60 @@ def test_source_aggregation_normalizes_owning_team_prefixes(): assert "Eng SRE" not in groups +def test_source_aggregation_counts_decision_rows_by_owner(): + now = time.time() + prompt = "Across the design reviews, count decisions by DRI." + profile = infer_context_preparation_profile(prompt) + artefacts = [ + Artefact( + key="file_summary:search-review.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/design-reviews/search-review.md", + source_mtime=now, + generated_at=now, + model="test", + content=( + "## Decisions\n" + "- Adopt hybrid search for recall-sensitive queries. DRI: Search Platform.\n" + "- Keep lexical fallback for exact references. DRI: Search Platform.\n" + ), + ), + Artefact( + key="file_summary:billing-review.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/design-reviews/billing-review.md", + source_mtime=now, + generated_at=now, + model="test", + content=( + "## Decisions\n" + "- Defer metering schema migration until export tests pass. DRI: Billing Infra.\n" + ), + ), + Artefact( + key="file_summary:notes.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/design-reviews/notes.md", + source_mtime=now, + generated_at=now, + model="test", + content="Meeting notes with no decision owner.", + ), + ] + + aggregate, trace = build_source_aggregation(prompt, artefacts, profile) + + assert aggregate is not None + assert "count_basis:extracted_item_count" in trace.notes + assert aggregate.metadata["aggregation_extraction_spec"]["item_type"] == "decision" + assert aggregate.metadata["aggregation_extraction_spec"]["group_by"] == "owner" + assert aggregate.metadata["extracted_row_count"] == 3 + groups = {group["value"]: group for group in aggregate.metadata["aggregation_groups"]} + assert groups["Search Platform"]["count"] == 2 + assert groups["Billing Infra"]["count"] == 1 + assert groups["Search Platform"]["extracted_rows"][0]["item_type"] == "decision" + + def test_scheduling_evidence_prepares_confirmed_time_source(): now = time.time() prompt = ( From 8cf44183a5be83989d6b2bf8bf48cfc7fac35ecd Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 14:08:16 +0200 Subject: [PATCH 13/22] Add scoped aggregation diagnostics --- src/vaner/broker/aggregation.py | 159 ++++++++++++++++++++++++++--- tests/test_broker/test_selector.py | 72 +++++++++++++ 2 files changed, 219 insertions(+), 12 deletions(-) diff --git a/src/vaner/broker/aggregation.py b/src/vaner/broker/aggregation.py index bb31857..8156a92 100644 --- a/src/vaner/broker/aggregation.py +++ b/src/vaner/broker/aggregation.py @@ -36,12 +36,19 @@ def build_source_aggregation( resolved_provenance_budget = max_sources if provenance_budget is None else provenance_budget extraction_spec = _infer_extraction_spec(prompt) - grouped, extracted_rows = _group_findings(prompt, source_candidates, extraction_spec) - facets = _facet_coverage(profile, source_candidates) - source_classes = Counter(_source_class(artefact.source_path) for artefact in source_candidates) - source_refs = _source_refs(source_candidates, source_by_key or {}, provenance_budget=resolved_provenance_budget) - gaps = _coverage_gaps(prompt, profile, source_candidates, grouped) - provenance_truncated = len(source_candidates) > resolved_provenance_budget + candidate_scope_spec = _infer_candidate_scope_spec(prompt, source_candidates) + eligible_sources, excluded_sources = _apply_candidate_scope(source_candidates, candidate_scope_spec) + aggregation_sources = eligible_sources or source_candidates + grouped, extracted_rows = _group_findings(prompt, aggregation_sources, extraction_spec) + extraction_source_keys = list(dict.fromkeys(row.source_key for row in extracted_rows)) + extraction_source_paths = list(dict.fromkeys(row.source_path for row in extracted_rows)) + facets = _facet_coverage(profile, aggregation_sources) + source_classes = Counter(_source_class(artefact.source_path) for artefact in aggregation_sources) + source_refs = _source_refs(aggregation_sources, source_by_key or {}, provenance_budget=resolved_provenance_budget) + gaps = _coverage_gaps(prompt, profile, aggregation_sources, grouped) + if not eligible_sources and excluded_sources: + gaps.append("scope filter produced no eligible sources; used unfiltered candidates") + provenance_truncated = len(aggregation_sources) > resolved_provenance_budget if provenance_truncated: gaps.append("provenance truncated to budget") @@ -51,9 +58,11 @@ def build_source_aggregation( "", "Coverage:", f"- sources_considered: {len(source_candidates)}", + f"- eligible_sources: {len(aggregation_sources)}", + f"- excluded_sources: {len(excluded_sources)}", "- source_classes: " + _format_counter(source_classes, limit=8), "- facets_covered: " + (", ".join(facets) if facets else "none detected"), - f"- provenance_coverage: {min(len(source_candidates), resolved_provenance_budget)}/{len(source_candidates)}", + f"- provenance_coverage: {min(len(aggregation_sources), resolved_provenance_budget)}/{len(aggregation_sources)}", "- gaps: " + ("; ".join(gaps) if gaps else "none detected"), "", "Extraction:", @@ -69,12 +78,19 @@ def build_source_aggregation( *source_refs, ] content = "\n".join(sections).strip() - digest = hashlib.sha256((prompt + "\n" + "\n".join(a.key for a in source_candidates)).encode("utf-8")).hexdigest()[:16] + digest = hashlib.sha256((prompt + "\n" + "\n".join(a.key for a in aggregation_sources)).encode("utf-8")).hexdigest()[:16] metadata = { "context_sources": ["aggregate_sources"], - "aggregation_source_count": len(source_candidates), - "aggregation_source_keys": [artefact.key for artefact in source_candidates[:resolved_provenance_budget]], - "aggregation_source_paths": [artefact.source_path for artefact in source_candidates[:resolved_provenance_budget]], + "aggregation_source_count": len(aggregation_sources), + "aggregation_source_keys": [artefact.key for artefact in aggregation_sources[:resolved_provenance_budget]], + "aggregation_source_paths": [artefact.source_path for artefact in aggregation_sources[:resolved_provenance_budget]], + "candidate_pool_count": len(source_candidates), + "eligible_source_count": len(aggregation_sources), + "eligible_source_keys": [artefact.key for artefact in aggregation_sources[:resolved_provenance_budget]], + "eligible_source_paths": [artefact.source_path for artefact in aggregation_sources[:resolved_provenance_budget]], + "excluded_source_count": len(excluded_sources), + "excluded_sources": [_excluded_source_metadata(item) for item in excluded_sources[:resolved_provenance_budget]], + "candidate_scope_spec": _candidate_scope_spec_metadata(candidate_scope_spec), "aggregation_groups": [ _group_metadata(group, evidence_budget=min(12, resolved_provenance_budget)) for group in grouped[:12] ], @@ -84,16 +100,22 @@ def build_source_aggregation( "extracted_row_count": len(extracted_rows), "extracted_rows": [_extracted_row_metadata(row) for row in extracted_rows[:resolved_provenance_budget]], "extraction_confidence": _mean_confidence(extracted_rows), - "provenance_coverage_count": min(len(source_candidates), resolved_provenance_budget), + "extraction_source_count": len(extraction_source_keys), + "extraction_source_keys": extraction_source_keys[:resolved_provenance_budget], + "extraction_source_paths": extraction_source_paths[:resolved_provenance_budget], + "provenance_coverage_count": min(len(aggregation_sources), resolved_provenance_budget), "provenance_truncated": provenance_truncated, "provenance": "prepared_context_aggregation", } trace.output_count = 1 trace.notes.append(f"sources_considered:{len(source_candidates)}") + trace.notes.append(f"eligible_sources:{len(aggregation_sources)}") + trace.notes.append(f"excluded_sources:{len(excluded_sources)}") trace.notes.append(f"groups:{len(grouped)}") trace.notes.append(f"count_basis:{extraction_spec.count_basis}") trace.notes.append(f"item_type:{extraction_spec.item_type}") trace.notes.append(f"extracted_rows:{len(extracted_rows)}") + trace.notes.append(f"extraction_sources:{len(extraction_source_keys)}") return ( Artefact( key=f"prepared_context:source_aggregation:{digest}", @@ -135,6 +157,98 @@ def _facet_coverage(profile: ContextPreparationProfile, candidates: list[Artefac return list(dict.fromkeys(covered))[:16] +def _infer_candidate_scope_spec(prompt: str, candidates: list[Artefact]) -> CandidateScopeSpec: + required_facets = _required_scope_facets(prompt) + source_class = _dominant_source_class(candidates) + confidence = 0.72 if required_facets else 0.55 + return CandidateScopeSpec( + source_class=source_class, + required_facets=required_facets, + optional_facets=extract_query_keywords(prompt, limit=8), + exclusion_rules=["missing_required_facets"] if required_facets else [], + confidence=confidence, + ) + + +def _required_scope_facets(prompt: str) -> list[str]: + lower = prompt.lower() + facets: list[str] = [] + project_match = re.search( + r"\b(?:project|initiative|program|customer|client|account|system|service|component|area)\s+([A-Z][A-Za-z0-9_-]{2,})\b", + prompt, + ) + if project_match: + facets.append(project_match.group(1)) + quoted = re.findall(r"['\"]([^'\"]{3,64})['\"]", prompt) + facets.extend(quoted) + if "postmortem" in lower or "postmortems" in lower: + facets.append("postmortem") + if "design review" in lower or "design reviews" in lower: + facets.append("design review") + if "customer feedback" in lower: + facets.append("customer feedback") + if "meeting notes" in lower: + facets.append("meeting notes") + return list(dict.fromkeys(facets))[:8] + + +def _dominant_source_class(candidates: list[Artefact]) -> str | None: + if not candidates: + return None + counts = Counter(_source_class(artefact.source_path) for artefact in candidates) + source_class, count = counts.most_common(1)[0] + return source_class if count >= max(2, len(candidates) // 3) else None + + +def _apply_candidate_scope( + candidates: list[Artefact], + scope_spec: CandidateScopeSpec, +) -> tuple[list[Artefact], list[ExcludedAggregationSource]]: + eligible: list[Artefact] = [] + excluded: list[ExcludedAggregationSource] = [] + for artefact in candidates: + reasons = _scope_exclusion_reasons(artefact, scope_spec) + if reasons: + excluded.append(ExcludedAggregationSource(source_key=artefact.key, source_path=artefact.source_path, reasons=reasons)) + else: + eligible.append(artefact) + return eligible, excluded + + +def _scope_exclusion_reasons(artefact: Artefact, scope_spec: CandidateScopeSpec) -> list[str]: + reasons: list[str] = [] + haystack = f"{artefact.source_path}\n{artefact.content}".lower() + for facet in scope_spec.required_facets: + if facet.lower() not in haystack: + reasons.append(f"missing_required_facet:{facet}") + if scope_spec.source_class and _source_class(artefact.source_path) != scope_spec.source_class: + reasons.append(f"outside_source_class:{scope_spec.source_class}") + return reasons + + +def _candidate_scope_spec_metadata(spec: CandidateScopeSpec) -> dict[str, object]: + return { + "source_class": spec.source_class, + "required_facets": spec.required_facets, + "optional_facets": spec.optional_facets, + "time_window": spec.time_window, + "entities": spec.entities, + "source_status": spec.source_status, + "canonicality": spec.canonicality, + "exclusion_rules": spec.exclusion_rules, + "max_scope_strategy": spec.max_scope_strategy, + "confidence": spec.confidence, + } + + +def _excluded_source_metadata(item: ExcludedAggregationSource) -> dict[str, object]: + return { + "source_key": item.source_key, + "source_path": item.source_path, + "reasons": item.reasons, + } + + @dataclass(frozen=True) class AggregationExtractionSpec: item_type: str = "generic_theme" @@ -153,6 +267,27 @@ class AggregationExtractionSpec: ) +@dataclass(frozen=True) +class CandidateScopeSpec: + source_class: str | None = None + required_facets: list[str] = field(default_factory=list) + optional_facets: list[str] = field(default_factory=list) + time_window: str | None = None + entities: list[str] = field(default_factory=list) + source_status: str | None = None + canonicality: str = "prefer_canonical" + exclusion_rules: list[str] = field(default_factory=list) + max_scope_strategy: str = "facet_intersection" + confidence: float = 0.55 + + +@dataclass(frozen=True) +class ExcludedAggregationSource: + source_key: str + source_path: str + reasons: list[str] + + @dataclass(frozen=True) class AggregationEvidence: source_key: str diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 90e7226..59eb1e8 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -380,6 +380,78 @@ def test_source_aggregation_counts_decision_rows_by_owner(): assert groups["Search Platform"]["extracted_rows"][0]["item_type"] == "decision" +def test_source_aggregation_scopes_extraction_to_requested_project(): + now = time.time() + prompt = "Across project Atlas design reviews, count decisions by DRI." + profile = infer_context_preparation_profile(prompt) + artefacts = [ + Artefact( + key="file_summary:atlas-search.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/design-reviews/atlas-search.md", + source_mtime=now, + generated_at=now, + model="test", + content=( + "Project Atlas design review\n" + "## Decisions\n" + "- Adopt hybrid search for Atlas recall. DRI: Search Platform.\n" + "- Keep lexical fallback for Atlas exact references. DRI: Search Platform.\n" + ), + ), + Artefact( + key="file_summary:atlas-billing.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/design-reviews/atlas-billing.md", + source_mtime=now, + generated_at=now, + model="test", + content=( + "Project Atlas design review\n" + "## Decisions\n" + "- Defer metering migration until export tests pass. DRI: Billing Infra.\n" + ), + ), + Artefact( + key="file_summary:zephyr-search.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/design-reviews/zephyr-search.md", + source_mtime=now, + generated_at=now, + model="test", + content=( + "Project Zephyr design review\n" + "## Decisions\n" + "- Replace indexing backend for Zephyr. DRI: Search Platform.\n" + "- Move Zephyr rollout ownership. DRI: Release Eng.\n" + ), + ), + ] + + aggregate, trace = build_source_aggregation(prompt, artefacts, profile) + + assert aggregate is not None + assert "eligible_sources:2" in trace.notes + assert "excluded_sources:1" in trace.notes + assert aggregate.metadata["candidate_scope_spec"]["required_facets"] == ["Atlas", "design review"] + assert aggregate.metadata["candidate_pool_count"] == 3 + assert aggregate.metadata["eligible_source_count"] == 2 + assert aggregate.metadata["excluded_source_count"] == 1 + assert aggregate.metadata["extraction_source_count"] == 2 + assert aggregate.metadata["extraction_source_keys"] == [ + "file_summary:atlas-search.md", + "file_summary:atlas-billing.md", + ] + assert aggregate.metadata["extracted_row_count"] == 3 + excluded = aggregate.metadata["excluded_sources"][0] + assert excluded["source_key"] == "file_summary:zephyr-search.md" + assert "missing_required_facet:Atlas" in excluded["reasons"] + groups = {group["value"]: group for group in aggregate.metadata["aggregation_groups"]} + assert groups["Search Platform"]["count"] == 2 + assert groups["Billing Infra"]["count"] == 1 + assert "Release Eng" not in groups + + def test_scheduling_evidence_prepares_confirmed_time_source(): now = time.time() prompt = ( From 66977b3755892a4ff11d733310345d70d3a1307d Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 14:12:01 +0200 Subject: [PATCH 14/22] Track counted aggregation rows --- src/vaner/broker/aggregation.py | 20 ++++++++++++++++++++ tests/test_broker/test_selector.py | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/src/vaner/broker/aggregation.py b/src/vaner/broker/aggregation.py index 8156a92..3a006c2 100644 --- a/src/vaner/broker/aggregation.py +++ b/src/vaner/broker/aggregation.py @@ -42,6 +42,10 @@ def build_source_aggregation( grouped, extracted_rows = _group_findings(prompt, aggregation_sources, extraction_spec) extraction_source_keys = list(dict.fromkeys(row.source_key for row in extracted_rows)) extraction_source_paths = list(dict.fromkeys(row.source_path for row in extracted_rows)) + counted_rows = [row for row in extracted_rows if row.scope_status == "counted"] + counted_source_keys = list(dict.fromkeys(row.source_key for row in counted_rows)) + counted_source_paths = list(dict.fromkeys(row.source_path for row in counted_rows)) + dropped_rows = [row for row in extracted_rows if row.scope_status != "counted"] facets = _facet_coverage(profile, aggregation_sources) source_classes = Counter(_source_class(artefact.source_path) for artefact in aggregation_sources) source_refs = _source_refs(aggregation_sources, source_by_key or {}, provenance_budget=resolved_provenance_budget) @@ -70,6 +74,8 @@ def build_source_aggregation( f"- group_by: {extraction_spec.group_by}", f"- count_basis: {extraction_spec.count_basis}", f"- extracted_row_count: {len(extracted_rows)}", + f"- counted_row_count: {len(counted_rows)}", + f"- dropped_row_count: {len(dropped_rows)}", "", "Grouped findings:", *_finding_lines(grouped), @@ -99,10 +105,17 @@ def build_source_aggregation( "count_basis": extraction_spec.count_basis, "extracted_row_count": len(extracted_rows), "extracted_rows": [_extracted_row_metadata(row) for row in extracted_rows[:resolved_provenance_budget]], + "counted_row_count": len(counted_rows), + "counted_rows": [_extracted_row_metadata(row) for row in counted_rows[:resolved_provenance_budget]], + "dropped_row_count": len(dropped_rows), + "dropped_rows": [_extracted_row_metadata(row) for row in dropped_rows[:resolved_provenance_budget]], "extraction_confidence": _mean_confidence(extracted_rows), "extraction_source_count": len(extraction_source_keys), "extraction_source_keys": extraction_source_keys[:resolved_provenance_budget], "extraction_source_paths": extraction_source_paths[:resolved_provenance_budget], + "counted_source_count": len(counted_source_keys), + "counted_source_keys": counted_source_keys[:resolved_provenance_budget], + "counted_source_paths": counted_source_paths[:resolved_provenance_budget], "provenance_coverage_count": min(len(aggregation_sources), resolved_provenance_budget), "provenance_truncated": provenance_truncated, "provenance": "prepared_context_aggregation", @@ -116,6 +129,9 @@ def build_source_aggregation( trace.notes.append(f"item_type:{extraction_spec.item_type}") trace.notes.append(f"extracted_rows:{len(extracted_rows)}") trace.notes.append(f"extraction_sources:{len(extraction_source_keys)}") + trace.notes.append(f"counted_rows:{len(counted_rows)}") + trace.notes.append(f"counted_sources:{len(counted_source_keys)}") + trace.notes.append(f"dropped_rows:{len(dropped_rows)}") return ( Artefact( key=f"prepared_context:source_aggregation:{digest}", @@ -309,6 +325,8 @@ class ExtractedAggregationRow: status: str | None = None date: str | None = None confidence: float = 0.7 + scope_status: str = "counted" + scope_exclusion_reason: str | None = None @dataclass(frozen=True) @@ -751,6 +769,8 @@ def _extracted_row_metadata(row: ExtractedAggregationRow) -> dict[str, object]: }, "status": row.status, "date": row.date, + "scope_status": row.scope_status, + "scope_exclusion_reason": row.scope_exclusion_reason, "confidence": row.confidence, } diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 59eb1e8..4b1f876 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -443,12 +443,20 @@ def test_source_aggregation_scopes_extraction_to_requested_project(): "file_summary:atlas-billing.md", ] assert aggregate.metadata["extracted_row_count"] == 3 + assert aggregate.metadata["counted_row_count"] == 3 + assert aggregate.metadata["dropped_row_count"] == 0 + assert aggregate.metadata["counted_source_count"] == 2 + assert aggregate.metadata["counted_source_keys"] == [ + "file_summary:atlas-search.md", + "file_summary:atlas-billing.md", + ] excluded = aggregate.metadata["excluded_sources"][0] assert excluded["source_key"] == "file_summary:zephyr-search.md" assert "missing_required_facet:Atlas" in excluded["reasons"] groups = {group["value"]: group for group in aggregate.metadata["aggregation_groups"]} assert groups["Search Platform"]["count"] == 2 assert groups["Billing Infra"]["count"] == 1 + assert groups["Search Platform"]["extracted_rows"][0]["scope_status"] == "counted" assert "Release Eng" not in groups From c539eac5f8e3ccac7d91a67c5615d91d6bd80c4b Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 15:34:56 +0200 Subject: [PATCH 15/22] Add artefact structure semantic context source --- src/vaner/broker/context_preparation.py | 97 ++++++++++++++++++++++++- src/vaner/broker/selector.py | 41 +++++++++-- src/vaner/semantic_aliases.py | 20 +++++ tests/test_broker/test_selector.py | 73 ++++++++++++++++++- 4 files changed, 221 insertions(+), 10 deletions(-) diff --git a/src/vaner/broker/context_preparation.py b/src/vaner/broker/context_preparation.py index 0b7a232..fd69ecc 100644 --- a/src/vaner/broker/context_preparation.py +++ b/src/vaner/broker/context_preparation.py @@ -113,10 +113,10 @@ def source_path_hint_candidates(prompt: str, available_paths: list[str], *, limi hint_terms.update({"calendar", "invite", "booking", "schedule", "meeting", "gmail"}) if re.search(r"\b(architecture|technical|deep dive|review|workshop)\b", lowered): hint_terms.update({"architecture", "review", "deepdive", "deep-dive", "technical", "workshop"}) - if re.search(r"\b(healthcare|health care|clinical|medical|hospital)\b", lowered): - hint_terms.update({"health", "healthcare", "clinical", "medical"}) - if re.search(r"\b(isolated network|private|vpc|on-prem|own network)\b", lowered): - hint_terms.update({"private", "privhost", "vpc", "network", "hosting"}) + if re.search(r"\b(healthcare|health care|clinical|medical|hospital|patient|phi)\b", lowered): + hint_terms.update({"health", "healthcare", "clinical", "medical", "hospital", "patient", "phi"}) + if re.search(r"\b(isolated network|locked-down|locked down|data center|datacenter|private|vpc|on-prem|onprem|own network)\b", lowered): + hint_terms.update({"private", "privhost", "vpc", "network", "hosting", "onprem", "on-prem", "datacenter", "implementation"}) if re.search(r"\b(issue|ticket|bug|support escalation|incident)\b", lowered): hint_terms.update({"jira", "linear", "support", "incidents"}) if not hint_terms: @@ -142,6 +142,52 @@ def source_path_hint_candidates(prompt: str, available_paths: list[str], *, limi return [path for _, path in ranked[:limit]] +def semantic_path_hint_candidates( + prompt: str, + available_paths: list[str], + *, + profile: ContextPreparationProfile | None = None, + limit: int = 64, +) -> list[str]: + """Return paths/titles whose canonical terms bridge an indirect prompt. + + This is an artefact-structure signal, not a benchmark shortcut. It helps + when a user asks in operational language but durable sources are named with + canonical implementation, project, customer, or workflow terms. + """ + + if not available_paths: + return [] + profile = profile or infer_context_preparation_profile(prompt) + terms = _semantic_path_terms(prompt, profile) + if not terms: + return [] + term_set = set(terms) + ranked: list[tuple[float, str]] = [] + for path in available_paths: + path_terms = set(extract_query_keywords(path.replace("/", " ").replace(".", " "), limit=48)) + if not path_terms: + continue + direct_hits = term_set & path_terms + if not direct_hits: + continue + score = 0.0 + for term in direct_hits: + score += 3.0 if _is_high_signal_path_term(term) else 1.0 + # Reward multiple independent bridges more than repeated single-word + # matches. A path that names two or three canonical concepts is a + # stronger source candidate than a broad folder match. + score += min(8.0, len(direct_hits) * len(direct_hits) * 0.65) + path_lower = path.lower() + for source_hint in profile.source_hints: + if source_hint and source_hint in path_lower: + score += 2.0 + if len(direct_hits) >= 2 or score >= 5.0: + ranked.append((score, path)) + ranked.sort(key=lambda item: (-item[0], item[1])) + return [path for _, path in ranked[: max(1, int(limit))]] + + def extract_query_keywords(prompt: str, *, limit: int = 24) -> list[str]: """Extract FTS-friendly terms from a conversational prompt.""" @@ -163,6 +209,49 @@ def extract_query_keywords(prompt: str, *, limit: int = 24) -> list[str]: return keywords[:limit] +def _semantic_path_terms(prompt: str, profile: ContextPreparationProfile, *, limit: int = 48) -> list[str]: + keywords = extract_query_keywords(prompt, limit=32) + facets = [facet.value for facet in profile.facets if len(facet.value) > 2] + aliases = sorted(engineering_semantic_aliases(prompt, [*keywords, *facets], stopwords=_QUERY_STOPWORDS)) + terms: list[str] = [] + seen: set[str] = set() + for term in [*keywords, *facets, *aliases]: + for part in extract_query_keywords(term.replace("/", " ").replace(".", " "), limit=8): + if part not in seen and part not in _QUERY_STOPWORDS: + seen.add(part) + terms.append(part) + if len(terms) >= limit: + break + return terms[:limit] + + +def _is_high_signal_path_term(term: str) -> bool: + if len(term) >= 7: + return True + return term in { + "annealing", + "auditlog", + "audit", + "billing", + "bifurcation", + "checkpoint", + "egress", + "escrow", + "failover", + "healthcare", + "kernel", + "latency", + "onprem", + "precision", + "rehearse", + "residency", + "routing", + "telemetry", + "vector", + "embedding", + } + + def _looks_like_scheduling_query(prompt: str) -> bool: lowered = prompt.lower() return bool( diff --git a/src/vaner/broker/selector.py b/src/vaner/broker/selector.py index 83f1d82..3908571 100644 --- a/src/vaner/broker/selector.py +++ b/src/vaner/broker/selector.py @@ -16,6 +16,7 @@ hard_constraints_satisfied, infer_context_preparation_profile, query_variants, + semantic_path_hint_candidates, source_path_hint_candidates, ) from vaner.broker.preparation_policy import choose_preparation_plan, plan_tool_names, plan_uses_tool @@ -582,18 +583,35 @@ async def select_artefacts_fts( available_paths = [] exact_paths = set(exact_reference_candidates(prompt, available_paths, limit=min(32, max(8, top_n * 3)))) source_hint_paths = set(source_path_hint_candidates(prompt, available_paths, limit=min(96, max(16, top_n * 8)))) + structure_hint_paths = set( + semantic_path_hint_candidates( + prompt, + available_paths, + profile=profile, + limit=min(96, max(16, top_n * 8)), + ) + ) if exact_paths: tool_traces.append( ContextToolTrace(tool="exact_reference_lookup", input_count=len(available_paths), output_count=len(exact_paths)) ) - if source_hint_paths: + if source_hint_paths or structure_hint_paths: tool_traces.append( - ContextToolTrace(tool="source_class_search", input_count=len(available_paths), output_count=len(source_hint_paths)) + ContextToolTrace( + tool="source_class_search", + input_count=len(available_paths), + output_count=len(source_hint_paths | structure_hint_paths), + notes=[ + f"source_class_paths:{len(source_hint_paths)}", + f"artefact_structure_paths:{len(structure_hint_paths)}", + ], + ) ) else: available_paths = [] exact_paths = set() source_hint_paths = set() + structure_hint_paths = set() semantic_trace = ContextToolTrace(tool="semantic_search") if semantic_memory_enabled and semantic_embed is not None and hasattr(store, "select_artefacts_semantic"): semantic_variants = variants if context_enabled else [prompt] @@ -623,7 +641,7 @@ async def select_artefacts_fts( loaded: list[Artefact] = [] load_keys = set(fused_keys) | preferred loaded.extend(await store.list_by_keys(load_keys, limit=max(retrieval_limit, len(load_keys)))) # type: ignore[union-attr] - path_loads = set(preferred_paths_set) | exact_paths | source_hint_paths + path_loads = set(preferred_paths_set) | exact_paths | source_hint_paths | structure_hint_paths if path_loads: if exact_paths: exact_loaded = await store.list_by_source_paths(exact_paths, limit=max(retrieval_limit, len(exact_paths))) # type: ignore[union-attr] @@ -633,7 +651,19 @@ async def select_artefacts_fts( hint_loaded = await store.list_by_source_paths(source_hint_paths, limit=max(retrieval_limit, len(source_hint_paths))) # type: ignore[union-attr] loaded.extend(hint_loaded) _record_source_ranking(source_rankings, source_by_key, "source_metadata", [artefact.key for artefact in hint_loaded]) - preferred_path_loads = preferred_paths_set - exact_paths - source_hint_paths + if structure_hint_paths: + structure_loaded = await store.list_by_source_paths( + structure_hint_paths, + limit=max(retrieval_limit, len(structure_hint_paths)), + ) # type: ignore[union-attr] + loaded.extend(structure_loaded) + _record_source_ranking( + source_rankings, + source_by_key, + "artefact_structure", + [artefact.key for artefact in structure_loaded], + ) + preferred_path_loads = preferred_paths_set - exact_paths - source_hint_paths - structure_hint_paths if preferred_path_loads: loaded.extend( await store.list_by_source_paths(preferred_path_loads, limit=max(retrieval_limit, len(preferred_path_loads))) # type: ignore[union-attr] @@ -818,6 +848,7 @@ def _fuse_source_rankings(source_rankings: dict[str, list[str]], *, limit: int) "semantic_memory": 0.75, "semantic_query_variants": 0.68, "source_metadata": 0.82, + "artefact_structure": 0.88, "coverage_floor": 0.85, } scores: dict[str, float] = {} @@ -912,7 +943,7 @@ def _aggregation_candidate_pool( continue sources = source_by_key.get(artefact.key, set()) source_signal = 0.7 if sources else 0.0 - if sources & {"source_metadata", "semantic_memory", "semantic_query_variants", "coverage_floor"}: + if sources & {"source_metadata", "artefact_structure", "semantic_memory", "semantic_query_variants", "coverage_floor"}: source_signal += 1.0 canonical_bonus = _canonical_source_class_bonus(prompt, artefact, profile) if canonical_source_class_enabled else 0.0 score = score_artefact(prompt, artefact) + canonical_bonus + source_signal diff --git a/src/vaner/semantic_aliases.py b/src/vaner/semantic_aliases.py index aef6401..9079af1 100644 --- a/src/vaner/semantic_aliases.py +++ b/src/vaner/semantic_aliases.py @@ -57,6 +57,26 @@ def has_any(*phrases: str) -> bool: if has_any("isolated network", "own network", "private network", "private hosting", "private deploy", "privnet", "vpc"): aliases.update({"private", "privnet", "vpc", "vnet", "isolated", "hosting", "deploy", "deployment"}) + if has_any("locked down", "locked-down", "data center", "datacenter", "on prem", "on-prem", "onprem", "air gapped", "air-gapped"): + aliases.update({"private", "onprem", "datacenter", "vpc", "airgap", "air-gapped", "implementation", "capture", "deployment"}) + + if has_any("failover", "takeover", "active location", "active locations", "resume credential", "resume token", "full authentication"): + aliases.update({"gateway", "bifurcation", "checkpoint", "checkpointing", "kv", "token", "grace", "takeover", "failover"}) + if has_any("bifurcation", "checkpointing", "token grace", "grace window", "kv checkpoint"): + aliases.update({"failover", "takeover", "resume", "credential", "authentication", "active", "region", "location"}) + + if has_any( + "wrong source", + "wrong source records", + "verify charges", + "charges", + "billing", + "telemetry never arrived", + "telemetry missing", + ): + aliases.update({"audit", "auditlog", "audit-log", "billing", "reconciliation", "mismatch", "gap", "telemetry", "observability"}) + if has_any("vector writes", "vector write", "semantic search", "source records", "dense index", "embedding index"): + aliases.update({"embedding", "embeddings", "embed", "index", "dense", "reconciliation", "vector", "vectors"}) if has_any("makes things up", "made things up", "hallucination", "hallucinate", "joke", "meme"): aliases.update({"hallucination", "hallucinate", "meme", "memes", "satire", "tagging", "joke", "jokes"}) diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index 4b1f876..e2f8242 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -7,7 +7,12 @@ import pytest from vaner.broker.aggregation import build_source_aggregation -from vaner.broker.context_preparation import infer_context_preparation_profile, query_variants, source_path_hint_candidates +from vaner.broker.context_preparation import ( + infer_context_preparation_profile, + query_variants, + semantic_path_hint_candidates, + source_path_hint_candidates, +) from vaner.broker.preparation_policy import choose_preparation_plan, plan_tool_names from vaner.broker.scheduling import build_scheduling_evidence from vaner.broker.selector import select_artefacts, select_artefacts_fts @@ -157,6 +162,72 @@ def test_source_path_hints_surface_source_class_candidates(): ] +def test_semantic_path_hints_bridge_indirect_prompt_to_canonical_source_names(): + paths = [ + "github/pr-39751-kernel-stability-thresholds-precision-annealing-kv-sync-guard.json", + "github/pr-874321-traffic-escrow-and-rehearse-proxy-for-staged-promotes.json", + "docs/team-handbook/weekly-planning.md", + ] + prompt = "What made low bit math safer before a machine steps down from the safest numeric mode?" + profile = infer_context_preparation_profile(prompt) + + selected = semantic_path_hint_candidates(prompt, paths, profile=profile, limit=2) + + assert selected[0] == "github/pr-39751-kernel-stability-thresholds-precision-annealing-kv-sync-guard.json" + + +def test_source_and_semantic_path_hints_support_private_healthcare_deployments(): + paths = [ + "google_drive/users/valehealth-implementation-capture.json", + "google_drive/users/acme-public-roadmap-notes.json", + "slack/random/lunch-thread.json", + ] + prompt = "Find the hospital implementation notes for running an intake chatbot inside a locked-down data center." + profile = infer_context_preparation_profile(prompt) + + source_hints = source_path_hint_candidates(prompt, paths) + semantic_hints = semantic_path_hint_candidates(prompt, paths, profile=profile) + + assert "google_drive/users/valehealth-implementation-capture.json" in source_hints + assert "google_drive/users/valehealth-implementation-capture.json" in semantic_hints + + +@pytest.mark.asyncio +async def test_select_artefacts_fts_loads_artefact_structure_candidates(tmp_path): + store = ArtefactStore(tmp_path / "store.db") + await store.initialize() + now = time.time() + target = Artefact( + key="file_summary:precision-annealing.json", + kind=ArtefactKind.FILE_SUMMARY, + source_path="github/pr-39751-kernel-stability-thresholds-precision-annealing-kv-sync-guard.json", + source_mtime=now, + generated_at=now, + model="test", + content="Default stability pass threshold is 0.995 before leaving safe fp32 mode.", + ) + distractor = Artefact( + key="file_summary:planning.md", + kind=ArtefactKind.FILE_SUMMARY, + source_path="docs/team-handbook/weekly-planning.md", + source_mtime=now, + generated_at=now, + model="test", + content="Weekly planning notes and team rituals.", + ) + await store.upsert(target) + await store.upsert(distractor) + + selected = await select_artefacts_fts( + "What made low bit math safer before a machine steps down from the safest numeric mode?", + store, + top_n=1, + ) + + assert selected[0].key == "file_summary:precision-annealing.json" + assert "artefact_structure" in selected[0].metadata["context_sources"] + + def test_source_path_hints_rank_specific_scheduling_threads(): paths = [ "gmail/team/generic-deepdive-scheduler.json", From 3eacc28e44a26748018f829662be8c5b457ad7cf Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Thu, 7 May 2026 22:36:00 +0200 Subject: [PATCH 16/22] Tighten context need inference for compound facts --- src/vaner/broker/context_preparation.py | 2 +- tests/test_broker/test_selector.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/vaner/broker/context_preparation.py b/src/vaner/broker/context_preparation.py index fd69ecc..993b0b4 100644 --- a/src/vaner/broker/context_preparation.py +++ b/src/vaner/broker/context_preparation.py @@ -448,7 +448,7 @@ def _infer_need(lowered: str, archetype: str): if archetype == "researcher": return "research_mapping" if re.search( - r"\b(all|every|list|compare|across|summarize|synthesize|count|which .* and|what .* and|which .* most|highest number)\b", + r"\b(all|every|list|compare|across|summarize|synthesize|count|which .* most|highest number)\b", lowered, ): return "multi_source_synthesis" diff --git a/tests/test_broker/test_selector.py b/tests/test_broker/test_selector.py index e2f8242..63e43bd 100644 --- a/tests/test_broker/test_selector.py +++ b/tests/test_broker/test_selector.py @@ -1618,6 +1618,14 @@ def test_context_profile_infers_multi_source_synthesis_without_benchmark_labels( assert any(constraint.kind == "restrictive_language" and constraint.value == "after" for constraint in profile.constraints) +def test_context_profile_does_not_treat_plain_compound_fact_question_as_multi_source(): + profile = infer_context_preparation_profile( + "What failover sequence and recovery targets did MedThink specify for handling an EU region outage?" + ) + + assert profile.need != "multi_source_synthesis" + + def test_select_artefacts_disables_global_gate_for_multi_source_context_need(): artefacts = [ Artefact( From 2ce0ac0ce22609aaa2872819ad37d9ed4b50eede Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Fri, 8 May 2026 16:52:21 +0200 Subject: [PATCH 17/22] Add generic agentic context preparation primitives --- src/vaner/broker/__init__.py | 12 ++ src/vaner/broker/agentic_preparation.py | 178 ++++++++++++++++++ src/vaner/models/agentic_preparation.py | 44 +++++ src/vaner/models/context_preparation.py | 2 + tests/test_broker/test_agentic_preparation.py | 75 ++++++++ 5 files changed, 311 insertions(+) create mode 100644 src/vaner/broker/agentic_preparation.py create mode 100644 src/vaner/models/agentic_preparation.py create mode 100644 tests/test_broker/test_agentic_preparation.py diff --git a/src/vaner/broker/__init__.py b/src/vaner/broker/__init__.py index 66dcda6..a7d5c9d 100644 --- a/src/vaner/broker/__init__.py +++ b/src/vaner/broker/__init__.py @@ -1,5 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 +from vaner.broker.agentic_preparation import ( + build_evidence_selection_prompt, + build_recall_planning_prompt, + coerce_recall_plan, + coerce_support_selection, + support_item_budget, +) from vaner.broker.answerable import build_answerable_briefing, build_answerable_briefing_from_text from vaner.broker.assembler import assemble_context_package from vaner.broker.compressor import compress_context @@ -12,12 +19,17 @@ __all__ = [ "assemble_context_package", + "build_evidence_selection_prompt", + "build_recall_planning_prompt", "build_answerable_briefing", "build_answerable_briefing_from_text", "builtin_retrieval_floor", "compress_context", + "coerce_recall_plan", + "coerce_support_selection", "detect_floor_conflicts", "score_artefact", "select_artefacts", "should_invoke_retrieval_floor", + "support_item_budget", ] diff --git a/src/vaner/broker/agentic_preparation.py b/src/vaner/broker/agentic_preparation.py new file mode 100644 index 0000000..58d92ad --- /dev/null +++ b/src/vaner/broker/agentic_preparation.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Iterable, Mapping + +from vaner.broker.context_preparation import infer_context_preparation_profile +from vaner.models.agentic_preparation import AgenticRecallPlan, EvidenceCandidate, EvidenceSupportSelection +from vaner.models.context_preparation import ContextPreparationProfile + +_MULTI_SOURCE_MARKERS = ( + " across ", + " all ", + " compare", + " each ", + " every ", + " list ", + " which documents", + " which sources", + " conflict", + " conflicting", + " summarize", + " themes", + " patterns", + " how many", + " count ", + " counts ", + " by team", + " by owner", + " by dri", + " end-to-end", +) + + +def support_item_budget( + prompt: str, + *, + profile: ContextPreparationProfile | None = None, + default_max: int = 12, +) -> int: + """Return a minimal direct-support budget for a context request. + + Single-fact requests should usually preserve one direct source. Broad + synthesis/comparison requests need a larger support set. The decision is + derived from general task language and the inferred context need. + """ + + profile = profile or infer_context_preparation_profile(prompt) + if profile.need in {"multi_source_synthesis", "conflict_resolution", "research_mapping"}: + return max(1, default_max) + text = f" {prompt.strip().lower()} " + if any(marker in text for marker in _MULTI_SOURCE_MARKERS): + return max(1, default_max) + if " and " in text and any(token in text for token in ("what ", "which ", "who ", "when ", "where ", "why ", "how ")): + return max(1, min(default_max, 3)) + return 1 + + +def excerpt_text(text: str, *, max_chars: int = 900) -> str: + compact = " ".join(text.split()) + if len(compact) <= max_chars: + return compact + return compact[: max_chars - 1].rstrip() + "..." + + +def candidate_rows(candidates: Iterable[EvidenceCandidate], *, max_excerpt_chars: int = 900) -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + for index, candidate in enumerate(candidates, start=1): + rows.append( + { + "rank": candidate.rank or index, + "key": candidate.key, + "path": candidate.path, + "title": candidate.title, + "excerpt": excerpt_text(candidate.text, max_chars=max_excerpt_chars), + } + ) + return rows + + +def build_recall_planning_prompt( + prompt: str, + candidates: Iterable[EvidenceCandidate], + *, + max_queries: int = 6, + max_seed_candidates: int = 24, +) -> str: + rows = candidate_rows(list(candidates)[:max_seed_candidates], max_excerpt_chars=500) + return ( + "You are Vaner's bounded context preparation agent. Your job is to improve evidence recall before a final " + "user-facing answer or action model runs.\n" + "Generate search queries that could find direct supporting context for the user's request. Use general " + "work-context reasoning: exact entities, aliases, likely source vocabulary, code/config names, dates, owners, " + "incidents, meetings, and paraphrases. Do not use benchmark IDs, gold answers, hidden labels, or dataset " + "shortcuts.\n" + "Return JSON only. Keep queries short and lexical enough for search.\n\n" + f"User request: {prompt}\n\n" + f"Current candidate summaries:\n{rows}\n\n" + f"Return at most {max_queries} queries." + ) + + +def build_evidence_selection_prompt( + prompt: str, + candidates: Iterable[EvidenceCandidate], + *, + support_budget: int, + max_candidates: int = 96, + max_excerpt_chars: int = 900, +) -> str: + rows = candidate_rows(list(candidates)[:max_candidates], max_excerpt_chars=max_excerpt_chars) + return ( + "You are Vaner's evidence preparation agent. Select the smallest set of candidate items that directly support " + "the user's request. Prefer exact evidence over topical similarity. Include multiple items only when the " + "request asks for comparison, completeness, conflicts, synthesis, counts, or multiple facts. Do not include " + "corroborating, background, nearby, follow-up, or merely topical items. For a one-fact request, select exactly " + "one item only if it directly contains the answer. Return no items if the candidates do not contain the answer.\n" + "This is a general context-preparation step, not a benchmark shortcut. Do not infer hidden expected sources; " + "use only the request and candidate content.\n\n" + f"User request: {prompt}\n\n" + f"Candidate items:\n{rows}\n\n" + f"Return at most {support_budget} support keys ordered by support strength." + ) + + +def coerce_recall_plan(payload: Mapping[str, object] | None, *, max_queries: int = 6, original_prompt: str = "") -> AgenticRecallPlan: + raw_queries = payload.get("queries") if payload else None + queries: list[str] = [] + if isinstance(raw_queries, list): + for value in raw_queries: + query = str(value).strip() + if query and query.lower() != original_prompt.lower() and query not in queries: + queries.append(query) + if len(queries) >= max_queries: + break + reason = str(payload.get("reason", "")) if payload else "" + return AgenticRecallPlan(queries=queries, reason=reason) + + +def coerce_support_selection( + payload: Mapping[str, object] | None, + *, + allowed_keys: set[str], + support_budget: int, +) -> EvidenceSupportSelection: + raw_keys = None + if payload: + raw_keys = payload.get("support_keys") + if raw_keys is None: + raw_keys = payload.get("document_ids") + support_keys: list[str] = [] + if isinstance(raw_keys, list): + for value in raw_keys: + key = str(value).strip() + if key in allowed_keys and key not in support_keys: + support_keys.append(key) + if len(support_keys) >= support_budget: + break + answerability = str(payload.get("answerability", "none")) if payload else "none" + if answerability not in {"full", "partial", "none"}: + answerability = "none" + coverage_note = str(payload.get("coverage_note", "")) if payload else "" + return EvidenceSupportSelection( + support_keys=support_keys, + coverage_note=coverage_note, + answerability=answerability, # type: ignore[arg-type] + ) + + +def promote_support_candidates( + candidates: Iterable[EvidenceCandidate], + support_keys: Iterable[str], +) -> list[EvidenceCandidate]: + support_order = list(dict.fromkeys(str(key) for key in support_keys)) + by_key = {candidate.key: candidate for candidate in candidates} + promoted = [by_key[key] for key in support_order if key in by_key] + promoted.extend(candidate for candidate in candidates if candidate.key not in set(support_order)) + return promoted diff --git a/src/vaner/models/agentic_preparation.py b/src/vaner/models/agentic_preparation.py new file mode 100644 index 0000000..52607ea --- /dev/null +++ b/src/vaner/models/agentic_preparation.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field + + +class EvidenceCandidate(BaseModel): + key: str + path: str = "" + title: str = "" + text: str = "" + rank: int = 0 + score: float = 0.0 + + +class AgenticRecallPlan(BaseModel): + queries: list[str] = Field(default_factory=list) + reason: str = "" + + +class EvidenceSupportSelection(BaseModel): + support_keys: list[str] = Field(default_factory=list) + coverage_note: str = "" + answerability: Literal["full", "partial", "none"] = "none" + + +class AgenticPreparationTrace(BaseModel): + planned_queries: list[str] = Field(default_factory=list) + expanded_candidate_keys: list[str] = Field(default_factory=list) + support_keys: list[str] = Field(default_factory=list) + answerability: Literal["full", "partial", "none"] = "none" + coverage_note: str = "" + failed: bool = False + failure_reason: str = "" + + +class AgenticPreparationConfig(BaseModel): + max_queries: int = 6 + max_candidates: int = 96 + max_support_items: int = 12 + max_excerpt_chars: int = 900 diff --git a/src/vaner/models/context_preparation.py b/src/vaner/models/context_preparation.py index 9c1cd86..d8dbea5 100644 --- a/src/vaner/models/context_preparation.py +++ b/src/vaner/models/context_preparation.py @@ -39,6 +39,8 @@ "test_lookup", "risk_check", "time_entity_extraction", + "agentic_recall_plan", + "agentic_evidence_selection", ] diff --git a/tests/test_broker/test_agentic_preparation.py b/tests/test_broker/test_agentic_preparation.py new file mode 100644 index 0000000..42435d4 --- /dev/null +++ b/tests/test_broker/test_agentic_preparation.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from vaner.broker.agentic_preparation import ( + build_evidence_selection_prompt, + build_recall_planning_prompt, + coerce_recall_plan, + coerce_support_selection, + promote_support_candidates, + support_item_budget, +) +from vaner.models.agentic_preparation import EvidenceCandidate + + +def test_support_item_budget_keeps_direct_fact_minimal() -> None: + assert support_item_budget("What launch code is required for the Apollo handoff?", default_max=12) == 1 + + +def test_support_item_budget_expands_for_broad_synthesis() -> None: + assert support_item_budget("Across all incident notes, count follow-up actions by owner.", default_max=12) == 12 + + +def test_agentic_prompts_are_product_general() -> None: + candidates = [ + EvidenceCandidate( + key="source:1", + path="notes/apollo.md", + title="Apollo handoff", + text="The launch code is LCH-427.", + ) + ] + + recall = build_recall_planning_prompt("What is the launch code?", candidates) + selection = build_evidence_selection_prompt("What is the launch code?", candidates, support_budget=1) + + assert "Vaner's bounded context preparation agent" in recall + assert "EnterpriseRAG" not in recall + assert "document_id" not in selection + assert "support keys" in selection + + +def test_coerce_recall_plan_filters_original_prompt_and_caps_queries() -> None: + plan = coerce_recall_plan( + {"queries": ["What is the launch code?", "Apollo launch code", "Apollo launch code", "handoff LCH"], "reason": "x"}, + max_queries=2, + original_prompt="What is the launch code?", + ) + + assert plan.queries == ["Apollo launch code", "handoff LCH"] + + +def test_coerce_support_selection_caps_and_filters_allowed_keys() -> None: + selection = coerce_support_selection( + { + "support_keys": ["source:2", "source:bad", "source:1", "source:2"], + "coverage_note": "source:2 is direct", + "answerability": "full", + }, + allowed_keys={"source:1", "source:2"}, + support_budget=1, + ) + + assert selection.support_keys == ["source:2"] + assert selection.answerability == "full" + + +def test_promote_support_candidates_keeps_support_first() -> None: + candidates = [ + EvidenceCandidate(key="source:1"), + EvidenceCandidate(key="source:2"), + EvidenceCandidate(key="source:3"), + ] + + promoted = promote_support_candidates(candidates, ["source:3", "source:1"]) + + assert [candidate.key for candidate in promoted] == ["source:3", "source:1", "source:2"] From 38a851a400cd0de2944c9eb7460c5cb73b1e48da Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Fri, 8 May 2026 17:08:31 +0200 Subject: [PATCH 18/22] Improve agentic evidence selection context --- src/vaner/broker/agentic_preparation.py | 6 ++++-- src/vaner/models/agentic_preparation.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/vaner/broker/agentic_preparation.py b/src/vaner/broker/agentic_preparation.py index 58d92ad..7b6d8dc 100644 --- a/src/vaner/broker/agentic_preparation.py +++ b/src/vaner/broker/agentic_preparation.py @@ -106,7 +106,7 @@ def build_evidence_selection_prompt( *, support_budget: int, max_candidates: int = 96, - max_excerpt_chars: int = 900, + max_excerpt_chars: int = 2400, ) -> str: rows = candidate_rows(list(candidates)[:max_candidates], max_excerpt_chars=max_excerpt_chars) return ( @@ -114,7 +114,9 @@ def build_evidence_selection_prompt( "the user's request. Prefer exact evidence over topical similarity. Include multiple items only when the " "request asks for comparison, completeness, conflicts, synthesis, counts, or multiple facts. Do not include " "corroborating, background, nearby, follow-up, or merely topical items. For a one-fact request, select exactly " - "one item only if it directly contains the answer. Return no items if the candidates do not contain the answer.\n" + "one item only if it directly contains the answer. If a candidate is clearly the requested source but the " + "available excerpt is incomplete, select it as partial support instead of dropping it. Return no items only " + "when no candidate is a plausible direct source for the request.\n" "This is a general context-preparation step, not a benchmark shortcut. Do not infer hidden expected sources; " "use only the request and candidate content.\n\n" f"User request: {prompt}\n\n" diff --git a/src/vaner/models/agentic_preparation.py b/src/vaner/models/agentic_preparation.py index 52607ea..fa25219 100644 --- a/src/vaner/models/agentic_preparation.py +++ b/src/vaner/models/agentic_preparation.py @@ -41,4 +41,4 @@ class AgenticPreparationConfig(BaseModel): max_queries: int = 6 max_candidates: int = 96 max_support_items: int = 12 - max_excerpt_chars: int = 900 + max_excerpt_chars: int = 2400 From d047b45a6b4cd30b4363d2f0cefdd82dabec87af Mon Sep 17 00:00:00 2001 From: abolsen <6937877+abolsen@users.noreply.github.com> Date: Mon, 25 May 2026 21:20:50 +0200 Subject: [PATCH 19/22] feat: add external-state finance preparation --- src/vaner/cli/commands/app.py | 15 +- src/vaner/cli/commands/bundles.py | 85 +++ src/vaner/cli/commands/config.py | 4 + src/vaner/cli/commands/external_state.py | 204 +++++++ src/vaner/cli/commands/mcp_clients.py | 24 +- src/vaner/cli/commands/supervisor.py | 2 +- src/vaner/daemon/http.py | 145 ++++- src/vaner/daemon/precompute_worker.py | 13 + src/vaner/engine.py | 60 ++- src/vaner/external_state/__init__.py | 27 + src/vaner/external_state/configurator.py | 505 ++++++++++++++++++ src/vaner/external_state/manager.py | 445 +++++++++++++++ src/vaner/external_state/model_search.py | 236 ++++++++ src/vaner/external_state/models.py | 99 ++++ src/vaner/external_state/telemetry.py | 23 + src/vaner/integrations/capability.py | 93 ++++ .../integrations/mcp_consumer/__init__.py | 13 + src/vaner/integrations/mcp_consumer/client.py | 194 +++++++ src/vaner/integrations/mcp_consumer/safety.py | 71 +++ src/vaner/intent/bundles.py | 80 +++ src/vaner/intent/features.py | 16 + src/vaner/intent/prepared_work.py | 51 +- src/vaner/intent/taxonomy.py | 21 +- src/vaner/intent/trainer.py | 17 +- src/vaner/intent/work_products.py | 383 +++++++++++++ src/vaner/intent/work_style_priors.py | 12 + src/vaner/mcp/apps/__init__.py | 44 +- src/vaner/mcp/apps/prepared_work.html | 245 ++++++++- src/vaner/mcp/contracts.py | 2 +- src/vaner/mcp/prompts.py | 243 +++++++++ src/vaner/mcp/server.py | 198 ++++++- src/vaner/models/config.py | 44 ++ src/vaner/models/prepared_work.py | 9 + src/vaner/models/work_product.py | 42 ++ src/vaner/setup/enums.py | 3 +- src/vaner/setup/model_recommendation.py | 2 +- src/vaner/setup/questions.py | 1 + src/vaner/setup/select.py | 1 + src/vaner/store/artefacts.py | 185 ++++++- tests/test_cli/test_clients_subcommand.py | 7 + tests/test_cli/test_config.py | 31 ++ tests/test_cli/test_up.py | 5 + tests/test_daemon/test_http.py | 62 +++ .../test_prepared_work_endpoint.py | 47 ++ .../test_work_products_endpoint.py | 18 +- tests/test_engine.py | 62 +++ tests/test_engine/test_work_product_pass.py | 51 ++ .../test_external_state/test_configurator.py | 172 ++++++ tests/test_external_state/test_manager.py | 443 +++++++++++++++ .../test_external_state/test_model_search.py | 135 +++++ tests/test_external_state/test_snapshots.py | 72 +++ tests/test_integrations/test_capability.py | 66 +++ .../test_mcp_consumer_client.py | 14 + .../test_mcp_consumer_safety.py | 80 +++ tests/test_intent/test_bundles.py | 52 ++ .../test_finance_feature_schema.py | 45 ++ tests/test_intent/test_taxonomy.py | 1 + .../test_work_product_generation.py | 89 ++- tests/test_mcp/test_tool_registry_parity.py | 19 + .../test_mcp_apps/test_dashboard_fallback.py | 115 +++- .../test_prepared_work_resource.py | 34 +- tests/test_mcp_apps/test_prompts.py | 159 ++++++ tests/test_mcp_v2/test_work_products_tool.py | 80 +++ tests/test_setup/test_enums.py | 1 + tests/test_store/test_work_products.py | 47 ++ ui/cockpit/src/App.tsx | 147 ++++- ui/cockpit/src/api/client.ts | 54 +- ui/cockpit/src/api/usePreparedWork.ts | 2 +- ui/cockpit/src/components/CockpitViews.tsx | 241 ++++++++- .../src/components/PreparedWorkPanel.test.tsx | 41 ++ .../src/components/PreparedWorkPanel.tsx | 104 +++- ui/cockpit/src/components/SystemVitals.tsx | 11 +- ui/cockpit/src/components/chrome.tsx | 268 +++++++++- ui/cockpit/src/types.ts | 63 ++- ui/cockpit/vite.config.ts | 42 ++ 75 files changed, 6581 insertions(+), 156 deletions(-) create mode 100644 src/vaner/cli/commands/bundles.py create mode 100644 src/vaner/cli/commands/external_state.py create mode 100644 src/vaner/external_state/__init__.py create mode 100644 src/vaner/external_state/configurator.py create mode 100644 src/vaner/external_state/manager.py create mode 100644 src/vaner/external_state/model_search.py create mode 100644 src/vaner/external_state/models.py create mode 100644 src/vaner/external_state/telemetry.py create mode 100644 src/vaner/integrations/mcp_consumer/__init__.py create mode 100644 src/vaner/integrations/mcp_consumer/client.py create mode 100644 src/vaner/integrations/mcp_consumer/safety.py create mode 100644 src/vaner/intent/bundles.py create mode 100644 src/vaner/mcp/prompts.py create mode 100644 tests/test_external_state/test_configurator.py create mode 100644 tests/test_external_state/test_manager.py create mode 100644 tests/test_external_state/test_model_search.py create mode 100644 tests/test_external_state/test_snapshots.py create mode 100644 tests/test_integrations/test_mcp_consumer_client.py create mode 100644 tests/test_integrations/test_mcp_consumer_safety.py create mode 100644 tests/test_intent/test_bundles.py create mode 100644 tests/test_intent/test_finance_feature_schema.py create mode 100644 tests/test_mcp_apps/test_prompts.py diff --git a/src/vaner/cli/commands/app.py b/src/vaner/cli/commands/app.py index 082108e..8b99bcb 100644 --- a/src/vaner/cli/commands/app.py +++ b/src/vaner/cli/commands/app.py @@ -33,6 +33,7 @@ from vaner import VERSION, api from vaner.broker.prompting import build_evidence_bound_context_prompt from vaner.cli.commands import mcp_clients +from vaner.cli.commands.bundles import bundle_app from vaner.cli.commands.clients import clients_app from vaner.cli.commands.config import load_config, set_compute_value, set_config_value from vaner.cli.commands.daemon import ( @@ -50,6 +51,7 @@ from vaner.cli.commands.deep_run import deep_run_app from vaner.cli.commands.distill import distill_skill_file from vaner.cli.commands.explain import render_human, render_json +from vaner.cli.commands.external_state import external_state_app from vaner.cli.commands.focus import focus_app, jobs_app, resources_app from vaner.cli.commands.guidance import guidance_app from vaner.cli.commands.hooks import HOOK_SURFACES, write_hooks @@ -1227,6 +1229,11 @@ def mcp_server( transport: str = typer.Option("stdio", "--transport", "-t", help="Transport: stdio | sse"), host: str = typer.Option("127.0.0.1", "--host", help="SSE server host (sse transport only)"), port: int = typer.Option(8472, "--port", "-p", help="SSE server port (sse transport only)"), + tool_name_format: str = typer.Option( + "canonical", + "--tool-name-format", + help="MCP tool naming: canonical dotted names, or strict regex-safe aliases for Claude Desktop.", + ), ) -> None: """Start the Vaner MCP server for native IDE integration. @@ -1251,16 +1258,18 @@ def mcp_server( _fail(f"MCP not available: {exc}", hint="Install optional extras: pip install 'vaner[mcp]'.") repo_root = _repo_root(path) + if tool_name_format not in {"canonical", "strict"}: + _fail("Invalid --tool-name-format value.", hint="Use `canonical` or `strict`.") if transport == "sse": typer.echo(f"Starting Vaner MCP server (SSE) on {host}:{port} repo={repo_root}") try: - _asyncio.run(run_sse(repo_root, host=host, port=port)) + _asyncio.run(run_sse(repo_root, host=host, port=port, tool_name_format=tool_name_format)) except RuntimeError as exc: _fail(str(exc), hint="Install optional extras: pip install 'vaner[mcp]'.") else: try: - _asyncio.run(run_stdio(repo_root)) + _asyncio.run(run_stdio(repo_root, tool_name_format=tool_name_format)) except RuntimeError as exc: _fail(str(exc), hint="Install optional extras: pip install 'vaner[mcp]'.") @@ -2436,7 +2445,9 @@ def upgrade() -> None: app.add_typer(sources_app, name="sources", rich_help_panel="Background and local") app.add_typer(guidance_app, name="guidance", rich_help_panel="Use with an agent") app.add_typer(integrations_app, name="integrations", rich_help_panel="Inspect and debug") +app.add_typer(external_state_app, name="external-state", rich_help_panel="Configure") app.add_typer(clients_app, name="clients", rich_help_panel="Connect MCP clients") +app.add_typer(bundle_app, name="bundle", rich_help_panel="Configure") app.add_typer(setup_app, name="setup", rich_help_panel="Get started") diff --git a/src/vaner/cli/commands/bundles.py b/src/vaner/cli/commands/bundles.py new file mode 100644 index 0000000..d79f582 --- /dev/null +++ b/src/vaner/cli/commands/bundles.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Annotated + +import typer + +from vaner.engine import VanerEngine +from vaner.intent.adapter import CodeRepoAdapter +from vaner.intent.bundles import bundle_public_summary, bundle_rejection_reason + +bundle_app = typer.Typer(help="Install and inspect trained Vaner bundles", no_args_is_help=True) + + +@bundle_app.command("inspect") +def inspect_bundle( + bundle_dir: Annotated[Path, typer.Argument(exists=True, file_okay=False, dir_okay=True, readable=True)], + as_json: Annotated[bool, typer.Option("--json", help="Print machine-readable JSON.")] = False, +) -> None: + summary = bundle_public_summary(bundle_dir) + if as_json: + typer.echo(json.dumps(summary, indent=2, sort_keys=True)) + return + typer.echo(f"Bundle: {summary['bundle_id']}") + typer.echo(f"Records: {summary.get('records_used')}") + typer.echo(f"Backend: {summary.get('scorer_backend')}") + typer.echo(f"Trained: {summary.get('trained')}") + if summary.get("rejection_reason"): + typer.secho(f"Rejected: {summary['rejection_reason']}", fg=typer.colors.YELLOW) + else: + typer.secho("Installable: yes", fg=typer.colors.GREEN) + + +@bundle_app.command("install") +def install_bundle( + bundle_dir: Annotated[Path, typer.Argument(exists=True, file_okay=False, dir_okay=True, readable=True)], + path: Annotated[Path, typer.Option("--path", "-p", help="Workspace/repository root to install into.")] = Path("."), + allow_rejected: Annotated[ + bool, + typer.Option("--allow-rejected", help="Install a rejected bundle for local diagnostics."), + ] = False, + allow_experimental_data: Annotated[ + bool, + typer.Option( + "--allow-experimental-data", + help="Install a locally trained bundle whose data is not release-eligible but passed training and temporal gates.", + ), + ] = False, + as_json: Annotated[bool, typer.Option("--json", help="Print machine-readable JSON.")] = False, +) -> None: + reason = bundle_rejection_reason(bundle_dir, allow_experimental_data=allow_experimental_data) + if reason and not allow_rejected: + payload = {"installed": False, "reason": reason, "bundle": bundle_public_summary(bundle_dir)} + if as_json: + typer.echo(json.dumps(payload, indent=2, sort_keys=True)) + else: + typer.secho(f"Refusing rejected bundle: {reason}", fg=typer.colors.RED, err=True) + typer.echo("Use --allow-rejected only for local diagnostics.") + raise typer.Exit(code=2) + + async def _install() -> bool: + engine = VanerEngine(adapter=CodeRepoAdapter(path.resolve())) + return await engine.load_bundle( + bundle_dir, + allow_rejected=allow_rejected, + allow_experimental_data=allow_experimental_data, + ) + + installed = asyncio.run(_install()) + payload = { + "installed": installed, + "experimental_data": bool(allow_experimental_data and not allow_rejected), + "bundle": bundle_public_summary(bundle_dir), + } + if as_json: + typer.echo(json.dumps(payload, indent=2, sort_keys=True)) + raise typer.Exit(code=0 if installed else 1) + if not installed: + typer.secho("Bundle was not installed; no compatible scorer model was loaded.", fg=typer.colors.RED, err=True) + raise typer.Exit(code=1) + typer.secho("Bundle installed.", fg=typer.colors.GREEN) diff --git a/src/vaner/cli/commands/config.py b/src/vaner/cli/commands/config.py index 9f85191..f8fba8f 100644 --- a/src/vaner/cli/commands/config.py +++ b/src/vaner/cli/commands/config.py @@ -15,6 +15,7 @@ BackendConfig, ComputeConfig, ExplorationConfig, + ExternalStateConfig, GatewayConfig, GenerationConfig, IntegrationsConfig, @@ -108,6 +109,7 @@ def load_config(repo_root: Path) -> VanerConfig: sources_section = _section_dict(parsed, "sources") refinement_section = _section_dict(parsed, "refinement") integrations_section = _section_dict(parsed, "integrations") + external_state_section = _section_dict(parsed, "external_state") setup_section = _section_dict(parsed, "setup") policy_section = _section_dict(parsed, "policy") limits_section = _section_dict(parsed, "limits") @@ -175,6 +177,7 @@ def load_config(repo_root: Path) -> VanerConfig: sources = _build_section(SourcesConfig, "sources", sources_section) refinement = _build_section(RefinementConfig, "refinement", refinement_section) integrations = _build_section(IntegrationsConfig, "integrations", integrations_section) + external_state = _build_section(ExternalStateConfig, "external_state", external_state_section) setup = _build_section(SetupConfig, "setup", setup_section) policy = _build_section(PolicyConfig, "policy", policy_section) @@ -201,6 +204,7 @@ def load_config(repo_root: Path) -> VanerConfig: sources=sources, refinement=refinement, integrations=integrations, + external_state=external_state, setup=setup, policy=policy, ) diff --git a/src/vaner/cli/commands/external_state.py b/src/vaner/cli/commands/external_state.py new file mode 100644 index 0000000..bbe9202 --- /dev/null +++ b/src/vaner/cli/commands/external_state.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +import shlex +from pathlib import Path +from typing import Annotated + +import typer +from rich.console import Console +from rich.table import Table + +from vaner.cli.commands.config import load_config +from vaner.external_state.configurator import ( + ProviderConfigInput, + discover_provider_sync, + external_state_payload, + save_external_state_enabled, + save_finance_settings, + save_provider_config, +) + +external_state_app = typer.Typer(help="External data provider controls") +providers_app = typer.Typer(help="Manage MCP external-state providers") +finance_app = typer.Typer(help="Finance external-state permissions") +external_state_app.add_typer(providers_app, name="providers") +external_state_app.add_typer(finance_app, name="finance") +console = Console() + + +def _repo_root(path: str | None) -> Path: + if path: + return Path(path).expanduser().resolve() + env_path = os.environ.get("VANER_PATH", "").strip() + return Path(env_path).expanduser().resolve() if env_path else Path.cwd() + + +def _parse_args(raw: list[str]) -> list[str]: + if len(raw) == 1: + return shlex.split(raw[0]) + return list(raw) + + +def _parse_env(values: list[str]) -> dict[str, str]: + env: dict[str, str] = {} + for item in values: + if "=" not in item: + raise typer.BadParameter("--env values must use KEY=VALUE") + key, value = item.split("=", 1) + key = key.strip() + if not key: + raise typer.BadParameter("--env key must not be empty") + env[key] = value + return env + + +@external_state_app.command("show") +def show_external_state( + repo: Annotated[str | None, typer.Option("--repo", help="Repository root")] = None, + as_json: Annotated[bool, typer.Option("--json", help="Print JSON")] = False, +) -> None: + """Show configured external-state providers without exposing env values.""" + + config = load_config(_repo_root(repo)) + payload = external_state_payload(config) + if as_json: + console.print_json(json.dumps(payload.model_dump(mode="json"))) + return + table = Table("Provider", "Transport", "Command/URL", "Tools", "Capabilities") + for provider in payload.providers: + endpoint = provider.url if provider.transport == "streamable_http" else " ".join([provider.command, *provider.args]).strip() + table.add_row( + provider.id, + provider.transport, + endpoint or "-", + str(len(provider.allowed_tools)), + str(len(provider.capability_tools)), + ) + console.print(f"external_state: {'enabled' if payload.enabled else 'disabled'}") + finance = payload.finance + console.print( + "finance: " + f"{'enabled' if finance.get('enabled') else 'disabled'}, " + f"market_data={'on' if finance.get('market_data_enabled') else 'off'}, " + f"account_state={'on' if finance.get('account_state_enabled') else 'off'}, " + f"provider={finance.get('provider') or '-'}" + ) + console.print(table) + + +@external_state_app.command("enable") +def enable_external_state( + repo: Annotated[str | None, typer.Option("--repo", help="Repository root")] = None, +) -> None: + save_external_state_enabled(_repo_root(repo), enabled=True) + console.print("External-state access enabled.") + + +@external_state_app.command("disable") +def disable_external_state( + repo: Annotated[str | None, typer.Option("--repo", help="Repository root")] = None, +) -> None: + save_external_state_enabled(_repo_root(repo), enabled=False) + console.print("External-state access disabled.") + + +@providers_app.command("add") +def add_provider( + provider_id: Annotated[str, typer.Argument(help="Provider id, e.g. local_broker")], + command: Annotated[str, typer.Option("--command", "-c", help="MCP server command")] = "", + arg: Annotated[list[str] | None, typer.Option("--arg", help="MCP server argument. Repeat or pass a quoted arg string.")] = None, + transport: Annotated[str, typer.Option("--transport", help="stdio or streamable_http")] = "stdio", + url: Annotated[str, typer.Option("--url", help="streamable_http URL")] = "", + env: Annotated[list[str] | None, typer.Option("--env", help="Environment override KEY=VALUE. Repeatable.")] = None, + timeout_ms: Annotated[int, typer.Option("--timeout-ms", min=100)] = 10000, + repo: Annotated[str | None, typer.Option("--repo", help="Repository root")] = None, +) -> None: + """Register a provider connection. Discovery happens separately.""" + + save_provider_config( + _repo_root(repo), + ProviderConfigInput( + provider_id=provider_id, + transport=transport, + command=command, + args=_parse_args(arg or []), + url=url, + env=_parse_env(env or []), + timeout_ms=timeout_ms, + ), + ) + console.print(f"Provider `{provider_id}` saved. Run `vaner external-state providers discover {provider_id}`.") + + +@providers_app.command("discover") +def discover_provider_command( + provider_id: Annotated[str, typer.Argument(help="Configured provider id")], + apply: Annotated[bool, typer.Option("--apply", help="Persist safe discovered read tools and capability mappings")] = False, + trust_unknown_read: Annotated[ + bool, + typer.Option( + "--trust-unknown-read", + help="Treat finance-looking tools with missing annotations as read-only provider metadata.", + ), + ] = False, + repo: Annotated[str | None, typer.Option("--repo", help="Repository root")] = None, + as_json: Annotated[bool, typer.Option("--json", help="Print JSON")] = False, +) -> None: + """List provider tools, classify safety, and propose finance capabilities.""" + + config = load_config(_repo_root(repo)) + payload = discover_provider_sync(config, provider_id, apply=apply, trust_unknown_read=trust_unknown_read) + if as_json: + console.print_json(json.dumps(payload.model_dump(mode="json"))) + return + table = Table("Tool", "Safety", "Capability", "Scope", "Selected") + for tool in payload.tools: + table.add_row( + tool.name, + tool.safety_reason, + tool.proposed_capability or "-", + tool.access_scope, + "yes" if tool.auto_selected else "", + ) + console.print(table) + if apply: + console.print( + f"Applied {len(payload.applied_allowed_tools)} tools and " + f"{len(payload.applied_capability_tools)} finance capability mappings." + ) + + +@finance_app.command("enable") +def enable_finance( + provider: Annotated[str, typer.Option("--provider", "-p", help="Provider id to use for finance")] = "", + market_data: Annotated[bool, typer.Option("--market-data/--no-market-data")] = True, + account_state: Annotated[bool, typer.Option("--account-state/--no-account-state")] = False, + repo: Annotated[str | None, typer.Option("--repo", help="Repository root")] = None, +) -> None: + """Enable finance recognition plus opt-in external finance data access.""" + + repo_root = _repo_root(repo) + save_external_state_enabled(repo_root, enabled=True) + save_finance_settings( + repo_root, + enabled=True, + provider=provider or None, + market_data_enabled=market_data, + account_state_enabled=account_state, + ) + console.print( + "Finance external data enabled " + f"(market_data={'on' if market_data else 'off'}, account_state={'on' if account_state else 'off'})." + ) + + +@finance_app.command("disable") +def disable_finance( + repo: Annotated[str | None, typer.Option("--repo", help="Repository root")] = None, +) -> None: + save_finance_settings(_repo_root(repo), enabled=False, market_data_enabled=False, account_state_enabled=False) + console.print("Finance external data disabled.") diff --git a/src/vaner/cli/commands/mcp_clients.py b/src/vaner/cli/commands/mcp_clients.py index ae7a59b..5fc5c90 100644 --- a/src/vaner/cli/commands/mcp_clients.py +++ b/src/vaner/cli/commands/mcp_clients.py @@ -334,6 +334,21 @@ def _json_entry(container_key: str, launcher_cmd: str, launcher_args: list[str]) return {"command": launcher_cmd, "args": launcher_args} +def _launcher_args_for_client(client_id: str, launcher_args: list[str]) -> list[str]: + if client_id != "claude-desktop": + return list(launcher_args) + if "--tool-name-format" in launcher_args: + return list(launcher_args) + adjusted = list(launcher_args) + try: + path_index = adjusted.index("--path") + except ValueError: + adjusted.extend(["--tool-name-format", "strict"]) + else: + adjusted[path_index:path_index] = ["--tool-name-format", "strict"] + return adjusted + + def _merge_json_server( *, client_id: str, @@ -517,6 +532,7 @@ def write_client( ) -> WriteResult: spec = detected.spec target_path = path_override or detected.path + effective_launcher_args = _launcher_args_for_client(spec.id, launcher_args) if spec.kind == "json-mcpServers": assert target_path is not None return _merge_json_server( @@ -524,7 +540,7 @@ def write_client( path=target_path, container_key="mcpServers", launcher_cmd=launcher_cmd, - launcher_args=launcher_args, + launcher_args=effective_launcher_args, server_key=server_key, dry_run=dry_run, force=force, @@ -536,7 +552,7 @@ def write_client( path=target_path, container_key="servers", launcher_cmd=launcher_cmd, - launcher_args=launcher_args, + launcher_args=effective_launcher_args, server_key=server_key, dry_run=dry_run, force=force, @@ -548,7 +564,7 @@ def write_client( path=target_path, container_key="context_servers", launcher_cmd=launcher_cmd, - launcher_args=launcher_args, + launcher_args=effective_launcher_args, server_key=server_key, dry_run=dry_run, force=force, @@ -559,7 +575,7 @@ def write_client( client_id=spec.id, path=target_path, launcher_cmd=launcher_cmd, - launcher_args=launcher_args, + launcher_args=effective_launcher_args, dry_run=dry_run, ) if spec.kind == "cli-claude": diff --git a/src/vaner/cli/commands/supervisor.py b/src/vaner/cli/commands/supervisor.py index f40e750..bb7f922 100644 --- a/src/vaner/cli/commands/supervisor.py +++ b/src/vaner/cli/commands/supervisor.py @@ -205,7 +205,7 @@ def run_up( "--path", str(repo_root), "--interval-seconds", - "45", + str(max(5, int(interval_seconds))), "--startup-delay-seconds", "10", ], diff --git a/src/vaner/daemon/http.py b/src/vaner/daemon/http.py index f32585f..7cabe19 100644 --- a/src/vaner/daemon/http.py +++ b/src/vaner/daemon/http.py @@ -21,6 +21,15 @@ from vaner.daemon.live_work import read_live_work_events from vaner.daemon.precompute_worker import read_prediction_snapshot, read_worker_status, request_precompute_wake, request_worker_control from vaner.events.bus import build_stage_payloads +from vaner.external_state.configurator import ( + ProviderConfigInput, + discover_provider, + external_state_payload, + save_external_state_enabled, + save_finance_settings, + save_model_native_search_settings, + save_provider_config, +) from vaner.focus import FocusManager from vaner.intent.prediction_serialization import compact_serialized_prediction from vaner.mcp.contracts import EvidenceItem, Provenance, Resolution @@ -1351,6 +1360,134 @@ async def backend_presets() -> JSONResponse: } ) + @app.get("/external-state") + async def get_external_state() -> JSONResponse: + return JSONResponse(external_state_payload(config).model_dump(mode="json")) + + @app.post("/external-state") + async def update_external_state(payload: dict[str, Any]) -> JSONResponse: + nonlocal config + allowed = {"enabled", "max_calls_per_cycle", "max_cycle_ms"} + for key in payload: + if key not in allowed: + raise HTTPException(status_code=400, detail=f"Unsupported external-state key: {key}") + save_external_state_enabled( + config.repo_root, + enabled=bool(payload["enabled"]) if "enabled" in payload else None, + max_calls_per_cycle=int(payload["max_calls_per_cycle"]) if "max_calls_per_cycle" in payload else None, + max_cycle_ms=int(payload["max_cycle_ms"]) if "max_cycle_ms" in payload else None, + ) + config = load_config(config.repo_root) + focus_manager.config = config + return JSONResponse(external_state_payload(config).model_dump(mode="json")) + + @app.post("/external-state/providers") + async def upsert_external_state_provider(payload: dict[str, Any]) -> JSONResponse: + nonlocal config + provider_id = str(payload.get("id") or payload.get("provider_id") or "").strip() + if not provider_id: + raise HTTPException(status_code=400, detail="provider id is required") + args = payload.get("args", []) + if isinstance(args, str): + args = [item for item in args.split(" ") if item] + if not isinstance(args, list): + raise HTTPException(status_code=400, detail="args must be a list") + env = payload.get("env", {}) + if not isinstance(env, dict): + raise HTTPException(status_code=400, detail="env must be an object") + try: + save_provider_config( + config.repo_root, + ProviderConfigInput( + provider_id=provider_id, + transport=str(payload.get("transport") or "stdio"), + command=str(payload.get("command") or ""), + args=[str(item) for item in args], + url=str(payload.get("url") or ""), + env={str(key): str(value) for key, value in env.items()}, + timeout_ms=int(payload.get("timeout_ms") or 10000), + ), + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + config = load_config(config.repo_root) + focus_manager.config = config + return JSONResponse(external_state_payload(config).model_dump(mode="json")) + + @app.post("/external-state/providers/{provider_id}/discover") + async def discover_external_state_provider(provider_id: str, payload: dict[str, Any] | None = None) -> JSONResponse: + nonlocal config + body = payload or {} + try: + discovery = await discover_provider( + config, + provider_id, + apply=bool(body.get("apply", False)), + trust_unknown_read=bool(body.get("trust_unknown_read", False)), + ) + except KeyError as exc: + raise HTTPException(status_code=404, detail="provider not found") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=502, detail=f"provider discovery failed: {exc}") from exc + if discovery.applied: + config = load_config(config.repo_root) + focus_manager.config = config + return JSONResponse(discovery.model_dump(mode="json")) + + @app.post("/external-state/finance") + async def update_external_state_finance(payload: dict[str, Any]) -> JSONResponse: + nonlocal config + allowed = {"enabled", "provider", "market_data_enabled", "account_state_enabled", "capability_tools"} + for key in payload: + if key not in allowed: + raise HTTPException(status_code=400, detail=f"Unsupported finance external-state key: {key}") + capability_tools = payload.get("capability_tools") + if capability_tools is not None and not isinstance(capability_tools, dict): + raise HTTPException(status_code=400, detail="capability_tools must be an object") + try: + save_finance_settings( + config.repo_root, + enabled=bool(payload["enabled"]) if "enabled" in payload else None, + provider=str(payload["provider"]) if "provider" in payload and payload.get("provider") is not None else None, + market_data_enabled=bool(payload["market_data_enabled"]) if "market_data_enabled" in payload else None, + account_state_enabled=bool(payload["account_state_enabled"]) if "account_state_enabled" in payload else None, + capability_tools={str(key): str(value) for key, value in capability_tools.items()} + if isinstance(capability_tools, dict) + else None, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + config = load_config(config.repo_root) + focus_manager.config = config + return JSONResponse(external_state_payload(config).model_dump(mode="json")) + + @app.post("/external-state/model-native-search") + async def update_model_native_search(payload: dict[str, Any]) -> JSONResponse: + nonlocal config + allowed = {"enabled", "provider", "base_url", "api_key_env", "max_results", "timeout_seconds"} + for key in payload: + if key not in allowed: + raise HTTPException(status_code=400, detail=f"Unsupported model-native search key: {key}") + try: + save_model_native_search_settings( + config.repo_root, + enabled=bool(payload["enabled"]) if "enabled" in payload else None, + provider=str(payload["provider"]) if "provider" in payload and payload.get("provider") is not None else None, + base_url=str(payload["base_url"]) if "base_url" in payload and payload.get("base_url") is not None else None, + api_key_env=str(payload["api_key_env"]) + if "api_key_env" in payload and payload.get("api_key_env") is not None + else None, + max_results=int(payload["max_results"]) if "max_results" in payload else None, + timeout_seconds=float(payload["timeout_seconds"]) if "timeout_seconds" in payload else None, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + config = load_config(config.repo_root) + focus_manager.config = config + return JSONResponse(external_state_payload(config).model_dump(mode="json")) + @app.get("/skills") async def list_skills() -> JSONResponse: from vaner.intent.skills_discovery import discover_skills @@ -1623,7 +1760,13 @@ async def _work_product_store() -> Any: else: from vaner.store.artefacts import ArtefactStore - store = ArtefactStore(config.repo_root / ".vaner" / "artefacts.db") + # Runtime worker state lives in config.store_path. Older + # cockpit-only tests and pre-0.9 installs may still seed the + # legacy artefacts DB directly, so keep that as a fallback + # only when the configured engine store has not been created. + legacy_path = config.repo_root / ".vaner" / "artefacts.db" + store_path = config.store_path if config.store_path.exists() else legacy_path + store = ArtefactStore(store_path) await store.initialize() work_product_store = store return store diff --git a/src/vaner/daemon/precompute_worker.py b/src/vaner/daemon/precompute_worker.py index 669de24..c9e726f 100644 --- a/src/vaner/daemon/precompute_worker.py +++ b/src/vaner/daemon/precompute_worker.py @@ -197,6 +197,17 @@ async def run_forever(self) -> None: engine = build_default_engine(self.repo_root, config) await engine.initialize() + external_state_manager = None + if bool(getattr(config.external_state, "enabled", False)): + from vaner.external_state import ExternalStateManager, model_search_provider_from_config + + external_state_manager = ExternalStateManager( + config.external_state, + model_search_provider=model_search_provider_from_config(config.external_state), + ) + await external_state_manager.start() + if hasattr(engine, "set_external_state_manager"): + engine.set_external_state_manager(external_state_manager) if hasattr(engine, "set_prediction_event_listener"): engine.set_prediction_event_listener(self._record_prediction_event) if hasattr(engine, "set_live_work_event_listener"): @@ -216,6 +227,8 @@ async def run_forever(self) -> None: finally: scheduler.cancel() controller.cancel() + if external_state_manager is not None: + await external_state_manager.stop() self._set_state("stopping", explanation="worker stopping") async def _schedule_periodic(self, focus_manager: FocusManager) -> None: diff --git a/src/vaner/engine.py b/src/vaner/engine.py index 76fddc1..b7d3eab 100644 --- a/src/vaner/engine.py +++ b/src/vaner/engine.py @@ -31,6 +31,7 @@ from vaner.intent.allocator import PortfolioAllocator from vaner.intent.arcs import ArcObservation, ConversationArcModel, classify_query_category, derive_prompt_macro from vaner.intent.briefing import BriefingAssembler +from vaner.intent.bundles import bundle_rejection_reason from vaner.intent.cache import TieredPredictionCache from vaner.intent.deep_run import ( DeepRunFocus, @@ -93,7 +94,7 @@ from vaner.intent.trainer import IntentTrainer from vaner.intent.transfer import bootstrap_transfer_priors from vaner.intent.volatility import semantic_volatility_profile -from vaner.intent.work_products import generate_work_products +from vaner.intent.work_products import generate_external_finance_work_products, generate_work_products from vaner.intent.work_style_priors import ( IntentPriorAdjustments, ) @@ -533,6 +534,7 @@ def __init__( self._last_explored_scenarios: list[ExploredScenario] = [] self._last_no_scenario_reason: str = "" self._last_refinement_outcomes: list[dict[str, object]] = [] + self._external_state_manager: Any | None = None self._core_group_rotation_index = 0 # Phase 4 / WS6: prediction registry persists across cycles. First # created on the first ``precompute_cycle``; thereafter reused and @@ -3015,6 +3017,9 @@ def get_last_refinement_outcomes(self) -> list[dict[str, object]]: return [dict(item) for item in self._last_refinement_outcomes] + def set_external_state_manager(self, manager: Any | None) -> None: + self._external_state_manager = manager + async def _run_work_product_pass(self, *, cycle_deadline: float | None) -> int: """Prepare a small set of Vaner-owned artifacts during idle cycles. @@ -3036,6 +3041,28 @@ async def _run_work_product_pass(self, *, cycle_deadline: float | None) -> int: artefacts=artefacts, max_products=4, ) + manager = self._external_state_manager + if ( + manager is not None + and bool(getattr(getattr(self.config, "external_state", None), "enabled", False)) + and (cycle_deadline is None or time.monotonic() < cycle_deadline) + ): + try: + if hasattr(manager, "reset_cycle_budget"): + manager.reset_cycle_budget() + snapshots = await manager.collect_finance_snapshots(recent_queries) + for snapshot in snapshots: + await self.store.upsert_external_state_snapshot(snapshot) + products.extend( + generate_external_finance_work_products( + repo_root=self.config.repo_root, + recent_queries=recent_queries, + snapshots=snapshots, + max_products=max(1, 4 - len(products)), + ) + ) + except Exception: + logger.debug("external finance work product pass skipped", exc_info=True) await self.store.refresh_work_product_staleness(self.config.repo_root) written = 0 for product in products: @@ -5786,7 +5813,13 @@ async def train_batch(self, output_dir: Path | None = None) -> Path | None: await self._persist_learning_state() return model_path - async def load_bundle(self, bundle_dir: Path | str) -> bool: + async def load_bundle( + self, + bundle_dir: Path | str, + *, + allow_rejected: bool = False, + allow_experimental_data: bool = False, + ) -> bool: """Apply a pre-trained bundle to this engine's store. A bundle is a directory produced by ``eval/train_policy.py`` containing: @@ -5798,7 +5831,11 @@ async def load_bundle(self, bundle_dir: Path | str) -> bool: the engine and persisted to the store so future ``initialize()`` calls load them automatically. - Returns True if the bundle was applied successfully, False on failure. + Rejected training bundles are refused by default. Use + ``allow_rejected=True`` only for local diagnostics. + + Returns True if the bundle's scorer model was applied successfully, + False on failure or when the bundle is rejected. """ import json as _json @@ -5806,6 +5843,14 @@ async def load_bundle(self, bundle_dir: Path | str) -> bool: bundle_path = Path(bundle_dir) if not bundle_path.exists(): return False + if not allow_rejected and bundle_rejection_reason( + bundle_path, + allow_experimental_data=allow_experimental_data, + ): + return False + + previous_policy = self._scoring_policy + previous_cache_policy = self._cache.scoring_policy # Load scoring policy policy_file = bundle_path / "scoring_policy.json" @@ -5826,6 +5871,10 @@ async def load_bundle(self, bundle_dir: Path | str) -> bool: model_path = Path(model_path_str) if not model_path.is_absolute(): model_path = bundle_path / model_path_str + elif not model_path.exists(): + bundled_model = bundle_path / model_path.name + if bundled_model.exists(): + model_path = bundled_model if model_path.exists(): loaded = self._intent_scorer.load_model( model_path, @@ -5849,6 +5898,11 @@ async def load_bundle(self, bundle_dir: Path | str) -> bool: if isinstance(influence, (int, float)) and model_loaded: self._intent_scorer.set_model_influence(float(influence)) + if not model_loaded: + self._scoring_policy = previous_policy + self._cache.scoring_policy = previous_cache_policy + return False + self._mark_policy_state_dirty() await self._persist_learning_state(force=True) return model_loaded diff --git a/src/vaner/external_state/__init__.py b/src/vaner/external_state/__init__.py new file mode 100644 index 0000000..64d5733 --- /dev/null +++ b/src/vaner/external_state/__init__.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 + +from vaner.external_state.manager import ExternalStateManager +from vaner.external_state.model_search import ( + ModelNativeSearchProvider, + build_model_search_news_snapshot, + model_search_provider_from_config, +) +from vaner.external_state.models import ( + ExternalStateFreshnessClass, + ExternalStateSensitivity, + ExternalStateSnapshot, + FinanceCapability, +) +from vaner.external_state.telemetry import redacted_snapshot_telemetry + +__all__ = [ + "ExternalStateFreshnessClass", + "ExternalStateManager", + "ExternalStateSensitivity", + "ExternalStateSnapshot", + "FinanceCapability", + "ModelNativeSearchProvider", + "build_model_search_news_snapshot", + "model_search_provider_from_config", + "redacted_snapshot_telemetry", +] diff --git a/src/vaner/external_state/configurator.py b/src/vaner/external_state/configurator.py new file mode 100644 index 0000000..347548b --- /dev/null +++ b/src/vaner/external_state/configurator.py @@ -0,0 +1,505 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""User-facing configuration helpers for external-state MCP providers.""" + +from __future__ import annotations + +import asyncio +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field + +from vaner.external_state.models import ExternalStateSensitivity, FinanceCapability +from vaner.integrations.mcp_consumer.client import discover_mcp_tools +from vaner.integrations.mcp_consumer.safety import McpToolDescriptor, ReadOnlyToolPolicy +from vaner.models.config import ExternalStateConfig, McpConsumerConfig, VanerConfig +from vaner.setup.config_io import update_toml_section + +_PROVIDER_ID_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$") + +_ACCOUNT_CAPABILITIES = { + FinanceCapability.LIST_ACCOUNTS.value, + FinanceCapability.GET_BALANCE.value, + FinanceCapability.LIST_POSITIONS.value, + FinanceCapability.LIST_ORDERS.value, + FinanceCapability.LIST_ALERTS.value, + FinanceCapability.REVIEW_STRATEGY_POSITIONS.value, +} + +_CAPABILITY_PATTERNS: tuple[tuple[FinanceCapability, tuple[str, ...]], ...] = ( + (FinanceCapability.ANALYZE_OPTION_STRATEGY, ("analyze_option_strategy", "analyse_option_strategy", "option_strategy_analy")), + (FinanceCapability.SCREEN_OPTION_STRATEGIES, ("screen_option_strateg", "scan_option_strateg", "find_option_strateg")), + (FinanceCapability.GET_OPTION_CHAIN, ("option_chain", "options_chain", "chain_options", "get_chain")), + (FinanceCapability.LIST_OPTION_EXPIRIES, ("option_expir", "option_expiries", "expiration", "expiry")), + (FinanceCapability.LIST_POSITIONS, ("list_positions", "positions", "holdings", "portfolio_positions")), + (FinanceCapability.LIST_ORDERS, ("list_orders", "orders", "order_activity")), + (FinanceCapability.GET_BALANCE, ("balance", "balances", "cash", "margin")), + (FinanceCapability.LIST_ALERTS, ("alerts", "watch_alerts")), + (FinanceCapability.LIST_ACCOUNTS, ("accounts", "account_summary")), + (FinanceCapability.GET_CHART, ("chart", "candles", "candle", "ohlc", "bars")), + ( + FinanceCapability.SEARCH_NEWS, + ("search_news", "market_news", "news_search", "news_sentiment", "web_search", "search_web", "headlines"), + ), + (FinanceCapability.SCREEN_MARKET, ("screen_market", "market_screen", "scanner", "scan_market", "search_market", "gainers", "losers")), + (FinanceCapability.GET_QUOTE, ("quote", "quotes", "infoprice", "info_price", "market_price", "snapshot_price")), +) + + +class DiscoveredExternalTool(BaseModel): + name: str + annotations: dict[str, Any] = Field(default_factory=dict) + read_only_hint: bool | None = None + destructive_hint: bool | None = None + open_world_hint: bool | None = None + provider_risk: str | None = None + safety_allowed: bool + safety_reason: str + proposed_capability: str | None = None + sensitivity_class: str = ExternalStateSensitivity.PUBLIC_MARKET_ONLY.value + access_scope: str = "market_data" + auto_selected: bool = False + + +class ExternalStateProviderPayload(BaseModel): + id: str + transport: str + command: str = "" + args: list[str] = Field(default_factory=list) + url: str = "" + env_keys: list[str] = Field(default_factory=list) + timeout_ms: int + allowed_tools: list[str] = Field(default_factory=list) + tool_risks: dict[str, str] = Field(default_factory=dict) + capability_tools: dict[str, str] = Field(default_factory=dict) + + +class ExternalStateSettingsPayload(BaseModel): + enabled: bool + max_calls_per_cycle: int + max_cycle_ms: int + providers: list[ExternalStateProviderPayload] = Field(default_factory=list) + finance: dict[str, Any] = Field(default_factory=dict) + model_native_search: dict[str, Any] = Field(default_factory=dict) + + +class ProviderDiscoveryPayload(BaseModel): + provider_id: str + tools: list[DiscoveredExternalTool] + applied: bool = False + applied_allowed_tools: list[str] = Field(default_factory=list) + applied_capability_tools: dict[str, str] = Field(default_factory=dict) + blocked_count: int = 0 + + +@dataclass(frozen=True, slots=True) +class ProviderConfigInput: + provider_id: str + transport: str = "stdio" + command: str = "" + args: list[str] | None = None + url: str = "" + env: dict[str, str] | None = None + timeout_ms: int = 10000 + + +def validate_provider_id(provider_id: str) -> str: + provider_id = provider_id.strip() + if not _PROVIDER_ID_RE.match(provider_id): + raise ValueError("provider id must match ^[a-zA-Z0-9_-]{1,64}$") + return provider_id + + +def external_state_payload(config: VanerConfig) -> ExternalStateSettingsPayload: + finance = config.external_state.finance + providers: list[ExternalStateProviderPayload] = [] + for provider_id, consumer in sorted(config.external_state.consumers.items()): + capabilities = { + capability: tool + for capability, tool in finance.capability_tools.items() + if tool in set(consumer.allowed_tools) or provider_id == finance.provider + } + providers.append( + ExternalStateProviderPayload( + id=provider_id, + transport=consumer.transport, + command=consumer.command, + args=list(consumer.args), + url=consumer.url, + env_keys=sorted(consumer.env), + timeout_ms=consumer.timeout_ms, + allowed_tools=list(consumer.allowed_tools), + tool_risks=dict(consumer.tool_risks), + capability_tools=capabilities, + ) + ) + return ExternalStateSettingsPayload( + enabled=config.external_state.enabled, + max_calls_per_cycle=config.external_state.max_calls_per_cycle, + max_cycle_ms=config.external_state.max_cycle_ms, + providers=providers, + finance=finance.model_dump(mode="json"), + model_native_search=config.external_state.model_native_search.model_dump(mode="json"), + ) + + +def provider_input_from_config(config: VanerConfig, provider_id: str) -> ProviderConfigInput: + provider_id = validate_provider_id(provider_id) + consumer = config.external_state.consumers.get(provider_id) + if consumer is None: + raise KeyError(provider_id) + return ProviderConfigInput( + provider_id=provider_id, + transport=consumer.transport, + command=consumer.command, + args=list(consumer.args), + url=consumer.url, + env=dict(consumer.env), + timeout_ms=consumer.timeout_ms, + ) + + +async def discover_provider( + config: VanerConfig, + provider_id: str, + *, + apply: bool = False, + trust_unknown_read: bool = False, +) -> ProviderDiscoveryPayload: + provider_id = validate_provider_id(provider_id) + consumer = config.external_state.consumers.get(provider_id) + if consumer is None: + raise KeyError(provider_id) + descriptors = await discover_mcp_tools( + transport=consumer.transport, + command=consumer.command, + args=list(consumer.args), + url=consumer.url, + env=dict(consumer.env), + timeout_ms=consumer.timeout_ms, + ) + payload = build_discovery_payload( + provider_id, + descriptors, + consumer, + finance_market_enabled=config.external_state.finance.market_data_enabled, + finance_account_enabled=config.external_state.finance.account_state_enabled, + trust_unknown_read=trust_unknown_read, + ) + if apply: + apply_discovery(config.repo_root, config.external_state, provider_id, payload, trust_unknown_read=trust_unknown_read) + payload.applied = True + return payload + + +def discover_provider_sync( + config: VanerConfig, + provider_id: str, + *, + apply: bool = False, + trust_unknown_read: bool = False, +) -> ProviderDiscoveryPayload: + return asyncio.run(discover_provider(config, provider_id, apply=apply, trust_unknown_read=trust_unknown_read)) + + +def build_discovery_payload( + provider_id: str, + descriptors: list[McpToolDescriptor], + consumer: McpConsumerConfig, + *, + finance_market_enabled: bool, + finance_account_enabled: bool, + trust_unknown_read: bool = False, +) -> ProviderDiscoveryPayload: + names = [descriptor.name for descriptor in descriptors] + policy_tool_risks = dict(consumer.tool_risks) + if trust_unknown_read: + for descriptor in descriptors: + if _infer_finance_capability(descriptor.name) is not None: + policy_tool_risks.setdefault(descriptor.name, "read") + policy = ReadOnlyToolPolicy(allowed_tools=set(names), tool_risks=policy_tool_risks) + tools: list[DiscoveredExternalTool] = [] + applied_allowed: list[str] = [] + applied_caps: dict[str, str] = {} + applied_cap_scores: dict[str, int] = {} + for descriptor in sorted(descriptors, key=lambda item: item.name): + descriptor = McpToolDescriptor( + name=descriptor.name, + annotations=dict(descriptor.annotations), + provider_risk=policy_tool_risks.get(descriptor.name), + ) + decision = policy.validate(descriptor) + capability = _infer_finance_capability(descriptor.name) + scope = _capability_scope(capability.value if capability is not None else None) + # A discovery/apply action is itself the explicit market-data opt-in. + # Account-state tools remain separately gated because positions, + # balances, orders, and alerts are materially more sensitive. + scope_enabled = finance_account_enabled if scope == "account_state" else True + auto_selected = bool(decision.allowed and capability is not None and scope_enabled) + sensitivity = _capability_sensitivity(capability.value if capability is not None else None) + if auto_selected: + applied_allowed.append(descriptor.name) + score = _capability_tool_score(capability, descriptor.name) + if score > applied_cap_scores.get(capability.value, -1): + applied_caps[capability.value] = descriptor.name + applied_cap_scores[capability.value] = score + tools.append( + DiscoveredExternalTool( + name=descriptor.name, + annotations=dict(descriptor.annotations), + read_only_hint=_bool_or_none(descriptor.annotations.get("readOnlyHint")), + destructive_hint=_bool_or_none(descriptor.annotations.get("destructiveHint")), + open_world_hint=_bool_or_none(descriptor.annotations.get("openWorldHint")), + provider_risk=descriptor.provider_risk, + safety_allowed=decision.allowed, + safety_reason=decision.reason, + proposed_capability=capability.value if capability is not None else None, + sensitivity_class=sensitivity, + access_scope=scope, + auto_selected=auto_selected, + ) + ) + return ProviderDiscoveryPayload( + provider_id=provider_id, + tools=tools, + applied_allowed_tools=applied_allowed, + applied_capability_tools=applied_caps, + blocked_count=len([tool for tool in tools if not tool.safety_allowed]), + ) + + +def save_provider_config(repo_root: Path, provider: ProviderConfigInput) -> Path: + provider_id = validate_provider_id(provider.provider_id) + config_path = _ensure_config(repo_root) + text = config_path.read_text(encoding="utf-8") + text = update_toml_section( + text, + f"external_state.consumers.{provider_id}", + { + "transport": provider.transport, + "command": provider.command, + "args": list(provider.args or []), + "url": provider.url, + "env": dict(provider.env or {}), + "timeout_ms": int(provider.timeout_ms), + }, + ) + config_path.write_text(text, encoding="utf-8") + return config_path + + +def save_external_state_enabled( + repo_root: Path, + *, + enabled: bool | None = None, + max_calls_per_cycle: int | None = None, + max_cycle_ms: int | None = None, +) -> Path: + config_path = _ensure_config(repo_root) + values: dict[str, object] = {} + if enabled is not None: + values["enabled"] = bool(enabled) + if max_calls_per_cycle is not None: + values["max_calls_per_cycle"] = int(max_calls_per_cycle) + if max_cycle_ms is not None: + values["max_cycle_ms"] = int(max_cycle_ms) + text = update_toml_section(config_path.read_text(encoding="utf-8"), "external_state", values) + config_path.write_text(text, encoding="utf-8") + return config_path + + +def save_finance_settings( + repo_root: Path, + *, + enabled: bool | None = None, + provider: str | None = None, + market_data_enabled: bool | None = None, + account_state_enabled: bool | None = None, + capability_tools: dict[str, str] | None = None, +) -> Path: + if provider: + provider = validate_provider_id(provider) + config_path = _ensure_config(repo_root) + values: dict[str, object] = {} + if enabled is not None: + values["enabled"] = bool(enabled) + if provider is not None: + values["provider"] = provider + if market_data_enabled is not None: + values["market_data_enabled"] = bool(market_data_enabled) + if account_state_enabled is not None: + values["account_state_enabled"] = bool(account_state_enabled) + if capability_tools is not None: + values["capability_tools"] = dict(capability_tools) + text = update_toml_section(config_path.read_text(encoding="utf-8"), "external_state.finance", values) + config_path.write_text(text, encoding="utf-8") + return config_path + + +def save_model_native_search_settings( + repo_root: Path, + *, + enabled: bool | None = None, + provider: str | None = None, + base_url: str | None = None, + api_key_env: str | None = None, + max_results: int | None = None, + timeout_seconds: float | None = None, +) -> Path: + config_path = _ensure_config(repo_root) + values: dict[str, object] = {} + if enabled is not None: + values["enabled"] = bool(enabled) + if provider is not None: + if provider not in {"ollama", "brave"}: + raise ValueError("model-native search provider must be 'ollama' or 'brave'") + values["provider"] = provider + if api_key_env is None and provider == "brave": + values["api_key_env"] = "BRAVE_SEARCH_API_KEY" + if base_url is None and provider == "brave": + values["base_url"] = "https://api.search.brave.com/res/v1" + if api_key_env is None and provider == "ollama": + values["api_key_env"] = "OLLAMA_API_KEY" + if base_url is None and provider == "ollama": + values["base_url"] = "https://ollama.com/api" + if base_url is not None: + values["base_url"] = base_url + if api_key_env is not None: + values["api_key_env"] = api_key_env + if max_results is not None: + values["max_results"] = int(max_results) + if timeout_seconds is not None: + values["timeout_seconds"] = float(timeout_seconds) + text = update_toml_section(config_path.read_text(encoding="utf-8"), "external_state.model_native_search", values) + config_path.write_text(text, encoding="utf-8") + return config_path + + +def apply_discovery( + repo_root: Path, + external_state: ExternalStateConfig, + provider_id: str, + discovery: ProviderDiscoveryPayload, + *, + trust_unknown_read: bool = False, +) -> Path: + provider_id = validate_provider_id(provider_id) + consumer = external_state.consumers.get(provider_id, McpConsumerConfig()) + allowed_tools = sorted(set(discovery.applied_allowed_tools)) + tool_risks = dict(consumer.tool_risks) + if trust_unknown_read: + for tool in discovery.tools: + if tool.auto_selected and tool.provider_risk == "read": + tool_risks[tool.name] = "read" + config_path = _ensure_config(repo_root) + text = config_path.read_text(encoding="utf-8") + text = update_toml_section( + text, + f"external_state.consumers.{provider_id}", + { + "transport": consumer.transport, + "command": consumer.command, + "args": list(consumer.args), + "url": consumer.url, + "env": dict(consumer.env), + "timeout_ms": int(consumer.timeout_ms), + "allowed_tools": allowed_tools, + "tool_risks": tool_risks, + }, + ) + capability_tools = dict(external_state.finance.capability_tools) + capability_tools.update(discovery.applied_capability_tools) + applied_market_data = any(_capability_scope(capability) == "market_data" for capability in discovery.applied_capability_tools) + applied_primary_finance = any( + capability != FinanceCapability.SEARCH_NEWS.value and _capability_scope(capability) == "market_data" + for capability in discovery.applied_capability_tools + ) or any(_capability_scope(capability) == "account_state" for capability in discovery.applied_capability_tools) + finance_provider = ( + provider_id + if not external_state.finance.provider or applied_primary_finance + else external_state.finance.provider + ) + text = update_toml_section( + text, + "external_state", + {"enabled": True}, + ) + text = update_toml_section( + text, + "external_state.finance", + { + "enabled": True, + "provider": finance_provider, + "market_data_enabled": bool(external_state.finance.market_data_enabled or applied_market_data), + "account_state_enabled": bool(external_state.finance.account_state_enabled), + "capability_tools": capability_tools, + }, + ) + config_path.write_text(text, encoding="utf-8") + return config_path + + +def _ensure_config(repo_root: Path) -> Path: + config_path = repo_root / ".vaner" / "config.toml" + config_path.parent.mkdir(parents=True, exist_ok=True) + if not config_path.exists(): + config_path.write_text("", encoding="utf-8") + return config_path + + +def _infer_finance_capability(tool_name: str) -> FinanceCapability | None: + normalized = re.sub(r"[^a-z0-9]+", "_", tool_name.lower()).strip("_") + if "alert" in normalized or "user_settings" in normalized: + return None + if "spread_quote" in normalized: + return None + if "stock_strateg" in normalized: + return None + for capability, patterns in _CAPABILITY_PATTERNS: + if any(pattern in normalized for pattern in patterns): + return capability + return None + + +def _capability_scope(capability: str | None) -> str: + if capability in _ACCOUNT_CAPABILITIES: + return "account_state" + return "market_data" + + +def _capability_sensitivity(capability: str | None) -> str: + if capability == FinanceCapability.LIST_ORDERS.value: + return ExternalStateSensitivity.ORDER_ACTIVITY.value + if capability in {FinanceCapability.LIST_POSITIONS.value, FinanceCapability.REVIEW_STRATEGY_POSITIONS.value}: + return ExternalStateSensitivity.POSITION_SPECIFIC.value + if capability in {FinanceCapability.LIST_ACCOUNTS.value, FinanceCapability.GET_BALANCE.value}: + return ExternalStateSensitivity.ACCOUNT_SUMMARY.value + if capability == FinanceCapability.LIST_ALERTS.value: + return ExternalStateSensitivity.USER_WATCHLIST.value + return ExternalStateSensitivity.PUBLIC_MARKET_ONLY.value + + +def _capability_tool_score(capability: FinanceCapability, tool_name: str) -> int: + normalized = re.sub(r"[^a-z0-9]+", "_", tool_name.lower()).strip("_") + exact_suffix = capability.value + if normalized.endswith(exact_suffix): + return 100 + if capability == FinanceCapability.LIST_POSITIONS: + if "closed" in normalized: + return 10 + if "net_positions" in normalized: + return 80 + if "review_strategy_positions" in normalized: + return 20 + if "list_positions" in normalized: + return 100 + if capability == FinanceCapability.LIST_OPTION_EXPIRIES and "standard_option_expiries" in normalized: + return 80 + return 50 + + +def _bool_or_none(value: Any) -> bool | None: + return value if isinstance(value, bool) else None diff --git a/src/vaner/external_state/manager.py b/src/vaner/external_state/manager.py new file mode 100644 index 0000000..6965dea --- /dev/null +++ b/src/vaner/external_state/manager.py @@ -0,0 +1,445 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import json +import time +from typing import Any + +from vaner.external_state.model_search import ModelNativeSearchProvider +from vaner.external_state.models import ( + ExternalStateFreshnessClass, + ExternalStateSensitivity, + ExternalStateSnapshot, + FinanceCapability, +) +from vaner.integrations.mcp_consumer.client import McpConsumerClient +from vaner.integrations.mcp_consumer.safety import ReadOnlyToolPolicy +from vaner.models.config import ExternalStateConfig + +_ACCOUNT_CAPABILITIES = { + FinanceCapability.LIST_ACCOUNTS.value, + FinanceCapability.GET_BALANCE.value, + FinanceCapability.LIST_POSITIONS.value, + FinanceCapability.LIST_ORDERS.value, + FinanceCapability.LIST_ALERTS.value, + FinanceCapability.REVIEW_STRATEGY_POSITIONS.value, +} + +_NEWS_TERMS = { + "catalyst", + "earnings", + "fed", + "headline", + "headlines", + "macro", + "news", + "sentiment", +} + +_OPTION_TERMS = { + "call", + "covered call", + "delta", + "expiry", + "greek", + "iv", + "option", + "premium", + "put", + "spread", + "strike", + "theta", + "vega", + "volatility", +} + +_TRADING_EXPLORATION_TERMS = { + "branch", + "branches", + "explore", + "setup", + "setups", + "trade", + "trades", + "trading", +} + +_MARKET_EXPLORATION_BRANCHES: tuple[dict[str, Any], ...] = ( + {"preset": "top_gainers", "market": "us", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, + {"preset": "top_losers", "market": "us", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, + {"preset": "premarket_gainers", "market": "us", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, + {"preset": "premarket_losers", "market": "us", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, + {"preset": "top_gainers", "market": "us_nasdaq", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, + {"preset": "top_losers", "market": "us_nasdaq", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, + {"preset": "top_gainers", "market": "us_nyse", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, + {"preset": "top_losers", "market": "us_nyse", "assetType": "Stock", "limit": 10, "maxInstruments": 100}, +) + + +class ExternalStateManager: + """Owns configured external-state MCP consumers for one Vaner engine.""" + + def __init__(self, config: ExternalStateConfig, *, model_search_provider: ModelNativeSearchProvider | None = None) -> None: + self.config = config + self._model_search_provider = model_search_provider + self._clients: dict[str, McpConsumerClient] = {} + self._cycle_calls = 0 + + @property + def enabled(self) -> bool: + return bool(self.config.enabled) + + async def start(self) -> None: + if not self.enabled: + return + for name, consumer in self.config.consumers.items(): + policy = ReadOnlyToolPolicy(allowed_tools=set(consumer.allowed_tools), tool_risks=dict(consumer.tool_risks)) + client = McpConsumerClient( + server_name=name, + transport=consumer.transport, + command=consumer.command, + args=consumer.args, + url=consumer.url, + env=consumer.env, + timeout_ms=consumer.timeout_ms, + policy=policy, + ) + await client.start() + self._clients[name] = client + + async def stop(self) -> None: + for client in list(self._clients.values()): + await client.stop() + self._clients.clear() + + def reset_cycle_budget(self) -> None: + self._cycle_calls = 0 + + def _check_cycle_budget(self) -> bool: + if self.config.max_calls_per_cycle <= 0: + return False + if self._cycle_calls >= self.config.max_calls_per_cycle: + return False + self._cycle_calls += 1 + return True + + async def call_finance_capability( + self, + capability: FinanceCapability | str, + args: dict[str, Any] | None = None, + *, + query_key: str = "", + freshness_class: ExternalStateFreshnessClass = ExternalStateFreshnessClass.MARKET_SNAPSHOT, + sensitivity_class: ExternalStateSensitivity = ExternalStateSensitivity.PUBLIC_MARKET_ONLY, + ttl_seconds: float | None = 300.0, + ) -> ExternalStateSnapshot | None: + if not self.enabled or not self.config.finance.enabled: + return None + capability_name = capability.value if isinstance(capability, FinanceCapability) else str(capability) + if capability_name in _ACCOUNT_CAPABILITIES and not self.config.finance.account_state_enabled: + return None + if capability_name not in _ACCOUNT_CAPABILITIES and not self.config.finance.market_data_enabled: + return None + tool_name = self.config.finance.capability_tools.get(capability_name, "") + if not tool_name: + return None + provider = self._provider_for_tool(tool_name) + if not provider or provider not in self._clients: + return None + if not self._check_cycle_budget(): + return None + result = await self._clients[provider].call_tool(tool_name, args or {}) + if not result.ok: + return None + payload = _payload_to_jsonable(result.payload) + if _payload_is_error(payload): + return None + return ExternalStateSnapshot.build( + provider_id=provider, + capability=capability_name, + query_key=query_key, + source_tool=tool_name, + freshness_class=freshness_class, + sensitivity_class=sensitivity_class, + payload=payload, + captured_at=time.time(), + ttl_seconds=ttl_seconds, + ) + + async def collect_finance_snapshots(self, recent_queries: list[str]) -> list[ExternalStateSnapshot]: + query_text = " ".join(recent_queries[-5:]).strip() + if not query_text: + return [] + snapshots: list[ExternalStateSnapshot] = [] + screen_criteria = _screen_market_criteria(query_text) + screen = await self.call_finance_capability( + FinanceCapability.SCREEN_MARKET, + screen_criteria, + query_key=_safe_query_key(query_text), + freshness_class=ExternalStateFreshnessClass.MARKET_SNAPSHOT, + sensitivity_class=ExternalStateSensitivity.PUBLIC_MARKET_ONLY, + ttl_seconds=300.0, + ) + if screen is not None: + snapshots.append(screen) + for branch_index, branch_criteria in enumerate(_market_exploration_branches(query_text, screen_criteria)): + branch = await self.call_finance_capability( + FinanceCapability.SCREEN_MARKET, + branch_criteria, + query_key=_safe_query_key(f"market-branch:{branch_index}:{query_text}"), + freshness_class=ExternalStateFreshnessClass.MARKET_SNAPSHOT, + sensitivity_class=ExternalStateSensitivity.PUBLIC_MARKET_ONLY, + ttl_seconds=300.0, + ) + if branch is not None: + snapshots.append(branch) + if _looks_option_related(query_text) and FinanceCapability.SCREEN_OPTION_STRATEGIES.value in self.config.finance.capability_tools: + account_key: str | None = None + if self.config.finance.account_state_enabled: + accounts = await self.call_finance_capability( + FinanceCapability.LIST_ACCOUNTS, + {}, + query_key="account_accounts", + freshness_class=ExternalStateFreshnessClass.ACCOUNT_SNAPSHOT, + sensitivity_class=ExternalStateSensitivity.ACCOUNT_SUMMARY, + ttl_seconds=120.0, + ) + if accounts is not None: + snapshots.append(accounts) + account_key = _first_account_key(accounts.payload) + if account_key is None: + return snapshots + strategies = await self.call_finance_capability( + FinanceCapability.SCREEN_OPTION_STRATEGIES, + _option_strategy_criteria(query_text, account_key=account_key), + query_key=_safe_query_key(f"option-strategies:{query_text}"), + freshness_class=ExternalStateFreshnessClass.DERIVED_ANALYSIS, + sensitivity_class=ExternalStateSensitivity.MIXED_SENSITIVE, + ttl_seconds=300.0, + ) + if strategies is not None: + snapshots.append(strategies) + news = await self._collect_news_snapshot(query_text) + if news is not None: + snapshots.append(news) + if self.config.finance.account_state_enabled: + positions = await self.call_finance_capability( + FinanceCapability.LIST_POSITIONS, + {}, + query_key="account_positions", + freshness_class=ExternalStateFreshnessClass.ACCOUNT_SNAPSHOT, + sensitivity_class=ExternalStateSensitivity.POSITION_SPECIFIC, + ttl_seconds=120.0, + ) + if positions is not None: + snapshots.append(positions) + return snapshots + + async def _collect_news_snapshot(self, query_text: str) -> ExternalStateSnapshot | None: + if not _looks_news_related(query_text): + return None + if not self.enabled or not self.config.finance.enabled or not self.config.finance.market_data_enabled: + return None + criteria = _news_search_criteria(query_text) + if FinanceCapability.SEARCH_NEWS.value in self.config.finance.capability_tools: + news = await self.call_finance_capability( + FinanceCapability.SEARCH_NEWS, + criteria, + query_key=_safe_query_key(f"news:{query_text}"), + freshness_class=ExternalStateFreshnessClass.NEWS_SNAPSHOT, + sensitivity_class=ExternalStateSensitivity.PUBLIC_MARKET_ONLY, + ttl_seconds=900.0, + ) + if news is not None: + return news + if self._model_search_provider is None or not self._check_cycle_budget(): + return None + try: + return await self._model_search_provider.search_news( + str(criteria["query"]), + topics=list(criteria["topics"]), + lookback_days=int(criteria["lookback_days"]), + limit=int(criteria["limit"]), + ) + except Exception: + return None + + def _provider_for_tool(self, tool_name: str) -> str: + preferred = self.config.finance.provider + if preferred and preferred in self._clients: + consumer = self.config.consumers.get(preferred) + if consumer is None or tool_name in set(consumer.allowed_tools): + return preferred + for provider, consumer in self.config.consumers.items(): + if provider in self._clients and tool_name in set(consumer.allowed_tools): + return provider + return preferred + + +def _payload_to_jsonable(payload: Any) -> dict[str, Any]: + if hasattr(payload, "model_dump"): + data = payload.model_dump(mode="json") + elif isinstance(payload, dict): + data = payload + else: + data = {"result": repr(payload)} + return data if isinstance(data, dict) else {"result": data} + + +def _payload_is_error(payload: dict[str, Any]) -> bool: + return payload.get("isError") is True + + +def _first_account_key(payload: dict[str, Any]) -> str | None: + data = _payload_data(payload) + queue: list[Any] = [data] + while queue: + current = queue.pop(0) + if isinstance(current, dict): + for key in ("AccountKey", "accountKey", "account_key"): + value = current.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + queue.extend(current.values()) + elif isinstance(current, list): + queue.extend(current) + return None + + +def _payload_data(payload: dict[str, Any]) -> Any: + content = payload.get("content") + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and item.get("type") == "text" and isinstance(item.get("text"), str): + text = item["text"].strip() + if not text: + continue + try: + return json.loads(text) + except json.JSONDecodeError: + return payload + return payload + + +def _safe_query_key(query: str) -> str: + return hashlib.sha1(query.encode("utf-8")).hexdigest()[:16] # noqa: S324 + + +def _looks_option_related(query: str) -> bool: + text = query.lower() + return any(term in text for term in _OPTION_TERMS) + + +def _looks_news_related(query: str) -> bool: + text = query.lower() + return any(term in text for term in _NEWS_TERMS) + + +def _looks_trading_exploration_related(query: str) -> bool: + text = query.lower() + return any(term in text for term in _TRADING_EXPLORATION_TERMS) + + +def _market_exploration_branches(query: str, primary: dict[str, Any]) -> list[dict[str, Any]]: + if not _looks_trading_exploration_related(query): + return [] + primary_key = (str(primary.get("preset") or ""), str(primary.get("market") or "")) + candidates = [ + dict(branch) + for branch in _MARKET_EXPLORATION_BRANCHES + if (str(branch.get("preset") or ""), str(branch.get("market") or "")) != primary_key + ] + if not candidates: + return [] + # Rotate the exploration branch every cycle instead of replaying the same + # branch set forever. This keeps a stable refresh path while adding novelty + # pressure across adjacent market regimes. + bucket = int(time.time() // 120) + offset = bucket % len(candidates) + rotated = candidates[offset:] + candidates[:offset] + return rotated[:2] + + +def _screen_market_criteria(query: str) -> dict[str, Any]: + """Build provider-neutral structured criteria for the market-screen capability.""" + + text = query.lower() + premarket = "premarket" in text or "pre-market" in text + downside = any(term in text for term in ("decliner", "decliners", "down", "loser", "losers", "selloff")) + if premarket: + preset = "premarket_losers" if downside else "premarket_gainers" + else: + preset = "top_losers" if downside else "top_gainers" + + market = "us" + market_terms = ( + ("denmark", ("denmark", "danish", "copenhagen")), + ("nordics", ("nordic", "nordics")), + ("sweden", ("sweden", "swedish", "stockholm")), + ("norway", ("norway", "norwegian", "oslo")), + ("finland", ("finland", "finnish", "helsinki")), + ("europe", ("europe", "european")), + ("us_nasdaq", ("nasdaq",)), + ("us_nyse", ("nyse",)), + ) + for candidate, terms in market_terms: + if any(term in text for term in terms): + market = candidate + break + + return { + "preset": preset, + "market": market, + "assetType": "Stock", + "limit": 10, + "maxInstruments": 100, + } + + +def _news_search_criteria(query: str) -> dict[str, Any]: + return { + "query": query[:300], + "topics": ["markets", "finance"], + "lookback_days": 7, + "limit": 10, + } + + +def _option_strategy_criteria(query: str, *, account_key: str) -> dict[str, Any]: + screen = _screen_market_criteria(query) + market = screen["market"] + if market not in {"us", "us_nasdaq", "us_nyse"}: + market = "us" + text = query.lower() + strategies: list[str] + playbook = "income_30_60d" + if "condor" in text: + strategies = ["iron_condor"] + elif any(term in text for term in ("bear", "downside", "put spread", "credit spread")): + strategies = ["put_credit_spread", "call_credit_spread"] + elif any(term in text for term in ("leap", "long call", "debit")): + strategies = ["long_call", "debit_spread"] + playbook = "long_term_directional" + else: + strategies = ["cash_secured_put", "put_credit_spread", "iron_condor"] + return { + "accountKey": account_key, + "market": market, + "underlyingUniverse": "auto", + "underlyingPreset": screen["preset"], + "playbook": playbook, + "riskProfile": "balanced", + "strategies": strategies, + "maxUnderlyings": 10, + "maxUnderlyingScan": 120, + "maxSymbolsToPlan": 3, + "maxPlans": 6, + "includeAccountContext": True, + "includeTechnicalContext": True, + "includeVolatilityContext": True, + "includeNewsContext": False, + "requireGreeks": False, + "riskBudgetPercent": 1, + } diff --git a/src/vaner/external_state/model_search.py b/src/vaner/external_state/model_search.py new file mode 100644 index 0000000..c635092 --- /dev/null +++ b/src/vaner/external_state/model_search.py @@ -0,0 +1,236 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import time +from typing import Any, Protocol + +import httpx + +from vaner.external_state.models import ( + ExternalStateFreshnessClass, + ExternalStateSensitivity, + ExternalStateSnapshot, + FinanceCapability, +) +from vaner.models.config import ExternalStateConfig + + +class ModelNativeSearchProvider(Protocol): + """Runtime hook for model/client-native search tools. + + Vaner core deliberately does not know whether the implementation is a + hosted model tool, a browser/search MCP server, or a local search gateway. + The runtime may inject an implementation when that capability is available. + """ + + async def search_news( + self, + query: str, + *, + topics: list[str], + lookback_days: int, + limit: int, + ) -> ExternalStateSnapshot | None: + ... + + +def build_model_search_news_snapshot( + *, + provider_id: str, + query: str, + payload: dict[str, Any], + source_tool: str = "model_native_search", + ttl_seconds: float = 900.0, +) -> ExternalStateSnapshot: + return ExternalStateSnapshot.build( + provider_id=provider_id, + capability=FinanceCapability.SEARCH_NEWS.value, + query_key=f"news:{query[:200]}", + source_tool=source_tool, + freshness_class=ExternalStateFreshnessClass.NEWS_SNAPSHOT, + sensitivity_class=ExternalStateSensitivity.PUBLIC_MARKET_ONLY, + payload=payload, + captured_at=time.time(), + ttl_seconds=ttl_seconds, + ) + + +class OllamaWebSearchProvider: + """Adapter for Ollama's web search API. + + This uses Ollama's provider-side search endpoint, not the local model chat + endpoint. It is opt-in and requires an API key configured through env. + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://ollama.com/api", + max_results: int = 5, + timeout_seconds: float = 15.0, + ) -> None: + self.api_key = api_key + self.base_url = base_url.rstrip("/") + self.max_results = max(1, min(int(max_results), 10)) + self.timeout_seconds = float(timeout_seconds) + + async def search_news( + self, + query: str, + *, + topics: list[str], + lookback_days: int, + limit: int, + ) -> ExternalStateSnapshot | None: + search_query = _search_query(query, topics=topics, lookback_days=lookback_days) + async with httpx.AsyncClient(timeout=self.timeout_seconds) as client: + response = await client.post( + f"{self.base_url}/web_search", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={"query": search_query, "max_results": min(int(limit), self.max_results)}, + ) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + payload = {"result": payload} + return build_model_search_news_snapshot( + provider_id="ollama_web_search", + query=query, + source_tool="ollama.web_search", + payload={ + "query": search_query, + "topics": topics, + "lookback_days": lookback_days, + "results": payload.get("results", payload), + }, + ) + + +class BraveWebSearchProvider: + """Adapter for Brave Search's web endpoint.""" + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.search.brave.com/res/v1", + max_results: int = 5, + timeout_seconds: float = 15.0, + ) -> None: + self.api_key = api_key + self.base_url = base_url.rstrip("/") + self.max_results = max(1, min(int(max_results), 10)) + self.timeout_seconds = float(timeout_seconds) + + async def search_news( + self, + query: str, + *, + topics: list[str], + lookback_days: int, + limit: int, + ) -> ExternalStateSnapshot | None: + search_query = _search_query(query, topics=topics, lookback_days=lookback_days) + async with httpx.AsyncClient(timeout=self.timeout_seconds) as client: + response = await client.get( + f"{self.base_url}/web/search", + headers={ + "Accept": "application/json", + "X-Subscription-Token": self.api_key, + }, + params={ + "q": search_query, + "count": min(int(limit), self.max_results), + "search_lang": "en", + "spellcheck": 1, + }, + ) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + payload = {"result": payload} + return build_model_search_news_snapshot( + provider_id="brave_search", + query=query, + source_tool="brave.web_search", + payload={ + "query": search_query, + "topics": topics, + "lookback_days": lookback_days, + "results": _brave_results(payload), + }, + ) + + +def model_search_provider_from_config(config: ExternalStateConfig) -> ModelNativeSearchProvider | None: + search = config.model_native_search + if not search.enabled: + return None + provider = search.provider + api_key_env = _api_key_env(provider, search.api_key_env) + api_key = os.environ.get(api_key_env, "").strip() + if not api_key: + return None + if provider == "ollama": + base_url = search.base_url or "https://ollama.com/api" + if "api.search.brave.com" in base_url: + base_url = "https://ollama.com/api" + return OllamaWebSearchProvider( + api_key=api_key, + base_url=base_url, + max_results=search.max_results, + timeout_seconds=search.timeout_seconds, + ) + if provider == "brave": + base_url = search.base_url or "https://api.search.brave.com/res/v1" + if "ollama.com" in base_url: + base_url = "https://api.search.brave.com/res/v1" + return BraveWebSearchProvider( + api_key=api_key, + base_url=base_url, + max_results=search.max_results, + timeout_seconds=search.timeout_seconds, + ) + return None + + +def _api_key_env(provider: str, configured: str) -> str: + if provider == "brave" and (not configured or configured == "OLLAMA_API_KEY"): + return "BRAVE_SEARCH_API_KEY" + if provider == "ollama" and not configured: + return "OLLAMA_API_KEY" + return configured + + +def _brave_results(payload: dict[str, Any]) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for bucket in ("news", "web"): + section = payload.get(bucket) + if not isinstance(section, dict): + continue + items = section.get("results") + if not isinstance(items, list): + continue + for item in items: + if not isinstance(item, dict): + continue + results.append( + { + "type": bucket, + "title": item.get("title"), + "url": item.get("url"), + "description": item.get("description"), + "age": item.get("age"), + "page_age": item.get("page_age"), + } + ) + return results or [payload] + + +def _search_query(query: str, *, topics: list[str], lookback_days: int) -> str: + terms = ", ".join(topic for topic in topics if topic) + suffix = f" recent {terms} news last {max(1, int(lookback_days))} days".strip() + return f"{query[:300]} {suffix}".strip()[:380] diff --git a/src/vaner/external_state/models.py b/src/vaner/external_state/models.py new file mode 100644 index 0000000..481e45b --- /dev/null +++ b/src/vaner/external_state/models.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import json +import time +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, Field + + +class ExternalStateFreshnessClass(StrEnum): + STATIC_REFERENCE = "static_reference" + SLOW_MARKET_CONTEXT = "slow_market_context" + MARKET_SNAPSHOT = "market_snapshot" + ACCOUNT_SNAPSHOT = "account_snapshot" + NEWS_SNAPSHOT = "news_snapshot" + DERIVED_ANALYSIS = "derived_analysis" + + +class ExternalStateSensitivity(StrEnum): + PUBLIC_MARKET_ONLY = "public_market_only" + USER_WATCHLIST = "user_watchlist" + ACCOUNT_SUMMARY = "account_summary" + POSITION_SPECIFIC = "position_specific" + ORDER_ACTIVITY = "order_activity" + MIXED_SENSITIVE = "mixed_sensitive" + + +class FinanceCapability(StrEnum): + GET_QUOTE = "get_quote" + GET_CHART = "get_chart" + SEARCH_NEWS = "search_news" + SCREEN_MARKET = "screen_market" + GET_OPTION_CHAIN = "get_option_chain" + LIST_OPTION_EXPIRIES = "list_option_expiries" + ANALYZE_OPTION_STRATEGY = "analyze_option_strategy" + SCREEN_OPTION_STRATEGIES = "screen_option_strategies" + REVIEW_STRATEGY_POSITIONS = "review_strategy_positions" + LIST_ACCOUNTS = "list_accounts" + GET_BALANCE = "get_balance" + LIST_POSITIONS = "list_positions" + LIST_ORDERS = "list_orders" + LIST_ALERTS = "list_alerts" + + +class ExternalStateSnapshot(BaseModel): + id: str + provider_id: str + capability: str + query_key: str = "" + source_tool: str = "" + freshness_class: ExternalStateFreshnessClass + sensitivity_class: ExternalStateSensitivity + captured_at: float = Field(default_factory=time.time) + expires_at: float | None = None + payload_fingerprint: str = "" + payload: dict[str, Any] = Field(default_factory=dict) + + @classmethod + def build( + cls, + *, + provider_id: str, + capability: str, + freshness_class: ExternalStateFreshnessClass, + sensitivity_class: ExternalStateSensitivity, + payload: dict[str, Any], + query_key: str = "", + source_tool: str = "", + captured_at: float | None = None, + ttl_seconds: float | None = None, + ) -> ExternalStateSnapshot: + ts = time.time() if captured_at is None else float(captured_at) + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + fingerprint = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + digest = hashlib.sha1( # noqa: S324 + f"{provider_id}\n{capability}\n{query_key}\n{fingerprint}\n{ts:.6f}".encode() + ).hexdigest()[:20] + return cls( + id=f"ext-{digest}", + provider_id=provider_id, + capability=capability, + query_key=query_key, + source_tool=source_tool, + freshness_class=freshness_class, + sensitivity_class=sensitivity_class, + captured_at=ts, + expires_at=(ts + float(ttl_seconds)) if ttl_seconds is not None else None, + payload_fingerprint=fingerprint, + payload=payload, + ) + + def is_stale(self, *, now: float | None = None) -> bool: + if self.expires_at is None: + return False + return (time.time() if now is None else float(now)) >= self.expires_at diff --git a/src/vaner/external_state/telemetry.py b/src/vaner/external_state/telemetry.py new file mode 100644 index 0000000..d2584f4 --- /dev/null +++ b/src/vaner/external_state/telemetry.py @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any + +from vaner.external_state.models import ExternalStateSnapshot + + +def redacted_snapshot_telemetry(snapshot: ExternalStateSnapshot, *, status: str, latency_ms: float) -> dict[str, Any]: + """Return telemetry-safe metadata for an external-state snapshot.""" + + return { + "provider_id": snapshot.provider_id, + "capability": snapshot.capability, + "status": status, + "latency_ms": round(max(0.0, float(latency_ms)), 3), + "sensitivity_class": snapshot.sensitivity_class.value, + "freshness_class": snapshot.freshness_class.value, + "captured_at": snapshot.captured_at, + "expires_at": snapshot.expires_at, + "payload_fingerprint": snapshot.payload_fingerprint, + } diff --git a/src/vaner/integrations/capability.py b/src/vaner/integrations/capability.py index bb5e83e..4446dc7 100644 --- a/src/vaner/integrations/capability.py +++ b/src/vaner/integrations/capability.py @@ -33,10 +33,94 @@ class ClientCapabilityTier(IntEnum): TIER_4 = 4 +class ClientFamily(IntEnum): + """Normalised host family derived from ``clientInfo.name``. + + Tier detection answers *what the host can render*. ClientFamily answers + *which host is rendering it* — useful for tiny per-host UX tweaks + (markdown vs plain, slash-command hints, dashboard wording) without + branching on capability flags. ``UNKNOWN`` covers everything we don't + have a tested heuristic for; treat it the same as a generic Tier-aware + host. + """ + + UNKNOWN = 0 + CLAUDE_CODE = 1 + CLAUDE_DESKTOP = 2 + CLAUDE_WEB = 3 + CHATGPT = 4 + CODEX = 5 + CURSOR = 6 + VSCODE = 7 + ZED = 8 + GOOSE = 9 + WINDSURF = 10 + CLINE = 11 + CONTINUE = 12 + + _UI_EXT_KEY = "io.modelcontextprotocol/ui" _INJECTION_EXT_KEY = "vaner.context_injection" +def classify_client(name: str | None) -> ClientFamily: + """Map a ``clientInfo.name`` string into a :class:`ClientFamily`. + + Matching is case-insensitive and substring-tolerant because hosts are + inconsistent ("claude-code" vs "Claude Code" vs "claude-code-cli"). New + hosts default to ``UNKNOWN`` — adding one is intentional, not automatic, + so we can document the heuristic per host. + """ + + if not isinstance(name, str) or not name.strip(): + return ClientFamily.UNKNOWN + raw = name.strip().lower() + # Normalise whitespace and underscores to hyphens so "Claude Code", + # "claude_code", and "claude-code" all classify identically. + n = raw.replace("_", "-") + n_compact = " ".join(n.split()) + n = n_compact.replace(" ", "-") + # Order matters: more specific prefixes first. + if "claude-code" in n: + return ClientFamily.CLAUDE_CODE + if "claude-desktop" in n: + return ClientFamily.CLAUDE_DESKTOP + if n in {"claude.ai", "claude-ai", "claude-web", "claudeai"}: + return ClientFamily.CLAUDE_WEB + if n.startswith("chatgpt") or "openai-chatgpt" in n: + return ClientFamily.CHATGPT + if "codex" in n: + return ClientFamily.CODEX + if "cursor" in n: + return ClientFamily.CURSOR + if "vscode" in n or "vs-code" in n or "visual-studio-code" in n or "copilot" in n: + return ClientFamily.VSCODE + if n.startswith("zed"): + return ClientFamily.ZED + if "goose" in n: + return ClientFamily.GOOSE + if "windsurf" in n: + return ClientFamily.WINDSURF + if "cline" in n: + return ClientFamily.CLINE + if n.startswith("continue") or "continue.dev" in n: + return ClientFamily.CONTINUE + return ClientFamily.UNKNOWN + + +def is_terminal_host(family: ClientFamily) -> bool: + """Return True for hosts that render output as terminal text (no iframes).""" + + return family in { + ClientFamily.CLAUDE_CODE, + ClientFamily.CODEX, + ClientFamily.ZED, + ClientFamily.CLINE, + ClientFamily.CONTINUE, + ClientFamily.WINDSURF, + } + + @dataclass(frozen=True) class TierDetection: """Explainable tier classification — includes the signals used.""" @@ -193,6 +277,15 @@ def current_detection(session: Any) -> TierDetection | None: return _CACHE.get(session) +def current_client_family(session: Any) -> ClientFamily: + """Return the :class:`ClientFamily` for *session*, or UNKNOWN if not seen.""" + + detection = _CACHE.get(session) + if detection is None: + return ClientFamily.UNKNOWN + return classify_client(detection.client_name) + + def reset_cache() -> None: """Test-only helper — clear the per-session cache.""" _CACHE._by_id.clear() diff --git a/src/vaner/integrations/mcp_consumer/__init__.py b/src/vaner/integrations/mcp_consumer/__init__.py new file mode 100644 index 0000000..4460339 --- /dev/null +++ b/src/vaner/integrations/mcp_consumer/__init__.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 + +from vaner.integrations.mcp_consumer.safety import ( + McpToolDescriptor, + ReadOnlyToolPolicy, + ToolSafetyDecision, +) + +__all__ = [ + "McpToolDescriptor", + "ReadOnlyToolPolicy", + "ToolSafetyDecision", +] diff --git a/src/vaner/integrations/mcp_consumer/client.py b/src/vaner/integrations/mcp_consumer/client.py new file mode 100644 index 0000000..a0689aa --- /dev/null +++ b/src/vaner/integrations/mcp_consumer/client.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import os +from dataclasses import dataclass +from typing import Any + +from vaner.integrations.mcp_consumer.safety import McpToolDescriptor, ReadOnlyToolPolicy, ToolSafetyDecision + + +@dataclass(frozen=True, slots=True) +class McpToolCallResult: + ok: bool + payload: Any = None + error: str = "" + safety_reason: str = "" + + +class McpConsumerClient: + """Managed MCP client for proactive read-only external-state calls.""" + + def __init__( + self, + *, + server_name: str, + transport: str, + command: str = "", + args: list[str] | None = None, + url: str = "", + env: dict[str, str] | None = None, + timeout_ms: int = 10000, + policy: ReadOnlyToolPolicy, + ) -> None: + self.server_name = server_name + self.transport = transport + self.command = command + self.args = list(args or []) + self.url = url + self.env = dict(env or {}) + self.timeout_ms = timeout_ms + self.policy = policy + self._session: Any = None + self._stdio_cm: Any = None + self._streams_cm: Any = None + self._tool_descriptors: dict[str, McpToolDescriptor] = {} + + async def start(self) -> None: + try: + from mcp import ClientSession + from mcp.client.stdio import StdioServerParameters, stdio_client + from mcp.client.streamable_http import streamablehttp_client + except ImportError as exc: # pragma: no cover - optional dependency guard + raise RuntimeError("mcp consumer support requires the 'mcp' optional dependency") from exc + + if self.transport == "stdio": + params = StdioServerParameters(command=self.command, args=self.args, env=_merged_env(self.env)) + self._stdio_cm = stdio_client(params) + read, write = await self._stdio_cm.__aenter__() + elif self.transport == "streamable_http": + self._streams_cm = streamablehttp_client(self.url) + read, write, _ = await self._streams_cm.__aenter__() + else: + raise ValueError(f"unsupported MCP consumer transport: {self.transport}") + try: + self._session = ClientSession(read, write) + await self._session.__aenter__() + await self._session.initialize() + decisions = await self.refresh_tools() + missing = sorted(self.policy.allowed_tools - set(self._tool_descriptors)) + if missing: + raise RuntimeError(f"MCP consumer {self.server_name} allowlist references unknown tools: {', '.join(missing)}") + unsafe = [decision for decision in decisions if decision.tool_name in self.policy.allowed_tools and not decision.allowed] + if unsafe: + details = ", ".join(f"{decision.tool_name}:{decision.reason}" for decision in unsafe) + raise RuntimeError(f"MCP consumer {self.server_name} allowlist contains unsafe tools: {details}") + except Exception: + await self.stop() + raise + + async def stop(self) -> None: + if self._session is not None: + await self._session.__aexit__(None, None, None) + self._session = None + if self._stdio_cm is not None: + await self._stdio_cm.__aexit__(None, None, None) + self._stdio_cm = None + if self._streams_cm is not None: + await self._streams_cm.__aexit__(None, None, None) + self._streams_cm = None + + async def refresh_tools(self) -> list[ToolSafetyDecision]: + if self._session is None: + raise RuntimeError("MCP consumer client is not started") + response = await asyncio.wait_for(self._session.list_tools(), timeout=self.timeout_ms / 1000) + descriptors: dict[str, McpToolDescriptor] = {} + for tool in getattr(response, "tools", []) or []: + annotations = getattr(tool, "annotations", None) + if annotations is None: + annotation_payload: dict[str, Any] = {} + elif hasattr(annotations, "model_dump"): + annotation_payload = annotations.model_dump(exclude_none=True) + else: + annotation_payload = dict(annotations) + descriptors[str(getattr(tool, "name", ""))] = McpToolDescriptor( + name=str(getattr(tool, "name", "")), + annotations=annotation_payload, + ) + self._tool_descriptors = descriptors + return self.policy.validate_all(list(descriptors.values())) + + async def call_tool(self, tool_name: str, args: dict[str, Any] | None = None) -> McpToolCallResult: + if self._session is None: + return McpToolCallResult(ok=False, error="client_not_started") + descriptor = self._tool_descriptors.get(tool_name, McpToolDescriptor(name=tool_name)) + decision = self.policy.validate(descriptor) + if not decision.allowed: + return McpToolCallResult(ok=False, error="tool_blocked", safety_reason=decision.reason) + try: + payload = await asyncio.wait_for(self._session.call_tool(tool_name, args or {}), timeout=self.timeout_ms / 1000) + except Exception as exc: # pragma: no cover - transport boundary + return McpToolCallResult(ok=False, error=str(exc), safety_reason=decision.reason) + return McpToolCallResult(ok=True, payload=payload, safety_reason=decision.reason) + + +async def discover_mcp_tools( + *, + transport: str, + command: str = "", + args: list[str] | None = None, + url: str = "", + env: dict[str, str] | None = None, + timeout_ms: int = 10000, +) -> list[McpToolDescriptor]: + """Connect to an MCP server long enough to list tools. + + Discovery never calls provider tools and intentionally does not apply a + consumer allowlist. The caller is responsible for running the returned + descriptors through :class:`ReadOnlyToolPolicy` before persisting any tool + as callable external state. + """ + + try: + from mcp import ClientSession + from mcp.client.stdio import StdioServerParameters, stdio_client + from mcp.client.streamable_http import streamablehttp_client + except ImportError as exc: # pragma: no cover - optional dependency guard + raise RuntimeError("mcp consumer support requires the 'mcp' optional dependency") from exc + + stdio_cm: Any = None + streams_cm: Any = None + session: Any = None + try: + if transport == "stdio": + params = StdioServerParameters(command=command, args=list(args or []), env=_merged_env(env)) + stdio_cm = stdio_client(params) + read, write = await stdio_cm.__aenter__() + elif transport == "streamable_http": + streams_cm = streamablehttp_client(url) + read, write, _ = await streams_cm.__aenter__() + else: + raise ValueError(f"unsupported MCP consumer transport: {transport}") + + session = ClientSession(read, write) + await session.__aenter__() + await session.initialize() + response = await asyncio.wait_for(session.list_tools(), timeout=timeout_ms / 1000) + descriptors: list[McpToolDescriptor] = [] + for tool in getattr(response, "tools", []) or []: + annotations = getattr(tool, "annotations", None) + if annotations is None: + annotation_payload: dict[str, Any] = {} + elif hasattr(annotations, "model_dump"): + annotation_payload = annotations.model_dump(exclude_none=True) + else: + annotation_payload = dict(annotations) + name = str(getattr(tool, "name", "")).strip() + if name: + descriptors.append(McpToolDescriptor(name=name, annotations=annotation_payload)) + return descriptors + finally: + if session is not None: + await session.__aexit__(None, None, None) + if stdio_cm is not None: + await stdio_cm.__aexit__(None, None, None) + if streams_cm is not None: + await streams_cm.__aexit__(None, None, None) + + +def _merged_env(env: dict[str, str] | None) -> dict[str, str]: + merged = dict(os.environ) + merged.update(dict(env or {})) + return merged diff --git a/src/vaner/integrations/mcp_consumer/safety.py b/src/vaner/integrations/mcp_consumer/safety.py new file mode 100644 index 0000000..8d8c997 --- /dev/null +++ b/src/vaner/integrations/mcp_consumer/safety.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +_DENY_NAME_RE = re.compile( + r"(^|[_\-.])(" + r"auth|oauth|token|login|logout|" + r"precheck|place|modify|cancel|create|update|delete|submit|execute|write" + r")([_\-.]|$)", + re.IGNORECASE, +) + + +@dataclass(frozen=True, slots=True) +class McpToolDescriptor: + name: str + annotations: dict[str, Any] = field(default_factory=dict) + provider_risk: str | None = None + + +@dataclass(frozen=True, slots=True) +class ToolSafetyDecision: + allowed: bool + reason: str + tool_name: str + + +class ReadOnlyToolPolicy: + """Provider-neutral guard for proactive MCP consumer calls. + + The policy is intentionally strict. A tool must be explicitly allowed and + must be positively classified as read-only by annotations or provider risk + metadata. Name patterns are a final deny layer, not the primary signal. + """ + + def __init__( + self, + *, + allowed_tools: set[str] | list[str] | tuple[str, ...], + tool_risks: dict[str, str] | None = None, + ) -> None: + self.allowed_tools = frozenset(str(tool) for tool in allowed_tools) + self.tool_risks = {str(name): str(risk).strip().lower() for name, risk in (tool_risks or {}).items()} + + def validate(self, descriptor: McpToolDescriptor) -> ToolSafetyDecision: + name = descriptor.name + if name not in self.allowed_tools: + return ToolSafetyDecision(False, "tool_not_in_allowlist", name) + if _DENY_NAME_RE.search(name): + return ToolSafetyDecision(False, "tool_name_matches_deny_pattern", name) + risk = (descriptor.provider_risk or self.tool_risks.get(name) or "").strip().lower() + if risk and risk != "read": + return ToolSafetyDecision(False, f"provider_risk_not_read:{risk}", name) + annotations = descriptor.annotations or {} + read_hint = annotations.get("readOnlyHint") + destructive_hint = annotations.get("destructiveHint") + open_world_hint = annotations.get("openWorldHint") + if destructive_hint is True: + return ToolSafetyDecision(False, "destructive_hint_true", name) + if open_world_hint is True and read_hint is not True: + return ToolSafetyDecision(False, "open_world_without_readonly_hint", name) + if read_hint is not True and risk != "read": + return ToolSafetyDecision(False, "missing_positive_readonly_classification", name) + return ToolSafetyDecision(True, "read_only_allowlisted", name) + + def validate_all(self, descriptors: list[McpToolDescriptor]) -> list[ToolSafetyDecision]: + return [self.validate(descriptor) for descriptor in descriptors] diff --git a/src/vaner/intent/bundles.py b/src/vaner/intent/bundles.py new file mode 100644 index 0000000..7ccc3a1 --- /dev/null +++ b/src/vaner/intent/bundles.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def _load_json(path: Path) -> dict[str, Any]: + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def bundle_rejection_reason(bundle_dir: Path | str, *, allow_experimental_data: bool = False) -> str | None: + """Return why a training bundle should not be installed by default.""" + + bundle_path = Path(bundle_dir) + manifest = _load_json(bundle_path / "manifest.json") + metrics = manifest.get("training_metrics") + if isinstance(metrics, dict): + if metrics.get("rejected_for_threshold") is True: + return "bundle was rejected by the training promotion threshold" + if metrics.get("trained") is False: + return "bundle manifest says the scorer was not promoted" + validation = manifest.get("finance_validation") + if isinstance(validation, dict) and validation.get("passed") is False: + return "bundle finance temporal validation did not pass" + governance = manifest.get("dataset_governance") + if ( + isinstance(governance, dict) + and governance.get("release_eligible") is False + and not allow_experimental_data + ): + return "bundle dataset governance is not release eligible" + + metadata = _load_json(bundle_path / "intent_scorer_metadata.json") + if str(metadata.get("reason", "")).strip().lower() == "rejected": + return "bundle scorer metadata marks the model as rejected" + return None + + +def bundle_public_summary(bundle_dir: Path | str) -> dict[str, Any]: + """Small non-sensitive summary for CLI/status output.""" + + bundle_path = Path(bundle_dir) + manifest = _load_json(bundle_path / "manifest.json") + metrics = manifest.get("training_metrics") if isinstance(manifest.get("training_metrics"), dict) else {} + quality = manifest.get("training_quality") if isinstance(manifest.get("training_quality"), dict) else {} + return { + "bundle_id": manifest.get("bundle_id") or bundle_path.name, + "created_at": manifest.get("created_at"), + "feature_schema_version": manifest.get("feature_schema_version"), + "records_used": manifest.get("records_used"), + "scorer_backend": manifest.get("scorer_backend"), + "trained": metrics.get("trained") if isinstance(metrics, dict) else None, + "rejected_for_threshold": metrics.get("rejected_for_threshold") if isinstance(metrics, dict) else None, + "mae": metrics.get("mae") if isinstance(metrics, dict) else None, + "baseline_mae": metrics.get("baseline_mae") if isinstance(metrics, dict) else None, + "improvement": metrics.get("improvement") if isinstance(metrics, dict) else None, + "feature_coverage_pct": quality.get("feature_coverage_pct") if isinstance(quality, dict) else None, + "release_eligible": ( + manifest.get("dataset_governance", {}).get("release_eligible") + if isinstance(manifest.get("dataset_governance"), dict) + else None + ), + "finance_temporal_passed": ( + manifest.get("finance_validation", {}).get("passed") if isinstance(manifest.get("finance_validation"), dict) else None + ), + "finance_temporal_improvement": ( + manifest.get("finance_validation", {}).get("improvement") + if isinstance(manifest.get("finance_validation"), dict) + else None + ), + "rejection_reason": bundle_rejection_reason(bundle_path), + "experimental_data_installable": bundle_rejection_reason(bundle_path, allow_experimental_data=True) is None, + } diff --git a/src/vaner/intent/features.py b/src/vaner/intent/features.py index f08686b..b5a0465 100644 --- a/src/vaner/intent/features.py +++ b/src/vaner/intent/features.py @@ -154,6 +154,22 @@ def feature_vector_for_artefact(features: dict[str, float], artefact: Artefact) features.get("skill_presence", 0.0), features.get("skill_kind_match", 0.0), features.get("follow_up_offer_strength", 0.0), + # Provider-neutral finance preparation signals (FEATURE_SCHEMA_VERSION v5). + # These are ex-ante inputs only; realized returns and keep/drop labels stay + # in training labels and are intentionally not part of the runtime vector. + features.get("finance_public_market_plane", 0.0), + features.get("finance_account_state_plane", 0.0), + features.get("finance_horizon_days", 0.0), + features.get("finance_option_strategy_entry_debit", 0.0), + features.get("finance_option_strategy_leg_count", 0.0), + features.get("finance_option_strategy_is_single_leg", 0.0), + features.get("finance_option_strategy_is_debit_spread", 0.0), + features.get("finance_option_strategy_is_call", 0.0), + features.get("finance_option_strategy_is_put", 0.0), + features.get("finance_option_strategy_min_abs_delta", 0.0), + features.get("finance_option_strategy_max_abs_delta", 0.0), + features.get("finance_option_strategy_mean_iv", 0.0), + features.get("finance_liquidity_penalty", 0.0), # Artefact-level fields (always last, matches trainer.py FEATURE_KEYS[-2:]) float(artefact.access_count), float(max(0.0, time.time() - artefact.generated_at)), diff --git a/src/vaner/intent/prepared_work.py b/src/vaner/intent/prepared_work.py index 6c3a4cb..dc48b65 100644 --- a/src/vaner/intent/prepared_work.py +++ b/src/vaner/intent/prepared_work.py @@ -35,10 +35,16 @@ WorkProductType.DOCS_DRIFT: PreparedWorkKind.DOCS, WorkProductType.VIRTUAL_DIFF: PreparedWorkKind.DIFF, WorkProductType.RESEARCH_BRIEF: PreparedWorkKind.BRIEF, + WorkProductType.FINANCE_MORNING_BRIEF: PreparedWorkKind.FINANCE, + WorkProductType.FINANCE_POSITION_BRIEF: PreparedWorkKind.FINANCE, + WorkProductType.FINANCE_SCREENING_RESULT: PreparedWorkKind.FINANCE, + WorkProductType.FINANCE_OPTION_STRATEGY_PLAN: PreparedWorkKind.FINANCE, + WorkProductType.FINANCE_TRADE_BRIEF: PreparedWorkKind.FINANCE, } _KIND_PRIORITY: dict[PreparedWorkKind, float] = { PreparedWorkKind.DIFF: 1.0, + PreparedWorkKind.FINANCE: 0.96, PreparedWorkKind.BRIEF: 0.92, PreparedWorkKind.REVIEW: 0.82, PreparedWorkKind.DRAFT: 0.8, @@ -248,8 +254,8 @@ def _card_from_work_product( source_id=product.id, source_type=PreparedWorkSourceType.WORK_PRODUCT, kind=kind, - title=str(sanitize_no_absolute_paths(product.title)), - summary=str(sanitize_no_absolute_paths(product.summary)), + title=_display_title(product), + summary=_display_summary(product), badge=_badge_for_kind(kind), confidence_label=_confidence_label(product.confidence), freshness_label=_freshness_label(product.updated_at, now), @@ -257,6 +263,10 @@ def _card_from_work_product( target_label=target_label, why_prepared=_why_prepared(product, target_label), action_note=_action_note(product), + sensitivity_class=product.sensitivity_class.value, + fresh_precheck_required=product.fresh_precheck_required, + external_input_count=len(product.external_inputs), + prohibited_actions=list(product.prohibited_actions), evidence_count=len(product.evidence_refs), created_at=product.created_at, updated_at=product.updated_at, @@ -516,6 +526,10 @@ def build_work_product_inspection(product: WorkProduct, *, now: float | None = N confidence_label=_confidence_label(product.confidence), freshness_label=_freshness_label(product.updated_at, ts), freshness_state=_freshness_state(product, ts), + sensitivity_class=product.sensitivity_class.value, + fresh_precheck_required=product.fresh_precheck_required, + external_input_count=len(product.external_inputs), + prohibited_actions=list(product.prohibited_actions), target_label=target_label, evidence_count=len(product.evidence_refs), evidence_refs=evidence_refs, @@ -564,6 +578,8 @@ def _why_prepared_prediction(prompt: Any) -> str: def _action_note(product: WorkProduct) -> str: + if product.fresh_precheck_required: + return "Fresh external-state precheck required before use; execution is prohibited." if product.type == WorkProductType.VIRTUAL_DIFF: if product.can_export(now=time.time()): return "Export returns the prepared diff only; Vaner will not apply it automatically." @@ -573,8 +589,38 @@ def _action_note(product: WorkProduct) -> str: return "Inspect this lead before using it." +def _display_title(product: WorkProduct) -> str: + if product.type == WorkProductType.FINANCE_POSITION_BRIEF: + return "Trading strategy prep" + if product.type == WorkProductType.FINANCE_SCREENING_RESULT: + return "Trading screen results" + if product.type == WorkProductType.FINANCE_OPTION_STRATEGY_PLAN: + return "Options strategy plan" + if product.type == WorkProductType.FINANCE_TRADE_BRIEF: + return "Trade brief" + if product.type == WorkProductType.FINANCE_MORNING_BRIEF: + return "Market morning brief" + return str(sanitize_no_absolute_paths(product.title)) + + +def _display_summary(product: WorkProduct) -> str: + if product.type.value.startswith("finance_"): + capabilities = sorted({item.capability for item in product.external_inputs}) + suffix = f" Uses {len(product.external_inputs)} fresh external input{'s' if len(product.external_inputs) != 1 else ''}." + if capabilities: + suffix += f" Capabilities: {', '.join(capabilities)}." + return str(sanitize_no_absolute_paths(product.summary + suffix)) + return str(sanitize_no_absolute_paths(product.summary)) + + def _inspection_warnings(product: WorkProduct) -> list[str]: warnings: list[str] = [] + if product.fresh_precheck_required: + warnings.append("Fresh external-state precheck is required before using this prepared work.") + if product.prohibited_actions: + warnings.append("Prohibited action: " + ", ".join(sorted(set(product.prohibited_actions))) + ".") + if product.sensitivity_class.value not in {"general", "public_market_only"}: + warnings.append(f"Sensitivity class: {product.sensitivity_class.value}.") if product.freshness == WorkProductFreshness.STALE: warnings.append("This prepared item is stale. Regenerate it before export or adoption.") elif product.freshness == WorkProductFreshness.UNKNOWN: @@ -665,6 +711,7 @@ def _badge_for_kind(kind: PreparedWorkKind) -> str: PreparedWorkKind.DOCS: "Docs", PreparedWorkKind.DIFF: "Diff", PreparedWorkKind.BRIEF: "Brief", + PreparedWorkKind.FINANCE: "Trading", PreparedWorkKind.DRAFT: "Draft", PreparedWorkKind.PREDICTION: "Ready", }[kind] diff --git a/src/vaner/intent/taxonomy.py b/src/vaner/intent/taxonomy.py index 5f3b605..f497b20 100644 --- a/src/vaner/intent/taxonomy.py +++ b/src/vaner/intent/taxonomy.py @@ -5,7 +5,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass -DOMAINS: tuple[str, ...] = ("coding", "research", "writing", "ops") +DOMAINS: tuple[str, ...] = ("coding", "research", "writing", "ops", "finance") MODES: tuple[str, ...] = ("understand", "implement", "debug", "validate", "plan", "explain") _DOMAIN_KEYWORDS: dict[str, tuple[str, ...]] = { @@ -13,6 +13,25 @@ "research": ("research", "paper", "benchmark", "evidence", "study", "compare", "analysis"), "writing": ("write", "copy", "draft", "tone", "grammar", "edit", "summary"), "ops": ("deploy", "infra", "incident", "monitor", "latency", "runtime", "oncall", "k8s"), + "finance": ( + "option", + "strike", + "expiry", + "delta", + "vega", + "theta", + "iv", + "underlying", + "position", + "portfolio", + "ticker", + "spread", + "premium", + "assignment", + "hedge", + "watchlist", + "earnings", + ), } _MODE_KEYWORDS: dict[str, tuple[str, ...]] = { diff --git a/src/vaner/intent/trainer.py b/src/vaner/intent/trainer.py index 3e0137b..7026896 100644 --- a/src/vaner/intent/trainer.py +++ b/src/vaner/intent/trainer.py @@ -10,7 +10,7 @@ from vaner.learning.reward import RewardInput, compute_reward from vaner.store.artefacts import ArtefactStore -FEATURE_SCHEMA_VERSION = "v4" +FEATURE_SCHEMA_VERSION = "v5" FEATURE_KEYS: tuple[str, ...] = ( "signal_count_recent_15m", "query_count_total", @@ -39,6 +39,19 @@ "skill_presence", "skill_kind_match", "follow_up_offer_strength", + "finance_public_market_plane", + "finance_account_state_plane", + "finance_horizon_days", + "finance_option_strategy_entry_debit", + "finance_option_strategy_leg_count", + "finance_option_strategy_is_single_leg", + "finance_option_strategy_is_debit_spread", + "finance_option_strategy_is_call", + "finance_option_strategy_is_put", + "finance_option_strategy_min_abs_delta", + "finance_option_strategy_max_abs_delta", + "finance_option_strategy_mean_iv", + "finance_liquidity_penalty", "access_count", "artefact_age_seconds", ) @@ -172,7 +185,7 @@ async def train_batch(self, output_dir: Path) -> Path | None: backend = self.config.scorer_backend extension = ".txt" - if backend == "xgboost": + if backend in {"auto", "xgboost"}: extension = ".json" elif backend == "catboost": extension = ".cbm" diff --git a/src/vaner/intent/work_products.py b/src/vaner/intent/work_products.py index a75bc96..a288a09 100644 --- a/src/vaner/intent/work_products.py +++ b/src/vaner/intent/work_products.py @@ -4,19 +4,23 @@ import difflib import hashlib +import json import re import time from pathlib import Path from typing import Any from vaner.daemon.signals.git_reader import read_content_hashes, read_git_state, read_head_sha +from vaner.external_state.models import ExternalStateSnapshot from vaner.models.artefact import Artefact from vaner.models.work_product import ( WorkProduct, WorkProductAdoptability, WorkProductEvidenceRef, + WorkProductExternalInput, WorkProductFreshness, WorkProductSelfEval, + WorkProductSensitivity, WorkProductSourceSnapshot, WorkProductStatus, WorkProductType, @@ -38,6 +42,31 @@ } _DOC_SUFFIXES = {".md", ".mdx", ".rst", ".txt"} _WORK_PRODUCT_VERSION = "work_product.v1" +_FINANCE_TERMS = { + "account", + "assignment", + "balance", + "delta", + "earnings", + "expiry", + "greek", + "hedge", + "iv", + "option", + "order", + "portfolio", + "position", + "premium", + "screen", + "spread", + "strike", + "theta", + "ticker", + "underlying", + "vega", + "volatility", + "watchlist", +} def _stable_id(kind: WorkProductType, target_key: str, body: str) -> str: @@ -352,6 +381,353 @@ def _generate_research_brief(repo_root: Path, recent_queries: list[str], artefac ) +def _finance_signal(text: str) -> bool: + lowered = text.lower() + return any(term in lowered for term in _FINANCE_TERMS) + + +def _finance_sensitivity(text: str) -> WorkProductSensitivity: + lowered = text.lower() + if any(term in lowered for term in ("order", "fill", "activity", "cancel", "modify")): + return WorkProductSensitivity.ORDER_ACTIVITY + if any(term in lowered for term in ("position", "holding", "assignment", "short leg", "long leg")): + return WorkProductSensitivity.POSITION_SPECIFIC + if any(term in lowered for term in ("account", "balance", "margin", "cash", "portfolio")): + return WorkProductSensitivity.ACCOUNT_SUMMARY + if "watchlist" in lowered: + return WorkProductSensitivity.USER_WATCHLIST + return WorkProductSensitivity.PUBLIC_MARKET_ONLY + + +def _finance_kind(query_text: str, evidence_text: str) -> WorkProductType: + lowered = f"{query_text}\n{evidence_text}".lower() + if any(term in lowered for term in ("position", "portfolio", "balance", "order", "assignment")): + return WorkProductType.FINANCE_POSITION_BRIEF + if any(term in lowered for term in ("option", "spread", "strike", "expiry", "delta", "theta", "vega", "iv")): + return WorkProductType.FINANCE_OPTION_STRATEGY_PLAN + if any(term in lowered for term in ("screen", "watchlist", "candidate")): + return WorkProductType.FINANCE_SCREENING_RESULT + if any(term in lowered for term in ("trade", "entry", "exit", "hedge", "rebalance")): + return WorkProductType.FINANCE_TRADE_BRIEF + return WorkProductType.FINANCE_MORNING_BRIEF + + +def _decode_external_payload(payload: dict[str, Any]) -> Any: + content = payload.get("content") + if isinstance(content, list): + decoded: list[Any] = [] + for item in content: + text = item.get("text") if isinstance(item, dict) else None + if not isinstance(text, str) or not text.strip(): + continue + try: + decoded.append(json.loads(text)) + except json.JSONDecodeError: + decoded.append({"text": text}) + if decoded: + return decoded[0] if len(decoded) == 1 else decoded + return payload + + +def _candidate_lists(value: Any, *, depth: int = 0) -> list[list[dict[str, Any]]]: + if depth > 4: + return [] + if isinstance(value, list): + if value and all(isinstance(item, dict) for item in value): + return [list(value)] + lists: list[list[dict[str, Any]]] = [] + for item in value: + lists.extend(_candidate_lists(item, depth=depth + 1)) + return lists + if not isinstance(value, dict): + return [] + preferred_keys = ( + "candidates", + "results", + "items", + "instruments", + "securities", + "strategies", + "quotes", + "data", + "rows", + ) + lists = [] + for key in preferred_keys: + if key in value: + lists.extend(_candidate_lists(value[key], depth=depth + 1)) + if lists: + return lists + for nested in value.values(): + lists.extend(_candidate_lists(nested, depth=depth + 1)) + return lists + + +def _field_value(item: dict[str, Any], names: tuple[str, ...]) -> Any: + lowered = {str(key).lower(): value for key, value in item.items()} + for name in names: + if name.lower() in lowered: + return lowered[name.lower()] + for value in item.values(): + if isinstance(value, dict): + nested = _field_value(value, names) + if nested not in (None, ""): + return nested + return None + + +def _candidate_label(item: dict[str, Any]) -> str: + value = _field_value( + item, + ( + "symbol", + "ticker", + "displaySymbol", + "identifier", + "uic", + "name", + "description", + "instrument", + ), + ) + if isinstance(value, dict): + value = _field_value(value, ("symbol", "ticker", "name", "description", "uic")) + label = str(value or "candidate").strip() + return sanitize_no_absolute_paths(label[:80]) + + +def _score_finance_candidate(item: dict[str, Any], query_terms: set[str], *, snapshot_stale: bool) -> tuple[float, list[str]]: + label = _candidate_label(item).lower() + present_fields = 0 + field_groups: tuple[tuple[str, ...], ...] = ( + ("symbol", "ticker", "uic", "identifier"), + ("name", "description"), + ("price", "lastPrice", "last", "close"), + ("change", "changePercent", "percentChange"), + ("volume", "turnover"), + ("score", "rank", "rankScore"), + ) + for group in field_groups: + if _field_value(item, group) not in (None, ""): + present_fields += 1 + relevance_hits = sum(1 for term in query_terms if len(term) > 2 and term in label) + score = 0.25 + min(0.4, present_fields * 0.075) + min(0.2, relevance_hits * 0.08) + reasons = ["complete-enough evidence"] if present_fields >= 3 else ["thin evidence"] + if relevance_hits: + reasons.append("matches recent finance intent") + if not snapshot_stale: + score += 0.1 + reasons.append("fresh snapshot") + else: + score -= 0.2 + reasons.append("stale snapshot") + return (max(0.0, min(1.0, score)), reasons) + + +def _finance_exploration_summary(snapshots: list[ExternalStateSnapshot], query_text: str) -> dict[str, Any]: + query_terms = set(re.findall(r"[a-zA-Z0-9_.$-]+", query_text.lower())) + candidate_rows: list[tuple[float, str, str, list[str]]] = [] + for snapshot in snapshots: + payload = _decode_external_payload(snapshot.payload) + snapshot_stale = snapshot.is_stale() + for candidate_list in _candidate_lists(payload): + for item in candidate_list[:100]: + score, reasons = _score_finance_candidate(item, query_terms, snapshot_stale=snapshot_stale) + candidate_rows.append((score, snapshot.capability, _candidate_label(item), reasons)) + candidate_rows.sort(key=lambda row: row[0], reverse=True) + kept = candidate_rows[:5] + pruning_rules = [ + "prefer fresh snapshots over stale branches", + "prefer candidates with enough comparable market fields", + "prefer candidates aligned with recent user finance intent", + "prune low-evidence branches before requesting deeper provider data", + ] + return { + "objective": "risk_adjusted_preparation", + "candidate_count": len(candidate_rows), + "kept_candidate_count": len(kept), + "pruned_candidate_count": max(0, len(candidate_rows) - len(kept)), + "kept_candidates": [ + {"label": label, "score": round(score, 3), "capability": capability, "reasons": reasons} + for score, capability, label, reasons in kept + ], + "pruning_rules": pruning_rules, + "abstentions": [] if candidate_rows else ["external snapshot did not expose a comparable candidate list"], + } + + +def _generate_finance_brief(repo_root: Path, recent_queries: list[str], artefacts: list[Artefact]) -> WorkProduct | None: + query_text = " ".join(recent_queries[-5:]) + finance_artefacts = [ + artefact + for artefact in artefacts + if Path(str(artefact.source_path)).suffix.lower() in _DOC_SUFFIXES + and _finance_signal(f"{artefact.source_path}\n{artefact.content}") + ] + if not finance_artefacts and not _finance_signal(query_text): + return None + selected = finance_artefacts[:3] + evidence_text = "\n".join(str(artefact.content) for artefact in selected) + if not selected: + return None + paths = [str(artefact.source_path) for artefact in selected] + sensitivity = _finance_sensitivity(f"{query_text}\n{evidence_text}") + kind = _finance_kind(query_text, evidence_text) + bullets: list[str] = [] + for artefact in selected: + excerpt = " ".join(str(artefact.content).split())[:260] + bullets.append(f"- `{artefact.source_path}`: {excerpt}") + body = ( + "Finance preparation from local workspace evidence only.\n\n" + + "\n".join(bullets) + + "\n\nFreshness note: no external market/account snapshot was used. " + "Treat this as planning context, not an execution-ready recommendation." + ) + target_digest = hashlib.sha1(f"{kind.value}\n{query_text}\n{','.join(paths)}".encode()).hexdigest()[:12] # noqa: S324 + product = _make_product( + repo_root=repo_root, + kind=kind, + title="Prepared finance brief", + summary="Local finance notes prepared without external market or account data.", + body=body, + paths=paths, + evidence_reason="local finance evidence; external data access was not required", + confidence=0.62 if query_text else 0.55, + adoptability=WorkProductAdoptability.ADVISORY, + target_key=f"{kind.value}:local:{target_digest}", + provenance={ + "finance": { + "external_state_enabled": False, + "abstention_or_downgrade": "local_only_no_external_snapshot", + } + }, + ) + product.sensitivity_class = sensitivity + product.fresh_precheck_required = False + product.stale_after = product.expires_at + product.prohibited_actions = ["execution"] + product.self_eval.stale_risk = max(product.self_eval.stale_risk, 0.35) + return product + + +def generate_external_finance_work_products( + *, + repo_root: Path, + recent_queries: list[str], + snapshots: list[ExternalStateSnapshot], + max_products: int = 2, +) -> list[WorkProduct]: + if not snapshots: + return [] + now = time.time() + strict_expiry = min((snapshot.expires_at for snapshot in snapshots if snapshot.expires_at is not None), default=now + 300) + sensitivity = WorkProductSensitivity.PUBLIC_MARKET_ONLY + if any(snapshot.sensitivity_class.value in {"position_specific", "order_activity"} for snapshot in snapshots): + sensitivity = WorkProductSensitivity.POSITION_SPECIFIC + elif any(snapshot.sensitivity_class.value == "account_summary" for snapshot in snapshots): + sensitivity = WorkProductSensitivity.ACCOUNT_SUMMARY + query_text = " ".join(recent_queries[-5:]) + capabilities = ", ".join(sorted({snapshot.capability for snapshot in snapshots})) + exploration = _finance_exploration_summary(snapshots, query_text) + candidate_lines = "" + if exploration["kept_candidates"]: + bullets = [ + f"- {item['label']} ({item['capability']}, score {item['score']}): {', '.join(item['reasons'])}" + for item in exploration["kept_candidates"] + ] + candidate_lines = "\n\nCandidate branches kept for review:\n" + "\n".join(bullets) + else: + candidate_lines = ( + "\n\nNo comparable candidate list was available from the external snapshots, " + "so Vaner abstained from candidate-level ranking." + ) + body = ( + "Finance preparation from fresh external-state snapshots.\n\n" + f"Capabilities observed: {capabilities}.\n\n" + "Exploration summary: " + f"evaluated {exploration['candidate_count']} candidate records, " + f"kept {exploration['kept_candidate_count']}, " + f"pruned {exploration['pruned_candidate_count']} lower-evidence branches." + f"{candidate_lines}\n\n" + "Pruning rules: " + + "; ".join(exploration["pruning_rules"]) + + ".\n\n" + "This prepared work is advisory. It ranks branches for risk-adjusted preparation, summarizes read-only provider data, " + "and requires a fresh precheck before any action. It is not execution guidance." + ) + kind = ( + WorkProductType.FINANCE_POSITION_BRIEF + if sensitivity in {WorkProductSensitivity.POSITION_SPECIFIC, WorkProductSensitivity.ACCOUNT_SUMMARY} + else WorkProductType.FINANCE_SCREENING_RESULT + ) + target_digest = hashlib.sha1( # noqa: S324 + f"{kind.value}\n{query_text}\n{','.join(snapshot.id for snapshot in snapshots)}".encode() + ).hexdigest()[:12] + product = WorkProduct( + id=_stable_id(kind, f"{kind.value}:external:{target_digest}", body), + type=kind, + title="Prepared finance snapshot brief", + summary="Read-only external finance snapshots prepared for inspection.", + body=body, + evidence_refs=[ + WorkProductEvidenceRef( + kind="record", + reason=f"external-state snapshot for {snapshot.capability}", + confidence=0.72, + ) + for snapshot in snapshots + ], + source_snapshot=build_source_snapshot( + repo_root, + [], + generated_at=now, + generator_version=f"{_WORK_PRODUCT_VERSION}.external_finance", + ), + confidence=0.72, + freshness=WorkProductFreshness.FRESH, + expires_at=strict_expiry, + status=WorkProductStatus.SURFACED, + adoptability=WorkProductAdoptability.ADVISORY, + provenance={ + "generator": f"{_WORK_PRODUCT_VERSION}.external_finance", + "finance": { + "fresh_precheck_required": True, + "external_state_enabled": True, + "snapshot_count": len(snapshots), + "exploration": exploration, + }, + }, + self_eval=_self_eval( + evidence_count=len(snapshots), + confidence=0.72, + stale_risk=0.45, + contradiction_risk=0.12, + reason="external finance snapshots decay and remain advisory", + ), + feedback_state="none", + sensitivity_class=sensitivity, + fresh_precheck_required=True, + external_inputs=[ + WorkProductExternalInput( + provider_id=snapshot.provider_id, + capability=snapshot.capability, + snapshot_id=snapshot.id, + freshness_class=snapshot.freshness_class.value, + captured_at=snapshot.captured_at, + expires_at=snapshot.expires_at, + payload_fingerprint=snapshot.payload_fingerprint, + ) + for snapshot in snapshots + ], + stale_after=strict_expiry, + prohibited_actions=["execution"], + created_at=now, + updated_at=now, + target_key=f"{kind.value}:external:{target_digest}", + ) + return [product][: max(1, int(max_products))] + + def generate_work_products( *, repo_root: Path, @@ -382,4 +758,11 @@ def generate_work_products( research = None if research is not None: products.append(research) + if len(products) < max_products: + try: + finance = _generate_finance_brief(repo_root, recent_queries, artefacts) + except Exception: + finance = None + if finance is not None: + products.append(finance) return products[:max_products] diff --git a/src/vaner/intent/work_style_priors.py b/src/vaner/intent/work_style_priors.py index 255c1e5..451f8d4 100644 --- a/src/vaner/intent/work_style_priors.py +++ b/src/vaner/intent/work_style_priors.py @@ -161,6 +161,18 @@ class IntentPriorAdjustments: preferred_artefact_templates=("debugging_brief", "patch_proposal"), composer_signal_weight_multiplier=1.0, ), + "trading": IntentPriorAdjustments( + # Trading/finance is preparation-oriented: favour strong evidence, + # broad branch pruning, and conservative draft surfacing because + # external market/account state can decay quickly. + artefact_alignment_weight_multiplier=1.4, + long_horizon_bonus_multiplier=1.1, + possible_branch_bonus_multiplier=1.4, + drafting_evidence_floor=0.6, + drafting_volatility_ceiling_multiplier=0.7, + preferred_artefact_templates=("finance_morning_brief", "position_brief", "option_strategy_plan"), + composer_signal_weight_multiplier=1.2, + ), "general": IntentPriorAdjustments( # General-purpose: very gentle nudges, no template preference. artefact_alignment_weight_multiplier=1.1, diff --git a/src/vaner/mcp/apps/__init__.py b/src/vaner/mcp/apps/__init__.py index 8e15229..1ffd9c7 100644 --- a/src/vaner/mcp/apps/__init__.py +++ b/src/vaner/mcp/apps/__init__.py @@ -33,8 +33,47 @@ ACTIVE_PREDICTIONS_HTML: str = _ACTIVE_PREDICTIONS_PATH.read_text(encoding="utf-8") """Full HTML bundle as a string. Consumers serve this verbatim.""" -PREPARED_WORK_HTML: str = _PREPARED_WORK_PATH.read_text(encoding="utf-8") -"""Full Prepared Work HTML bundle as a string.""" + + +_SDK_START_SENTINEL = "// === @modelcontextprotocol/ext-apps@0.4.0" +_SDK_END_SENTINEL = ";const App = gc;" + + +def _extract_ext_apps_sdk(html: str) -> str: + """Pull the vendored ext-apps SDK + ``const App = gc`` rebind from a bundle. + + ``active_predictions.html`` is the single source of truth for the + vendored SDK; other bundles (currently ``prepared_work.html``) inline + the same block via a ``__EXT_APPS_SDK__`` placeholder substitution at + import time. Keeping one copy avoids drift on re-vendor and means we + only ship the ~300KB SDK twice if a host actually loads both bundles. + """ + start = html.find(_SDK_START_SENTINEL) + if start < 0: + raise RuntimeError("ext-apps SDK start sentinel missing from active_predictions.html") + end = html.find(_SDK_END_SENTINEL, start) + if end < 0: + raise RuntimeError("ext-apps SDK end sentinel missing from active_predictions.html") + end += len(_SDK_END_SENTINEL) + return html[start:end] + + +EXT_APPS_SDK_JS: str = _extract_ext_apps_sdk(ACTIVE_PREDICTIONS_HTML) +"""Vendored ``@modelcontextprotocol/ext-apps`` SDK + ``const App = gc`` rebind. + +Sliced from ``active_predictions.html`` between the vendor banner and the +trailing ``const App = gc`` rebinding. Bundles that need the SDK include +``__EXT_APPS_SDK__`` as a placeholder; :func:`_inline_sdk` replaces it at +import time. +""" + + +def _inline_sdk(template: str) -> str: + return template.replace("__EXT_APPS_SDK__", EXT_APPS_SDK_JS) + + +PREPARED_WORK_HTML: str = _inline_sdk(_PREPARED_WORK_PATH.read_text(encoding="utf-8")) +"""Full Prepared Work HTML bundle as a string (SDK inlined at import time).""" ACTIVE_PREDICTIONS_SHA256: str = hashlib.sha256(ACTIVE_PREDICTIONS_HTML.encode("utf-8")).hexdigest() """Stable content hash — useful for cache validation + CSP pinning.""" @@ -50,6 +89,7 @@ "ACTIVE_PREDICTIONS_TITLE", "ACTIVE_PREDICTIONS_URI", "CSP_RESOURCE_DOMAINS", + "EXT_APPS_SDK_JS", "PREPARED_WORK_DESCRIPTION", "PREPARED_WORK_HTML", "PREPARED_WORK_MIME", diff --git a/src/vaner/mcp/apps/prepared_work.html b/src/vaner/mcp/apps/prepared_work.html index ecb2ce1..603e70b 100644 --- a/src/vaner/mcp/apps/prepared_work.html +++ b/src/vaner/mcp/apps/prepared_work.html @@ -4,6 +4,15 @@ Vaner Prepared Work +