From fb73f9bda17d2a2ccfc70df65f4b7a595896278d Mon Sep 17 00:00:00 2001 From: magnus919 Date: Fri, 16 Oct 2026 11:48:37 -0400 Subject: [PATCH 1/2] fix(research): honor completion budget and bound construction input --- agent-svc/agent/experimental/durable_plan.py | 3 +- agent-svc/agent/experimental/live_runner.py | 10 +- .../agent/experimental/query_construction.py | 88 ++++++++++++++--- agent-svc/agent/experimental/query_runner.py | 4 + agent-svc/agent/experimental/real_journey.py | 5 + .../agent/routes/experimental_research.py | 2 +- .../adr/0070-execute-live-durable-research.md | 6 +- tests/service/test_live_research_recovery.py | 16 ++- tests/unit/test_query_construction.py | 97 +++++++++++++++++++ 9 files changed, 209 insertions(+), 22 deletions(-) diff --git a/agent-svc/agent/experimental/durable_plan.py b/agent-svc/agent/experimental/durable_plan.py index 661def10..a1d78252 100644 --- a/agent-svc/agent/experimental/durable_plan.py +++ b/agent-svc/agent/experimental/durable_plan.py @@ -26,6 +26,7 @@ async def plan_queries( model: str, selected_sources: tuple[CapturedSource, ...] = (), as_of: str | None = None, + max_output_tokens: int = 2048, ) -> tuple[str, ...]: reply = await complete( ReviewRequest( @@ -44,7 +45,7 @@ async def plan_queries( } ).encode(), model, - 2048, + max_output_tokens, QueryPlan.model_json_schema(), ) ) diff --git a/agent-svc/agent/experimental/live_runner.py b/agent-svc/agent/experimental/live_runner.py index 49de8687..c0cb02d0 100644 --- a/agent-svc/agent/experimental/live_runner.py +++ b/agent-svc/agent/experimental/live_runner.py @@ -33,7 +33,11 @@ async def run_live(record: Any, app: Any): raise ValueError("admission unavailable") evaluated = datetime.fromisoformat(admitted.payload["evaluation_time"]) model = admitted.payload["model"] - planned_policy = admitted.payload.get("policy_version") == "durable-live/2" + bounded_evidence_policy = admitted.payload.get("policy_version") == "durable-live/3" + planned_policy = admitted.payload.get("policy_version") in { + "durable-live/2", + "durable-live/3", + } selected_sources = tuple( CapturedSource(**source) for source in admitted.payload.get("selected_sources", []) @@ -198,6 +202,7 @@ async def fetch(): token_parameter, token_budget = next( iter(admitted.payload["token_params"].items()) ) + completion_budget = min(token_budget, 8192) if bounded_evidence_policy else 2048 transport = ReviewTransport( client, base_url=app.state.llm_base_url, @@ -243,6 +248,7 @@ async def fetch(): model=model, selected_sources=selected_sources, as_of=admitted.payload.get("as_of"), + max_output_tokens=completion_budget, ) if planned_policy and len(selected_sources) < 8 else None @@ -265,4 +271,6 @@ async def fetch(): queries=queries, source_limit=8 if queries is not None else 3, allow_partial=planned_policy, + max_output_tokens=completion_budget, + evidence_budget_chars=32000 if bounded_evidence_policy else None, ) diff --git a/agent-svc/agent/experimental/query_construction.py b/agent-svc/agent/experimental/query_construction.py index 28ffb071..950ed079 100644 --- a/agent-svc/agent/experimental/query_construction.py +++ b/agent-svc/agent/experimental/query_construction.py @@ -6,7 +6,7 @@ from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, datetime -from typing import Annotated, Literal +from typing import Annotated, Literal, NotRequired, TypedDict from uuid import NAMESPACE_URL, uuid4, uuid5 from pydantic import Field @@ -159,6 +159,57 @@ def _edges(content: ConstructedContent, revision_id: str) -> list[dict[str, obje return edges +class ConstructionLine(TypedDict): + line: int + text: str + + +class ConstructionSource(TypedDict): + source_index: int + url: str | None + lines: list[ConstructionLine] + omitted_lines: NotRequired[int] + + +def construction_sources( + sources: tuple[CapturedSource, ...], objective: str, budget: int | None = None +) -> list[ConstructionSource]: + """Select whole original lines; retain full snapshots separately for citation reads.""" + from ..research.evidence import select_passages, validate_evidence_budget + + if budget is not None: + validate_evidence_budget(budget) + remaining = budget + result: list[ConstructionSource] = [] + for index, source in enumerate(sources, 1): + lines = source.text.splitlines(keepends=True) + rows: list[ConstructionLine] + if remaining is None: + rows = [{"line": n, "text": line} for n, line in enumerate(lines, 1)] + else: + allowance = remaining // (len(sources) - index + 1) + spans = select_passages(source.text, objective, allowance)["spans"] + rows = [] + offset = 0 + for n, line in enumerate(lines, 1): + end = offset + len(line) + if any( + offset >= span["start"] and end <= span["end"] for span in spans + ): + rows.append({"line": n, "text": line}) + offset = end + remaining -= sum(len(row["text"]) for row in rows) + entry: ConstructionSource = { + "source_index": index, + "url": source.url, + "lines": rows, + } + if budget is not None: + entry["omitted_lines"] = len(lines) - len(rows) + result.append(entry) + return result + + async def construct_research( objective: str, sources: tuple[CapturedSource, ...], @@ -171,6 +222,8 @@ async def construct_research( revision_id: str | None = None, as_of: str | None = None, acquisition_failures: tuple[str, ...] = (), + max_output_tokens: int = 2048, + evidence_budget_chars: int | None = None, ) -> ConstructedResearch: """One model call, no retries; returned knowledge is NOT verified or publishable.""" if ( @@ -214,23 +267,16 @@ async def construct_research( reference, source.text.encode(), "utf8-exact/1", "text/plain" ) ) + if type(max_output_tokens) is not int or not 1 <= max_output_tokens <= 8192: + raise ValueError("invalid construction completion budget") + presented_sources = construction_sources(sources, objective, evidence_budget_chars) payload = json.dumps( { "objective": objective, "as_of": as_of, "acquisition_failures": acquisition_failures, "as_of_semantics": "Evidence evaluation target only; newly captured text is not an archived capture. Do not invent dates or historical validity.", - "sources": [ - { - "source_index": index, - "url": s.url, - "lines": [ - {"line": n, "text": line} - for n, line in enumerate(s.text.splitlines(keepends=True), 1) - ], - } - for index, s in enumerate(sources, 1) - ], + "sources": presented_sources, "schema": ConstructedContent.model_json_schema(), } ).encode() @@ -240,10 +286,15 @@ async def construct_research( reply = await asyncio.ensure_future( complete( ReviewRequest( - CONSTRUCTION_PROMPT, + CONSTRUCTION_PROMPT + + ( + "\nSources are selected excerpts with original line numbers and possible gaps. Select only contiguous lines actually supplied. Never infer absence or exhaustive coverage from omitted lines." + if evidence_budget_chars is not None + else "" + ), payload, model, - 2048, + max_output_tokens, ConstructedContent.model_json_schema(), ) ) @@ -255,6 +306,15 @@ async def construct_research( reply.content, schema_version="research-construction/4" ) content = ConstructedContent.model_validate_json(document.data) + if evidence_budget_chars is not None: + for item in content.evidence: + if not 1 <= item.source_index <= len(presented_sources): + raise ValueError("model evidence was not supplied") + shown = { + row["line"] for row in presented_sources[item.source_index - 1]["lines"] + } + if not all(n in shown for n in range(item.start_line, item.end_line + 1)): + raise ValueError("model evidence was not supplied") answer = _selected((content.answer_claim_index,), len(content.claims), "claim")[0] now = clock() offset = now.utcoffset() diff --git a/agent-svc/agent/experimental/query_runner.py b/agent-svc/agent/experimental/query_runner.py index c3fd2604..42d3909d 100644 --- a/agent-svc/agent/experimental/query_runner.py +++ b/agent-svc/agent/experimental/query_runner.py @@ -31,6 +31,8 @@ async def run_query( as_of: str | None = None, queries: tuple[str, ...] | None = None, allow_partial: bool = False, + max_output_tokens: int = 2048, + evidence_budget_chars: int | None = None, ) -> JourneyResult: """Run a bounded query plan, retained acquisitions, construction and checks. @@ -150,4 +152,6 @@ async def run_query( artifact_set_id=artifact_set_id, as_of=as_of, acquisition_failures=tuple(failures), + max_output_tokens=max_output_tokens, + evidence_budget_chars=evidence_budget_chars, ) diff --git a/agent-svc/agent/experimental/real_journey.py b/agent-svc/agent/experimental/real_journey.py index 0ff93f5c..dcd829fe 100644 --- a/agent-svc/agent/experimental/real_journey.py +++ b/agent-svc/agent/experimental/real_journey.py @@ -190,6 +190,8 @@ async def research_from_sources( artifact_set_id: str | None = None, as_of: str | None = None, acquisition_failures: tuple[str, ...] = (), + max_output_tokens: int = 2048, + evidence_budget_chars: int | None = None, ) -> JourneyResult: """Construct, check and audit a bounded root; no retained publication yet.""" execution_clock = clock or (lambda: datetime.now(UTC)) @@ -204,12 +206,15 @@ async def research_from_sources( revision_id=revision_id, as_of=as_of, acquisition_failures=acquisition_failures, + max_output_tokens=max_output_tokens, + evidence_budget_chars=evidence_budget_chars, ) adapter = ModelReviewAdapter( provider="configured-litellm", model=model, complete=complete, timeout_seconds=90, + max_output_tokens=max_output_tokens, ) mechanical_reviewer = ToolReviewer( kind="tool", diff --git a/agent-svc/agent/routes/experimental_research.py b/agent-svc/agent/routes/experimental_research.py index d76d3e86..4421e72a 100644 --- a/agent-svc/agent/routes/experimental_research.py +++ b/agent-svc/agent/routes/experimental_research.py @@ -1127,7 +1127,7 @@ async def admit_research_run( "objective": payload.objective, "research_id": str(uuid4()), "execution_mode": "fixture" if fixture else "live", - "policy_version": "durable-live/2", + "policy_version": "durable-live/3", "model": model, "token_params": _token_budget_params(), "selected_sources": [asdict(source) for source in selected_sources], diff --git a/docs/adr/0070-execute-live-durable-research.md b/docs/adr/0070-execute-live-durable-research.md index 8b4b8736..22fedec1 100644 --- a/docs/adr/0070-execute-live-durable-research.md +++ b/docs/adr/0070-execute-live-durable-research.md @@ -23,7 +23,11 @@ Startup scans reclaimable admissions and tracks worker shutdown. Terminal failur ## Bounded discovery, time and callbacks -New admissions pin policy `durable-live/2`: one recorded model planning call produces at most three distinct queries, followed by round-robin source selection across their results. At most eight sources, 256000 source bytes, 64 model calls and an absolute 180-second execution deadline are permitted. Selected retained sources consume the same budgets; eight retained sources need no new search. A known failed capture can leave a checked report with insufficient coverage, never an unqualified complete result. Ambiguous dispatch still stops immediately. Older admissions keep their one-search/three-fresh-source policy so restart does not alter retained operation identities. These bounds are not equivalent to the existing iterative deep-research loop. +New admissions pin policy `durable-live/3`: one recorded model planning call produces at most three distinct queries, followed by round-robin source selection across their results. At most eight sources, 256000 source bytes, 64 model calls and an absolute 180-second execution deadline are permitted. Selected retained sources consume the same budgets; eight retained sources need no new search. A known failed capture can leave a checked report with insufficient coverage, never an unqualified complete result. Ambiguous dispatch still stops immediately. Older admissions keep their one-search/three-fresh-source policy so restart does not alter retained operation identities. These bounds are not equivalent to the existing iterative deep-research loop. + +Policy version 3 also uses the admitted completion allowance, capped at 8192 tokens, for planning, construction and review. A production smoke test showed that the former 2048-token construction ceiling could be consumed entirely by model reasoning, leaving no answer. Raising that ceiling alone did not resolve the observed latency. Construction therefore selects relevant whole lines within a shared 32000-character allowance, divided across sources. Full captured source text remains retained. Original line numbers are preserved, and a model citation that crosses omitted lines is rejected. The prompt explicitly warns that excerpts cannot establish exhaustive coverage. These are request limits, not performance targets; measured completion times remain observations. + +Admissions under `durable-live/2` keep their original full-source construction and 2048-token request ceiling. Their recorded request identities and receipts must remain replayable. Version 3 changes only newly admitted work and does not authorize retries of ambiguous external calls. `as_of` means a timezone-aware, nonfuture evidence evaluation target. It does not request historical search or fabricate archived captures. Original URLs and retrieval dates remain null when unknown. Freshness checks compare actual retained evidence times with that target; unsupported past validity fails as `historical_validity_unestablished`, even if a model returns a passing opinion. Evidence captured before the target remains eligible for the other checks. diff --git a/tests/service/test_live_research_recovery.py b/tests/service/test_live_research_recovery.py index cc966725..9059d815 100644 --- a/tests/service/test_live_research_recovery.py +++ b/tests/service/test_live_research_recovery.py @@ -36,7 +36,7 @@ def ledger(): value.redis.close() -def admitted(ledger, policy=None): +def admitted(ledger, policy=None, max_tokens=2048): value = ledger.admit( "test-owner", str(uuid4()), @@ -46,7 +46,7 @@ def admitted(ledger, policy=None): "research_id": str(uuid4()), "execution_mode": "live", "model": "configured-test-model", - "token_params": {"max_tokens": 2048}, + "token_params": {"max_tokens": max_tokens}, "evaluation_time": datetime.now(UTC).isoformat(), "policy_version": policy, }, @@ -249,7 +249,7 @@ async def blocked(*_args): @pytest.mark.asyncio -@pytest.mark.parametrize("policy", [None, "durable-live/2"]) +@pytest.mark.parametrize("policy", [None, "durable-live/2", "durable-live/3"]) async def test_real_construction_and_review_replay_without_new_model_calls( ledger, monkeypatch, policy ): @@ -257,7 +257,7 @@ async def test_real_construction_and_review_replay_without_new_model_calls( from tests.unit.test_query_construction import TEXT, batch_decisions, proposal - record = admitted(ledger, policy) + record = admitted(ledger, policy, max_tokens=8192) model_calls = [] search_calls = [] @@ -283,7 +283,15 @@ def __init__(self, *_args, **_kwargs): pass async def __call__(self, request): + assert request.max_output_tokens == ( + 8192 if policy == "durable-live/3" else 2048 + ) value = json.loads(request.payload) + if "schema" in value: + assert all( + ("omitted_lines" in source) == (policy == "durable-live/3") + for source in value["sources"] + ) model_calls.append(value) reply = ( { diff --git a/tests/unit/test_query_construction.py b/tests/unit/test_query_construction.py index 3a868291..ec896e1b 100644 --- a/tests/unit/test_query_construction.py +++ b/tests/unit/test_query_construction.py @@ -518,3 +518,100 @@ async def complete(_request): allow_partial=True, ) assert captured == ["https://source.test/unknown"] + + +@pytest.mark.asyncio +async def test_bounded_construction_keeps_original_lines_and_configured_budget(): + from agent.experimental.query_construction import construction_sources + + text = "".join(f"Unrelated filler {i}.\n" for i in range(1000)) + text += "Alder cancellation requires thirty days notice.\n" + source = CapturedSource("https://example.test/terms", text, "2026-09-06T00:00:00Z") + shown = construction_sources((source,), "Alder cancellation notice", 1000) + assert sum(len(row["text"]) for row in shown[0]["lines"]) <= 1000 + assert shown[0]["omitted_lines"] > 0 + assert any(row["line"] == 1001 for row in shown[0]["lines"]) + value = proposal() + value["evidence"][0].update(start_line=1001, end_line=1001) + + async def complete(request): + assert request.max_output_tokens == 8192 + assert json.loads(request.payload)["sources"] == shown + return ModelReply(json.dumps(value).encode(), "local", 100, 2500) + + result = await construct_research( + "Alder cancellation notice", + (source,), + complete=complete, + scope_id="owner", + clock=journey_clock, + max_output_tokens=8192, + evidence_budget_chars=1000, + ) + assert result.context.evidence[0].quote == text.splitlines(keepends=True)[1000] + resolved = await result.resolve(result.context.snapshots[0].content_ref) + assert resolved.body == text.encode() + + +@pytest.mark.asyncio +async def test_bounded_construction_rejects_evidence_from_omitted_lines(): + text = "".join(f"Unrelated filler {i}.\n" for i in range(1000)) + text += "Alder cancellation requires thirty days notice.\n" + + async def complete(request): + shown = { + row["line"] for row in json.loads(request.payload)["sources"][0]["lines"] + } + omitted = next(n for n in range(1, 1002) if n not in shown) + value = proposal() + value["evidence"][0].update(start_line=omitted, end_line=omitted) + return ModelReply(json.dumps(value).encode(), "local", 100, 100) + + with pytest.raises(ValueError, match="not supplied"): + await construct_research( + "Alder cancellation notice", + ( + CapturedSource( + "https://example.test/terms", text, "2026-09-06T00:00:00Z" + ), + ), + complete=complete, + scope_id="owner", + clock=journey_clock, + evidence_budget_chars=1000, + ) + + +def test_construction_excerpt_budget_is_shared_across_sources(): + from agent.experimental.query_construction import construction_sources + + sources = tuple( + CapturedSource( + f"https://example.test/{i}", + "irrelevant background\n" * 200 + + f"Research outcome: trial {i} reduced losses.\n", + None, + ) + for i in range(8) + ) + selected = construction_sources(sources, "Research outcome reduced losses", 8000) + assert len(selected) == 8 + assert ( + sum(len(row["text"]) for source in selected for row in source["lines"]) <= 8000 + ) + for original, source in zip(sources, selected, strict=True): + assert source["lines"] + assert any("Research outcome" in row["text"] for row in source["lines"]) + for row in source["lines"]: + assert ( + row["text"] == original.text.splitlines(keepends=True)[row["line"] - 1] + ) + + +def test_construction_does_not_present_truncated_line_as_complete(): + from agent.experimental.query_construction import construction_sources + + source = CapturedSource(None, "relevant " * 1000, None) + selected = construction_sources((source,), "relevant", 1000) + assert selected[0]["lines"] == [] + assert selected[0]["omitted_lines"] == 1 From 13bd833ab21b271fdf03709fe2da2258c1d1d487 Mon Sep 17 00:00:00 2001 From: magnus919 Date: Fri, 16 Oct 2026 12:50:01 -0400 Subject: [PATCH 2/2] fix(ci): pass supported reasoning setting to Luna reviewer --- .github/workflows/droid-review.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/droid-review.yml b/.github/workflows/droid-review.yml index eb0be228..632fe7b1 100644 --- a/.github/workflows/droid-review.yml +++ b/.github/workflows/droid-review.yml @@ -149,6 +149,7 @@ jobs: automatic_review: true review_depth: ${{ needs.assess-depth.outputs.depth }} review_model: custom:Nous-GPT-6-Luna-0 + reasoning_effort: none security_model: custom:Nous-GPT-6-Luna-0 automatic_security_review: ${{ needs.assess-depth.outputs.security }} allowed_bots: "dependabot[bot]"