From b9690aaa772302a75bcec250d250e2b453aacc50 Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 21:25:53 -0400 Subject: [PATCH 01/11] fix(research): require explicit fixture run admission Signed-off-by: magnus919 --- .env.experimental-candidate.sample | 2 + .env.sample | 4 + .../agent/routes/experimental_research.py | 23 +++++- compose.experimental-candidate.yml | 1 + docker-compose.yml | 1 + .../client-protocol-conformance.md | 8 +- docs/guides/api.md | 14 +++- docs/guides/research-workspace.md | 5 ++ docs/reference/public-surface.md | 1 + .../experimental-replacement-candidate.md | 13 +++ scripts/verify_experimental_candidate.py | 21 +++-- ...test_experimental_research_capabilities.py | 36 ++++++++- .../test_experimental_research_durable.py | 3 + ...xperimental_research_postgres_artifacts.py | 6 +- .../test_experimental_research_runs.py | 79 ++++++++++++++++++- tests/service/test_followup_preview.py | 1 + .../test_workspace_document_contract.py | 1 + ...xperimental_candidate_fixture_admission.py | 55 +++++++++++++ tests/unit/test_mainline_migration_compose.py | 1 + 19 files changed, 257 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_experimental_candidate_fixture_admission.py diff --git a/.env.experimental-candidate.sample b/.env.experimental-candidate.sample index fbd5a860..c99dd1a4 100644 --- a/.env.experimental-candidate.sample +++ b/.env.experimental-candidate.sample @@ -6,6 +6,8 @@ CANDIDATE_BIND_IP=127.0.0.1 CANDIDATE_AGENT_PORT=18080 CANDIDATE_MCP_PORT=18002 CANDIDATE_API_KEY=replace-with-random-api-key +# Development fixture verifier only; leave false for normal operation. +FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false # If CANDIDATE_BIND_IP is changed from loopback, admit the exact API/MCP host # values clients will send. The HTTP API remains protected by CANDIDATE_API_KEY. diff --git a/.env.sample b/.env.sample index 84bf6ba8..4ea253e8 100644 --- a/.env.sample +++ b/.env.sample @@ -88,6 +88,10 @@ # FEATURE_EXPERIMENTAL_RESEARCH_RUNS=false # FEATURE_EXPERIMENTAL_RESEARCH_DURABLE=false # FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS=false +# Development/demo only: new runs still execute fictional sources and authored +# judgments. Leave false in production; existing retained reads/recovery use +# the gates above and remain available without creating new fixture runs. +# FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false # Schema 16 is required before enabling PostgreSQL artifact authority. The # durable-research Compose profile provisions pgvector and initializes this # schema; runtime migrations advance an existing database safely. diff --git a/agent-svc/agent/routes/experimental_research.py b/agent-svc/agent/routes/experimental_research.py index 68c0aaed..40e825dc 100644 --- a/agent-svc/agent/routes/experimental_research.py +++ b/agent-svc/agent/routes/experimental_research.py @@ -1,8 +1,9 @@ """Opt-in experimental research protocol adapters. -The first executable adapter is deliberately fixture-backed and process-local. -It exercises the protocol lifecycle and exact artifact reads without claiming -durable execution, live-provider quality, or production authorization policy. +New runs use fictional sources and authored judgments and require an explicit +development fixture opt-in. Existing process-local, Valkey, and PostgreSQL +retained reads and recovery keep their separate gates; no live research adapter +is selected by this route. """ import asyncio @@ -715,10 +716,14 @@ def capability_document() -> dict[str, Any]: """Return the honest capability boundary for the current W6 slice.""" runs_available = is_enabled("experimental_research_runs") durable = runs_available and _durable_enabled() + fixture_creation = runs_available and is_enabled( + "experimental_research_fixture_runs" + ) return { "protocol_version": "research/1", "route_prefix": _ROUTE_PREFIX, - "implementation_stage": "postgres_artifact_authority_adapter" + "live_research_available": False, + "implementation_stage": "postgres_artifact_authority_fixture_adapter" if durable and _artifact_authority_enabled() else "durable_fixture_run_adapter" if durable @@ -749,6 +754,11 @@ def capability_document() -> dict[str, Any]: "runs": { "available": runs_available, "reason": None if runs_available else "public_adapters_pending", + "create_available": fixture_creation, + "creation_reason": None + if fixture_creation + else "development_fixture_opt_in_required", + "execution_mode": "fixture_only", }, "artifacts": { "available": runs_available, @@ -781,6 +791,11 @@ async def create_experimental_research_run( ) -> dict[str, Any]: """Admit one bounded fixture run with scoped idempotency.""" _require_runs() + if not is_enabled("experimental_research_fixture_runs"): + raise HTTPException( + status_code=404, + detail="Fixture research creation disabled; no live research adapter is available", + ) key = request.headers.get("Idempotency-Key", "").strip() if not key or len(key) > 200: raise HTTPException(status_code=400, detail="Idempotency-Key is required") diff --git a/compose.experimental-candidate.yml b/compose.experimental-candidate.yml index ff8d35ff..4476ff36 100644 --- a/compose.experimental-candidate.yml +++ b/compose.experimental-candidate.yml @@ -674,6 +674,7 @@ services: FEATURE_EXPERIMENTAL_RESEARCH_RUNS: "true" FEATURE_EXPERIMENTAL_RESEARCH_DURABLE: "true" FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS: "true" + FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS: ${FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS:-false} LOG_LEVEL: ${LOG_LEVEL:-INFO} HOST: 0.0.0.0 PORT: "8080" diff --git a/docker-compose.yml b/docker-compose.yml index cd43f630..3d6ce6a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -363,6 +363,7 @@ services: - FEATURE_EXPERIMENTAL_RESEARCH_RUNS=${FEATURE_EXPERIMENTAL_RESEARCH_RUNS:-false} - FEATURE_EXPERIMENTAL_RESEARCH_DURABLE=${FEATURE_EXPERIMENTAL_RESEARCH_DURABLE:-false} - FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS=${FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS:-false} + - FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=${FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS:-false} - VALKEY_URL=${VALKEY_URL:-redis://valkey:6379/0} - SEARXNG_URL=${SEARXNG_URL:-http://slopsearx:8080} - SLOPSEARX_PROVENANCE_PROFILE=${SLOPSEARX_PROVENANCE_PROFILE:-} diff --git a/docs/experiments/client-protocol-conformance.md b/docs/experiments/client-protocol-conformance.md index ec942f35..f4471156 100644 --- a/docs/experiments/client-protocol-conformance.md +++ b/docs/experiments/client-protocol-conformance.md @@ -11,8 +11,10 @@ and cancelled outcomes for cross-client parity checks. The opt-in `GET /experimental/research/v1/capabilities` adapter now exposes this boundary when `FEATURE_EXPERIMENTAL_RESEARCH=true`. A second opt-in gate, -`FEATURE_EXPERIMENTAL_RESEARCH_RUNS=true`, enables a fixture-backed run/status/cancel/event -adapter plus exact in-memory artifact and evidence reads. A third opt-in gate, +`FEATURE_EXPERIMENTAL_RESEARCH_RUNS=true`, enables retained run/status/cancel/event +and artifact/evidence reads. New fixture admission additionally requires the +development-only `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true` opt-in; +leaving it false preserves those reads and recovery without creating new runs. A third opt-in gate, `FEATURE_EXPERIMENTAL_RESEARCH_DURABLE=true`, wires admission, lease fencing, checkpoint identity, terminal status projection and cancellation authority to Valkey. The additional `FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS=true` gate requires @@ -23,7 +25,7 @@ reclaimed run uses a deterministic artifact-set identity, so a PostgreSQL commit followed by process loss can be replayed without creating a second artifact set; deletion reaches PostgreSQL before the Valkey tombstone. Capabilities report `fixture_run_adapter`/`process_local`, `durable_fixture_run_adapter`/`valkey_fenced`, -or `postgres_artifact_authority_adapter`/`postgres_artifacts_valkey_fenced` accordingly. Session attachment remains +or `postgres_artifact_authority_fixture_adapter`/`postgres_artifacts_valkey_fenced` accordingly. Session attachment remains `attachment_only`; all flags are off by default, and inherited `/v2` routes are unchanged. The tests cover completed, failed and cancelled terminal outcomes, duplicate replay diff --git a/docs/guides/api.md b/docs/guides/api.md index 55213a04..945c71c6 100644 --- a/docs/guides/api.md +++ b/docs/guides/api.md @@ -80,14 +80,22 @@ GroktoCrawl targets Firecrawl v2 request/response conventions for its compatible The opt-in research protocol is separate from `/v2/agent` and `/v2/answer`: `GET /experimental/research/v1/capabilities` reports enabled capabilities, and -`POST /experimental/research/v1/runs` starts a bounded run. Run status and events +`POST /experimental/research/v1/runs` admits only development fixture runs when +`FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true`; it returns 404 by default, +before storage or execution. This backend uses fictional sources and authored +judgments, not live web research. Run status and events are available under `/runs/{run_id}` and `/runs/{run_id}/events`; audited artifacts, evidence, and explicit workspace actions have routes under the same prefix. See the [retained research workspace guide](research-workspace.md) for named actions and state boundaries. -The protocol uses `FEATURE_EXPERIMENTAL_RESEARCH`; run creation, durable execution, -and PostgreSQL artifact authority have separate feature flags during rollout. A +The protocol uses `FEATURE_EXPERIMENTAL_RESEARCH`; run readability, durable execution, +and PostgreSQL artifact authority have separate feature flags during rollout. +The additional fixture-admission flag affects only new POST admissions. Leave it +false in production while preserving the existing flags for retained status, +artifacts, evidence, workspace access, and recovery of already admitted runs. +Capabilities distinguish readable runs from `create_available` and always identify +the fixture execution backend, including with PostgreSQL artifact authority. A disabled capability returns 404. PostgreSQL is included in the standard Compose deployment; durable operation requires its configured connection and explicit ordered schema migrations. Existing inline jobs, sessions, research-memory records, and `/v2/*` diff --git a/docs/guides/research-workspace.md b/docs/guides/research-workspace.md index 29c18f0b..bf13f1a8 100644 --- a/docs/guides/research-workspace.md +++ b/docs/guides/research-workspace.md @@ -3,6 +3,11 @@ The experimental workspace browses existing independent roots. It does not create chat history or replay research implicitly. Capabilities report the real fixture, process-local, Valkey, or PostgreSQL boundary; existing feature gates still apply. +New run creation is disabled by default and requires the development-only +`FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true` opt-in. It still executes +fictional sources and authored judgments. Keep that flag false in production; +existing run, artifact, evidence, workspace, and recovery gates remain unchanged. +The workspace does not provide a live research execution adapter. Use `groktocrawl research workspace --json` to list permitted roots and `groktocrawl research workspace RUN_ID --json` to inspect one. Read its revision, diff --git a/docs/reference/public-surface.md b/docs/reference/public-surface.md index a9f8a17e..f63d0fce 100644 --- a/docs/reference/public-surface.md +++ b/docs/reference/public-surface.md @@ -180,6 +180,7 @@ configured private model endpoint; it does not permit private scrape targets. - ENGINE_SEMANTICSCHOLAR_API_KEY - FEATURE_EXPERIMENTAL_RESEARCH - FEATURE_EXPERIMENTAL_RESEARCH_DURABLE +- FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS - FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS - FEATURE_EXPERIMENTAL_RESEARCH_RUNS - FLARE_SOLVERR_URL diff --git a/docs/runbooks/experimental-replacement-candidate.md b/docs/runbooks/experimental-replacement-candidate.md index a2587d91..92b0d7e7 100644 --- a/docs/runbooks/experimental-replacement-candidate.md +++ b/docs/runbooks/experimental-replacement-candidate.md @@ -173,6 +173,19 @@ capture-readiness signal. ## Verify and freeze the receipt +This verifier creates a fictional research run. Use it only on an isolated +development/demo stack. Explicitly opt into fixture admission before verification: + +```sh +FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true docker compose \ + --env-file "$CANDIDATE_ENV" -f "$CANDIDATE_COMPOSE" \ + up -d --no-deps candidate-agent +``` + +Keep the new flag false for normal operation. After the fixture check, recreate +only that demo service with `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false` +using the same Compose command. Existing retained reads remain available. + Load only the candidate API key into the verifier process and write the receipt outside the repository first: diff --git a/scripts/verify_experimental_candidate.py b/scripts/verify_experimental_candidate.py index b03886c8..86f610ef 100644 --- a/scripts/verify_experimental_candidate.py +++ b/scripts/verify_experimental_candidate.py @@ -17,6 +17,21 @@ import httpx +def _require_fixture_capabilities(capabilities: dict[str, Any]) -> None: + runs = capabilities.get("operations", {}).get("runs", {}) + if ( + capabilities.get("implementation_stage") + != "postgres_artifact_authority_fixture_adapter" + or capabilities.get("live_research_available") is not False + or runs.get("execution_mode") != "fixture_only" + or runs.get("create_available") is not True + ): + raise RuntimeError( + "Isolated fixture verification requires PostgreSQL fixture authority and " + "FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true; never enable it for production research" + ) + + def _run(command: list[str], *, env: dict[str, str] | None = None) -> str: result = subprocess.run( command, @@ -150,11 +165,7 @@ async def verify(args: argparse.Namespace) -> dict[str, Any]: capability_response = await client.get("/experimental/research/v1/capabilities") capability_response.raise_for_status() capabilities = capability_response.json() - if ( - capabilities.get("implementation_stage") - != "postgres_artifact_authority_adapter" - ): - raise RuntimeError("PostgreSQL artifact authority is not active") + _require_fixture_capabilities(capabilities) admitted_response = await client.post( "/experimental/research/v1/runs", diff --git a/tests/service/test_experimental_research_capabilities.py b/tests/service/test_experimental_research_capabilities.py index aac25b06..8efabb38 100644 --- a/tests/service/test_experimental_research_capabilities.py +++ b/tests/service/test_experimental_research_capabilities.py @@ -61,5 +61,39 @@ def test_capabilities_advertise_postgres_artifact_authority(monkeypatch) -> None monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_DURABLE", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS", "true") payload = _client().get("/experimental/research/v1/capabilities").json() - assert payload["implementation_stage"] == "postgres_artifact_authority_adapter" + assert ( + payload["implementation_stage"] == "postgres_artifact_authority_fixture_adapter" + ) assert payload["recovery_mode"] == "postgres_artifacts_valkey_fenced" + + +def test_retained_capabilities_distinguish_fixture_creation_from_reads(monkeypatch): + for flag in ( + "EXPERIMENTAL_RESEARCH", + "EXPERIMENTAL_RESEARCH_RUNS", + "EXPERIMENTAL_RESEARCH_DURABLE", + "EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS", + ): + monkeypatch.setenv("FEATURE_" + flag, "true") + monkeypatch.delenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", raising=False) + payload = _client().get("/experimental/research/v1/capabilities").json() + assert payload["live_research_available"] is False + assert ( + payload["implementation_stage"] == "postgres_artifact_authority_fixture_adapter" + ) + assert payload["operations"]["workspace"]["available"] is True + assert payload["operations"]["artifacts"]["available"] is True + runs = payload["operations"]["runs"] + assert runs["available"] is True + assert runs["create_available"] is False + assert runs["creation_reason"] == "development_fixture_opt_in_required" + assert runs["execution_mode"] == "fixture_only" + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "true") + enabled = _client().get("/experimental/research/v1/capabilities").json() + assert enabled["operations"]["runs"]["create_available"] is True + assert enabled["operations"]["runs"]["creation_reason"] is None + assert enabled["live_research_available"] is False + # The fixture opt-in cannot bypass the existing route/readability gate. + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_RUNS", "false") + disabled = _client().get("/experimental/research/v1/capabilities").json() + assert disabled["operations"]["runs"]["create_available"] is False diff --git a/tests/service/test_experimental_research_durable.py b/tests/service/test_experimental_research_durable.py index c8e7e7ab..7b4a5b9e 100644 --- a/tests/service/test_experimental_research_durable.py +++ b/tests/service/test_experimental_research_durable.py @@ -25,6 +25,7 @@ def app(monkeypatch: pytest.MonkeyPatch) -> Iterator[FastAPI]: raise RuntimeError("DURABLE_RESEARCH_REDIS_URL or VALKEY_URL is required") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_RUNS", "true") + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_DURABLE", "true") namespace = f"groktocrawl:test:service:{uuid4()}" redis = Redis.from_url(url, decode_responses=True) @@ -190,6 +191,7 @@ async def pause(_record: object) -> None: ) assert created.status_code == 202 run_id = created.json()["run_id"] + monkeypatch.delenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS") experimental._RUNS.clear() recovered = await client.get(created.json()["status_url"]) assert recovered.status_code == 200 @@ -241,6 +243,7 @@ async def run(self) -> object: original.durable_ledger.redis.delete(original.durable_ledger._lease_key(run_id)) original_task.cancel() await asyncio.gather(original_task, return_exceptions=True) + monkeypatch.delenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS") experimental._RUNS.clear() started.clear() recovered = await client.get(created.json()["status_url"]) diff --git a/tests/service/test_experimental_research_postgres_artifacts.py b/tests/service/test_experimental_research_postgres_artifacts.py index 8d40c19c..4d2cdd7b 100644 --- a/tests/service/test_experimental_research_postgres_artifacts.py +++ b/tests/service/test_experimental_research_postgres_artifacts.py @@ -186,6 +186,7 @@ def namespaced_ledger_factory(redis_url: str) -> DurableResearchLedger: for name in ( "EXPERIMENTAL_RESEARCH", "EXPERIMENTAL_RESEARCH_RUNS", + "EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "EXPERIMENTAL_RESEARCH_DURABLE", "EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS", ): @@ -438,7 +439,9 @@ async def test_workspace_recovers_cited_evidence_without_process_memory(authorit @pytest.mark.asyncio -async def test_postgres_workspace_outlives_ledger_and_deletes_durably(authority_app): +async def test_postgres_workspace_outlives_ledger_and_deletes_durably( + authority_app, monkeypatch +): app, authority, _ = authority_app async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://test" @@ -452,6 +455,7 @@ async def test_postgres_workspace_outlives_ledger_and_deletes_durably(authority_ ).json() status = await _wait(client, admitted["status_url"]) run_id = status["run_id"] + monkeypatch.delenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS") record = experimental._RUNS[run_id] record.durable_ledger.redis.delete(record.durable_ledger._run_key(run_id)) experimental._RUNS.clear() diff --git a/tests/service/test_experimental_research_runs.py b/tests/service/test_experimental_research_runs.py index 4658bfc4..61cbaa8c 100644 --- a/tests/service/test_experimental_research_runs.py +++ b/tests/service/test_experimental_research_runs.py @@ -14,6 +14,7 @@ def app(monkeypatch: pytest.MonkeyPatch) -> FastAPI: monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_RUNS", "true") + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "true") experimental._RUNS.clear() experimental._IDEMPOTENCY.clear() experimental._SESSION_ATTACHMENTS.clear() @@ -251,7 +252,9 @@ async def delayed_run(): assert (await client.get(admission["status_url"])).status_code == 410 assert (await client.get(admission["events_url"])).status_code == 410 - assert (await client.get(f"/experimental/research/v1/workspace/{run_id}")).status_code == 410 + assert ( + await client.get(f"/experimental/research/v1/workspace/{run_id}") + ).status_code == 410 attached = await client.post( "/experimental/research/v1/sessions/deleted-session/attachments", json={"run_id": run_id, "expected_revision": 0}, @@ -377,3 +380,77 @@ async def test_deleting_one_root_does_not_block_other_roots(app: FastAPI): assert ( await client.get("/experimental/research/v1/artifacts/unknown") ).status_code == 404 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fixture_flag", [None, "false", "invalid"]) +async def test_production_flags_do_not_admit_fixture_work( + app, monkeypatch, fixture_flag +): + # Enabling retained readers must not authorize new fictional research. + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_DURABLE", "true") + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS", "true") + if fixture_flag is None: + monkeypatch.delenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS") + else: + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", fixture_flag) + + def unexpected_work(*args, **kwargs): + raise AssertionError("fixture admission reached storage or execution") + + monkeypatch.setattr(experimental, "_durable_ledger", unexpected_work) + monkeypatch.setattr(experimental, "_artifact_authority", unexpected_work) + monkeypatch.setattr(experimental, "example_journey", unexpected_work) + async with await _client(app) as client: + response = await client.post( + "/experimental/research/v1/runs", + headers={"Idempotency-Key": "production-must-not-create"}, + json={"objective": "A real user research request"}, + ) + assert response.status_code == 404 + assert "Fixture research creation disabled" in response.json()["detail"] + assert experimental._RUNS == {} + assert experimental._IDEMPOTENCY == {} + + +@pytest.mark.asyncio +async def test_disabling_fixture_admission_preserves_existing_reads(app, monkeypatch): + async with await _client(app) as client: + created = await client.post( + "/experimental/research/v1/runs", + headers={"Idempotency-Key": "retained-before-disable"}, + json={"objective": "Explicit development fixture"}, + ) + assert created.status_code == 202 + admitted = created.json() + before = await _wait_for_terminal(client, admitted["run_id"]) + result = before["result"] + artifact_before = await client.get(result["artifacts"]["summary"]) + monkeypatch.delenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS") + assert (await client.get(admitted["status_url"])).json() == before + assert (await client.get(admitted["events_url"])).status_code == 200 + assert (await client.get(result["manifest_url"])).status_code == 200 + assert ( + await client.get(result["artifacts"]["summary"]) + ).content == artifact_before.content + assert ( + await client.get( + f"/experimental/research/v1/research/{result['research_id']}/evidence/source-1" + ) + ).status_code == 200 + listing = await client.get("/experimental/research/v1/workspace") + assert any( + item["run_id"] == admitted["run_id"] for item in listing.json()["items"] + ) + assert ( + await client.get( + f"/experimental/research/v1/workspace/{admitted['run_id']}" + ) + ).status_code == 200 + rejected = await client.post( + "/experimental/research/v1/runs", + headers={"Idempotency-Key": "new-after-disable"}, + json={"objective": "Must not create another fixture"}, + ) + assert rejected.status_code == 404 + assert len(experimental._RUNS) == 1 diff --git a/tests/service/test_followup_preview.py b/tests/service/test_followup_preview.py index b8880899..9d056e08 100644 --- a/tests/service/test_followup_preview.py +++ b/tests/service/test_followup_preview.py @@ -203,6 +203,7 @@ async def test_real_root_resolver_scope_identity_deletion_and_feature_gate(monke monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_RUNS", "true") + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "true") app = FastAPI() app.include_router(experimental.router) app.include_router(router) diff --git a/tests/service/test_workspace_document_contract.py b/tests/service/test_workspace_document_contract.py index a8cb5a52..3c5f2892 100644 --- a/tests/service/test_workspace_document_contract.py +++ b/tests/service/test_workspace_document_contract.py @@ -16,6 +16,7 @@ async def test_workspace_named_document_context_and_preview(document_app, monkey app, store = document_app monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_RUNS", "true") + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_DURABLE", "false") experimental._RUNS.clear() experimental._IDEMPOTENCY.clear() diff --git a/tests/unit/test_experimental_candidate_fixture_admission.py b/tests/unit/test_experimental_candidate_fixture_admission.py new file mode 100644 index 00000000..6126d55d --- /dev/null +++ b/tests/unit/test_experimental_candidate_fixture_admission.py @@ -0,0 +1,55 @@ +"""The isolated candidate verifier must explicitly opt into fictional runs.""" + +from copy import deepcopy +from pathlib import Path + +import pytest +import yaml + +from scripts import verify_experimental_candidate as verifier + + +def capabilities(): + return { + "implementation_stage": "postgres_artifact_authority_fixture_adapter", + "live_research_available": False, + "operations": { + "runs": {"execution_mode": "fixture_only", "create_available": True} + }, + } + + +def test_candidate_verifier_accepts_only_explicit_fixture_admission(): + verifier._require_fixture_capabilities(capabilities()) + + +@pytest.mark.parametrize( + "change", ["stage", "creation", "live", "execution", "missing"] +) +def test_candidate_verifier_refuses_incompatible_or_disabled_admission(change): + value = deepcopy(capabilities()) + if change == "stage": + value["implementation_stage"] = "postgres_artifact_authority_adapter" + elif change == "creation": + value["operations"]["runs"]["create_available"] = False + elif change == "live": + value["live_research_available"] = True + elif change == "execution": + value["operations"]["runs"]["execution_mode"] = "live" + else: + value = {} + with pytest.raises( + RuntimeError, match="FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true" + ): + verifier._require_fixture_capabilities(value) + + +def test_demo_compose_requires_explicit_fixture_opt_in(): + root = Path(__file__).parents[2] + model = yaml.safe_load((root / "compose.experimental-candidate.yml").read_text()) + assert ( + model["services"]["candidate-agent"]["environment"][ + "FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS" + ] + == "${FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS:-false}" + ) diff --git a/tests/unit/test_mainline_migration_compose.py b/tests/unit/test_mainline_migration_compose.py index 5597b0f1..adf39eb5 100644 --- a/tests/unit/test_mainline_migration_compose.py +++ b/tests/unit/test_mainline_migration_compose.py @@ -113,6 +113,7 @@ def test_durable_research_profile_wires_one_password_secret_and_keeps_default_of ) assert _env(agent)["FEATURE_EXPERIMENTAL_RESEARCH"].endswith("false}") assert _env(agent)["FEATURE_EXPERIMENTAL_RESEARCH_DURABLE"].endswith("false}") + assert _env(agent)["FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS"].endswith("false}") assert agent["depends_on"]["research-postgres"] == { "condition": "service_healthy", "required": False, From f6dfa764ebad0ded6c6ca744375c99cf1659f414 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Thu, 15 Oct 2026 21:20:52 -0400 Subject: [PATCH 02/11] docs: clarify live research migration boundary (cherry picked from commit f63d84c377436b6f949be19c072ae0dd14fca116) --- CHANGELOG.md | 151 ++++++++++++++++-- .../0069-adopt-bounded-research-substrate.md | 18 +-- docs/plans/research-architecture-migration.md | 50 ++++-- .../mainline-architecture-migration.md | 41 +++-- 4 files changed, 210 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 944fe43e..fa47e8ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,23 +4,150 @@ All notable changes to GroktoCrawl are documented in this file. ## [1.0.0](https://github.com/groktopus/groktocrawl/compare/v0.14.0...v1.0.0) (2026-10-16) -GroktoCrawl 1.0.0 adopts the research architecture developed and tested in GroktoCrawl X. Future development continues in the main repository. -### What is new +GroktoCrawl 1.0 brings the architecture developed in GroktoCrawl X into the main project, together with improvements to research, document handling, search, capture recovery, and agent integrations since 0.14. Development now continues in one codebase. -- Durable research records, retained source evidence, and research workspaces. -- PostgreSQL storage and pgvector similarity search, with a documented migration from Qdrant. -- Automatic capture recovery and an optional isolated capture deployment. -- More complete API, CLI, and MCP research tools, while preserving existing session, plan, and memory operations. -- Publication-date filters that report when a search backend cannot enforce them, plus exports of already-fetched source content. -- Runtime revision information for checking which release is deployed. -- Reliable AMD64 and ARM64 image publishing, with verified release tags and a fix for cleanup that could break image downloads. +**A significant limitation:** the new durable research endpoint still runs a scripted demonstration with fictional sources. It is not yet connected to live search and model-backed research. Its storage, recovery, evidence, and workspace machinery are included as a foundation; they should not be mistaken for a finished durable web-research agent. The existing `/v2` research paths continue to perform live research. Earlier short release notes did not make this distinction clear enough. -### Upgrading +GroktoCrawl remains self-hosted, MIT-licensed, and compatible with its existing Firecrawl-style API surface. -Follow the [migration runbook](https://github.com/groktopus/groktocrawl/blob/main/docs/runbooks/mainline-architecture-migration.md) before enabling PostgreSQL or changing vector storage. Back up existing data, validate the restore, and check backfill parity before switching serving modes. Qdrant rollback writes remain available during the transition. +## What changes for users today -Existing endpoints and client settings remain supported. The search-engine service definitions are unchanged. Historical experiment reports and decisions remain in the repository for reference. +### Use the whole source, not just its opening paragraphs + +Existing research, grounded answers, and session queries can select relevant passages from across complete acquired source bodies. A useful fact halfway down a page or near the end of a document is no longer excluded simply because it falls outside an opening-text slice. + +Responses expose source hashes, selected character ranges, and omitted-text coverage. Answer and session callers can choose an evidence budget. This gives you a way to inspect the material supplied to the model and adjust the request when too little was included. Selection is bounded and lexical; it improves access to acquired text without claiming that every relevant passage will be found. + +### Know what the investigation actually covered + +Deep research no longer stops at the earlier three-source threshold when its planned queries support more evidence. It reports planned and executed queries, candidates, capture attempts, reused sources, and remaining gaps. + +That matters when a convincing answer rests on a narrow search. Clients can see whether a configured source target was met and what work remains, rather than interpreting a source count as proof that the topic was covered exhaustively. + +### Bring your own documents alongside web evidence + +Attach a local document to an owned research session and ask questions across selected document and web references. The workflow uses the existing local-first Parse service and exposes extracted text, source identities, and exact passage references through API, CLI, and MCP. + +This is useful when the question depends on a report, specification, or other material that a web search cannot find. You choose the references used for the question and can inspect the extracted text afterward. + +Attachments follow the session's expiry and deletion rules. They retain extracted text, not an indefinite archive of the original uploaded file. Use authenticated sessions for private material. + +[Private document guide](https://github.com/groktopus/groktocrawl/blob/v1.0.0/docs/guides/private-documents.md) + +### Let GroktoCrawl handle page recovery + +When a supported capture path encounters a challenge or another recognized barrier, the shared scraper recovery policy tries applicable recovery methods within its configured limits. Calling agents no longer need to choose the next internal recovery tool for those paths. + +Recovery outcomes and barrier classifications stay visible to callers. A site may still be unavailable, but the result includes a clearer account of what happened after the applicable recovery steps. Operators can also configure an optional protected-capture deployment with explicit network isolation. + +### Get more out of your agent's existing connection + +The MCP surface now includes typed session operations, research-plan review and execution, and research-memory lifecycle tools alongside the adopted research interfaces. The combined surface has 73 tools; individual capabilities still depend on their configuration and maturity. + +For live workflows, agents can maintain a research session, inspect a plan before executing it, resolve source references, and manage reusable research memory through MCP. The CLI and API preserve corresponding coverage and source metadata. Firefox users also get a fix for Deep Research and recent-query submissions that could abort streaming responses. + +### Keep the evidence you already fetched + +Opt-in pyramid exports preserve complete fetched Markdown and source metadata instead of producing empty source dossiers. You can move from a summary to its source material without making another capture just to recover text the service already had. + +Fast, non-streaming keyword search also gains explicit page/offset continuation. Publication-date filters require trustworthy enforcement information from the backend; an unsupported filter produces an explicit error rather than silently returning an unfiltered result. + +## The architectural changes + +### PostgreSQL and a reversible pgvector migration + +PostgreSQL is integrated into the Compose architecture for retained research artifacts and evidence. After a verified migration, pgvector can serve the existing vector index. Valkey retains its execution-coordination and existing application responsibilities. + +The migration path backfills existing Qdrant data, checks identifiers, content, vectors, and retrieval parity, then switches reads. Qdrant stays current through rollback writes during the transition. Installing 1.0 does not automatically move or delete existing data, and unmigrated installations continue to use Qdrant. + +### Separate execution, evidence, verification, and reports + +The new architecture defines explicit ownership of run state, budgets, cancellation, retries, and completion. Separate records describe source captures, claims, supporting or conflicting evidence, verification outcomes, and rendered reports. Reports identify the knowledge revision they represent. + +This separation is intended to let models and workflow implementations evolve without making them the authority for permissions, budgets, or stored evidence. It also provides a foundation for summaries, deeper analysis, and source reports that remain tied to the same evidence. + +**What is implemented versus integrated:** the new route family exercises these contracts using a fixed example. Its durable ledger, PostgreSQL artifact storage, and retained workspace have lifecycle and recovery tests, but the endpoint does not yet acquire real sources for the submitted objective. Connecting live acquisition, claim construction, verification, and rendering is still required before this can replace the existing live research path. + +### A workspace and recovery protocol for the new foundation + +The new workspace can browse retained roots, inspect reports and citation bytes, export results, and perform explicit attachment, cancellation, and deletion actions. It exposes coverage, revision, and retention information through portal, CLI, and MCP interfaces. At this release, roots created by the new run endpoint contain the scripted example described above. + +The durable protocol has defined recovery and cancellation behavior for that path. It does not make existing inline `/v2` jobs restart-safe. Private session documents also have a separate lifecycle from PostgreSQL-retained research artifacts. + +[Research workspace guide](https://github.com/groktopus/groktocrawl/blob/v1.0.0/docs/guides/research-workspace.md) + +### Keep orchestration replaceable + +The release includes the reference workflow and optional LangGraph comparison work. LangGraph is not installed as a mandatory runtime for every request. These experiments inform future workflow development; they are not a claim that advanced LangGraph research is enabled in the shipped service. + +## Upgrading from 0.14 + +This is a major release because the storage and research architecture have changed substantially. Existing endpoints and client settings remain supported, but enabling durable research and switching vector storage require an operator-led migration. + +1. Read the [mainline migration runbook](https://github.com/groktopus/groktocrawl/blob/v1.0.0/docs/runbooks/mainline-architecture-migration.md). +2. Back up the existing stores and configuration, and verify that the backups can be restored. +3. Apply the ordered PostgreSQL schema migrations only as part of the documented migration. Keep the fixture-backed research run endpoint disabled for production research; enabling its flags does not turn it into a live investigator. +4. Backfill your existing vector data, check identifiers, content, vectors, and retrieval parity, then switch serving modes. Keep the documented Qdrant rollback path available. +5. Verify API, CLI, MCP, retained data, and runtime revision after the update. Update the CLI from the release source if you install it as a standalone script. + +The new research API retains the `/experimental/research/v1` path for compatibility. Its capabilities remain opt-in, and the run endpoint is fixture-backed as described above. A separate repository or stack is not required to work on this foundation. Retention remains explicit and bounded. Private session documents and retained research artifacts have different lifecycles, described in their guides. + +## Changelog: v0.14.0 → v1.0.0 + +The following changelog groups the user-facing and operational changes in this release. The complete commit-level comparison is linked at the end. + +### Research architecture and retained evidence foundation + +The following new-route capabilities are exercised through the scripted adapter in 1.0; they do not yet provide live durable research. + +- Adopt the GroktoCrawl X research architecture in mainline, with an explicit compatibility and migration decision in [ADR-0069](https://github.com/groktopus/groktocrawl/blob/v1.0.0/docs/adr/0069-adopt-bounded-research-substrate.md). ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Add durable research execution records, checkpoint recovery, persisted cancellation, and bounded retry handling for the new research workflow. +- Add versioned claims, source provenance, evidence references, verification records, and report-to-knowledge revision tracking. +- Add PostgreSQL authority for retained artifact sets, source text, and audited knowledge, with ordered schema migrations and explicit expiry/deletion behavior. +- Add retained research workspace operations across the portal, API, CLI, and MCP, including report reading, export, and explicit follow-up actions. +- Preserve the experimental ADRs and evaluation records as historical evidence while establishing mainline as the home for future development. + +### Search, answers, and source exports + +- Select bounded, verbatim evidence across complete acquired source bodies; expose source hashes, selected ranges, and omitted-text coverage. ([#661](https://github.com/groktopus/groktocrawl/pull/661)) +- Add caller-selectable evidence budgets for grounded answers and session queries, and preserve coverage metadata in streaming completion and cached answers. ([#661](https://github.com/groktopus/groktocrawl/pull/661)) +- Add page/offset continuation for fast, non-streaming keyword search, with explicit rejection of unsupported combinations. ([#661](https://github.com/groktopus/groktocrawl/pull/661)) +- Keep deep research acquisition going beyond the earlier three-source stopping behavior when the planned query set supports more evidence; expose discovery attempts, outcomes, reuse, and gaps. ([#644](https://github.com/groktopus/groktocrawl/pull/644)) +- Preserve fetched Markdown and source metadata in opt-in pyramid exports instead of producing empty source dossiers. ([#643](https://github.com/groktopus/groktocrawl/pull/643), [#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Add publication-date bounds that require trustworthy backend enforcement metadata; unsupported backends return an explicit error rather than pretending the filter was applied. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) + +### Agent tools and user interfaces + +- Add owned private-document attachments, document/web session questions, exact extracted-text passage reads, and document lifecycle operations. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) + +- Expose research-session creation, steps, retrieval, export, resolution, and deletion through typed MCP tools. ([#646](https://github.com/groktopus/groktocrawl/pull/646)) +- Expose research-plan creation, inspection, and explicitly approved execution through MCP. ([#647](https://github.com/groktopus/groktocrawl/pull/647)) +- Expose research-memory query, store, retrieval, deletion, batch, and sweep operations through MCP, preserving freshness and compatibility metadata. ([#648](https://github.com/groktopus/groktocrawl/pull/648)) +- Reconcile the existing and adopted research tools into a 73-tool MCP surface. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Fix Firefox Deep Research and recent-query submissions that could abort streaming requests. ([#657](https://github.com/groktopus/groktocrawl/pull/657)) +- Preserve CLI crawl outcome counts and zero-page reasons, streaming argument handling, and typed barrier errors through the adopted fixes. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) + +### Capture, model compatibility, and integrations + +- Centralize bounded capture recovery so supported callers share the scraper's recovery policy and diagnostics. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Add optional protected capture profiles and their network-boundary tests. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Allow explicit selection of `max_tokens` or `max_completion_tokens`, with separate generation and health budgets, for OpenAI-compatible providers. ([#655](https://github.com/groktopus/groktocrawl/pull/655)) +- Pass `ENGINE_SEMANTICSCHOLAR_API_KEY` into the search service from the stack configuration. ([#652](https://github.com/groktopus/groktocrawl/pull/652)) +- Add optional Jev configuration through `TYPESAFE_API_KEY` for the configured SlopSearX services. Jev remains an operator opt-in, not a mandatory inference dependency. ([#651](https://github.com/groktopus/groktocrawl/pull/651)) +- Improve SearXNG health fallback and reduce sweep noise; update CloakBrowser to 0.5.9. ([#649](https://github.com/groktopus/groktocrawl/pull/649), [#618](https://github.com/groktopus/groktocrawl/pull/618)) + +### Storage, deployment, and release reliability + +- Add pgvector serving, Qdrant backfill/parity checks, shadow comparison, and rollback writes as part of the documented storage migration. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Expose runtime revision and model readiness information to make deployed versions easier to verify. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Add the workflow for official AMD64 and ARM64 images with source-revision metadata and immutable digest references. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Add verified release-image promotion and remove registry cleanup that could delete platform manifests still referenced by multi-architecture images. ([#665](https://github.com/groktopus/groktocrawl/pull/665)) +- Make research-storage, restore, and protected-capture checks part of the required runtime gate. Both full integration suites passed 2,700 tests with zero failures on the [release PR CI run](https://github.com/groktopus/groktocrawl/actions/runs/38830497723); each recorded 153 conditional skips. + +**Full commit history:** [v0.14.0…v1.0.0](https://github.com/groktopus/groktocrawl/compare/v0.14.0...v1.0.0) + +**Release source:** [`f2ebbbb4e8da42c96f054e19d5d86fe556d62a6e`](https://github.com/groktopus/groktocrawl/commit/f2ebbbb4e8da42c96f054e19d5d86fe556d62a6e) ## [0.14.0](https://github.com/groktopus/groktocrawl/compare/v0.13.0...v0.14.0) (2026-09-07) diff --git a/docs/adr/0069-adopt-bounded-research-substrate.md b/docs/adr/0069-adopt-bounded-research-substrate.md index 27acd2db..5200ec43 100644 --- a/docs/adr/0069-adopt-bounded-research-substrate.md +++ b/docs/adr/0069-adopt-bounded-research-substrate.md @@ -3,20 +3,20 @@ - Status: accepted - Deciders: Groktopus maintainers - Date: 2026-10-15 -- Scope: research execution and retained-evidence substrate in the standard GroktoCrawl deployment; capability flags are temporary rollout controls +- Scope: research execution and retained-evidence substrate in the standard GroktoCrawl deployment; scripted fixture-run admission remains disabled in deployments - Provenance: adopted in part from the bounded architecture records in [`docs/adr/experimental/groktocrawl-x/`](experimental/groktocrawl-x/README.md), based on experiment commit `9c7744b66f205080a44bfd89808494dbe80eb5a4` - Supersedes: ADR-0026's Qdrant serving-store choice for deployments that complete the verified pgvector backfill and cutover. Qdrant remains a rollback-write target. This does not revise ADR-0047 or change the existing public API, job, session, or cache contracts. ## Context and Problem Statement -The research architecture work developed an explicit execution controller, versioned claims-and-evidence records, verification outcomes, and audited artifact rendering. The mainline standard deployment includes PostgreSQL for durable research state, making these capabilities usable in the same deployment as deterministic retrieval, scraping, and existing clients. +The research architecture work developed an explicit execution controller, versioned claims-and-evidence records, verification outcomes, and audited artifact rendering. PostgreSQL is integrated into the standard Compose deployment for durable retained records and the separate vector migration. The run endpoint remains a scripted example until the live acquisition and publication adapter is implemented; a shared database alone does not make those research capabilities operational. The experimental records include decisions accepted for bounded prototypes, evaluations, and use in the former experimental repository. Those acceptances do not establish that every experimental feature is production ready or that existing user data has been migrated. Mainline needs a clear adoption boundary that preserves deployed interfaces and makes state changes reversible. ## Decision Drivers - Keep existing API, CLI, MCP, job, session, and research-memory behavior compatible by default. -- Keep research routes and pgvector serving disabled during the initial rollout checks, then enable the adopted capabilities in the standard deployment after the documented migration and verification gates pass. +- Keep scripted fixture-run admission disabled in deployments with `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false`. Existing four research/workspace/artifact flags retain their read, status, workspace, and recovery behavior and may remain enabled to serve retained records. This development-only flag is not a production live-research switch; a future live adapter must be integrated and separately verified. Pgvector serving is a separate vector-store migration and does not depend on research flags. - Separate execution ownership, evidence, verified knowledge, and rendered artifacts so each can be inspected and versioned. - Avoid implicit conversion, deletion, or reinterpretation of existing persisted state. - Permit rollback to the existing research path without losing data or changing existing schemas. @@ -29,7 +29,7 @@ The experimental records include decisions accepted for bounded prototypes, eval ## Decision Outcome -Adopt option 3. The new research substrate provides explicit bounded execution, versioned knowledge and evidence, verification records, and audited renderings in the standard deployment architecture. The `/experimental/research/v1` route family and its existing feature flags remain the rollout boundary while implementation and operations are verified. Its path remains stable for compatibility; the word `experimental` does not create a separate repository or a permanent separate deployment mode. After the required checks pass, the standard deployment enables the supported durable research capabilities without changing the route or moving the service to a different stack. +Adopt option 3. The research substrate defines explicit bounded execution, versioned knowledge and evidence, verification records, and audited renderings in the standard deployment architecture. The current `/experimental/research/v1` run adapter is a scripted fixture demonstration; it does not acquire live sources for the submitted objective. Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false` in deployments, while preserving the existing research/workspace/artifact flags and read access for retained records as required. This opt-in is only for development/demo fixture admission, not production live research. A live adapter must connect the existing research acquisition path to this execution, evidence, verification, and publication flow and pass separate product acceptance checks before production run admission is considered. The route path remains stable for compatibility; its name does not require a separate repository or deployment. Pgvector serving is independent of the fixture route and follows the vector migration below. For vector search, pgvector becomes the serving store only after a complete mainline-Qdrant backfill and the parity and operational checks in the [migration runbook](../runbooks/mainline-architecture-migration.md). The cutover keeps Qdrant online and dual-writes to it so operators can return reads to Qdrant without rebuilding the old index. This narrows ADR-0026 only for deployments that complete this migration; unmigrated deployments continue using Qdrant. @@ -37,13 +37,13 @@ The execution owner is the sole authority for run transitions, budgets, cancella All loops, retries, source acquisition, and rendering repair remain subject to explicit limits. A model response cannot increase budgets or permissions. Missing or incompatible evidence, state, or verification versions fail closed or produce an explicitly qualified result according to the operation contract. -The new route family does not replace the existing research, `/v2/answer`, session, job, or research-memory path and does not change their request or response schemas. PostgreSQL is part of the standard Compose deployment and is the durable store for the research capabilities that require it; the feature flags control route and capability availability during rollout. They are temporary rollout controls, not a requirement to operate a separate experimental deployment. A flag being present in an environment inventory does not imply the feature is enabled; after migration checks pass, the deployment enables the supported durable capabilities. +The new route family does not replace the existing live research, `/v2/answer`, session, job, or research-memory paths and does not change their request or response schemas. PostgreSQL is part of the standard Compose deployment and supports retained research artifacts as well as the independent pgvector migration. Database availability or schema version 16 does not make the fixture adapter production research. Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false` in deployments; the existing four research flags may remain enabled for retained reads and recovery. A pgvector cutover can proceed independently of the research flags. ### Compatibility and state migration Existing inline jobs, sessions, research-memory records, API payloads, CLI flags, and MCP tools retain their current readers and writers. The new durable research ledger applies only to the research route family; it does not make legacy inline jobs restart-safe or change their best-effort contract in ADR-0047. No startup task automatically imports or rewrites legacy records into the new execution or evidence stores. New records must carry explicit schema and policy versions. Schema migrations must be additive and ordered, and must refuse an unknown or incompatible prior schema before modifying it. -Before enabling durable research capabilities, the deployment procedure must identify the affected PostgreSQL schema, take and verify a recoverable backup, record the source and target schema versions, run the explicit ordered migration, and verify representative reads and integrity checks. The vector-store cutover additionally requires a verified Qdrant backfill and parity checks; Qdrant remains current through dual writes for rollback. Any conversion of existing research records requires the resumable migration and reconciliation procedure in the migration runbook, with provenance, counts, integrity checks, and a documented abort path. This ADR does not authorize deletion or reinterpretation of legacy state. +The current fixture-backed run endpoint is not an operational capability to enable. A future live adapter requires a separately reviewed migration and acceptance plan that covers the ordered schema migration, backup and restore, objective-dependent HTTP/MCP execution, retained evidence and artifact reads, and crash, budget, and cancellation behavior. The vector-store cutover is independent: it requires a verified Qdrant backfill and parity checks, and Qdrant remains current through dual writes for rollback. Any conversion of existing research records requires the resumable migration and reconciliation procedure in the migration runbook, with provenance, counts, integrity checks, and a documented abort path. This ADR does not authorize deletion or reinterpretation of legacy state. ### Rollback @@ -54,8 +54,8 @@ Operators can disable the feature gates and route compatible work through the pr ### Positive Consequences - Research architecture can evolve in the mainline repository without silently changing existing API or storage contracts. -- Every experimental run can carry explicit budget, ownership, evidence, verification, and artifact-version information. -- Operators have a defined rollout, migration, and rollback boundary before default enablement; verified deployments converge on the durable research capabilities and pgvector serving. +- The research design gives runs explicit budget, ownership, evidence, verification, and artifact-version information when its live adapter is completed. +- Operators have separate migration and rollback paths for the live-research foundation and pgvector serving; vector cutover does not require enabling the fixture route. ### Negative Consequences @@ -65,7 +65,7 @@ Operators can disable the feature gates and route compatible work through the pr ## Confirmation -For each capability, its implementation review must identify the relevant feature flag, public-surface impact, schema/version behavior, migration and rollback evidence, and test coverage. Before enabling it by default in the standard deployment, confirm operator backup/restore evidence, deployment configuration, and required CI gates. Until those checks are complete, documentation must label the API capability experimental and must not present it as a default-enabled guarantee. +For pgvector, confirm backup/restore evidence, exact backfill parity, Qdrant rollback writes, deployment configuration, and required CI gates before cutover. Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false` while the route is fixture-backed; preserve the existing research flags needed for retained-record access. This opt-in must not be used for production research. A future live adapter requires separate HTTP/MCP, retained-byte, crash-recovery, budget, and cancellation acceptance before its production run admission is enabled. Documentation must continue to identify the current route as a scripted demonstration until then. ## Links diff --git a/docs/plans/research-architecture-migration.md b/docs/plans/research-architecture-migration.md index 69912a61..e5074ba5 100644 --- a/docs/plans/research-architecture-migration.md +++ b/docs/plans/research-architecture-migration.md @@ -2,10 +2,10 @@ ## Outcome -GroktoCrawl will include the research architecture developed in GroktoCrawl X. +GroktoCrawl has adopted the research architecture developed in GroktoCrawl X. Existing API, MCP, CLI, and deployment contracts remain supported. Future -implementation and issues belong in the main repository. The experimental -repository remains a historical record once migration is verified. +implementation and issues belong in the main repository. Release, deployment, +and repository consolidation remain tracked below. ## Source baseline @@ -36,19 +36,32 @@ tools, portal submission fixes, and release safeguards alongside the new code. including its reconciliation checks and abort path. Record the rehearsal and cutover results before declaring completion. 4. **Release and deploy.** Merge after required checks pass and published review - findings are addressed. Publish official images, update GroktoCrawl services - with those images, and verify API, MCP, CLI, stored research, and recovery. - Preserve the existing search-engine service and its configuration. + findings are addressed. Publish official images and update GroktoCrawl + services with those images. Verify the existing live `/v2` research, document, + and capture paths, plus MCP, CLI, and pgvector readiness. Keep scripted + fixture-run admission disabled with + `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false`: + the new durable run endpoint executes an example and is not connected to + objective-dependent live acquisition, verification, and retained + publication. The existing four research flags may remain enabled for + read-only access to retained records. Pgvector cutover is an independent storage change and does not + require enabling fixture-run admission. Preserve the existing search-engine + service and its configuration. 5. **Consolidate development.** Move remaining optional research issues to mainline, redirect the experimental repository to the supported project, and retain its history and recovery material. ## Completion checks -Completion requires an official mainline commit and published images, a verified -running deployment, preserved existing data and client contracts, a tested -recovery path, and reconciled documentation and issue ownership. Passing tests -alone does not establish deployment completion. +Completion of the mainline/vector migration requires an official mainline commit +and published images, a verified running deployment, preserved existing data and +client contracts, a tested vector rollback path, and reconciled documentation +and issue ownership. It does not make the fixture-backed run endpoint a live +research product. That requires separate product work to connect the existing +live research acquirer to the durable execution, verification, and publication +path, with its own objective-dependent HTTP/MCP and failure-recovery acceptance +tests. Passing migration tests alone does not establish that capability or +deployment completion. Date-bounded searches remain conditional on the backend returning trustworthy enforcement evidence. Unsupported backends must return an explicit error. @@ -57,8 +70,15 @@ results. ## Progress -- Both source baselines fetched and compared. -- Mainline-only runtime, documentation, and deployment contracts audited. -- Integration and compatibility checks passed on the prior integration head; final startup and health-check patch verification is in progress. -- Isolated restoration and shadow-backfill rehearsal completed: all 11,852 vector records matched across IDs, payload/content, and vectors. -- Release, live deployment verification, and repository consolidation remain pending. +- Mainline integration merged in PR #665 at `7741ddfe03773acae08e0cef47aa6ad6a457b481` after the required integration and compatibility checks passed. +- The isolated restore and shadow-backfill rehearsal matched all 11,852 records across IDs, payload/content, and vectors. +- Stable 1.0.0 was published on 2026-10-16 ([release](https://github.com/groktopus/groktocrawl/releases/tag/v1.0.0)); its release PR merged at `f2ebbbb4e8da42c96f054e19d5d86fe556d62a6e`. +- Official image publication and immutable image digests are still pending verification. +- The isolated restore and backfill verified pgvector parity; live vector-serving cutover and deployment verification are pending. Verify the existing live `/v2` research, document, capture, MCP, CLI, and semantic paths. Keep scripted fixture-run admission disabled with `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false`; the `/experimental/research/v1` run adapter is an example, not a live researcher. The existing four research flags may remain enabled for previously retained records. +- Transfer of remaining optional issues and retirement of the experimental repository are pending. + +The live adapter gap is separate product work: connect objective-dependent live +source acquisition, evidence and claim verification, and retained publication to +the durable run protocol. Acceptance must exercise meaningful HTTP and MCP +objectives and verify retained source/artifact bytes, as well as budget limits, +cancellation, and recovery across a forced process crash. diff --git a/docs/runbooks/mainline-architecture-migration.md b/docs/runbooks/mainline-architecture-migration.md index 40205bd8..18080c1f 100644 --- a/docs/runbooks/mainline-architecture-migration.md +++ b/docs/runbooks/mainline-architecture-migration.md @@ -4,8 +4,11 @@ This runbook adds the durable research PostgreSQL store and, after verified backfill and parity checks, makes pgvector the vector-serving store in the existing Compose project. Qdrant remains online and receives writes for read rollback. The existing Valkey job store, SlopSearX image/configuration, and -API/MCP ports are preserved. Research flags remain off during initial -preparation and are enabled after their migration and integrity checks pass. +API/MCP ports are preserved. This runbook does not enable scripted research +execution. Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false` throughout +the vector migration. +Existing research/workspace/artifact read flags may remain enabled to serve +retained records. Pgvector promotion is independent of research-run admission. The adoption decision is [ADR-0069](../adr/0069-adopt-bounded-research-substrate.md); the sequencing and release work are in the [research architecture migration plan](../plans/research-architecture-migration.md). @@ -37,9 +40,11 @@ down -v`. PostgreSQL gets a new volume named from the project and chmod 600 /secure/path/research-postgres-password ``` - Keep all four `FEATURE_EXPERIMENTAL_RESEARCH*` settings false and - `VECTOR_STORE_MODE=qdrant` during preparation. Leave Valkey DB 0 and Qdrant - as they are. + Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false` and + `VECTOR_STORE_MODE=qdrant` during preparation. Do not enable fixture-run + admission at any point in this vector migration. Existing research flags may + remain enabled when needed for read-only access to previously retained roots + and artifacts. Leave Valkey DB 0 and Qdrant as they are. 3. Back up the full Valkey dataset (the RDB includes DB 0 and DB 2) and take a Qdrant collection snapshot before enabling the new profile. Store the resulting files off-host and verify their checksums and restore procedure. @@ -150,11 +155,15 @@ down -v`. PostgreSQL gets a new volume named from the project and 8. Cut over only after the reconciliation and comparison gates pass. Set `VECTOR_STORE_MODE=pgvector` and retain Qdrant dual writes. If research - artifacts are in scope, enable all four research feature flags only after the - schema-16 and table-level reconciliation passes. Apply flags in dependency - order: `FEATURE_EXPERIMENTAL_RESEARCH`, then `_RUNS`, then `_DURABLE`, then - `_POSTGRES_ARTIFACTS`. Recreate the agent and semantic services and confirm - API, MCP, and semantic readiness on their existing ports. Monitor search + artifacts are in scope, keep the database schema and read-only artifact + authority available after the schema-16 and table-level reconciliation + passes. Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false`: the + current run endpoint executes a scripted example and is not connected to + live query acquisition and publication. Existing research flags may remain + enabled for retained reads. Pgvector cutover is independent of research-run + admission and must be validated on its own. Recreate the agent and semantic + services and confirm existing API, MCP, and semantic readiness on their + existing ports. Monitor search result parity, vector counts, PostgreSQL saturation, Qdrant write freshness, request errors, and latency for the agreed observation window. @@ -174,10 +183,14 @@ down -v`. PostgreSQL gets a new volume named from the project and store during pgvector serving. Verify readiness reports Qdrant ready and representative searches return results. If the current image is defective, restore the previously recorded agent/semantic image digests as well. -- **Research API rollback:** turn off the experimental feature flags and restore - the previous agent image. This leaves existing mainline jobs in Valkey DB 0 - and all research rows in PostgreSQL for diagnosis. Do not delete the new - database volume as part of application rollback. +- **Research API rollback:** scripted fixture-run admission should already be + disabled by `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false`. If it was + enabled accidentally, disable it and restore the previous agent image if + needed. Keep existing + research flags required to read retained roots until their owners have a + separate disposition. This leaves existing mainline jobs in Valkey DB 0 and + research rows in PostgreSQL intact. Do not delete the new database volume as + part of application rollback. - **Database recovery:** restore PostgreSQL only when the database itself is damaged or inconsistent. Take a forensic copy first. Restore the verified `pg_dump` into a new/empty PostgreSQL volume, run schema checks, and point the From af836116f6b9a5fdff00692f43d87808a57b6752 Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 21:29:52 -0400 Subject: [PATCH 03/11] docs: clarify the live research boundary in release overview --- CHANGELOG.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa47e8ae..7529363a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,7 +67,7 @@ The new architecture defines explicit ownership of run state, budgets, cancellat This separation is intended to let models and workflow implementations evolve without making them the authority for permissions, budgets, or stored evidence. It also provides a foundation for summaries, deeper analysis, and source reports that remain tied to the same evidence. -**What is implemented versus integrated:** the new route family exercises these contracts using a fixed example. Its durable ledger, PostgreSQL artifact storage, and retained workspace have lifecycle and recovery tests, but the endpoint does not yet acquire real sources for the submitted objective. Connecting live acquisition, claim construction, verification, and rendering is still required before this can replace the existing live research path. +**What is implemented versus integrated:** the new route family exercises these contracts using a fixed example. Its durable ledger, PostgreSQL artifact storage, and retained workspace have lifecycle and recovery tests, but the endpoint does not yet acquire real sources for the submitted objective. Connecting live acquisition, claim construction, verification, and rendering is still required before this can replace the existing live research path. That integration is tracked in [#666](https://github.com/groktopus/groktocrawl/issues/666). ### A workspace and recovery protocol for the new foundation diff --git a/README.md b/README.md index ef68b2a3..77ccb559 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ curl -N -X POST http://localhost:8080/v2/crawl \ ## Research architecture -GroktoCrawl includes a bounded research substrate for explicit execution, versioned evidence and knowledge, verification, and audited artifacts. Existing APIs, jobs, sessions, and research-memory contracts remain compatible. New durable research capabilities use PostgreSQL and are enabled in the standard deployment after the documented migration checks pass. The `/experimental/research/v1` path remains stable for compatibility; its feature flags are rollout controls, not a permanent opt-in deployment mode. Vector serving moves from Qdrant to pgvector only after a verified backfill and parity checks, while Qdrant remains current for rollback. See [ADR-0069](docs/adr/0069-adopt-bounded-research-substrate.md), the [migration plan](docs/plans/research-architecture-migration.md), and the [deployment migration runbook](docs/runbooks/mainline-architecture-migration.md). The protected capture topology is described in the [protected Compose override](docker-compose.protected.yml). Historical experiment decisions and their scope are recorded in the [GroktoCrawl X ADR archive](docs/adr/experimental/groktocrawl-x/README.md). +GroktoCrawl includes a bounded research substrate for explicit execution, versioned evidence and knowledge, verification, and audited artifacts. Existing APIs, jobs, sessions, and research-memory contracts remain compatible. PostgreSQL-backed retention and recovery are included, but new runs on `/experimental/research/v1` still execute a scripted example, not live web research. Creating those demonstration runs requires an explicit development-only opt-in and is disabled by default; existing retained reports remain readable when their storage capabilities are enabled. The existing `/v2` paths continue to perform live research. Connecting live acquisition to the durable endpoint is tracked in [#666](https://github.com/groktopus/groktocrawl/issues/666). Vector serving moves from Qdrant to pgvector only after a verified backfill and parity checks, while Qdrant remains current for rollback. See [ADR-0069](docs/adr/0069-adopt-bounded-research-substrate.md), the [migration plan](docs/plans/research-architecture-migration.md), and the [deployment migration runbook](docs/runbooks/mainline-architecture-migration.md). The protected capture topology is described in the [protected Compose override](docker-compose.protected.yml). Historical experiment decisions and their scope are recorded in the [GroktoCrawl X ADR archive](docs/adr/experimental/groktocrawl-x/README.md). ## What it provides From 20af6eb9f68856902085fd74e4160fd59304585d Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 21:38:17 -0400 Subject: [PATCH 04/11] docs: persist fixture opt-in through isolated checkpoints Signed-off-by: magnus919 --- .../client-protocol-conformance.md | 7 ++- .../experimental-replacement-candidate.md | 49 +++++++++++++++---- .../mainline-architecture-migration.md | 7 ++- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/docs/experiments/client-protocol-conformance.md b/docs/experiments/client-protocol-conformance.md index f4471156..30c78efe 100644 --- a/docs/experiments/client-protocol-conformance.md +++ b/docs/experiments/client-protocol-conformance.md @@ -14,7 +14,12 @@ boundary when `FEATURE_EXPERIMENTAL_RESEARCH=true`. A second opt-in gate, `FEATURE_EXPERIMENTAL_RESEARCH_RUNS=true`, enables retained run/status/cancel/event and artifact/evidence reads. New fixture admission additionally requires the development-only `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true` opt-in; -leaving it false preserves those reads and recovery without creating new runs. A third opt-in gate, +leaving it false preserves those reads and recovery without creating new runs. +For [isolated demo verification and checkpoints](../runbooks/experimental-replacement-candidate.md#verify-and-freeze-the-receipt), +persist the true setting in the dedicated private candidate env file used by +all Compose recreations, not just in a one-command shell prefix. After the full +sequence, reset that file to false and recreate the demo agent. Production +configuration keeps fixture admission false throughout. A third opt-in gate, `FEATURE_EXPERIMENTAL_RESEARCH_DURABLE=true`, wires admission, lease fencing, checkpoint identity, terminal status projection and cancellation authority to Valkey. The additional `FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS=true` gate requires diff --git a/docs/runbooks/experimental-replacement-candidate.md b/docs/runbooks/experimental-replacement-candidate.md index 92b0d7e7..eddd9f8a 100644 --- a/docs/runbooks/experimental-replacement-candidate.md +++ b/docs/runbooks/experimental-replacement-candidate.md @@ -174,17 +174,25 @@ capture-readiness signal. ## Verify and freeze the receipt This verifier creates a fictional research run. Use it only on an isolated -development/demo stack. Explicitly opt into fixture admission before verification: +development/demo stack with its dedicated private `$CANDIDATE_ENV`; never edit a +production environment file for this procedure. In that file, set the existing +`FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS` entry to `true` (add it if absent). +Keep it true through the entire verification and every pilot checkpoint below. +The verifier and checkpoint runner recreate services using `--env-file`, so a +flag prefixed to only the initial `docker compose up` command does not persist. + +Clear any shell override so subsequent commands use that dedicated file: ```sh -FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true docker compose \ - --env-file "$CANDIDATE_ENV" -f "$CANDIDATE_COMPOSE" \ - up -d --no-deps candidate-agent +unset FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS +docker compose --env-file "$CANDIDATE_ENV" -f "$CANDIDATE_COMPOSE" \ + up -d --no-deps --force-recreate candidate-agent ``` -Keep the new flag false for normal operation. After the fixture check, recreate -only that demo service with `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false` -using the same Compose command. Existing retained reads remain available. +Keep the flag false for normal operation. Reset the dedicated file to false and +recreate the demo agent only after the entire verification/checkpoint sequence +is finished or abandoned, as described below. Existing retained reads remain +available while fixture admission is disabled. Load only the candidate API key into the verifier process and write the receipt outside the repository first: @@ -225,6 +233,11 @@ their receipt records only readiness, HTTP class, and latency. ## Run a time-gated pilot checkpoint +Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=true` in the same dedicated +private `$CANDIDATE_ENV` for every checkpoint and retry; do not reset it after +only the initial verifier. Each checkpoint passes this file to later Compose +recreations and to the fixture verifier. + Use the checkpoint runner for restart checkpoint 0, checkpoint 1, and checkpoint 2. It reads the tracked pilot state, refuses an early or out-of-order checkpoint, resolves the effective @@ -291,14 +304,32 @@ elapsed, all three checkpoint numbers are present, and the packet total proves at least 30 successful operations. A failing report is retained evidence for a revise or reject decision; it must not be rewritten into a passing state. +## End isolated fixture verification + +After the final checkpoint, or when abandoning the isolated sequence, change +`FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS` to `false` in the same private +`$CANDIDATE_ENV`. Then apply that setting only to the demo agent: + +```sh +unset FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS +docker compose --env-file "$CANDIDATE_ENV" -f "$CANDIDATE_COMPOSE" \ + up -d --no-deps --force-recreate candidate-agent +``` + +Keep the dedicated file false afterward so any later normal Compose recreation +also disables new fictional runs. Retained artifacts and workspace reads keep +their existing feature gates. Do not enable fixture admission in production. + ## Stop without destroying evidence ```sh docker compose --env-file "$CANDIDATE_ENV" -f "$CANDIDATE_COMPOSE" stop ``` -Use `start` followed by the verifier for the restart check. `down` removes -containers and networks while preserving named volumes. Do not use `down -v`; +For another fixture restart check, repeat the isolated opt-in and agent +recreation above before using `start` and the verifier, and reset the dedicated +file after that sequence. `down` removes containers and networks while +preserving named volumes. Do not use `down -v`; volume removal belongs only to the later, separately reviewed retirement step. ## Failure boundary diff --git a/docs/runbooks/mainline-architecture-migration.md b/docs/runbooks/mainline-architecture-migration.md index 18080c1f..ab533411 100644 --- a/docs/runbooks/mainline-architecture-migration.md +++ b/docs/runbooks/mainline-architecture-migration.md @@ -160,8 +160,11 @@ down -v`. PostgreSQL gets a new volume named from the project and passes. Keep `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false`: the current run endpoint executes a scripted example and is not connected to live query acquisition and publication. Existing research flags may remain - enabled for retained reads. Pgvector cutover is independent of research-run - admission and must be validated on its own. Recreate the agent and semantic + enabled for retained reads. The separate [isolated demo verification](experimental-replacement-candidate.md#verify-and-freeze-the-receipt) + persists a true fixture flag only in its dedicated test environment through + all checkpoints, then resets that file to false and recreates the demo agent; + do not apply that test opt-in to this production migration. + Pgvector cutover is independent of research-run admission and must be validated on its own. Recreate the agent and semantic services and confirm existing API, MCP, and semantic readiness on their existing ports. Monitor search result parity, vector counts, PostgreSQL saturation, Qdrant write freshness, From 9f634debeee6877061fd18d5fcb86d95e4d81d76 Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 22:46:28 -0400 Subject: [PATCH 05/11] fix(research): execute live objectives with durable call receipts --- .env.sample | 6 +- .github/workflows/runtime.yml | 21 +- AGENTS.md | 4 + CONTRIBUTING.md | 4 + README.md | 6 +- agent-svc/agent/app.py | 13 + .../agent/experimental/artifact_authority.py | 22 + .../agent/experimental/consolidated_bundle.py | 21 +- .../agent/experimental/durable_research.py | 110 +++- .../agent/experimental/knowledge_checks.py | 2 +- .../agent/experimental/knowledge_context.py | 16 +- agent-svc/agent/experimental/live_runner.py | 242 +++++++ .../agent/experimental/model_transport.py | 20 +- .../agent/experimental/query_construction.py | 28 +- agent-svc/agent/experimental/query_runner.py | 44 +- agent-svc/agent/experimental/real_journey.py | 22 +- agent-svc/agent/experimental/source_store.py | 6 +- .../agent/routes/experimental_research.py | 597 +++++++++++++----- .../adr/0070-execute-live-durable-research.md | 41 ++ docs/adr/README.md | 3 +- .../test_live_research_http_adapters.py | 515 +++++++++++++++ ...test_experimental_research_capabilities.py | 16 +- ...xperimental_research_postgres_artifacts.py | 6 +- .../test_experimental_research_runs.py | 32 +- .../test_experimental_runtime_workflow.py | 10 +- tests/service/test_live_research_recovery.py | 359 +++++++++++ tests/unit/test_query_construction.py | 19 + 27 files changed, 1982 insertions(+), 203 deletions(-) create mode 100644 agent-svc/agent/experimental/live_runner.py create mode 100644 docs/adr/0070-execute-live-durable-research.md create mode 100644 tests/integration/test_live_research_http_adapters.py create mode 100644 tests/service/test_live_research_recovery.py diff --git a/.env.sample b/.env.sample index 4ea253e8..b897c987 100644 --- a/.env.sample +++ b/.env.sample @@ -88,9 +88,9 @@ # FEATURE_EXPERIMENTAL_RESEARCH_RUNS=false # FEATURE_EXPERIMENTAL_RESEARCH_DURABLE=false # FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS=false -# Development/demo only: new runs still execute fictional sources and authored -# judgments. Leave false in production; existing retained reads/recovery use -# the gates above and remain available without creating new fixture runs. +# Development/demo only: true selects fictional sources and authored judgments. +# Leave false for live durable research with all four gates above enabled. +# Live admission requires PostgreSQL schema16 and configured search/scraper/LLM. # FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false # Schema 16 is required before enabling PostgreSQL artifact authority. The # durable-research Compose profile provisions pgvector and initializes this diff --git a/.github/workflows/runtime.yml b/.github/workflows/runtime.yml index c053edaa..e56549c2 100644 --- a/.github/workflows/runtime.yml +++ b/.github/workflows/runtime.yml @@ -162,6 +162,13 @@ jobs: cp .env.sample .env echo "TWIN_RUN_ID=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> .env echo "DURABLE_RESEARCH_REDIS_URL=redis://valkey:6379/15" >> .env + echo "SEARXNG_URL=http://slopsearx-fixture:8080" >> .env + umask 077 + research_password_file="/tmp/groktocrawl-research-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password" + python3 -c 'from secrets import token_urlsafe; print(token_urlsafe(32))' > "$research_password_file" + chmod 600 "$research_password_file" + echo "DURABLE_RESEARCH_POSTGRES_PASSWORD_FILE=$research_password_file" >> .env + echo "DURABLE_RESEARCH_POSTGRES_DSN=postgresql://groktocrawl@research-postgres:5432/groktocrawl" >> .env echo "LLM_BASE_URL=http://llm-svc:8011/v1?run_id=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> .env echo "LLM_MODEL=fixture-model" >> .env echo "CAPTCHA_VISION_BASE_URL=http://llm-svc:8011/v1" >> .env @@ -187,6 +194,7 @@ jobs: --group runtime --profile indexing --profile fixture + --profile durable-research --receipt runtime-public-image-prefetch.json timeout-minutes: 10 @@ -199,7 +207,7 @@ jobs: if-no-files-found: warn - name: Build all application and fixture images from checkout - run: docker compose --profile indexing --profile fixture build + run: docker compose --profile indexing --profile fixture --profile durable-research build timeout-minutes: 25 - name: Build and probe the capture gateway image @@ -217,7 +225,7 @@ jobs: docker exec -i "$container" python - < tests/integration/capture_egress_image_probe.py - name: Start the Docker stack - run: docker compose --profile indexing --profile fixture up -d --no-build --pull never + run: docker compose --profile indexing --profile fixture --profile durable-research up -d --no-build --pull never timeout-minutes: 10 - name: Wait for services to be healthy @@ -592,9 +600,12 @@ jobs: set +e docker compose exec -T agent-svc env \ QA_OUTCOME_PATH=/app/qa-outcomes-integration.json \ + DURABLE_RESEARCH_REDIS_URL=redis://valkey:6379/15 \ + DURABLE_RESEARCH_POSTGRES_DSN=postgresql://groktocrawl@research-postgres:5432/groktocrawl \ + PGPASSFILE=/tmp/postgres.pgpass \ PYTHONPATH=/app/agent-svc:/app/scraper-svc:/app/llm-svc:/app/parse-svc:/app/portal-svc:/app/browser-svc:/app \ SCRAPER_BASE_URL=http://scraper-svc:8001 \ - SEARCH_BASE_URL=http://slopsearx:8080 \ + SEARCH_BASE_URL=http://slopsearx-fixture:8080 \ LLM_BASE_URL=http://llm-svc:8011/v1?run_id=${{ github.run_id }}-${{ github.run_attempt }} \ TEST_SITE_BASE_URL=http://test-site:8000 \ TIER3_FIXTURE_BASE_URL=http://tier3-fixture:8000 \ @@ -816,7 +827,9 @@ jobs: - name: Tear down stack if: always() - run: docker compose --profile indexing --profile fixture down --volumes --remove-orphans --timeout 30 + run: | + docker compose --profile indexing --profile fixture --profile durable-research down --volumes --remove-orphans --timeout 30 + rm -f "/tmp/groktocrawl-research-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password" protected-browser-profile: name: Protected Browser Egress Profile diff --git a/AGENTS.md b/AGENTS.md index 60c473f0..aff6ada6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,10 @@ Architecture Decision Records (ADRs) live in `docs/adr/` and capture the context Jobs are processed with `asyncio.create_task()` inside the API process. Valkey persists job records, not execution: interrupted jobs are not resumed after restart, partial artifacts can remain, and undelivered webhooks are not replayed. `TaskTracker` provides only a short graceful-shutdown window. Restart-safe execution is deferred until it is an explicit product requirement; any future design must define ownership, leases, retries, cancellation, artifact consistency, and webhook idempotency in an ADR before selecting queue technology. +### Durable research execution + +The separate `/experimental/research/v1` live run path uses server-configured search, acquisition, and model clients when its four existing feature flags are enabled and the explicit development fixture flag is false. PostgreSQL schema16 retains source bytes and audited report sets; Valkey retains scoped admission, owner generations, operation intents/receipts, and terminal projections. Do not replace a failed or unknown external operation with an unrecorded retry. The worker renews ownership; claim, publication, and cancellation serialize on a scoped PostgreSQL advisory lock, and deletion fences late publication. Recovery checks committed PostgreSQL results before invoking any adapter. Legacy `/v2` jobs keep their separate contract. Read [ADR-0070](docs/adr/0070-execute-live-durable-research.md) before changing these boundaries. + ### Three-tier smart scraper Tier 1: Check `/llms.txt` at the site root (one GET, whole site in markdown) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fdc67fda..de7669ec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,6 +109,10 @@ The Docker command above remains the integration-test path. All tests must pass before a PR is merged. +### Durable research acceptance + +`tests/service/test_live_research_recovery.py` exercises operation reuse, ambiguous-dispatch refusal, persistent failures and lease/cancel fencing with isolated Valkey. `tests/integration/test_live_research_http_adapters.py` additionally requires an isolated PostgreSQL schema16 database and runs actual HTTP search/capture/model digital twins. The runtime workflow provisions the durable profile and runs this test mandatorily. These twins require no provider credentials. Component tests alone do not establish objective-dependent retained research; changes to the durable adapter must preserve this full HTTP-to-retained-byte test and its recovery checks. Never point acceptance tests at a production database. + ## Development Setup You need Docker and Docker Compose. No other dependencies are required — everything runs in containers. diff --git a/README.md b/README.md index 77ccb559..1b45b348 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,11 @@ curl -N -X POST http://localhost:8080/v2/crawl \ ## Research architecture -GroktoCrawl includes a bounded research substrate for explicit execution, versioned evidence and knowledge, verification, and audited artifacts. Existing APIs, jobs, sessions, and research-memory contracts remain compatible. PostgreSQL-backed retention and recovery are included, but new runs on `/experimental/research/v1` still execute a scripted example, not live web research. Creating those demonstration runs requires an explicit development-only opt-in and is disabled by default; existing retained reports remain readable when their storage capabilities are enabled. The existing `/v2` paths continue to perform live research. Connecting live acquisition to the durable endpoint is tracked in [#666](https://github.com/groktopus/groktocrawl/issues/666). Vector serving moves from Qdrant to pgvector only after a verified backfill and parity checks, while Qdrant remains current for rollback. See [ADR-0069](docs/adr/0069-adopt-bounded-research-substrate.md), the [migration plan](docs/plans/research-architecture-migration.md), and the [deployment migration runbook](docs/runbooks/mainline-architecture-migration.md). The protected capture topology is described in the [protected Compose override](docker-compose.protected.yml). Historical experiment decisions and their scope are recorded in the [GroktoCrawl X ADR archive](docs/adr/experimental/groktocrawl-x/README.md). +The `/experimental/research/v1` endpoint can execute an objective through the server-configured search, scraper, and model, then retain source bytes, checked knowledge, and three report layers in PostgreSQL. Enable all four research flags and schema 16; leave the development-only `FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false`. Setting that flag true explicitly selects scripted demonstration runs. Existing retained records keep their readers. + +This durable endpoint currently uses one search and at most three freshly acquired sources, with a 180-second deadline. Completed external-call receipts are reused after restart; an interrupted dispatch without a receipt fails as `outcome_unknown` and is never automatically sent again. `as_of` and `webhook` are rejected before admission while their durable contracts are completed. This is separate from the existing `/v2` research loop and does not imply execution of workspace follow-up previews or source copying into sessions. Completion of the remaining release work is tracked in [#666](https://github.com/groktopus/groktocrawl/issues/666). See [ADR-0070](docs/adr/0070-execute-live-durable-research.md). + +Vector serving moves from Qdrant to pgvector after verified backfill and parity checks, with Qdrant retained for rollback. See [ADR-0069](docs/adr/0069-adopt-bounded-research-substrate.md), the [migration plan](docs/plans/research-architecture-migration.md), and the [deployment migration runbook](docs/runbooks/mainline-architecture-migration.md). Protected capture is described in the [protected Compose override](docker-compose.protected.yml). Historical decisions remain in the [GroktoCrawl X ADR archive](docs/adr/experimental/groktocrawl-x/README.md). ## What it provides diff --git a/agent-svc/agent/app.py b/agent-svc/agent/app.py index c6c5b1bf..17a5b1f8 100644 --- a/agent-svc/agent/app.py +++ b/agent-svc/agent/app.py @@ -371,6 +371,16 @@ async def startup_event() -> None: This runs inside a running event loop (unlike module-level ``asyncio.create_task()`` which would fail at import time). """ + from common.features import is_enabled + + from .routes.experimental_research import recover_research_runs + + if is_enabled("experimental_research_runs") and is_enabled( + "experimental_research_durable" + ): + app.state.task_tracker.create_background_task( + recover_research_runs(app, app.state.task_tracker.shutdown_event) + ) if app.state.slopsearx_provenance is not None: await app.state.slopsearx_provenance.check_profile() app.state.task_tracker.create_background_task( @@ -386,7 +396,10 @@ async def startup_event() -> None: @app.on_event("shutdown") async def shutdown_event() -> None: + from .routes.experimental_research import stop_research_runs + await app.state.task_tracker.shutdown(grace_period=5.0) + await stop_research_runs(app) await app.state.research_memory.close() await app.state.scraper_client.close() await app.state.searxng_client.close() diff --git a/agent-svc/agent/experimental/artifact_authority.py b/agent-svc/agent/experimental/artifact_authority.py index 55d07af9..7cad06fe 100644 --- a/agent-svc/agent/experimental/artifact_authority.py +++ b/agent-svc/agent/experimental/artifact_authority.py @@ -5,6 +5,7 @@ import hashlib import json from collections.abc import Mapping +from contextlib import asynccontextmanager from dataclasses import dataclass from pathlib import Path from typing import Any @@ -66,6 +67,26 @@ def __init__(self, status_code: int): class ArtifactAuthority(SourceStore): """Atomic, scoped storage boundary; callers establish scope authorization.""" + @staticmethod + async def _lock_execution(conn, scope: UUID, research: UUID) -> None: + key = int.from_bytes( + hashlib.sha256(f"{scope}:{research}".encode()).digest()[:8], + "big", + signed=True, + ) + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (key,)) + + @asynccontextmanager + async def execution_lock(self, scope: UUID, research: UUID): + """Serialize execution ownership, publication and cancellation for a root. + + No external research calls belong inside this short transaction. Artifact + writes use their own transaction while this connection holds the lock. + """ + async with self._transaction() as conn: + await self._lock_execution(conn, scope, research) + yield + async def migrate_artifact_authority(self) -> None: migration = ( Path(__file__).with_name("migrations") @@ -371,6 +392,7 @@ async def find_run(self, scope: UUID, run: UUID) -> RetainedArtifactSet | None: async def delete(self, scope: UUID, research: UUID) -> None: async with self._transaction() as conn: + await self._lock_execution(conn, scope, research) row = await ( await conn.execute( "SELECT deleted FROM research_staging.research_artifact_sets WHERE scope_id=%s AND research_id=%s FOR UPDATE", diff --git a/agent-svc/agent/experimental/consolidated_bundle.py b/agent-svc/agent/experimental/consolidated_bundle.py index 4e42ec01..edd2d715 100644 --- a/agent-svc/agent/experimental/consolidated_bundle.py +++ b/agent-svc/agent/experimental/consolidated_bundle.py @@ -135,7 +135,8 @@ async def admit_consolidated_bundle( not isinstance(snapshot_ids, list) or len(snapshot_ids) > MAX_SNAPSHOTS or any( - not isinstance(value, str) or not 1 <= len(value) <= 200 + not isinstance(value, str) + or not 1 <= len(value) <= 200 or "/" in value or "\\" in value or value in {".", ".."} @@ -149,7 +150,11 @@ async def admit_consolidated_bundle( names = { "knowledge.json", "manifest.json", - *(f"sources/{index}.{extension}" for index in range(len(snapshot_ids)) for extension in ("body", "json")), + *( + f"sources/{index}.{extension}" + for index in range(len(snapshot_ids)) + for extension in ("body", "json") + ), *(f"outputs/{layer}.md" for layer in LAYERS), } if not isinstance(members, dict) or set(members) != names: @@ -165,12 +170,16 @@ async def admit_consolidated_bundle( if actual_snapshot_ids != snapshot_ids: raise ValueError("consolidated snapshot identities differ from knowledge") sources = [] - for index, snapshot in enumerate(sorted(context.snapshots, key=lambda item: item.snapshot_id)): + for index, snapshot in enumerate( + sorted(context.snapshots, key=lambda item: item.snapshot_id) + ): body = decoded[f"sources/{index}.body"] descriptor = admit_canonical_json( decoded[f"sources/{index}.json"], schema_version=SCHEMA ) - expected = source_descriptor(body, snapshot.canonical_url) + expected = source_descriptor( + body, snapshot.canonical_url, allow_unknown_url=True + ) if descriptor.data != expected.data: raise ValueError("consolidated source descriptor differs from body") sources.append( @@ -244,7 +253,9 @@ async def build_consolidated_bundle( snapshots = sorted(knowledge.context.snapshots, key=lambda item: item.snapshot_id) for index, snapshot in enumerate(snapshots): source = sources[snapshot.snapshot_id] - descriptor = source_descriptor(source.body, snapshot.canonical_url) + descriptor = source_descriptor( + source.body, snapshot.canonical_url, allow_unknown_url=True + ) members[f"sources/{index}.json"] = bundle_member(descriptor.data) members[f"sources/{index}.body"] = bundle_member(source.body) for report in retained.reports: diff --git a/agent-svc/agent/experimental/durable_research.py b/agent-svc/agent/experimental/durable_research.py index 5fe12037..9e260523 100644 --- a/agent-svc/agent/experimental/durable_research.py +++ b/agent-svc/agent/experimental/durable_research.py @@ -31,6 +31,10 @@ class LeaseLostError(DurableConflictError): """The caller no longer owns the fencing generation.""" +class OutcomeUnknownError(DurableResearchError): + """An external dispatch has no durable receipt and must not be replayed.""" + + @dataclass(frozen=True) class DurableRun: """The persisted projection needed to recover one admitted run.""" @@ -180,6 +184,50 @@ def from_record(cls, record: dict[str, Any]) -> DurableRun: return {1, 'deleted'} """ +_STEP_SCRIPT = """ +local raw = redis.call('GET', KEYS[1]) +if not raw or redis.call('GET', KEYS[2]) ~= ARGV[1] then return {-1} end +local record = cjson.decode(raw) +if record.state ~= 'running' then return {-1} end +record.operations = record.operations or {} +local prior = record.operations[ARGV[2]] +if prior then + if prior.input_digest ~= ARGV[3] then return {-2} end + if ARGV[4] == 'begin' then + if prior.state == 'receipt' then return {1, cjson.encode(prior.output)} end + return {2} + end + if prior.state == 'receipt' then return {-2} end +elseif ARGV[4] ~= 'begin' then return {-2} +end +if ARGV[4] == 'begin' then + local count = 0 + for _ in pairs(record.operations) do count = count + 1 end + if count >= 72 then return {-2} end + record.operations[ARGV[2]] = {input_digest=ARGV[3], state='dispatch_intent'} +else + record.operations[ARGV[2]] = {input_digest=ARGV[3], state='receipt', output=cjson.decode(ARGV[5])} +end +local encoded = cjson.encode(record) +if string.len(encoded) > 1048576 then return {-2} end +redis.call('SET', KEYS[1], encoded, 'PX', ARGV[6]) +return {0} +""" + +_FAIL_SCRIPT = """ +local raw = redis.call('GET', KEYS[1]) +if not raw or redis.call('GET', KEYS[2]) ~= ARGV[1] then return 0 end +local record = cjson.decode(raw) +if record.state ~= 'running' then return 0 end +record.state = 'failed' +record.terminal_payload = cjson.decode(ARGV[2]) +record.owner_id = false +record.lease_expires_at_ms = false +redis.call('SET', KEYS[1], cjson.encode(record), 'PX', ARGV[3]) +redis.call('DEL', KEYS[2]) +return 1 +""" + def _digest(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() @@ -276,7 +324,9 @@ def get(self, run_id: str) -> DurableRun | None: raw = self.redis.get(self._run_key(run_id)) return None if raw is None else DurableRun.from_record(json.loads(raw)) - def claim(self, run_id: str, owner_id: str, *, attempt_id: str | None = None) -> DurableRun: + def claim( + self, run_id: str, owner_id: str, *, attempt_id: str | None = None + ) -> DurableRun: """Claim or reclaim work with a monotonic fencing generation.""" result = self.redis.eval( _CLAIM_SCRIPT, @@ -313,6 +363,64 @@ def heartbeat(self, run_id: str, owner_id: str, generation: int) -> None: if int(result) != 1: raise LeaseLostError("heartbeat rejected by the fencing authority") + def step( + self, + run_id: str, + owner_id: str, + generation: int, + operation_id: str, + input_digest: str, + *, + output: Any = None, + complete: bool = False, + ) -> tuple[bool, Any]: + """Persist intent before dispatch; reuse receipts, never replay ambiguity.""" + if not operation_id or len(operation_id) > 200: + raise DurableResearchError("invalid operation identity") + encoded = json.dumps(output, separators=(",", ":"), allow_nan=False) + if len(encoded.encode()) > 512_000: + raise DurableResearchError("operation receipt exceeds byte budget") + result = self.redis.eval( + _STEP_SCRIPT, + 2, + self._run_key(run_id), + self._lease_key(run_id), + f"{owner_id}:{generation}", + operation_id, + input_digest, + "complete" if complete else "begin", + encoded, + str(self.retention_ms), + ) + code = int(result[0]) + if code == -1: + raise LeaseLostError("operation rejected for stale owner") + if code == -2: + raise DurableConflictError("operation conflicts with retained execution") + if code == 2: + raise OutcomeUnknownError("external operation has no retained receipt") + return code == 1, json.loads(result[1]) if code == 1 else None + + def fail( + self, + run_id: str, + owner_id: str, + generation: int, + terminal_payload: dict[str, Any], + ) -> None: + """Persist a terminal failure only while this worker still owns the run.""" + result = self.redis.eval( + _FAIL_SCRIPT, + 2, + self._run_key(run_id), + self._lease_key(run_id), + f"{owner_id}:{generation}", + json.dumps(terminal_payload), + str(self.retention_ms), + ) + if int(result) != 1: + raise LeaseLostError("failure rejected for stale owner") + def checkpoint( self, run_id: str, diff --git a/agent-svc/agent/experimental/knowledge_checks.py b/agent-svc/agent/experimental/knowledge_checks.py index e3b8ac78..929d57bf 100644 --- a/agent-svc/agent/experimental/knowledge_checks.py +++ b/agent-svc/agent/experimental/knowledge_checks.py @@ -135,7 +135,7 @@ def freshness_allows_pass(self) -> bool: if basis.basis == "unknown": return False if basis.basis == "historical_snapshot": - if claim.temporal_scope != "historical": + if claim.temporal_scope != "historical" or source.retrieved_at is None: return False date = moment(source.retrieved_at) else: diff --git a/agent-svc/agent/experimental/knowledge_context.py b/agent-svc/agent/experimental/knowledge_context.py index 00cb85a5..09b61db0 100644 --- a/agent-svc/agent/experimental/knowledge_context.py +++ b/agent-svc/agent/experimental/knowledge_context.py @@ -46,10 +46,13 @@ class ContentReference(StrictRecord): class ReferencedSnapshot(StrictRecord): snapshot_id: Identity - canonical_url: Annotated[ - str, Field(max_length=8192, pattern=r"^https?://[^\s/]+(?:/[^\s]*)?$") - ] - retrieved_at: Timestamp + canonical_url: ( + Annotated[ + str, Field(max_length=8192, pattern=r"^https?://[^\s/]+(?:/[^\s]*)?$") + ] + | None + ) + retrieved_at: Timestamp | None normalization_version: Identity media_type: MediaType content_ref: ContentReference @@ -148,7 +151,10 @@ def references_and_chronology(self) -> Self: source.content_ref.snapshot_id, ) != (self.scope_id, self.research_id, source.snapshot_id): raise ValueError("source reference crosses logical context identity") - if moment(source.retrieved_at) > created: + if ( + source.retrieved_at is not None + and moment(source.retrieved_at) > created + ): raise ValueError("context predates acquisition") if any(e.snapshot_id not in snapshots for e in self.evidence): raise ValueError("evidence references an unavailable snapshot") diff --git a/agent-svc/agent/experimental/live_runner.py b/agent-svc/agent/experimental/live_runner.py new file mode 100644 index 00000000..a03b2faf --- /dev/null +++ b/agent-svc/agent/experimental/live_runner.py @@ -0,0 +1,242 @@ +"""Server-owned live adapters with persisted, non-replaying external operations.""" + +import asyncio +import hashlib +import json +from dataclasses import asdict +from datetime import UTC, datetime +from typing import Any +from uuid import NAMESPACE_URL, UUID, uuid5 + +import httpx + +from ..admission import get_admission +from ..barrier_guard import is_block_flagged +from ..searxng_client import SearXNGClient +from ..session_store import SessionStore +from ..settings import load_settings +from .durable_research import OutcomeUnknownError +from .model_review import ModelReply +from .model_transport import ReviewTransport +from .query_construction import CapturedSource +from .query_runner import run_query + + +async def run_live(record: Any, app: Any): + """Execute an admitted objective; credentials stay in server-owned clients.""" + ledger = record.durable_ledger + if ledger is None or record.durable_owner_id is None: + raise ValueError("live research requires durable ownership") + admitted = await asyncio.to_thread(ledger.get, record.run_id) + if admitted is None: + raise ValueError("admission unavailable") + evaluated = datetime.fromisoformat(admitted.payload["evaluation_time"]) + model = admitted.payload["model"] + selected_sources = tuple( + CapturedSource(**source) + for source in admitted.payload.get("selected_sources", []) + ) + + async def validate_selected(): + for source in selected_sources: + if not source.origin_id or not source.lineage_id: + raise ValueError("selected source has no retained authority") + kind, container, reference = source.origin_id.split(":", 2) + if kind == "root": + body, _ = await record.artifact_authority.read_evidence( + uuid5( + NAMESPACE_URL, f"groktocrawl-x:research-scope:{record.scope_id}" + ), + UUID(container), + reference, + ) + elif kind == "session": + settings = load_settings() + store = SessionStore( + redis_url=f"redis://{settings.valkey_host}:{settings.valkey_port}/{settings.valkey_db}" + ) + try: + session = await store.aget(container) + if ( + not session + or not session.get("owner_scope") + or not admitted.payload.get("session_scope") + or session.get("owner_scope") + != admitted.payload.get("session_scope") + ): + raise ValueError( + "selected session is unavailable to admitted scope" + ) + ref = await store.aget_ref(container, reference) + if not ref: + raise ValueError("selected session source is unavailable") + pointer = ref.get("research_source") + if pointer is not None: + if pointer.get("kind") != "root": + raise ValueError("invalid linked research source") + body, _ = await record.artifact_authority.read_evidence( + uuid5( + NAMESPACE_URL, + f"groktocrawl-x:research-scope:{record.scope_id}", + ), + UUID(pointer["container_id"]), + pointer["ref_id"], + ) + if hashlib.sha256(body).hexdigest() != ref.get( + "content_digest" + ): + raise ValueError("linked research source changed") + elif ref.get("markdown"): + body = ref["markdown"].encode() + else: + raise ValueError("selected session source is unavailable") + finally: + await asyncio.to_thread(store.redis.close) + else: + raise ValueError("selected source authority is unsupported") + actual = hashlib.sha256(body).hexdigest() + if actual != source.lineage_id or body != source.text.encode(): + raise ValueError("selected source changed or failed integrity") + + await validate_selected() + + async def effect(identity, inputs, call): + encoded = json.dumps( + inputs, sort_keys=True, separators=(",", ":"), allow_nan=False + ) + input_digest = hashlib.sha256(encoded.encode()).hexdigest() + replay, output = await asyncio.to_thread( + ledger.step, + record.run_id, + record.durable_owner_id, + record.durable_generation, + identity, + input_digest, + ) + if replay: + return output + # Every exception after intent is ambiguous until a bounded receipt is + # durable. Never convert it to a retry or silently select another model. + try: + output = await call() + except asyncio.CancelledError: + raise + except Exception as exc: + raise OutcomeUnknownError("external dispatch outcome is unknown") from exc + await asyncio.to_thread( + ledger.step, + record.run_id, + record.durable_owner_id, + record.durable_generation, + identity, + input_digest, + output=output, + complete=True, + ) + return output + + async def search(objective, limit): + async def fetch(): + search_client = SearXNGClient( + getattr(app.state, "searxng_url", None) + or app.state.searxng_client.base_url, + max_searches=1, + ) + try: + rows, health = await search_client.search( + objective, limit=limit, raise_on_rate_limit=True + ) + finally: + await search_client.close() + return { + "urls": list(dict.fromkeys(row["url"] for row in rows))[:limit], + "empty": health.empty_result, + } + + result = await effect("search", {"objective": objective, "limit": limit}, fetch) + if (result["empty"] or not result["urls"]) and not selected_sources: + raise ValueError("search returned no usable sources") + return tuple(result["urls"]) + + async def acquire(url): + async def fetch(): + result = await app.state.scraper_client.scrape(url, lightweight_only=True) + return { + "response": { + "success": result.get("success"), + "warning": bool(result.get("warning")), + "data": {"markdown": (result.get("data") or {}).get("markdown")}, + }, + "blocked": is_block_flagged(result), + "retrieved_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + } + + result = await effect( + "acquire:" + hashlib.sha256(url.encode()).hexdigest(), {"url": url}, fetch + ) + value = result["response"] + if not value.get("success") or value.get("warning") or result["blocked"]: + raise ValueError("source acquisition failed or was qualified") + text = (value.get("data") or {}).get("markdown") + if not isinstance(text, str) or not text.strip(): + raise ValueError("source acquisition returned no text") + return CapturedSource(url, text, result["retrieved_at"]) + + model_calls = 0 + async with httpx.AsyncClient() as client: + token_parameter, token_budget = next( + iter(admitted.payload["token_params"].items()) + ) + transport = ReviewTransport( + client, + base_url=app.state.llm_base_url, + api_key=app.state.llm_api_key, + token_parameter=token_parameter, + token_budget=token_budget, + ) + + async def complete(request): + nonlocal model_calls + model_calls += 1 + await validate_selected() + if model_calls > 64: + raise ValueError("model call budget exhausted") + + async def fetch(): + reply = await transport(request) + return {**asdict(reply), "content": reply.content.decode("utf-8")} + + admission = get_admission() + weight = admission.weight_for("llm") + await admission.acquire("llm", weight=weight) + try: + values = await effect( + f"model:{model_calls}", + {**asdict(request), "payload": request.payload.decode("utf-8")}, + fetch, + ) + finally: + admission.release("llm", weight=weight) + return ModelReply( + **{**values, "content": values["content"].encode("utf-8")} + ) + + remaining = admitted.created_at_ms / 1000 + 180 - datetime.now(UTC).timestamp() + if remaining <= 0: + raise ValueError("research execution deadline expired") + async with asyncio.timeout(remaining): + return await run_query( + record.objective, + search=search, + acquire=acquire, + complete=complete, + scope_id=record.scope_id, + model=model, + clock=lambda: evaluated, + research_id=record.research_id, + revision_id=str(uuid5(NAMESPACE_URL, f"{record.run_id}:revision")), + artifact_set_id=str( + uuid5(NAMESPACE_URL, f"groktocrawl-x:artifact-set:{record.run_id}") + ), + selected_sources=selected_sources, + ) diff --git a/agent-svc/agent/experimental/model_transport.py b/agent-svc/agent/experimental/model_transport.py index c782baae..14c403f3 100644 --- a/agent-svc/agent/experimental/model_transport.py +++ b/agent-svc/agent/experimental/model_transport.py @@ -12,7 +12,13 @@ class ReviewTransport: """Server-configured endpoint/key; no redirects, retries or logged response bodies.""" def __init__( - self, client: httpx.AsyncClient, *, base_url: str, api_key: str + self, + client: httpx.AsyncClient, + *, + base_url: str, + api_key: str, + token_parameter: str = "max_tokens", + token_budget: int | None = None, ) -> None: url = httpx.URL(base_url) if ( @@ -26,6 +32,14 @@ def __init__( self._url = str(url.copy_with(path=url.path.rstrip("/") + "/chat/completions")) self._client = client self._key = api_key + if token_parameter not in {"max_tokens", "max_completion_tokens"}: + raise ValueError("invalid configured token budget parameter") + if token_budget is not None and ( + type(token_budget) is not int or token_budget <= 0 + ): + raise ValueError("invalid configured token budget") + self._token_parameter = token_parameter + self._token_budget = token_budget async def __call__(self, request: ReviewRequest) -> ModelReply: payload = { @@ -35,7 +49,9 @@ async def __call__(self, request: ReviewRequest) -> ModelReply: {"role": "user", "content": request.payload.decode("utf-8")}, ], "stream": False, - "max_tokens": request.max_output_tokens, + self._token_parameter: min(request.max_output_tokens, self._token_budget) + if self._token_budget is not None + else request.max_output_tokens, "response_format": ( { "type": "json_schema", diff --git a/agent-svc/agent/experimental/query_construction.py b/agent-svc/agent/experimental/query_construction.py index 5a1da099..cab98b52 100644 --- a/agent-svc/agent/experimental/query_construction.py +++ b/agent-svc/agent/experimental/query_construction.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from datetime import UTC, datetime from typing import Annotated, Literal -from uuid import uuid4 +from uuid import NAMESPACE_URL, uuid4, uuid5 from pydantic import Field @@ -76,9 +76,11 @@ class ConstructedContent(StrictRecord): @dataclass(frozen=True) class CapturedSource: - url: str + url: str | None text: str - retrieved_at: str + retrieved_at: str | None + origin_id: str | None = None + lineage_id: str | None = None @dataclass(frozen=True) @@ -130,7 +132,7 @@ def _locate( return located -def _edges(content: ConstructedContent) -> list[dict[str, object]]: +def _edges(content: ConstructedContent, revision_id: str) -> list[dict[str, object]]: edges: list[dict[str, object]] = [] for index, claim in enumerate(content.claims, 1): for kind, selected in ( @@ -140,7 +142,12 @@ def _edges(content: ConstructedContent) -> list[dict[str, object]]: for evidence_id in _selected(selected, len(content.evidence), "evidence"): edges.append( { - "relationship_id": str(uuid4()), + "relationship_id": str( + uuid5( + NAMESPACE_URL, + f"{revision_id}:{index}:{kind}:{evidence_id}", + ) + ), "kind": kind, "source_id": evidence_id, "target_id": f"claim-{index}", @@ -160,6 +167,8 @@ async def construct_research( scope_id: str, model: str = "local", clock: Callable[[], datetime] = _utc, + research_id: str | None = None, + revision_id: str | None = None, ) -> ConstructedResearch: """One model call, no retries; returned knowledge is NOT verified or publishable.""" if ( @@ -175,7 +184,8 @@ async def construct_research( raise ValueError("construction source budget exceeded") if sum(len(s.text.splitlines()) for s in sources) > 10_000: raise ValueError("construction source line budget exceeded") - research_id, revision_id = str(uuid4()), str(uuid4()) + research_id = research_id or str(uuid4()) + revision_id = revision_id or str(uuid4()) snapshots, resolved = [], [] for index, source in enumerate(sources, 1): identity = f"source-{index}" @@ -193,8 +203,8 @@ async def construct_research( content_bytes=len(source.text.encode()), published_at=None, effective_at=None, - origin_id=None, - lineage_id=None, + origin_id=source.origin_id, + lineage_id=source.lineage_id, ) snapshots.append(snapshot) resolved.append( @@ -259,7 +269,7 @@ async def construct_research( } for index, c in enumerate(content.claims, 1) ], - "relationships": _edges(content), + "relationships": _edges(content, revision_id), "questions": [ { "question_id": "question-root", diff --git a/agent-svc/agent/experimental/query_runner.py b/agent-svc/agent/experimental/query_runner.py index 7462b123..bd03ef2f 100644 --- a/agent-svc/agent/experimental/query_runner.py +++ b/agent-svc/agent/experimental/query_runner.py @@ -2,7 +2,7 @@ import asyncio from collections.abc import Awaitable, Callable -from datetime import datetime +from datetime import UTC, datetime from urllib.parse import urlsplit from .consolidated_journey import JourneyResult @@ -24,6 +24,10 @@ async def run_query( model: str = "local", source_limit: int = 3, clock: Callable[[], datetime] | None = None, + research_id: str | None = None, + revision_id: str | None = None, + artifact_set_id: str | None = None, + selected_sources: tuple[CapturedSource, ...] = (), ) -> JourneyResult: """One search, up to three acquisitions, one construction, executed reviews. @@ -40,12 +44,27 @@ async def run_query( raise ValueError("invalid research objective") if type(source_limit) is not int or not 1 <= source_limit <= 3: raise ValueError("source limit must be between one and three") + if len(selected_sources) > 8 or any( + not source.text.strip() for source in selected_sources + ): + raise ValueError("invalid retained source context") + seed_bytes = sum(len(source.text.encode()) for source in selected_sources) + if seed_bytes > 256_000: + raise ValueError("retained context exceeds research byte budget") async with asyncio.timeout(180): - urls = await asyncio.ensure_future(search(objective, source_limit)) + fresh_limit = min(source_limit, 8 - len(selected_sources)) + urls = ( + await asyncio.ensure_future(search(objective, fresh_limit)) + if fresh_limit + else () + ) owner = asyncio.current_task() if owner is not None and owner.cancelling(): raise asyncio.CancelledError - if not isinstance(urls, tuple) or not 1 <= len(urls) <= source_limit: + if ( + not isinstance(urls, tuple) + or not (0 if selected_sources else 1) <= len(urls) <= source_limit + ): raise ValueError("search did not return a bounded source plan") if len(set(urls)) != len(urls): raise ValueError("search returned duplicate source URLs") @@ -58,9 +77,11 @@ async def run_query( or parsed.password ): raise ValueError("search returned invalid source URL") - sources = [] - retained_bytes = 0 + sources = list(selected_sources) + retained_bytes = seed_bytes for url in urls: + if any(source.url == url for source in selected_sources): + continue source = await asyncio.ensure_future(acquire(url)) task = asyncio.current_task() if task is not None and task.cancelling(): @@ -73,11 +94,22 @@ async def run_query( if retained_bytes > 256_000: raise ValueError("acquisition exceeded research byte budget") sources.append(source) + evaluated = max( + [clock() if clock is not None else datetime.now(UTC)] + + [ + datetime.fromisoformat(source.retrieved_at) + for source in sources + if source.retrieved_at is not None + ] + ) return await research_from_sources( objective, tuple(sources), complete=complete, scope_id=scope_id, model=model, - clock=clock, + clock=lambda: evaluated, + research_id=research_id, + revision_id=revision_id, + artifact_set_id=artifact_set_id, ) diff --git a/agent-svc/agent/experimental/real_journey.py b/agent-svc/agent/experimental/real_journey.py index 841e1f23..02da1d68 100644 --- a/agent-svc/agent/experimental/real_journey.py +++ b/agent-svc/agent/experimental/real_journey.py @@ -67,6 +67,7 @@ def review_plan( "snapshot_id": s.snapshot_id, "basis": "historical_snapshot" if claim.temporal_scope == "historical" + and s.retrieved_at is not None else "unknown", "max_age_seconds": 604800, "reason": "Captured document only; publication/effective dates are unknown.", @@ -80,7 +81,11 @@ def review_plan( json.dumps( { "schema_version": "knowledge-check-input-prototype/1", - "input_id": str(uuid4()), + "input_id": str( + uuid5( + NAMESPACE_URL, f"{context.revision_id}:{kind}:{subject}" + ) + ), "check_type": kind, "subject_id": subject, "policy_version": context.policy_version, @@ -138,10 +143,14 @@ async def render_research( source = next( s for s in context.snapshots if s.snapshot_id == passage.snapshot_id ) - text += f"{passage.evidence_id}: {source.canonical_url}\n\n{passage.quote}\n\n" + text += f"{passage.evidence_id}: {source.canonical_url or 'Original URL unknown; retained source ' + source.snapshot_id}\n\n{passage.quote}\n\n" text += ( "## Sources\n\n" - + "\n".join(s.canonical_url for s in context.snapshots) + + "\n".join( + s.canonical_url + or "Original URL unknown; retained source " + s.snapshot_id + for s in context.snapshots + ) + "\n" ) identity = str(uuid5(NAMESPACE_URL, f"{artifact_set_id}:{layer}")) @@ -176,6 +185,9 @@ async def research_from_sources( scope_id: str, model: str = "local", clock: Callable[[], datetime] | None = None, + research_id: str | None = None, + revision_id: str | None = None, + artifact_set_id: str | None = None, ) -> JourneyResult: """Construct, check and audit a bounded root; no retained publication yet.""" execution_clock = clock or (lambda: datetime.now(UTC)) @@ -186,6 +198,8 @@ async def research_from_sources( scope_id=scope_id, model=model, clock=execution_clock, + research_id=research_id, + revision_id=revision_id, ) adapter = ModelReviewAdapter( provider="configured-litellm", @@ -199,7 +213,7 @@ async def research_from_sources( version="1", configuration_digest=text_digest("knowledge-context-prototype/1"), ) - identity = str(uuid4()) + identity = artifact_set_id or str(uuid4()) callbacks = {} for source in constructed.sources: diff --git a/agent-svc/agent/experimental/source_store.py b/agent-svc/agent/experimental/source_store.py index 5510cd92..bbd4f9aa 100644 --- a/agent-svc/agent/experimental/source_store.py +++ b/agent-svc/agent/experimental/source_store.py @@ -42,12 +42,14 @@ class RetainedSource: descriptor: CanonicalDocument -def source_descriptor(body: bytes, url: str) -> CanonicalDocument: +def source_descriptor( + body: bytes, url: str | None, *, allow_unknown_url: bool = False +) -> CanonicalDocument: """Validate bounded exact UTF-8 source bytes before entering a transaction.""" if not isinstance(body, bytes) or len(body) > MAX_BODY: raise ValueError("source byte limit exceeded") body.decode("utf-8", errors="strict") - if ( + if not (allow_unknown_url and url is None) and ( not isinstance(url, str) or not url.startswith(("https://", "http://")) or len(url) > 8192 diff --git a/agent-svc/agent/routes/experimental_research.py b/agent-svc/agent/routes/experimental_research.py index 40e825dc..7f0b5e28 100644 --- a/agent-svc/agent/routes/experimental_research.py +++ b/agent-svc/agent/routes/experimental_research.py @@ -1,10 +1,4 @@ -"""Opt-in experimental research protocol adapters. - -New runs use fictional sources and authored judgments and require an explicit -development fixture opt-in. Existing process-local, Valkey, and PostgreSQL -retained reads and recovery keep their separate gates; no live research adapter -is selected by this route. -""" +"""Durable bounded live research and explicitly selected development fixtures.""" import asyncio import base64 @@ -12,8 +6,9 @@ import importlib.util import json import os -from dataclasses import dataclass, field -from datetime import datetime +from contextlib import asynccontextmanager, suppress +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime from typing import Any, Literal, cast from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 @@ -34,8 +29,14 @@ DurableResearchError, DurableResearchLedger, DurableRun, + LeaseLostError, + OutcomeUnknownError, ) +from ..experimental.live_runner import run_live +from ..experimental.query_construction import CapturedSource from ..experimental.source_store import StorageConflictError +from ..llm import _token_budget_params +from ..session_scope import request_scope router = APIRouter() @@ -44,7 +45,7 @@ class CreateResearchRunRequest(BaseModel): - """Bounded fixture-run admission request.""" + """Bounded objective-driven research admission request.""" protocol_version: Literal["research/1"] = "research/1" objective: str = Field(min_length=1, max_length=10_000) @@ -85,6 +86,8 @@ class _RunRecord: durable_evidence: dict[str, tuple[bytes, str]] = field(default_factory=dict) artifact_authority: ArtifactAuthority | None = None authority_set_digest: str | None = None + execution_mode: str = "fixture" + app: Any = None _RUNS: dict[str, _RunRecord] = {} @@ -385,6 +388,24 @@ def _check_retained_ledger(record: _RunRecord) -> None: raise HTTPException(status_code=410, detail="Research run expired") if current.state == "deleted": record.deleted = True + elif current.state in {"completed", "failed", "cancelled"} and record.state in { + "accepted", + "running", + "cancel_requested", + }: + restored = _restore_durable_record(current) + for name in ( + "state", + "result", + "error", + "events", + "durable_manifest", + "durable_artifacts", + "durable_knowledge", + "durable_evidence", + "authority_set_digest", + ): + setattr(record, name, getattr(restored, name)) if record.deleted: raise HTTPException(status_code=410, detail="Research run deleted") @@ -413,6 +434,7 @@ def _find_run(run_id: str, request: Request) -> _RunRecord: record.task = asyncio.create_task(_execute_run(record)) if record is None or record.scope_id != _scope_id(request): raise HTTPException(status_code=404, detail="Research run not found") + record.app = request.app _check_retained_ledger(record) return record @@ -499,6 +521,8 @@ def _restore_durable_record(durable: DurableRun) -> _RunRecord: authority_set_digest=authority_set_digest, durable_ledger=None, deleted=deleted, + execution_mode=payload.get("execution_mode", "fixture"), + error=terminal.get("error"), ) @@ -540,6 +564,11 @@ def _retained_artifact_result(retained: RetainedArtifactSet) -> dict[str, Any]: async def _reconcile_authority(record: _RunRecord) -> None: + async with _execution_lock(record): + await _reconcile_authority_locked(record) + + +async def _reconcile_authority_locked(record: _RunRecord) -> None: """Project an already committed PostgreSQL result into Valkey after loss.""" if ( record.state not in {"accepted", "running"} @@ -569,6 +598,16 @@ async def _reconcile_authority(record: _RunRecord) -> None: raise DurableResearchError("reconciliation owner unavailable") record.durable_owner_id = claimed.owner_id record.durable_generation = claimed.owner_generation + _project_retained_owned(record, retained) + + +def _project_retained_owned(record: _RunRecord, retained: RetainedArtifactSet) -> None: + if ( + record.durable_ledger is None + or record.durable_owner_id is None + or record.durable_generation is None + ): + raise LeaseLostError("retained projection requires owned execution") record.result = _retained_artifact_result(retained) record.authority_set_digest = retained.set_digest terminal_event = _build_event(record, "done", "completed", result=record.result) @@ -581,15 +620,15 @@ async def _reconcile_authority(record: _RunRecord) -> None: ).hexdigest() record.durable_ledger.checkpoint( record.run_id, - claimed.owner_id, - claimed.owner_generation, + record.durable_owner_id, + record.durable_generation, "terminal_projection", checkpoint_digest, ) record.durable_ledger.commit_result( record.run_id, - claimed.owner_id, - claimed.owner_generation, + record.durable_owner_id, + record.durable_generation, result_digest, terminal_payload=terminal_payload, ) @@ -597,119 +636,278 @@ async def _reconcile_authority(record: _RunRecord) -> None: record.events.append(terminal_event) -async def _execute_run(record: _RunRecord) -> None: - async with record.lock: - if record.deleted: - record.state = "cancelled" - return - if record.state not in {"accepted", "running"}: - return - if record.durable_ledger is not None: - try: - claimed = record.durable_ledger.claim( - record.run_id, - owner_id=f"worker:{uuid4()}", - attempt_id=f"attempt:{uuid4()}", - ) - except Exception: - return - record.durable_owner_id = claimed.owner_id - record.durable_generation = claimed.owner_generation - record.state = "running" - _event(record, "progress", "running", stage="acquisition") +@asynccontextmanager +async def _execution_lock(record: _RunRecord): + if record.execution_mode == "live" and record.artifact_authority is not None: + async with record.artifact_authority.execution_lock( + _scope_uuid(record.scope_id), UUID(record.research_id) + ): + yield + else: + yield + + +async def _renew_owner(record: _RunRecord, worker: asyncio.Task[Any]) -> None: + ledger = record.durable_ledger + if ( + ledger is None + or record.durable_owner_id is None + or record.durable_generation is None + ): + raise LeaseLostError("renewal requires owned execution") + owner, generation = record.durable_owner_id, record.durable_generation try: - journey = example_journey( - scope_id=record.scope_id, - research_id=record.research_id, - objective=record.objective, - artifact_set_id=str( - uuid5(NAMESPACE_URL, f"groktocrawl-x:artifact-set:{record.run_id}") - ), - ) - result = await journey.run() + while True: + await asyncio.sleep(ledger.lease_ms / 3000) + await asyncio.to_thread( + ledger.heartbeat, + record.run_id, + owner, + generation, + ) except asyncio.CancelledError: - async with record.lock: - if record.state not in {"completed", "failed", "cancelled"}: - record.state = "cancelled" - _event(record, "cancelled", "cancelled") raise - except Exception: # pragma: no cover - defensive terminal contract - async with record.lock: - if record.state == "cancel_requested": + except Exception: + # The old worker must stop dispatching when renewal is uncertain. + worker.cancel() + + +async def _execute_run(record: _RunRecord) -> None: + renewal = None + try: + async with record.lock, _execution_lock(record): + if record.deleted: + record.state = "cancelled" + return + if record.state not in {"accepted", "running"}: + return + if record.durable_ledger is not None: + try: + claimed = await asyncio.to_thread( + record.durable_ledger.claim, + record.run_id, + owner_id=f"worker:{uuid4()}", + attempt_id=f"attempt:{uuid4()}", + ) + except DurableConflictError: + return + record.durable_owner_id = claimed.owner_id + record.durable_generation = claimed.owner_generation + if ( + record.execution_mode == "live" + and record.artifact_authority is not None + ): + retained = await record.artifact_authority.find_run( + _scope_uuid(record.scope_id), UUID(record.run_id) + ) + if retained is not None: + _project_retained_owned(record, retained) + return + worker = asyncio.current_task() + assert worker is not None + renewal = asyncio.create_task(_renew_owner(record, worker)) + record.state = "running" + _event(record, "progress", "running", stage="acquisition") + if record.execution_mode == "live": + remaining = ( + claimed.created_at_ms / 1000 + 180 - datetime.now(UTC).timestamp() + ) + async with asyncio.timeout(max(0, remaining)): + result = await run_live(record, record.app) + else: + journey = example_journey( + scope_id=record.scope_id, + research_id=record.research_id, + objective=record.objective, + artifact_set_id=str( + uuid5(NAMESPACE_URL, f"groktocrawl-x:artifact-set:{record.run_id}") + ), + ) + result = await journey.run() + async with record.lock, _execution_lock(record): + if record.deleted or record.state in {"cancelled", "cancel_requested"}: record.state = "cancelled" _event(record, "cancelled", "cancelled") - elif record.state not in {"completed", "cancelled"}: - record.state = "failed" - record.error = {"code": "fixture_run_failed", "retryable": False} - _event(record, "error", "failed", error=record.error) - return - async with record.lock: - if record.deleted or record.state == "cancel_requested": + return + if record.durable_ledger is not None: + if record.durable_owner_id is None or record.durable_generation is None: + raise LeaseLostError("publication requires owned execution") + await asyncio.to_thread( + record.durable_ledger.heartbeat, + record.run_id, + record.durable_owner_id, + record.durable_generation, + ) + await _publish_run(record, result) + except asyncio.CancelledError: + # Shutdown/lease loss leaves running work reclaimable. A client cancel + # has its own durable fence; neither invents successful provider receipts. + if record.durable_ledger is None and record.state not in { + "completed", + "failed", + }: record.state = "cancelled" _event(record, "cancelled", "cancelled") - return - record.journey = result - record.result = _artifact_result(record) - terminal_event: ProtocolEvent | None = None - if ( - record.durable_ledger is not None - and record.durable_owner_id is not None - and record.durable_generation is not None - ): - terminal_event = _build_event( - record, "done", "completed", result=record.result - ) - retained = None - if record.artifact_authority is not None: - manifest = result.candidate.admitted.manifest - retained = await record.artifact_authority.commit( - _scope_uuid(record.scope_id), - UUID(record.research_id), - UUID(record.run_id), - UUID(manifest.artifact_set_id), - result.manifest_bytes, - { - report.artifact.layer: ( - report.artifact.artifact_id, - report.body, - ) - for report in result.reports - }, - knowledge=result.knowledge_bytes, - evidence={ - source.reference.snapshot_id: (source.body, source.media_type) - for source in result.sources - }, + raise + except LeaseLostError: + return + except Exception as exc: + if record.artifact_authority is not None: + try: + retained = await record.artifact_authority.find_run( + _scope_uuid(record.scope_id), UUID(record.run_id) ) - record.authority_set_digest = retained.set_digest - terminal_payload = _durable_terminal_payload( - record, terminal_event, retained - ) - checkpoint_digest = hashlib.sha256( - json.dumps(terminal_payload, sort_keys=True).encode("utf-8") - ).hexdigest() - result_digest = hashlib.sha256( - json.dumps(record.result, sort_keys=True).encode("utf-8") - ).hexdigest() - record.durable_ledger.checkpoint( - record.run_id, - record.durable_owner_id, - record.durable_generation, - "terminal_projection", - checkpoint_digest, - ) - record.durable_ledger.commit_result( - record.run_id, - record.durable_owner_id, - record.durable_generation, - result_digest, - terminal_payload=terminal_payload, + if retained is not None: + # PG committed but the projection was interrupted. A later + # owner/read reconciles the exact root without paid calls. + return + except Exception: + # Publication may have committed. Do not invent a failed root + # while the authority cannot settle that outcome. + return + async with record.lock: + if record.state in {"completed", "cancelled"} or record.deleted: + return + record.error = { + "code": "outcome_unknown" + if isinstance(exc, OutcomeUnknownError) + else "research_run_failed" + if record.execution_mode == "live" + else "fixture_run_failed", + "retryable": False, + } + failure = _build_event(record, "error", "failed", error=record.error) + if ( + record.durable_ledger is not None + and record.durable_owner_id is not None + and record.durable_generation is not None + ): + try: + await asyncio.to_thread( + record.durable_ledger.fail, + record.run_id, + record.durable_owner_id, + record.durable_generation, + { + "research_id": record.research_id, + "error": record.error, + "events": [ + event.model_dump(mode="json") + for event in (*record.events, failure) + ], + }, + ) + except DurableResearchError: + return + record.state = "failed" + record.events.append(failure) + finally: + if renewal is not None: + renewal.cancel() + await asyncio.gather(renewal, return_exceptions=True) + if record.task is asyncio.current_task(): + record.task = None + + +async def recover_research_runs(app: Any, shutdown: asyncio.Event) -> None: + """Reconcile/reclaim admitted work without requiring a client to poll it.""" + request = Request({"type": "http", "app": app}) + ledger = _durable_ledger(request) + while not shutdown.is_set(): + try: + pending = await asyncio.to_thread(ledger.reclaimable) + for durable in pending[:100]: + record = _RUNS.get(durable.run_id) + if record is None: + record = _restore_durable_record(durable) + _RUNS[durable.run_id] = record + record.durable_ledger = ledger + record.app = app + if _artifact_authority_enabled(): + record.artifact_authority = _artifact_authority(request) + if record.task is None or record.task.done(): + record.task = asyncio.create_task(_execute_run(record)) + except asyncio.CancelledError: + raise + except Exception: + # A transient storage outage is not permission to dispatch without + # ownership. The next scan may recover once authority returns. + pass + with suppress(TimeoutError): + await asyncio.wait_for(shutdown.wait(), timeout=0.5) + + +async def stop_research_runs(app: Any) -> None: + """Stop this process's workers; leave durable intents for safe recovery.""" + tasks = [ + record.task + for record in _RUNS.values() + if record.app is app and record.task is not None and not record.task.done() + ] + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +async def _publish_run(record: _RunRecord, result: Any) -> None: + record.journey = result + record.result = _artifact_result(record) + terminal_event: ProtocolEvent | None = None + if ( + record.durable_ledger is not None + and record.durable_owner_id is not None + and record.durable_generation is not None + ): + terminal_event = _build_event(record, "done", "completed", result=record.result) + retained = None + if record.artifact_authority is not None: + manifest = result.candidate.admitted.manifest + retained = await record.artifact_authority.commit( + _scope_uuid(record.scope_id), + UUID(record.research_id), + UUID(record.run_id), + UUID(manifest.artifact_set_id), + result.manifest_bytes, + { + report.artifact.layer: ( + report.artifact.artifact_id, + report.body, + ) + for report in result.reports + }, + knowledge=result.knowledge_bytes, + evidence={ + source.reference.snapshot_id: (source.body, source.media_type) + for source in result.sources + }, ) - record.state = "completed" - if terminal_event is not None: - record.events.append(terminal_event) - else: - _event(record, "done", "completed", result=record.result) + record.authority_set_digest = retained.set_digest + terminal_payload = _durable_terminal_payload(record, terminal_event, retained) + checkpoint_digest = hashlib.sha256( + json.dumps(terminal_payload, sort_keys=True).encode("utf-8") + ).hexdigest() + result_digest = hashlib.sha256( + json.dumps(record.result, sort_keys=True).encode("utf-8") + ).hexdigest() + record.durable_ledger.checkpoint( + record.run_id, + record.durable_owner_id, + record.durable_generation, + "terminal_projection", + checkpoint_digest, + ) + record.durable_ledger.commit_result( + record.run_id, + record.durable_owner_id, + record.durable_generation, + result_digest, + terminal_payload=terminal_payload, + ) + record.state = "completed" + if terminal_event is not None: + record.events.append(terminal_event) + else: + _event(record, "done", "completed", result=record.result) def capability_document() -> dict[str, Any]: @@ -719,11 +917,14 @@ def capability_document() -> dict[str, Any]: fixture_creation = runs_available and is_enabled( "experimental_research_fixture_runs" ) + live_creation = durable and _artifact_authority_enabled() and not fixture_creation return { "protocol_version": "research/1", "route_prefix": _ROUTE_PREFIX, - "live_research_available": False, - "implementation_stage": "postgres_artifact_authority_fixture_adapter" + "live_research_available": live_creation, + "implementation_stage": "postgres_artifact_authority_live_adapter" + if live_creation + else "postgres_artifact_authority_fixture_adapter" if durable and _artifact_authority_enabled() else "durable_fixture_run_adapter" if durable @@ -754,11 +955,21 @@ def capability_document() -> dict[str, Any]: "runs": { "available": runs_available, "reason": None if runs_available else "public_adapters_pending", - "create_available": fixture_creation, + "create_available": fixture_creation or live_creation, "creation_reason": None - if fixture_creation - else "development_fixture_opt_in_required", - "execution_mode": "fixture_only", + if fixture_creation or live_creation + else "durable_artifact_authority_required", + "execution_mode": "live" if live_creation else "fixture_only", + "policy": { + "searches": 1, + "sources": 3, + "source_bytes": 256_000, + "timeout_seconds": 180, + "model_calls": 64, + "automatic_ambiguous_call_retry": False, + "webhook": False, + "as_of": False, + }, }, "artifacts": { "available": runs_available, @@ -789,12 +1000,39 @@ async def get_experimental_research_capabilities() -> dict[str, Any]: async def create_experimental_research_run( payload: CreateResearchRunRequest, request: Request ) -> dict[str, Any]: - """Admit one bounded fixture run with scoped idempotency.""" + return await admit_research_run(payload, request) + + +async def admit_research_run( + payload: CreateResearchRunRequest, + request: Request, + *, + selected_sources: tuple[CapturedSource, ...] = (), +) -> dict[str, Any]: + """Admit live durable work or an explicitly selected development fixture.""" _require_runs() - if not is_enabled("experimental_research_fixture_runs"): + if ( + len(selected_sources) > 8 + or sum(len(source.text.encode()) for source in selected_sources) > 256_000 + ): raise HTTPException( - status_code=404, - detail="Fixture research creation disabled; no live research adapter is available", + status_code=422, detail="Selected evidence exceeds research context budget" + ) + if payload.webhook is not None or payload.as_of is not None: + raise HTTPException( + status_code=422, + detail="Durable research does not yet support webhook delivery or as_of evaluation", + ) + fixture = is_enabled("experimental_research_fixture_runs") + if not fixture and not (_durable_enabled() and _artifact_authority_enabled()): + raise HTTPException( + status_code=503, + detail="Live research requires durable execution and PostgreSQL artifact authority", + ) + model = getattr(request.app.state, "llm_model", None) + if not fixture and not model: + raise HTTPException( + status_code=503, detail="Configured research model unavailable" ) key = request.headers.get("Idempotency-Key", "").strip() if not key or len(key) > 200: @@ -810,7 +1048,12 @@ async def create_experimental_research_run( status_code=503, detail="Artifact evidence schema 16 required before admission", ) from exc - digest = hashlib.sha256(payload.model_dump_json().encode()).hexdigest() + admitted_input = payload.model_dump_json().encode() + if selected_sources: + admitted_input += json.dumps( + [asdict(source) for source in selected_sources], sort_keys=True + ).encode() + digest = hashlib.sha256(admitted_input).hexdigest() existing = _IDEMPOTENCY.get((scope, key)) if durable is None else None if durable is not None: selected_run_id = str(uuid4()) @@ -821,17 +1064,28 @@ async def create_experimental_research_run( event="accepted", state="accepted", ) - admitted = durable.admit( - scope, - key, - digest, - run_id=selected_run_id, - payload={ - "objective": payload.objective, - "research_id": str(uuid4()), - "events": [accepted_event.model_dump(mode="json")], - }, - ) + try: + admitted = durable.admit( + scope, + key, + digest, + run_id=selected_run_id, + payload={ + "objective": payload.objective, + "research_id": str(uuid4()), + "execution_mode": "fixture" if fixture else "live", + "model": model, + "token_params": _token_budget_params(), + "selected_sources": [asdict(source) for source in selected_sources], + "session_scope": request_scope(request), + "evaluation_time": datetime.now(UTC).isoformat(), + "events": [accepted_event.model_dump(mode="json")], + }, + ) + except DurableConflictError as exc: + raise HTTPException( + status_code=409, detail="Idempotency key conflict" + ) from exc research_id = str(admitted.payload.get("research_id", str(uuid4()))) record = _RUNS.get(admitted.run_id) if record is None: @@ -840,6 +1094,7 @@ async def create_experimental_research_run( record.durable_ledger = durable _RUNS[admitted.run_id] = record record.artifact_authority = authority + record.app = request.app if authority is not None: await authority.ensure_scope(_scope_uuid(scope)) if record.task is None: @@ -927,9 +1182,23 @@ async def cancel_experimental_research_run( """Request cancellation and return the current authoritative projection.""" _require_runs() record = _find_run(run_id, request) - async with record.lock: + async with record.lock, _execution_lock(record): if record.state in {"completed", "failed", "cancelled"}: return _record_projection(record) + if record.execution_mode == "live" and record.artifact_authority is not None: + retained = await record.artifact_authority.find_run( + _scope_uuid(record.scope_id), UUID(record.run_id) + ) + if retained is not None: + # Publication acquired the same lock first. It is committed, + # so cancellation cannot retrospectively claim it was stopped. + await _reconcile_authority_locked(record) + return { + **_record_projection(record), + "state": "completed", + "execution_outcome": "completed", + "result": _retained_artifact_result(retained), + } if record.durable_ledger is not None: terminal_event = _build_event(record, "cancelled", "cancelled") terminal_payload = { @@ -942,7 +1211,7 @@ async def cancel_experimental_research_run( durable = record.durable_ledger.cancel( record.run_id, terminal_payload=terminal_payload ) - if durable.state == "completed": + if durable.state in {"completed", "failed", "deleted"}: recovered = _restore_durable_record(durable) record.state = recovered.state record.result = recovered.result @@ -1162,7 +1431,7 @@ async def get_experimental_artifact(artifact_id: str, request: Request) -> Respo async def get_experimental_evidence( research_id: str, snapshot_id: str, request: Request ) -> dict[str, Any]: - """Return bounded exact fixture evidence for one scoped snapshot.""" + """Return exact scoped evidence with its recorded acquisition provenance.""" _require_runs() scope = _scope_id(request) for record in _RUNS.values(): @@ -1183,6 +1452,9 @@ async def get_experimental_evidence( "media_type": source.media_type, "body": source.body.decode("utf-8"), "content_digest": hashlib.sha256(source.body).hexdigest(), + **_evidence_provenance( + record.journey.knowledge_bytes, snapshot_id, source.body + ), } for record in _durable_read_records(request): if record.research_id != research_id or record.scope_id != scope: @@ -1203,6 +1475,7 @@ async def get_experimental_evidence( "media_type": media_type, "body": body.decode("utf-8"), "content_digest": hashlib.sha256(body).hexdigest(), + **_evidence_provenance(record.durable_knowledge, snapshot_id, body), } try: body, media_type = await record.artifact_authority.read_evidence( @@ -1216,6 +1489,13 @@ async def get_experimental_evidence( "media_type": media_type, "body": body.decode("utf-8"), "content_digest": hashlib.sha256(body).hexdigest(), + **_evidence_provenance( + await record.artifact_authority.read_knowledge( + _scope_uuid(scope), UUID(research_id) + ), + snapshot_id, + body, + ), } if _artifact_authority_enabled(): authority = _artifact_authority(request) @@ -1235,10 +1515,41 @@ async def get_experimental_evidence( "media_type": media_type, "body": body.decode("utf-8"), "content_digest": hashlib.sha256(body).hexdigest(), + **_evidence_provenance( + await authority.read_knowledge(_scope_uuid(scope), research), + snapshot_id, + body, + ), } raise HTTPException(status_code=404, detail="Evidence not found") +def _evidence_provenance( + knowledge: bytes | None, snapshot_id: str, body: bytes +) -> dict[str, Any]: + unknown = {"url": None, "retrieved_at": None, "origin_id": None, "lineage_id": None} + if knowledge is None: + return unknown + snapshots = json.loads(knowledge).get("context", {}).get("snapshots", []) + snapshot = next( + (item for item in snapshots if item.get("snapshot_id") == snapshot_id), None + ) + if ( + snapshot is None + or snapshot.get("content_digest") != hashlib.sha256(body).hexdigest() + ): + raise HTTPException( + status_code=503, detail="Evidence provenance integrity unavailable" + ) + return { + "url": snapshot.get("canonical_url"), + **{ + key: snapshot.get(key) + for key in ("retrieved_at", "origin_id", "lineage_id") + }, + } + + @router.delete(f"{_ROUTE_PREFIX}/research/{{research_id}}", status_code=202) async def delete_experimental_research( research_id: str, request: Request diff --git a/docs/adr/0070-execute-live-durable-research.md b/docs/adr/0070-execute-live-durable-research.md new file mode 100644 index 00000000..04dc1226 --- /dev/null +++ b/docs/adr/0070-execute-live-durable-research.md @@ -0,0 +1,41 @@ +# Execute live durable research with persisted external-call receipts + +- Status: proposed +- Date: 2026-10-16 +- Deciders: GroktoCrawl maintainers +- Scope: `/experimental/research/v1` run execution; legacy jobs are unchanged + +## Context + +ADR-0069 adopted storage and verification components, but the HTTP run worker selected a scripted example. Separately tested real-source construction did not prove that an HTTP objective acquired sources or retained those bytes. Connecting that function without lifecycle integration would repeat external calls after crashes: the old worker renewed no lease and checkpointed only its terminal result. + +## Decision + +Use the existing server-configured SlopSearX client, scraper client, and OpenAI-compatible review transport. Model/source text cannot select endpoints, credentials, models, or budgets. Four existing research flags enable the live durable path only with Valkey and PostgreSQL artifact authority schema16. The development-only fixture flag explicitly selects fictional execution. Historical admissions without an execution mode remain fixture admissions; recovery never silently upgrades them to live work. + +Admission fixes the scoped run/research IDs, configured model and token-budget policy, and evaluation clock. Construction, relationship and check identities are reproducible from admitted identities. Selected retained material is internal, scoped server context, included in idempotency and combined source budgets. + +Before each external operation, atomically persist its identity, input digest and dispatch intent under the current Valkey fencing generation. Persist its bounded exact output before starting dependent work. Recovery reuses completed responses. An intent without a receipt is ambiguous and produces the persistent nonretryable `outcome_unknown` failure; it never authorizes an automatic repeated paid call. This is not an exactly-once network guarantee. New explicit requests remain separate admissions. + +Renew the worker lease during execution. A short PostgreSQL advisory transaction lock keyed by scope/research serializes ownership claims, publication and cancellation; deletion uses the same key. No search/acquisition/model calls occur inside this lock. Within publication, check Valkey ownership, then atomically retain the complete report, knowledge and evidence set. Reconciliation projects an already committed PostgreSQL root after an interrupted Valkey terminal update without invoking adapters again. If authority is unavailable, do not invent an uncommitted failure for a potentially committed result. Cancellation records its fence before cancelling a local task; another process's stale worker cannot begin a subsequent ledger operation or publish. A committed root wins over a later cancellation. + +Startup scans reclaimable admissions and tracks worker shutdown. Terminal failures and cancellation survive process loss; process-local status is synchronized from the durable ledger. Errors are redacted stable codes, not source/provider content or credentials. + +## Current bounds and remaining release work + +The initial adapter policy is one search, at most three fresh sources, 256000 source bytes and an absolute 180-second execution deadline. Up to eight selected retained sources can supplement the plan within the same byte budget; fresh slots shrink to keep eight sources total. Unknown original URLs and retrieval dates remain null rather than fabricated. Construction and review calls have bounded count and output. These bounds are not equivalent to the existing iterative deep-research loop. + +`as_of` and `webhook` are explicitly rejected before admission in this initial core change. Their semantics, broader bounded discovery, executable workspace follow-ups and source handoff remain required release work tracked by issue #666. This ADR is not evidence that the complete 1.0.1 goal has shipped. + +## Confirmation + +Require objective-dependent HTTP search/capture/model tests against controlled local services and actual isolated Valkey/PostgreSQL, retained source/report integrity across a fresh application, scoped idempotency, completed-step reuse, refusal to replay ambiguous dispatch, persistent failure, renewed ownership and competing cancellation. Inject interruptions after external receipts and PostgreSQL commit. Existing fixture-only API tests cannot substitute for these checks. No provider calls or production deployment are required to run acceptance. + +## Consequences + +An ambiguous external dispatch can stop a run that may have incurred cost; this is deliberate and visible, rather than silently spending again. PostgreSQL and Valkey availability are both required for live admission. Recovery does not guarantee that lost Valkey operation receipts can be reconstructed from provider billing. Existing retained PostgreSQL readers and legacy fixture records remain compatible; no historical evidence is rewritten. + +## Links + +- [ADR-0069](0069-adopt-bounded-research-substrate.md) +- [Completion issue #666](https://github.com/groktopus/groktocrawl/issues/666) diff --git a/docs/adr/README.md b/docs/adr/README.md index bad6e8d4..4f3c521c 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -25,7 +25,7 @@ An Architecture Decision Record captures an important architectural decision mad **Current accepted decisions:** ADR-0001–0007, 0009–0012, 0014–0022, 0026 (as superseded for migrated deployments), 0029–0035, 0038, 0039, 0043–0057, 0059–0067, and 0069. -**Proposed work:** ADR-0023–0025, 0027, 0028, 0036, 0037, and 0040–0042. +**Proposed work:** ADR-0070 and ADR-0023–0025, 0027, 0028, 0036, 0037, and 0040–0042. | ADR | Title | Status | |-----|-------|--------| @@ -97,4 +97,5 @@ An Architecture Decision Record captures an important architectural decision mad | 0067 | [Select the LLM Token Budget Parameter Explicitly](0067-explicit-llm-token-budget-parameter.md) | accepted | | 0068 | [Select bounded evidence from complete sources](0068-bounded-full-source-evidence.md) | proposed | | 0069 | [Adopt a Bounded Research Substrate](0069-adopt-bounded-research-substrate.md) | accepted; rollout-gated adoption in standard deployment | +| 0070 | [Execute live durable research with persisted external-call receipts](0070-execute-live-durable-research.md) | proposed; implementation acceptance pending | See [CONTRIBUTING.md](../../CONTRIBUTING.md) for the full ADR workflow: when to write an ADR, how to number it, and how to get it reviewed in a PR. Experimental records are linked from the provenance archive, not included in the mainline sequence. diff --git a/tests/integration/test_live_research_http_adapters.py b/tests/integration/test_live_research_http_adapters.py new file mode 100644 index 00000000..206eed1d --- /dev/null +++ b/tests/integration/test_live_research_http_adapters.py @@ -0,0 +1,515 @@ +"""HTTP acceptance tests for durable objective-driven research. + +The search, capture, and OpenAI-compatible model endpoints here are local digital +twins. Valkey and PostgreSQL must be the isolated acceptance services configured +by the caller; no paid or public provider is contacted. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib +import json +import os +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from uuid import uuid4 + +import httpx +import pytest +import pytest_asyncio +from agent.experimental.durable_research import DurableResearchLedger +from agent.routes import experimental_research as experimental +from agent.scraper_client import ScraperClient +from agent.searxng_client import SearXNGClient +from fastapi import FastAPI +from redis import Redis + +OBJECTIVES = { + "What does the Aurora battery pilot report?": ( + "https://research.test/aurora", + "AURORA DIGITAL TWIN: The pilot measured 18 percent lower charging loss.", + ), + "What does the Birch heat-pump trial report?": ( + "https://research.test/birch", + "BIRCH DIGITAL TWIN: The trial measured 11 percent lower winter electricity use.", + ), +} + + +class LocalProviderTwin: + """Three local HTTP endpoints with request logs and deterministic responses.""" + + def __init__(self) -> None: + self.search_requests: list[dict] = [] + self.capture_requests: list[dict] = [] + self.model_requests: list[dict] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args) -> None: + return + + def _reply(self, status: int, value: dict) -> None: + body = json.dumps(value).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + if self.path.split("?", 1)[0] != "/search": + self._reply(404, {"error": "unknown path"}) + return + from urllib.parse import parse_qs, urlparse + + query = parse_qs(urlparse(self.path).query).get("q", [""])[0] + with owner._lock: + owner.search_requests.append({"query": query, "path": self.path}) + selected = OBJECTIVES.get(query) + rows = ( + [{"url": selected[0], "title": "Local source", "content": ""}] + if selected + else [] + ) + self._reply( + 200, + { + "query": query, + "number_of_results": len(rows), + "results": rows, + "engines": [{"name": "local-twin", "results": len(rows)}], + }, + ) + + def do_POST(self) -> None: + length = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(length) + try: + payload = json.loads(raw) + except Exception: + self._reply(400, {"error": "invalid JSON"}) + return + if self.path == "/scrape": + url = payload.get("url", "") + with owner._lock: + owner.capture_requests.append(payload) + body = next( + ( + text + for source_url, text in OBJECTIVES.values() + if source_url == url + ), + None, + ) + if body is None: + self._reply(404, {"success": False, "error": "unknown source"}) + return + self._reply( + 200, + { + "success": True, + "data": { + "url": url, + "markdown": body, + "source": "local-digital-twin", + }, + }, + ) + return + if self.path == "/chat/completions": + with owner._lock: + owner.model_requests.append(payload) + user = payload["messages"][1]["content"] + request = json.loads(user) + if "sources" in request and "objective" in request: + source = request["sources"][0] + line = source["lines"][0]["text"].rstrip("\n") + answer = { + "schema_version": "research-construction/4", + "evidence": [ + {"source_index": 1, "start_line": 1, "end_line": 1} + ], + "claims": [ + { + "text": f"The captured source reports: {line}", + "qualifiers": [ + "One source captured by the local acceptance twin" + ], + "temporal_scope": "historical", + "supported_by": [1], + "contradicted_by": [], + } + ], + "answer_claim_index": 1, + "answer_status": "answered", + "conflicts": [], + } + elif "checks" in request: + answer = { + "schema_version": "model-review-batch/1", + "decisions": [ + { + "check_index": item["check_index"], + "outcome": "supported" + if item["check_type"] == "assessment" + else "pass", + "reason": "Local deterministic acceptance response; not an independent quality judgment.", + } + for item in request["checks"] + ], + } + else: + assessment = request.get("check_type") == "assessment" + answer = { + "schema_version": "model-review-decision/1", + "input_digest": request["input_digest"], + "outcome": "supported" if assessment else "pass", + "reason": "Local deterministic acceptance response; not an independent quality judgment.", + } + self._reply( + 200, + { + "model": "acceptance-local-model", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + "choices": [ + { + "finish_reason": "stop", + "message": {"content": json.dumps(answer)}, + } + ], + }, + ) + return + self._reply(404, {"error": "unknown path"}) + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.base_url = f"http://127.0.0.1:{self.server.server_port}" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=2) + + +class LocalAdmission: + """Keep the test focused on the configured HTTP capture adapter.""" + + @staticmethod + def weight_for(_resource_class: str) -> int: + return 1 + + @staticmethod + async def acquire(_resource_class: str, *, weight: int) -> None: + assert weight == 1 + + @staticmethod + def release(_resource_class: str, *, weight: int) -> None: + assert weight == 1 + + +@pytest.fixture +def provider_twin(): + twin = LocalProviderTwin() + try: + yield twin + finally: + twin.close() + + +@pytest_asyncio.fixture +async def durable_apps( + monkeypatch: pytest.MonkeyPatch, provider_twin: LocalProviderTwin +): + redis_url = os.environ.get("DURABLE_RESEARCH_REDIS_URL") or os.environ.get( + "VALKEY_URL" + ) + postgres_dsn = os.environ.get("DURABLE_RESEARCH_POSTGRES_DSN") + if not redis_url or not postgres_dsn: + pytest.fail( + "acceptance requires isolated DURABLE_RESEARCH_REDIS_URL/VALKEY_URL and DURABLE_RESEARCH_POSTGRES_DSN" + ) + + for flag in ( + "EXPERIMENTAL_RESEARCH", + "EXPERIMENTAL_RESEARCH_RUNS", + "EXPERIMENTAL_RESEARCH_DURABLE", + "EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS", + ): + monkeypatch.setenv(f"FEATURE_{flag}", "true") + monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "false") + + namespace = f"groktocrawl:test:live-http:{uuid4()}" + redis = Redis.from_url(redis_url, decode_responses=True) + original_factory = experimental.DurableResearchLedger + created_ledgers: list[DurableResearchLedger] = [] + + def namespaced_ledger(url: str) -> DurableResearchLedger: + ledger = original_factory(url, namespace=namespace) + created_ledgers.append(ledger) + return ledger + + monkeypatch.setattr(experimental, "DurableResearchLedger", namespaced_ledger) + for cache in ( + experimental._RUNS, + experimental._IDEMPOTENCY, + experimental._DURABLE_LEDGERS, + experimental._ARTIFACT_AUTHORITIES, + ): + cache.clear() + + def make_app() -> FastAPI: + app = FastAPI() + app.state.valkey_url = redis_url + app.state.research_postgres_dsn = postgres_dsn + app.state.searxng_client = SearXNGClient(provider_twin.base_url, max_searches=5) + app.state.scraper_client = ScraperClient( + provider_twin.base_url, admission=LocalAdmission() + ) + app.state.llm_base_url = provider_twin.base_url + app.state.llm_api_key = "local-acceptance-only" + app.state.llm_model = "acceptance-local-model" + app.include_router(experimental.router) + return app + + apps = [make_app()] + cleanup_research_ids: list[str] = [] + cleanup_clients = [] + try: + yield ( + apps, + namespace, + redis, + created_ledgers, + make_app, + cleanup_research_ids, + cleanup_clients, + ) + finally: + if cleanup_research_ids and apps: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=apps[-1]), + base_url="http://acceptance", + headers={"Authorization": "Bearer live-http-acceptance-owner"}, + ) as cleanup_client: + for research_id in cleanup_research_ids: + await cleanup_client.delete( + f"/experimental/research/v1/research/{research_id}" + ) + for app in apps: + for name in ("searxng_client", "scraper_client"): + client = getattr(app.state, name, None) + if client is not None: + await client.close() + for client in cleanup_clients: + await client.close() + for ledger in created_ledgers: + ledger.redis.close() + for cache in ( + experimental._RUNS, + experimental._IDEMPOTENCY, + experimental._DURABLE_LEDGERS, + experimental._ARTIFACT_AUTHORITIES, + ): + cache.clear() + prefix = f"{namespace}:" + for key in redis.scan_iter(match=f"{prefix}*"): + redis.delete(key) + redis.close() + + +async def _terminal(client: httpx.AsyncClient, url: str) -> dict: + for _ in range(600): + response = await client.get(url) + assert response.status_code == 200, response.text + status = response.json() + if status["state"] in {"completed", "failed", "cancelled"}: + return status + await asyncio.sleep(0.05) + raise AssertionError("live research run did not reach a terminal state") + + +@pytest.mark.asyncio +async def test_two_objectives_use_http_adapters_and_recover_actual_bytes_through_mcp( + durable_apps, provider_twin: LocalProviderTwin, monkeypatch: pytest.MonkeyPatch +): + apps, _, _, _, make_app, cleanup_research_ids, cleanup_clients = durable_apps + app = apps[0] + owner = {"Authorization": "Bearer live-http-acceptance-owner"} + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://acceptance", + headers=owner, + ) as client: + first_objective = next(iter(OBJECTIVES)) + first = await client.post( + "/experimental/research/v1/runs", + headers={"Idempotency-Key": "aurora-objective"}, + json={"objective": first_objective}, + ) + assert first.status_code == 202, first.text + first_admission = first.json() + cleanup_research_ids.append(first_admission["research_id"]) + duplicate = await client.post( + "/experimental/research/v1/runs", + headers={"Idempotency-Key": "aurora-objective"}, + json={"objective": first_objective}, + ) + assert duplicate.status_code == 202 + assert duplicate.json()["run_id"] == first_admission["run_id"] + first_status = await _terminal(client, first_admission["status_url"]) + assert first_status["state"] == "completed", first_status + + mcp_dir = str(Path(__file__).resolve().parents[2] / "mcp-svc") + if mcp_dir not in sys.path: + sys.path.insert(0, mcp_dir) + mcp_server = importlib.import_module("mcp_server") + from groktocrawl_client import GroktocrawlClient + + mcp_client = GroktocrawlClient(base_url="http://acceptance") + cleanup_clients.append(mcp_client) + mcp_client._client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://acceptance", + headers=owner, + ) + prior_client = mcp_server._client + monkeypatch.setattr(mcp_server, "_client", mcp_client) + second_objective = tuple(OBJECTIVES)[1] + mcp_response, _structured = await mcp_server.mcp.call_tool( + "research_create", + {"objective": second_objective, "idempotency_key": "birch-objective"}, + ) + second_admission = json.loads(mcp_response[0].text) + cleanup_research_ids.append(second_admission["research_id"]) + second_status = await _terminal(client, second_admission["status_url"]) + assert second_status["state"] == "completed", second_status + assert second_status["result"]["research_id"] == second_admission["research_id"] + mcp_status_response, _structured = await mcp_server.mcp.call_tool( + "research_status", {"run_id": second_admission["run_id"]} + ) + assert json.loads(mcp_status_response[0].text)["state"] == "completed" + + assert [item["query"] for item in provider_twin.search_requests] == [ + first_objective, + second_objective, + ] + assert len(provider_twin.capture_requests) == 2 + assert all( + item.get("lightweight_only") is True + for item in provider_twin.capture_requests + ) + assert len(provider_twin.model_requests) >= 4 + assert all( + item["model"] == "acceptance-local-model" + for item in provider_twin.model_requests + ) + model_inputs = [ + json.loads(item["messages"][1]["content"]) + for item in provider_twin.model_requests + ] + for objective, (_url, source_body) in OBJECTIVES.items(): + construction = next( + item + for item in model_inputs + if item.get("objective") == objective and "sources" in item + ) + assert source_body in "".join( + line["text"] + for source in construction["sources"] + for line in source["lines"] + ) + + # Drop all route process state, create a distinct FastAPI app, and recover + # source bytes/artifacts through the durable ledger + PostgreSQL authority. + for current_app in apps: + for name in ("searxng_client", "scraper_client"): + await getattr(current_app.state, name).close() + experimental._RUNS.clear() + experimental._IDEMPOTENCY.clear() + experimental._DURABLE_LEDGERS.clear() + experimental._ARTIFACT_AUTHORITIES.clear() + recovered_app = make_app() + apps.append(recovered_app) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=recovered_app), + base_url="http://acceptance", + headers=owner, + ) as recovered: + observations = [] + for admission, objective, expected_url, expected_body, objective_marker in ( + ( + first_admission, + first_objective, + *OBJECTIVES[first_objective], + "AURORA", + ), + ( + second_admission, + second_objective, + *OBJECTIVES[second_objective], + "BIRCH", + ), + ): + workspace = await recovered.get( + f"/experimental/research/v1/workspace/{admission['run_id']}" + ) + assert workspace.status_code == 200, workspace.text + document = workspace.json() + assert document["state"] == "completed" + assert document["objective"] == objective + evidence_link = next( + item + for item in document["evidence"] + if item.get("source", {}).get("canonical_url") == expected_url + ) + evidence = await recovered.get(evidence_link["url"]) + assert evidence.status_code == 200, evidence.text + assert evidence.json()["body"].encode() == expected_body.encode() + assert ( + evidence.json()["content_digest"] + == hashlib.sha256(expected_body.encode()).hexdigest() + ) + summary_url = document["actions"]["render_summary"]["url"] + summary = await recovered.get(summary_url) + assert summary.status_code == 200 + assert objective_marker in summary.text + assert expected_body in summary.text + observations.append((evidence.json()["body"], summary.text)) + + assert observations[0][0] != observations[1][0] + assert observations[0][1] != observations[1][1] + monkeypatch.setattr(mcp_server, "_client", prior_client) + + +@pytest.mark.asyncio +async def test_live_search_failure_is_not_reported_as_success( + durable_apps, provider_twin +): + apps, _, _, _, _, cleanup_research_ids, _cleanup_clients = durable_apps + app = apps[0] + objective = "What does the unavailable local source establish?" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://acceptance" + ) as client: + created = await client.post( + "/experimental/research/v1/runs", + headers={"Idempotency-Key": "empty-search"}, + json={"objective": objective}, + ) + assert created.status_code == 202, created.text + cleanup_research_ids.append(created.json()["research_id"]) + status = await _terminal(client, created.json()["status_url"]) + assert status["state"] == "failed" + assert status["error"]["code"] in {"research_run_failed", "outcome_unknown"} + assert "result" not in status or status["result"] is None + assert any(item["query"] == objective for item in provider_twin.search_requests) + assert not provider_twin.capture_requests + assert not provider_twin.model_requests diff --git a/tests/service/test_experimental_research_capabilities.py b/tests/service/test_experimental_research_capabilities.py index 8efabb38..41187276 100644 --- a/tests/service/test_experimental_research_capabilities.py +++ b/tests/service/test_experimental_research_capabilities.py @@ -61,9 +61,7 @@ def test_capabilities_advertise_postgres_artifact_authority(monkeypatch) -> None monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_DURABLE", "true") monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS", "true") payload = _client().get("/experimental/research/v1/capabilities").json() - assert ( - payload["implementation_stage"] == "postgres_artifact_authority_fixture_adapter" - ) + assert payload["implementation_stage"] == "postgres_artifact_authority_live_adapter" assert payload["recovery_mode"] == "postgres_artifacts_valkey_fenced" @@ -77,17 +75,15 @@ def test_retained_capabilities_distinguish_fixture_creation_from_reads(monkeypat monkeypatch.setenv("FEATURE_" + flag, "true") monkeypatch.delenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", raising=False) payload = _client().get("/experimental/research/v1/capabilities").json() - assert payload["live_research_available"] is False - assert ( - payload["implementation_stage"] == "postgres_artifact_authority_fixture_adapter" - ) + assert payload["live_research_available"] is True + assert payload["implementation_stage"] == "postgres_artifact_authority_live_adapter" assert payload["operations"]["workspace"]["available"] is True assert payload["operations"]["artifacts"]["available"] is True runs = payload["operations"]["runs"] assert runs["available"] is True - assert runs["create_available"] is False - assert runs["creation_reason"] == "development_fixture_opt_in_required" - assert runs["execution_mode"] == "fixture_only" + assert runs["create_available"] is True + assert runs["creation_reason"] is None + assert runs["execution_mode"] == "live" monkeypatch.setenv("FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS", "true") enabled = _client().get("/experimental/research/v1/capabilities").json() assert enabled["operations"]["runs"]["create_available"] is True diff --git a/tests/service/test_experimental_research_postgres_artifacts.py b/tests/service/test_experimental_research_postgres_artifacts.py index 4d2cdd7b..01760196 100644 --- a/tests/service/test_experimental_research_postgres_artifacts.py +++ b/tests/service/test_experimental_research_postgres_artifacts.py @@ -289,8 +289,10 @@ def interrupt_once(self, *args, **kwargs): ) task = experimental._RUNS[created.json()["run_id"]].task assert task is not None - with pytest.raises(RuntimeError, match="projection interruption"): - await task + await task + # A committed PG root is neither failed nor rerun when the Valkey + # projection is interrupted. Recovery projects that exact root. + assert experimental._RUNS[created.json()["run_id"]].state == "running" ledger = DurableResearchLedger( url, namespace=app.state.research_durable_namespace ) diff --git a/tests/service/test_experimental_research_runs.py b/tests/service/test_experimental_research_runs.py index 61cbaa8c..31edf056 100644 --- a/tests/service/test_experimental_research_runs.py +++ b/tests/service/test_experimental_research_runs.py @@ -407,8 +407,8 @@ def unexpected_work(*args, **kwargs): headers={"Idempotency-Key": "production-must-not-create"}, json={"objective": "A real user research request"}, ) - assert response.status_code == 404 - assert "Fixture research creation disabled" in response.json()["detail"] + assert response.status_code == 503 + assert "Configured research model unavailable" in response.json()["detail"] assert experimental._RUNS == {} assert experimental._IDEMPOTENCY == {} @@ -452,5 +452,31 @@ async def test_disabling_fixture_admission_preserves_existing_reads(app, monkeyp headers={"Idempotency-Key": "new-after-disable"}, json={"objective": "Must not create another fixture"}, ) - assert rejected.status_code == 404 + assert rejected.status_code == 503 assert len(experimental._RUNS) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "unsupported", + [{"webhook": "https://receiver.test/hook"}, {"as_of": "2026-01-01T00:00:00Z"}], +) +async def test_unimplemented_options_rejected_before_admission( + app, monkeypatch, unsupported +): + def unexpected(*_args, **_kwargs): + raise AssertionError("unsupported option admitted work") + + monkeypatch.setattr(experimental, "_durable_ledger", unexpected) + monkeypatch.setattr(experimental, "_artifact_authority", unexpected) + async with await _client(app) as client: + response = await client.post( + "/experimental/research/v1/runs", + headers={"Idempotency-Key": "unsupported"}, + json={ + "objective": "Must not silently ignore supplied options", + **unsupported, + }, + ) + assert response.status_code == 422 + assert experimental._RUNS == {} diff --git a/tests/service/test_experimental_runtime_workflow.py b/tests/service/test_experimental_runtime_workflow.py index 12c33f39..b486828c 100644 --- a/tests/service/test_experimental_runtime_workflow.py +++ b/tests/service/test_experimental_runtime_workflow.py @@ -59,12 +59,20 @@ def test_stack_is_built_locally_with_fixture_search_and_owned_volumes(): job = WORKFLOW["jobs"]["integration-tests"] assert job["env"]["COMPOSE_FILE"] == "docker-compose.yml:docker-compose.ci.yml" commands = "\n".join(step.get("run", "") for step in job["steps"]) - assert "--profile indexing --profile fixture build" in commands + assert ( + "--profile indexing --profile fixture --profile durable-research build" + in commands + ) assert "up -d --no-build --pull never" in commands assert "scripts/ci_public_image_prefetch.py" in commands assert "--group runtime" in commands assert "--profile indexing" in commands assert "--profile fixture" in commands + assert "--profile durable-research" in commands + assert "DURABLE_RESEARCH_POSTGRES_DSN=" in commands + assert "PGPASSFILE=/tmp/postgres.pgpass" in commands + assert "SEARXNG_URL=http://slopsearx-fixture:8080" in commands + assert "SEARCH_BASE_URL=http://slopsearx-fixture:8080" in commands assert "LLM_BASE_URL=http://llm-svc:8011/v1?run_id=" in commands assert "docker rm -f" not in commands assert "down --volumes --remove-orphans" in commands diff --git a/tests/service/test_live_research_recovery.py b/tests/service/test_live_research_recovery.py new file mode 100644 index 00000000..900bce36 --- /dev/null +++ b/tests/service/test_live_research_recovery.py @@ -0,0 +1,359 @@ +"""Crash boundaries use isolated Valkey and controlled adapters, never providers.""" + +import asyncio +import hashlib +import json +import os +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from agent.experimental import live_runner +from agent.experimental.durable_research import ( + DurableConflictError, + DurableResearchLedger, + OutcomeUnknownError, +) +from agent.experimental.model_review import ModelReply, ReviewRequest +from agent.routes import experimental_research as routes + + +@pytest.fixture +def ledger(): + url = os.environ.get("DURABLE_RESEARCH_REDIS_URL") or os.environ.get("VALKEY_URL") + if not url: + raise RuntimeError("isolated durable Valkey URL required") + namespace = f"groktocrawl:test:live-recovery:{uuid4()}" + value = DurableResearchLedger(url, namespace=namespace) + try: + yield value + finally: + for key in value.redis.scan_iter(match=f"{namespace}:*"): + if key.startswith(namespace + ":"): + value.redis.delete(key) + value.redis.close() + + +def admitted(ledger): + value = ledger.admit( + "test-owner", + str(uuid4()), + "input-digest", + payload={ + "objective": "Bounded test question", + "research_id": str(uuid4()), + "execution_mode": "live", + "model": "configured-test-model", + "token_params": {"max_tokens": 2048}, + "evaluation_time": datetime.now(UTC).isoformat(), + }, + ) + record = routes._restore_durable_record(value) + record.durable_ledger = ledger + record.app = SimpleNamespace( + state=SimpleNamespace( + searxng_url="http://local-search.test", + llm_base_url="http://local-model.test", + llm_api_key="test-only-key", + llm_model="configured-test-model", + ) + ) + return record + + +def claim(record): + owned = record.durable_ledger.claim( + record.run_id, owner_id=str(uuid4()), attempt_id=str(uuid4()) + ) + record.durable_owner_id = owned.owner_id + record.durable_generation = owned.owner_generation + + +@pytest.mark.asyncio +async def test_restart_reuses_exact_external_receipts(ledger, monkeypatch): + record = admitted(ledger) + calls = {"search": 0, "acquire": 0, "model": 0} + + class Search: + def __init__(self, *_args, **_kwargs): + pass + + async def search(self, *_args, **_kwargs): + calls["search"] += 1 + return [{"url": "https://source.test/one"}], SimpleNamespace( + empty_result=False + ) + + async def close(self): + pass + + class Scraper: + async def scrape(self, *_args, **_kwargs): + calls["acquire"] += 1 + return { + "success": True, + "data": {"markdown": "Source-specific retained bytes"}, + } + + class Transport: + def __init__(self, *_args, **_kwargs): + pass + + async def __call__(self, request): + calls["model"] += 1 + return ModelReply( + b'{"answer":"from source"}', request.requested_model, None, 7 + ) + + first = True + observed = [] + + async def pipeline(objective, *, search, acquire, complete, **kwargs): + nonlocal first + urls = await search(objective, 3) + source = await acquire(urls[0]) + reply = await complete( + ReviewRequest( + "instruction", b'{"input":"stable"}', "configured-test-model", 2048 + ) + ) + observed.append((source, reply, {**kwargs, "clock": kwargs["clock"]()})) + if first: + first = False + raise asyncio.CancelledError # process lost after durable model receipt + return "completed without new external effects" + + monkeypatch.setattr(live_runner, "SearXNGClient", Search) + monkeypatch.setattr(live_runner, "ReviewTransport", Transport) + monkeypatch.setattr(live_runner, "run_query", pipeline) + record.app.state.scraper_client = Scraper() + claim(record) + with pytest.raises(asyncio.CancelledError): + await live_runner.run_live(record, record.app) + ledger.redis.delete(ledger._lease_key(record.run_id)) + recovered = routes._restore_durable_record(ledger.get(record.run_id)) + recovered.durable_ledger = ledger + recovered.app = record.app + claim(recovered) + assert ( + await live_runner.run_live(recovered, recovered.app) + == "completed without new external effects" + ) + assert calls == {"search": 1, "acquire": 1, "model": 1} + assert observed[0] == observed[1] + assert observed[1][1].input_tokens is None # unknown usage never becomes zero + + +@pytest.mark.asyncio +async def test_restart_does_not_replay_dispatch_without_receipt(ledger, monkeypatch): + record = admitted(ledger) + claim(record) + inputs = json.dumps( + {"objective": record.objective, "limit": 3}, + sort_keys=True, + separators=(",", ":"), + ) + ledger.step( + record.run_id, + record.durable_owner_id, + record.durable_generation, + "search", + hashlib.sha256(inputs.encode()).hexdigest(), + ) + ledger.redis.delete(ledger._lease_key(record.run_id)) + claim(record) + + async def pipeline(objective, *, search, **_kwargs): + await search(objective, 3) + raise AssertionError("ambiguous dispatch was replayed") + + def unexpected_dispatch(*_args, **_kwargs): + raise AssertionError("external client reached after ambiguous intent") + + monkeypatch.setattr(live_runner, "run_query", pipeline) + monkeypatch.setattr(live_runner, "SearXNGClient", unexpected_dispatch) + with pytest.raises(OutcomeUnknownError): + await live_runner.run_live(record, record.app) + + +class EmptyAuthority: + def __init__(self): + self.lock = asyncio.Lock() + self.commits = 0 + + @asynccontextmanager + async def execution_lock(self, *_args): + async with self.lock: + yield + + async def find_run(self, *_args): + return None + + async def commit(self, *_args, **_kwargs): + self.commits += 1 + raise AssertionError("cancelled or failed worker tried publishing") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure,code", + [ + (ValueError("bad evidence"), "research_run_failed"), + (OutcomeUnknownError("lost receipt"), "outcome_unknown"), + ], +) +async def test_terminal_failure_survives_worker_loss( + ledger, monkeypatch, failure, code +): + record = admitted(ledger) + record.artifact_authority = EmptyAuthority() + + async def fail(*_args): + raise failure + + monkeypatch.setattr(routes, "run_live", fail) + await routes._execute_run(record) + restored = routes._restore_durable_record(ledger.get(record.run_id)) + assert restored.state == "failed" + assert restored.error == {"code": code, "retryable": False} + assert restored.events[-1].event == "error" + assert record.artifact_authority.commits == 0 + + +@pytest.mark.asyncio +async def test_heartbeat_and_remote_cancel_fence_publication(ledger, monkeypatch): + record = admitted(ledger) + record.artifact_authority = EmptyAuthority() + started, release = asyncio.Event(), asyncio.Event() + + async def blocked(*_args): + started.set() + await release.wait() + return object() + + monkeypatch.setattr(routes, "run_live", blocked) + record.task = asyncio.create_task(routes._execute_run(record)) + await asyncio.wait_for(started.wait(), timeout=2) + await asyncio.sleep(2.2) # longer than the actual default lease + with pytest.raises(DurableConflictError): + ledger.claim(record.run_id, owner_id="other-process", attempt_id="duplicate") + ledger.cancel(record.run_id, terminal_payload={"research_id": record.research_id}) + release.set() + await asyncio.wait_for(record.task, timeout=2) + routes._check_retained_ledger(record) + assert record.state == "cancelled" + assert record.artifact_authority.commits == 0 + + +@pytest.mark.asyncio +async def test_real_construction_and_review_replay_without_new_model_calls( + ledger, monkeypatch +): + from agent.experimental import real_journey + + from tests.unit.test_query_construction import TEXT, batch_decisions, proposal + + record = admitted(ledger) + model_calls = [] + + class Search: + def __init__(self, *_args, **_kwargs): + pass + + async def search(self, *_args, **_kwargs): + return [{"url": "https://source.test/pilot"}], SimpleNamespace( + empty_result=False + ) + + async def close(self): + pass + + class Scraper: + async def scrape(self, *_args, **_kwargs): + return {"success": True, "data": {"markdown": TEXT}} + + class Transport: + def __init__(self, *_args, **_kwargs): + pass + + async def __call__(self, request): + value = json.loads(request.payload) + model_calls.append(value) + reply = proposal() if "schema" in value else batch_decisions(value) + return ModelReply(json.dumps(reply).encode(), "configured-test-model", 1, 1) + + original = real_journey.ConsolidatedJourney + completed = [] + + class InterruptAfterReview: + def __init__(self, **kwargs): + self.inner = original(**kwargs) + + async def run(self): + result = await self.inner.run() + completed.append(result) + if len(completed) == 1: + raise asyncio.CancelledError + return result + + monkeypatch.setattr(live_runner, "SearXNGClient", Search) + monkeypatch.setattr(live_runner, "ReviewTransport", Transport) + monkeypatch.setattr(real_journey, "ConsolidatedJourney", InterruptAfterReview) + record.app.state.scraper_client = Scraper() + claim(record) + with pytest.raises(asyncio.CancelledError): + await live_runner.run_live(record, record.app) + ledger.redis.delete(ledger._lease_key(record.run_id)) + claim(record) + result = await live_runner.run_live(record, record.app) + assert len(model_calls) == 2 # construction plus batched semantic review + assert result.candidate.admitted.manifest.research_id == record.research_id + assert result.sources == completed[0].sources + assert [report.body for report in result.reports] == [ + report.body for report in completed[0].reports + ] + assert not result.candidate.fixture_only + + +@pytest.mark.asyncio +@pytest.mark.parametrize("changed", [False, True]) +async def test_selected_root_liveness_rechecked_before_external_work( + ledger, monkeypatch, changed +): + from agent.experimental.source_store import StorageConflictError + + record = admitted(ledger) + durable = ledger.get(record.run_id) + body = b"Selected retained source" + durable.payload["selected_sources"] = [ + { + "url": None, + "text": body.decode(), + "retrieved_at": None, + "origin_id": f"root:{uuid4()}:source-1", + "lineage_id": hashlib.sha256(body).hexdigest(), + } + ] + raw = json.loads(ledger.redis.get(ledger._run_key(record.run_id))) + raw["payload"] = durable.payload + ledger.redis.set( + ledger._run_key(record.run_id), json.dumps(raw), px=ledger.retention_ms + ) + + class Authority(EmptyAuthority): + async def read_evidence(self, *_args): + if changed: + return b"Changed retained source", "text/plain" + raise StorageConflictError("selected root deleted") + + record.artifact_authority = Authority() + + def unexpected(*_args, **_kwargs): + raise AssertionError("deleted/changed source reached model transport") + + monkeypatch.setattr(live_runner, "ReviewTransport", unexpected) + claim(record) + with pytest.raises((ValueError, StorageConflictError)): + await live_runner.run_live(record, record.app) diff --git a/tests/unit/test_query_construction.py b/tests/unit/test_query_construction.py index 429d4e74..b9bcb5e1 100644 --- a/tests/unit/test_query_construction.py +++ b/tests/unit/test_query_construction.py @@ -374,3 +374,22 @@ async def test_invalid_line_ranges_are_rejected(start, end): value["evidence"][0].update(start_line=start, end_line=end) with pytest.raises(ValueError): await run(value) + + +@pytest.mark.asyncio +async def test_selected_source_preserves_unknown_original_metadata(): + async def complete(_request): + return ModelReply(json.dumps(proposal()).encode(), "local", None, None) + + result = await construct_research( + QUESTION, + (CapturedSource(None, TEXT, None, origin_id="retained-root:source-1"),), + complete=complete, + scope_id="owner", + clock=journey_clock, + ) + snapshot = result.context.snapshots[0] + assert snapshot.canonical_url is None + assert snapshot.retrieved_at is None + assert snapshot.origin_id == "retained-root:source-1" + assert (await result.resolve(snapshot.content_ref)).body == TEXT.encode() From 319a2fece9903a9d9a65990e472e3b0487f6bc75 Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 23:12:52 -0400 Subject: [PATCH 06/11] test: install pinned MCP dependency for runtime acceptance Signed-off-by: magnus919 --- pyproject.toml | 2 ++ uv.lock | 92 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c53f9b0c..63bec85f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,6 +223,8 @@ fast-tests = [ "readability-lxml==0.8.4.1", "tabulate>=0.9.0", "youtube_transcript_api>=1.0.0", + # Runtime acceptance dispatches the actual MCP tools, not a client mock. + "mcp==1.27.0", "rfc8785==0.1.4", "psycopg[binary]==3.3.5", ] diff --git a/uv.lock b/uv.lock index 5e804222..bb0de8cb 100644 --- a/uv.lock +++ b/uv.lock @@ -842,6 +842,7 @@ fast-tests = [ { name = "jsonschema" }, { name = "markdownify" }, { name = "markitdown" }, + { name = "mcp" }, { name = "numpy" }, { name = "openpyxl" }, { name = "pillow" }, @@ -900,6 +901,7 @@ fast-tests = [ { name = "jsonschema", specifier = ">=4.23.0" }, { name = "markdownify", specifier = "==1.2.3" }, { name = "markitdown", specifier = ">=0.1.0" }, + { name = "mcp", specifier = "==1.27.0" }, { name = "numpy", specifier = ">=1.26.0" }, { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pillow", specifier = ">=10.0.0" }, @@ -1088,6 +1090,15 @@ http2 = [ { name = "h2" }, ] +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, +] + [[package]] name = "huggingface-hub" version = "1.26.0" @@ -1432,6 +1443,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mcp" +version = "1.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/eb/c0cfc62075dc6e1ec1c64d352ae09ac051d9334311ed226f1f425312848a/mcp-1.27.0.tar.gz", hash = "sha256:d3dc35a7eec0d458c1da4976a48f982097ddaab87e278c5511d5a4a56e852b83", size = 607509, upload-time = "2026-04-02T14:48:08.88Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/46/f6b4ad632c67ef35209a66127e4bddc95759649dd595f71f13fba11bdf9a/mcp-1.27.0-py3-none-any.whl", hash = "sha256:5ce1fa81614958e267b21fb2aa34e0aea8e2c6ede60d52aba45fd47246b4d741", size = 215967, upload-time = "2026-04-02T14:48:07.24Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -1524,7 +1560,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -1563,7 +1599,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -1575,7 +1611,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -1605,9 +1641,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -1619,7 +1655,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -2058,6 +2094,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, ] +[[package]] +name = "pydantic-settings" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/3b/a5d2294799b53b448319978cfb5bd139d5a9d45e862af91661614f14c922/pydantic_settings-2.16.0.tar.gz", hash = "sha256:5b6c578049ede4db0e2ef3b4eaa4ad4069cfa9211f83fb38df899dfade50a614", size = 301031, upload-time = "2026-10-14T12:44:09.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/f4/b987bf8c51e5b19a95fa66d1ee596074141e085d9c2ddf97920803c7029b/pydantic_settings-2.16.0-py3-none-any.whl", hash = "sha256:7e73acf7f61936a15e5a3b6eedaea29f133357faf7272f2607ba479b049dd7f2", size = 80364, upload-time = "2026-10-14T12:44:08.233Z" }, +] + [[package]] name = "pyee" version = "13.0.1" @@ -2079,6 +2130,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pypdf" version = "6.14.2" @@ -2831,6 +2896,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] +[[package]] +name = "sse-starlette" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/be/0123026f719d1a7936f214a88b553bb5701e04ff2511147c1dab0c5035eb/sse_starlette-3.5.0.tar.gz", hash = "sha256:75de713aa8a9441513cc283220826da079d982770965b951e9437720e8bafdb2", size = 36794, upload-time = "2026-09-28T17:48:14.7Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/e4/cdda14023c316d71493bc54fdffc3dd006631b88866145c9d3cc33e0f1df/sse_starlette-3.5.0-py3-none-any.whl", hash = "sha256:3e6e1070df3f0f5d9cea81496de92dbb72f6721871d99748ece67441dd8b7997", size = 17407, upload-time = "2026-09-28T17:48:13.228Z" }, +] + [[package]] name = "starlette" version = "1.3.1" From 15df8c0f3426555d46ccd17caae7c8258846d552 Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 23:43:31 -0400 Subject: [PATCH 07/11] ci: start PostgreSQL for standard research acceptance tests Signed-off-by: magnus919 --- .github/workflows/docker.yml | 8 ++++---- tests/service/test_experimental_runtime_workflow.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b4645b0c..61a220e3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -395,7 +395,7 @@ jobs: run: docker ps -a --filter name=groktocrawl -q | xargs -r docker rm -f || true - name: Clean up stale containers from previous run - run: docker compose down --remove-orphans --timeout 30 || true + run: docker compose --profile durable-research down --remove-orphans --timeout 30 || true - name: Ensure HF model cache volume exists run: docker volume inspect hf-cache || docker volume create hf-cache @@ -431,7 +431,7 @@ jobs: # have a published tag yet (e.g. a newly added service), which fails # its manifest lookup; tolerate that and let the build/up steps build # the missing image from source instead. - docker compose --profile indexing pull || true + docker compose --profile indexing --profile durable-research pull || true timeout-minutes: 10 - name: Build service images (service-local PR build / tag-push full build) @@ -509,7 +509,7 @@ jobs: PY - name: Start the Docker stack - run: docker compose --profile indexing up -d + run: docker compose --profile indexing --profile durable-research up -d timeout-minutes: 10 - name: Start test fixtures @@ -1121,7 +1121,7 @@ jobs: - name: Tear down stack if: always() - run: docker compose down --remove-orphans --timeout 30 + run: docker compose --profile durable-research down --remove-orphans --timeout 30 research-runtime: uses: ./.github/workflows/runtime.yml diff --git a/tests/service/test_experimental_runtime_workflow.py b/tests/service/test_experimental_runtime_workflow.py index b486828c..3ddf588c 100644 --- a/tests/service/test_experimental_runtime_workflow.py +++ b/tests/service/test_experimental_runtime_workflow.py @@ -85,6 +85,16 @@ def test_stack_is_built_locally_with_fixture_search_and_owned_volumes(): assert f'docker cp {filename} "$svc":/app/{filename}' in commands +def test_standard_integration_provisions_durable_database(): + workflow = yaml.safe_load((ROOT / ".github/workflows/docker.yml").read_text()) + jobs = workflow["jobs"].values() + job = next(job for job in jobs if job.get("name") == "Integration Tests") + steps = {step.get("name"): step for step in job["steps"]} + assert "--profile durable-research up -d" in steps["Start the Docker stack"]["run"] + assert "--profile durable-research down" in steps["Tear down stack"]["run"] + assert "/app/tests/integration/" in steps["Run integration tests"]["run"] + + def test_protected_browser_profile_is_required_by_runtime_gate(): profile = WORKFLOW["jobs"]["protected-browser-profile"] gate = WORKFLOW["jobs"]["runtime-gate"] From a7ddca38d923b40a5d76e5b8fa10428eb505374e Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 23:45:12 -0400 Subject: [PATCH 08/11] test: align CI contracts with PostgreSQL acceptance setup Signed-off-by: magnus919 --- .../service/test_ci_change_classification.py | 20 ++++++++++++++----- .../test_resource_envelope_contract.py | 9 +++++++-- tests/service/test_workflow_contract.py | 5 ++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/service/test_ci_change_classification.py b/tests/service/test_ci_change_classification.py index 77640abe..e86cfcfb 100644 --- a/tests/service/test_ci_change_classification.py +++ b/tests/service/test_ci_change_classification.py @@ -451,7 +451,10 @@ def test_pr_lane_pulls_published_images_before_service_local_build(self) -> None # refreshed from the latest published images so unchanged services match a # known commit; GHCR login runs for pull_request events. self.assertIn("Pull freshly published service images", self.integration_tests) - self.assertIn("docker compose --profile indexing pull", self.integration_tests) + self.assertIn( + "docker compose --profile indexing --profile durable-research pull", + self.integration_tests, + ) self.assertIn( "github.event_name == 'push' && github.ref == 'refs/heads/main'", self.integration_tests, @@ -460,7 +463,10 @@ def test_pr_lane_pulls_published_images_before_service_local_build(self) -> None def test_main_push_pulls_published_images_without_rebuilding(self) -> None: self.assertIn("Pull freshly published service images", self.integration_tests) - self.assertIn("docker compose --profile indexing pull", self.integration_tests) + self.assertIn( + "docker compose --profile indexing --profile durable-research pull", + self.integration_tests, + ) self.assertIn( "github.event_name == 'push' && github.ref == 'refs/heads/main'", self.integration_tests, @@ -479,9 +485,13 @@ def test_tag_push_builds_full_stack_from_tagged_source(self) -> None: ) def test_start_stack_step_no_longer_uses_build(self) -> None: - self.assertIn("docker compose --profile indexing up -d", self.integration_tests) + self.assertIn( + "docker compose --profile indexing --profile durable-research up -d", + self.integration_tests, + ) self.assertNotIn( - "docker compose --profile indexing up --build -d", self.integration_tests + "docker compose --profile indexing --profile durable-research up --build -d", + self.integration_tests, ) # The only `up` is build-less: no event rebuilds the full stack on top of # an already-provisioned image set. @@ -574,7 +584,7 @@ def test_integration_test_staging_copies_required_contract_inputs( self.assertLess(probe_index, targeted_index) self.assertLess(targeted_index, critical_index) self.assertNotIn( - "SEARXNG_URL=http://slopsearx-fixture:8080 docker compose --profile indexing up -d", + "SEARXNG_URL=http://slopsearx-fixture:8080 docker compose --profile indexing --profile durable-research up -d", self.integration_tests, ) self.assertIn("--cov=/app/agent", self.integration_tests) diff --git a/tests/service/test_resource_envelope_contract.py b/tests/service/test_resource_envelope_contract.py index 41dc4486..f283b2dd 100644 --- a/tests/service/test_resource_envelope_contract.py +++ b/tests/service/test_resource_envelope_contract.py @@ -14,8 +14,13 @@ def test_integration_workflow_enables_indexing_profile(): # but no longer redundantly rebuilds the full stack on top of build-and-push # (#494): images are provisioned separately (pulled on push, service-local # build on PR) before an `up -d` with no `--build`. - assert "docker compose --profile indexing up -d" in workflow - assert "docker compose --profile indexing up --build -d" not in workflow + assert ( + "docker compose --profile indexing --profile durable-research up -d" in workflow + ) + assert ( + "docker compose --profile indexing --profile durable-research up --build -d" + not in workflow + ) assert "docker compose exec -T agent-svc env \\" in workflow assert "docker compose exec -T \\\\n -e" not in workflow diff --git a/tests/service/test_workflow_contract.py b/tests/service/test_workflow_contract.py index 32767a80..01e8df14 100644 --- a/tests/service/test_workflow_contract.py +++ b/tests/service/test_workflow_contract.py @@ -132,7 +132,10 @@ def test_mainline_ci_routes_the_real_agent_and_direct_search_clients_to_fixture( assert "payload['success'] and results" in probe # These inputs configure test clients and the agent without changing either # SlopSearX service's production contract. - assert "docker compose --profile indexing up -d" in steps["Start the Docker stack"] + assert ( + "docker compose --profile indexing --profile durable-research up -d" + in steps["Start the Docker stack"] + ) def test_published_service_images_target_amd64_and_arm64(): From 06aafa43f552357d613dbc9f45c75783b206f61a Mon Sep 17 00:00:00 2001 From: magnus919 Date: Thu, 15 Oct 2026 23:49:29 -0400 Subject: [PATCH 09/11] ci: isolate research acceptance database credentials and volumes Signed-off-by: magnus919 --- .github/workflows/docker.yml | 14 +++++++++++--- .../service/test_experimental_runtime_workflow.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 61a220e3..e37dee2b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -342,6 +342,7 @@ jobs: # kept so the job env documents the port map and the stack still starts with # sane defaults if the preflight step is ever removed or bypassed. env: + COMPOSE_PROJECT_NAME: groktocrawl-ci-${{ github.run_id }}-${{ github.run_attempt }} AGENT_PORT: "18080" SCRAPER_HOST_PORT: "18001" SLOPSEARX_HOST_PORT: "18081" @@ -369,6 +370,11 @@ jobs: cp .env.sample .env echo "TWIN_RUN_ID=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> .env echo "DURABLE_RESEARCH_REDIS_URL=redis://valkey:6379/15" >> .env + umask 077 + research_password_file="${RUNNER_TEMP}/groktocrawl-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password" + python3 -c 'from secrets import token_urlsafe; print(token_urlsafe(32))' > "$research_password_file" + echo "DURABLE_RESEARCH_POSTGRES_PASSWORD_FILE=$research_password_file" >> .env + echo "DURABLE_RESEARCH_POSTGRES_DSN=postgresql://groktocrawl@research-postgres:5432/groktocrawl" >> .env # The full suite uses the real agent API, whose upstream must be the # deterministic fixture rather than the live SlopSearX service. echo "SEARXNG_URL=http://slopsearx-fixture:8080" >> .env @@ -392,10 +398,10 @@ jobs: # the integration lane, so no token is needed here. - name: Force-remove all stale project containers - run: docker ps -a --filter name=groktocrawl -q | xargs -r docker rm -f || true + run: docker ps -a --filter "label=com.docker.compose.project=$COMPOSE_PROJECT_NAME" -q | xargs -r docker rm -f || true - name: Clean up stale containers from previous run - run: docker compose --profile durable-research down --remove-orphans --timeout 30 || true + run: docker compose --profile durable-research down --volumes --remove-orphans --timeout 30 || true - name: Ensure HF model cache volume exists run: docker volume inspect hf-cache || docker volume create hf-cache @@ -1121,7 +1127,9 @@ jobs: - name: Tear down stack if: always() - run: docker compose --profile durable-research down --remove-orphans --timeout 30 + run: | + trap 'rm -f "${RUNNER_TEMP}/groktocrawl-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password"' EXIT + docker compose --profile durable-research down --volumes --remove-orphans --timeout 30 research-runtime: uses: ./.github/workflows/runtime.yml diff --git a/tests/service/test_experimental_runtime_workflow.py b/tests/service/test_experimental_runtime_workflow.py index 3ddf588c..a4b3c029 100644 --- a/tests/service/test_experimental_runtime_workflow.py +++ b/tests/service/test_experimental_runtime_workflow.py @@ -93,6 +93,21 @@ def test_standard_integration_provisions_durable_database(): assert "--profile durable-research up -d" in steps["Start the Docker stack"]["run"] assert "--profile durable-research down" in steps["Tear down stack"]["run"] assert "/app/tests/integration/" in steps["Run integration tests"]["run"] + assert job["env"]["COMPOSE_PROJECT_NAME"].endswith( + "${{ github.run_id }}-${{ github.run_attempt }}" + ) + assert "token_urlsafe(32)" in steps["Set up .env"]["run"] + assert "umask 077" in steps["Set up .env"]["run"] + assert ( + "DURABLE_RESEARCH_POSTGRES_PASSWORD_FILE=$research_password_file" + in steps["Set up .env"]["run"] + ) + assert "--volumes" in steps["Tear down stack"]["run"] + assert ( + "label=com.docker.compose.project=$COMPOSE_PROJECT_NAME" + in steps["Force-remove all stale project containers"]["run"] + ) + assert "rm -f" in steps["Tear down stack"]["run"] def test_protected_browser_profile_is_required_by_runtime_gate(): From beadd738701afaca579253b4a39213857c0d740c Mon Sep 17 00:00:00 2001 From: magnus919 Date: Fri, 16 Oct 2026 00:12:22 -0400 Subject: [PATCH 10/11] ci: seed research inputs across the runner Docker boundary Signed-off-by: magnus919 --- .github/workflows/docker.yml | 33 ++++ .github/workflows/runtime.yml | 1 + compose.ci-research-inputs.yml | 33 ++++ .../test_experimental_runtime_workflow.py | 156 ++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 compose.ci-research-inputs.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e37dee2b..d7636f3f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -343,6 +343,7 @@ jobs: # sane defaults if the preflight step is ever removed or bypassed. env: COMPOSE_PROJECT_NAME: groktocrawl-ci-${{ github.run_id }}-${{ github.run_attempt }} + COMPOSE_FILE: docker-compose.yml:compose.ci-research-inputs.yml AGENT_PORT: "18080" SCRAPER_HOST_PORT: "18001" SLOPSEARX_HOST_PORT: "18081" @@ -514,6 +515,36 @@ jobs: print(f"exported {line}") PY + - name: Seed daemon-owned research inputs + run: | + set -euo pipefail + seed_container="${COMPOSE_PROJECT_NAME}-research-inputs" + trap 'docker rm -f "$seed_container" >/dev/null 2>&1 || true' EXIT + for volume in ci_research_secret ci_research_init; do + docker volume create \ + --label "com.docker.compose.project=$COMPOSE_PROJECT_NAME" \ + --label "com.docker.compose.volume=$volume" \ + "${COMPOSE_PROJECT_NAME}_${volume}" >/dev/null + done + research_image=$(uv run --no-sync python -c 'import yaml; print(yaml.safe_load(open("docker-compose.yml"))["services"]["research-postgres"]["image"])') + docker create --name "$seed_container" \ + --label "com.docker.compose.project=$COMPOSE_PROJECT_NAME" \ + --network none --user 0:0 --entrypoint /bin/sh \ + --mount "type=volume,source=${COMPOSE_PROJECT_NAME}_ci_research_secret,target=/run/secrets" \ + --mount "type=volume,source=${COMPOSE_PROJECT_NAME}_ci_research_init,target=/docker-entrypoint-initdb.d" \ + "$research_image" -ec ' + test -s /run/secrets/research_postgres_password + test -s /docker-entrypoint-initdb.d/016_artifact_evidence.sql + chmod 0755 /run/secrets /docker-entrypoint-initdb.d + chmod 0444 /run/secrets/research_postgres_password /docker-entrypoint-initdb.d/*.sql + ' >/dev/null + research_password_file="${RUNNER_TEMP}/groktocrawl-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password" + docker cp "$research_password_file" "$seed_container:/run/secrets/research_postgres_password" + docker cp agent-svc/agent/experimental/migrations/. "$seed_container:/docker-entrypoint-initdb.d/" + docker start --attach "$seed_container" + test "$(docker inspect -f '{{.State.ExitCode}}' "$seed_container")" = 0 + timeout-minutes: 5 + - name: Start the Docker stack run: docker compose --profile indexing --profile durable-research up -d timeout-minutes: 10 @@ -756,6 +787,7 @@ jobs: docker cp .github/workflows/fast-tests.yml "$svc":/app/.github/workflows/fast-tests.yml docker cp .github/workflows/. "$svc":/app/.github/workflows/ docker cp docker-compose.ci.yml "$svc":/app/docker-compose.ci.yml + docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml # Include the protected-profile inputs read by service contract tests. docker cp compose.experimental-candidate.yml "$svc":/app/compose.experimental-candidate.yml docker cp compose.protected-flare-ci.yml "$svc":/app/compose.protected-flare-ci.yml @@ -1129,6 +1161,7 @@ jobs: if: always() run: | trap 'rm -f "${RUNNER_TEMP}/groktocrawl-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password"' EXIT + docker rm -f "${COMPOSE_PROJECT_NAME}-research-inputs" >/dev/null 2>&1 || true docker compose --profile durable-research down --volumes --remove-orphans --timeout 30 research-runtime: diff --git a/.github/workflows/runtime.yml b/.github/workflows/runtime.yml index e56549c2..347b6409 100644 --- a/.github/workflows/runtime.yml +++ b/.github/workflows/runtime.yml @@ -458,6 +458,7 @@ jobs: docker cp .github/workflows/docker.yml "$svc":/app/.github/workflows/docker.yml docker cp .github/workflows/. "$svc":/app/.github/workflows/ docker cp docker-compose.ci.yml "$svc":/app/docker-compose.ci.yml + docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml # Include the protected-profile inputs read by service contract tests. docker cp compose.experimental-candidate.yml "$svc":/app/compose.experimental-candidate.yml docker cp compose.protected-flare-ci.yml "$svc":/app/compose.protected-flare-ci.yml diff --git a/compose.ci-research-inputs.yml b/compose.ci-research-inputs.yml new file mode 100644 index 00000000..49c02d2b --- /dev/null +++ b/compose.ci-research-inputs.yml @@ -0,0 +1,33 @@ +# CI runner and Docker daemon may have different filesystems. Seed these +# per-project volumes with docker cp; no runner paths become daemon bind mounts. +services: + research-postgres: + secrets: !reset [] + volumes: + - type: volume + source: ci_research_secret + target: /run/secrets + read_only: true + - type: volume + source: ci_research_init + target: /docker-entrypoint-initdb.d + read_only: true + agent-svc: + secrets: !reset [] + volumes: + - type: volume + source: ci_research_secret + target: /run/secrets + read_only: true + semantic-svc: + secrets: !reset [] + volumes: + - type: volume + source: ci_research_secret + target: /run/secrets + read_only: true + +secrets: !reset {} +volumes: + ci_research_secret: {} + ci_research_init: {} diff --git a/tests/service/test_experimental_runtime_workflow.py b/tests/service/test_experimental_runtime_workflow.py index a4b3c029..285da8f2 100644 --- a/tests/service/test_experimental_runtime_workflow.py +++ b/tests/service/test_experimental_runtime_workflow.py @@ -1,5 +1,6 @@ """Validate the experimental CI execution boundary and required-check outcomes.""" +import hashlib import ipaddress import itertools import json @@ -7,6 +8,7 @@ import re import shutil import subprocess +import sys import tempfile from pathlib import Path @@ -110,6 +112,160 @@ def test_standard_integration_provisions_durable_database(): assert "rm -f" in steps["Tear down stack"]["run"] +def _standard_integration_job(): + workflow = yaml.safe_load((ROOT / ".github/workflows/docker.yml").read_text()) + return next( + job + for job in workflow["jobs"].values() + if job.get("name") == "Integration Tests" + ) + + +def test_self_hosted_research_inputs_use_only_owned_named_volumes(): + class ComposeLoader(yaml.SafeLoader): + pass + + def reset(loader, node): + if isinstance(node, yaml.MappingNode): + return loader.construct_mapping(node, deep=True) + return loader.construct_sequence(node, deep=True) + + ComposeLoader.add_constructor("!reset", reset) + overlay = yaml.load( + (ROOT / "compose.ci-research-inputs.yml").read_text(), Loader=ComposeLoader + ) + assert set(overlay["services"]) == { + "research-postgres", + "agent-svc", + "semantic-svc", + } + assert overlay["secrets"] == {} + assert set(overlay["volumes"]) == {"ci_research_secret", "ci_research_init"} + for service in overlay["services"].values(): + assert service["secrets"] == [] + assert all( + mount["type"] == "volume" and mount["read_only"] is True + for mount in service["volumes"] + ) + assert ( + next( + mount + for mount in service["volumes"] + if mount["target"] == "/run/secrets" + )["source"] + == "ci_research_secret" + ) + # Compose merges volumes by container target, replacing the checkout bind. + assert ( + next( + mount + for mount in overlay["services"]["research-postgres"]["volumes"] + if mount["target"] == "/docker-entrypoint-initdb.d" + )["source"] + == "ci_research_init" + ) + job = _standard_integration_job() + assert ( + job["env"]["COMPOSE_FILE"] + == "docker-compose.yml:compose.ci-research-inputs.yml" + ) + names = [step.get("name") for step in job["steps"]] + assert ( + names.index("Clean up stale containers from previous run") + < names.index("Seed daemon-owned research inputs") + < names.index("Start the Docker stack") + ) + steps = {step.get("name"): step for step in job["steps"]} + assert steps["Seed daemon-owned research inputs"]["timeout-minutes"] == 5 + assert ( + 'docker rm -f "${COMPOSE_PROJECT_NAME}-research-inputs"' + in steps["Tear down stack"]["run"] + ) + for workflow in ("docker.yml", "runtime.yml"): + assert ( + 'docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml' + in (ROOT / ".github/workflows" / workflow).read_text() + ) + + +@pytest.mark.parametrize("fail_copy", [False, True]) +def test_research_input_seeding_streams_files_and_cleans_helper_on_failure( + tmp_path, fail_copy +): + job = _standard_integration_job() + command = next( + step["run"] + for step in job["steps"] + if step.get("name") == "Seed daemon-owned research inputs" + ) + project = "groktocrawl-ci-123-2" + secret = "test-only-value-must-not-appear-in-argv-or-logs" + password_file = tmp_path / "groktocrawl-postgres-123-2.password" + password_file.write_text(secret) + password_file.chmod(0o600) + log = tmp_path / "commands.jsonl" + docker = tmp_path / "docker" + docker.write_text( + f"#!{sys.executable}\n" + + """import hashlib, json, os, pathlib, sys +args = sys.argv[1:] +record = {"argv": args} +if args[0] == "cp": + source = pathlib.Path(args[1]) + if source.is_file(): + record["copied_digest"] = hashlib.sha256(source.read_bytes()).hexdigest() + else: + record["copied_sql"] = sorted(path.name for path in source.glob("*.sql")) +with open(os.environ["FAKE_DOCKER_LOG"], "a") as output: + output.write(json.dumps(record) + "\\n") +if args[0] == "cp" and "migrations" in args[1] and os.environ["FAIL_SQL_COPY"] == "true": + sys.exit(7) +if args[0] == "inspect": + print(0) +""" + ) + docker.chmod(0o700) + uv = tmp_path / "uv" + uv.write_text( + f"#!{sys.executable}\nimport os, sys\nos.execv(sys.executable, [sys.executable, *sys.argv[4:]])\n" + ) + uv.chmod(0o700) + result = subprocess.run( + ["bash", "-c", command], + cwd=ROOT, + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}", + "RUNNER_TEMP": str(tmp_path), + "COMPOSE_PROJECT_NAME": project, + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "2", + "FAKE_DOCKER_LOG": str(log), + "FAIL_SQL_COPY": str(fail_copy).lower(), + }, + timeout=20, + ) + assert result.returncode == (7 if fail_copy else 0), result.stderr + records = [json.loads(line) for line in log.read_text().splitlines()] + create = next(item["argv"] for item in records if item["argv"][0] == "create") + assert create[create.index("--network") + 1] == "none" + assert "type=bind" not in str(create) + assert str(tmp_path) not in str(create) + assert all( + item["argv"][-1].startswith(project + "_") + for item in records + if item["argv"][:2] == ["volume", "create"] + ) + copied = next(item for item in records if "copied_digest" in item) + assert copied["copied_digest"] == hashlib.sha256(secret.encode()).hexdigest() + sql = next(item["copied_sql"] for item in records if "copied_sql" in item) + assert "001_source_staging.sql" in sql and "016_artifact_evidence.sql" in sql + assert records[-1]["argv"] == ["rm", "-f", project + "-research-inputs"] + assert secret not in result.stdout + result.stderr + log.read_text() + + def test_protected_browser_profile_is_required_by_runtime_gate(): profile = WORKFLOW["jobs"]["protected-browser-profile"] gate = WORKFLOW["jobs"]["runtime-gate"] From 510ba810ec141542b8968360c0241767fa620fd7 Mon Sep 17 00:00:00 2001 From: magnus919 Date: Fri, 16 Oct 2026 00:35:13 -0400 Subject: [PATCH 11/11] ci: include migration SQL in workflow test containers Signed-off-by: magnus919 --- .github/workflows/docker.yml | 3 +++ .github/workflows/runtime.yml | 3 +++ tests/service/test_experimental_runtime_workflow.py | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d7636f3f..e8f4e8db 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -788,6 +788,9 @@ jobs: docker cp .github/workflows/. "$svc":/app/.github/workflows/ docker cp docker-compose.ci.yml "$svc":/app/docker-compose.ci.yml docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml + # Preserve checkout-relative SQL inputs for the seeding contract test. + docker compose exec -T agent-svc mkdir -p /app/agent-svc/agent/experimental/migrations + docker cp agent-svc/agent/experimental/migrations/. "$svc":/app/agent-svc/agent/experimental/migrations/ # Include the protected-profile inputs read by service contract tests. docker cp compose.experimental-candidate.yml "$svc":/app/compose.experimental-candidate.yml docker cp compose.protected-flare-ci.yml "$svc":/app/compose.protected-flare-ci.yml diff --git a/.github/workflows/runtime.yml b/.github/workflows/runtime.yml index 347b6409..3740b608 100644 --- a/.github/workflows/runtime.yml +++ b/.github/workflows/runtime.yml @@ -459,6 +459,9 @@ jobs: docker cp .github/workflows/. "$svc":/app/.github/workflows/ docker cp docker-compose.ci.yml "$svc":/app/docker-compose.ci.yml docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml + # Preserve checkout-relative SQL inputs for the seeding contract test. + docker compose exec -T agent-svc mkdir -p /app/agent-svc/agent/experimental/migrations + docker cp agent-svc/agent/experimental/migrations/. "$svc":/app/agent-svc/agent/experimental/migrations/ # Include the protected-profile inputs read by service contract tests. docker cp compose.experimental-candidate.yml "$svc":/app/compose.experimental-candidate.yml docker cp compose.protected-flare-ci.yml "$svc":/app/compose.protected-flare-ci.yml diff --git a/tests/service/test_experimental_runtime_workflow.py b/tests/service/test_experimental_runtime_workflow.py index 285da8f2..25eb3478 100644 --- a/tests/service/test_experimental_runtime_workflow.py +++ b/tests/service/test_experimental_runtime_workflow.py @@ -186,6 +186,10 @@ def reset(loader, node): 'docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml' in (ROOT / ".github/workflows" / workflow).read_text() ) + assert ( + 'docker cp agent-svc/agent/experimental/migrations/. "$svc":/app/agent-svc/agent/experimental/migrations/' + in (ROOT / ".github/workflows" / workflow).read_text() + ) @pytest.mark.parametrize("fail_copy", [False, True])