fix(research): execute live objectives with durable call receipts

.env.experimental-candidate.sample

@@ -6,6 +6,8 @@ CANDIDATE_BIND_IP=127.0.0.1
 CANDIDATE_AGENT_PORT=18080
 CANDIDATE_MCP_PORT=18002
 CANDIDATE_API_KEY=replace-with-random-api-key
+# Development fixture verifier only; leave false for normal operation.
+FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false
 
 # If CANDIDATE_BIND_IP is changed from loopback, admit the exact API/MCP host
 # values clients will send. The HTTP API remains protected by CANDIDATE_API_KEY.

.env.sample

@@ -88,6 +88,10 @@
 # FEATURE_EXPERIMENTAL_RESEARCH_RUNS=false
 # FEATURE_EXPERIMENTAL_RESEARCH_DURABLE=false
 # FEATURE_EXPERIMENTAL_RESEARCH_POSTGRES_ARTIFACTS=false
+# Development/demo only: true selects fictional sources and authored judgments.
+# Leave false for live durable research with all four gates above enabled.
+# Live admission requires PostgreSQL schema16 and configured search/scraper/LLM.
+# FEATURE_EXPERIMENTAL_RESEARCH_FIXTURE_RUNS=false
 # Schema 16 is required before enabling PostgreSQL artifact authority. The
 # durable-research Compose profile provisions pgvector and initializes this
 # schema; runtime migrations advance an existing database safely.

.github/workflows/docker.yml

@@ -342,6 +342,8 @@ jobs:
     # kept so the job env documents the port map and the stack still starts with
     # sane defaults if the preflight step is ever removed or bypassed.
     env:
+      COMPOSE_PROJECT_NAME: groktocrawl-ci-${{ github.run_id }}-${{ github.run_attempt }}
+      COMPOSE_FILE: docker-compose.yml:compose.ci-research-inputs.yml
       AGENT_PORT: "18080"
       SCRAPER_HOST_PORT: "18001"
       SLOPSEARX_HOST_PORT: "18081"
@@ -369,6 +371,11 @@ jobs:
           cp .env.sample .env
           echo "TWIN_RUN_ID=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> .env
           echo "DURABLE_RESEARCH_REDIS_URL=redis://valkey:6379/15" >> .env
+          umask 077
+          research_password_file="${RUNNER_TEMP}/groktocrawl-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password"
+          python3 -c 'from secrets import token_urlsafe; print(token_urlsafe(32))' > "$research_password_file"
+          echo "DURABLE_RESEARCH_POSTGRES_PASSWORD_FILE=$research_password_file" >> .env
+          echo "DURABLE_RESEARCH_POSTGRES_DSN=postgresql://groktocrawl@research-postgres:5432/groktocrawl" >> .env
           # The full suite uses the real agent API, whose upstream must be the
           # deterministic fixture rather than the live SlopSearX service.
           echo "SEARXNG_URL=http://slopsearx-fixture:8080" >> .env
@@ -392,10 +399,10 @@ jobs:
           # the integration lane, so no token is needed here.
 
       - name: Force-remove all stale project containers
-        run: docker ps -a --filter name=groktocrawl -q | xargs -r docker rm -f || true
+        run: docker ps -a --filter "label=com.docker.compose.project=$COMPOSE_PROJECT_NAME" -q | xargs -r docker rm -f || true
 
       - name: Clean up stale containers from previous run
-        run: docker compose down --remove-orphans --timeout 30 || true
+        run: docker compose --profile durable-research down --volumes --remove-orphans --timeout 30 || true
 
       - name: Ensure HF model cache volume exists
         run: docker volume inspect hf-cache || docker volume create hf-cache
@@ -431,7 +438,7 @@ jobs:
           # have a published tag yet (e.g. a newly added service), which fails
           # its manifest lookup; tolerate that and let the build/up steps build
           # the missing image from source instead.
-          docker compose --profile indexing pull || true
+          docker compose --profile indexing --profile durable-research pull || true
         timeout-minutes: 10
 
       - name: Build service images (service-local PR build / tag-push full build)
@@ -508,8 +515,38 @@ jobs:
               print(f"exported {line}")
           PY
 
+      - name: Seed daemon-owned research inputs
+        run: |
+          set -euo pipefail
+          seed_container="${COMPOSE_PROJECT_NAME}-research-inputs"
+          trap 'docker rm -f "$seed_container" >/dev/null 2>&1 || true' EXIT
+          for volume in ci_research_secret ci_research_init; do
+            docker volume create \
+              --label "com.docker.compose.project=$COMPOSE_PROJECT_NAME" \
+              --label "com.docker.compose.volume=$volume" \
+              "${COMPOSE_PROJECT_NAME}_${volume}" >/dev/null
+          done
+          research_image=$(uv run --no-sync python -c 'import yaml; print(yaml.safe_load(open("docker-compose.yml"))["services"]["research-postgres"]["image"])')
+          docker create --name "$seed_container" \
+            --label "com.docker.compose.project=$COMPOSE_PROJECT_NAME" \
+            --network none --user 0:0 --entrypoint /bin/sh \
+            --mount "type=volume,source=${COMPOSE_PROJECT_NAME}_ci_research_secret,target=/run/secrets" \
+            --mount "type=volume,source=${COMPOSE_PROJECT_NAME}_ci_research_init,target=/docker-entrypoint-initdb.d" \
+            "$research_image" -ec '
+              test -s /run/secrets/research_postgres_password
+              test -s /docker-entrypoint-initdb.d/016_artifact_evidence.sql
+              chmod 0755 /run/secrets /docker-entrypoint-initdb.d
+              chmod 0444 /run/secrets/research_postgres_password /docker-entrypoint-initdb.d/*.sql
+            ' >/dev/null
+          research_password_file="${RUNNER_TEMP}/groktocrawl-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password"
+          docker cp "$research_password_file" "$seed_container:/run/secrets/research_postgres_password"
+          docker cp agent-svc/agent/experimental/migrations/. "$seed_container:/docker-entrypoint-initdb.d/"
+          docker start --attach "$seed_container"
+          test "$(docker inspect -f '{{.State.ExitCode}}' "$seed_container")" = 0
+        timeout-minutes: 5
+
       - name: Start the Docker stack
-        run: docker compose --profile indexing up -d
+        run: docker compose --profile indexing --profile durable-research up -d
         timeout-minutes: 10
 
       - name: Start test fixtures
@@ -750,6 +787,10 @@ jobs:
           docker cp .github/workflows/fast-tests.yml "$svc":/app/.github/workflows/fast-tests.yml
           docker cp .github/workflows/. "$svc":/app/.github/workflows/
           docker cp docker-compose.ci.yml "$svc":/app/docker-compose.ci.yml
+          docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml
+          # Preserve checkout-relative SQL inputs for the seeding contract test.
+          docker compose exec -T agent-svc mkdir -p /app/agent-svc/agent/experimental/migrations
+          docker cp agent-svc/agent/experimental/migrations/. "$svc":/app/agent-svc/agent/experimental/migrations/
           # Include the protected-profile inputs read by service contract tests.
           docker cp compose.experimental-candidate.yml "$svc":/app/compose.experimental-candidate.yml
           docker cp compose.protected-flare-ci.yml "$svc":/app/compose.protected-flare-ci.yml
@@ -1121,7 +1162,10 @@ jobs:
 
       - name: Tear down stack
         if: always()
-        run: docker compose down --remove-orphans --timeout 30
+        run: |
+          trap 'rm -f "${RUNNER_TEMP}/groktocrawl-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password"' EXIT
+          docker rm -f "${COMPOSE_PROJECT_NAME}-research-inputs" >/dev/null 2>&1 || true
+          docker compose --profile durable-research down --volumes --remove-orphans --timeout 30
 
   research-runtime:
     uses: ./.github/workflows/runtime.yml

.github/workflows/runtime.yml

@@ -162,6 +162,13 @@ jobs:
           cp .env.sample .env
           echo "TWIN_RUN_ID=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> .env
           echo "DURABLE_RESEARCH_REDIS_URL=redis://valkey:6379/15" >> .env
+          echo "SEARXNG_URL=http://slopsearx-fixture:8080" >> .env
+          umask 077
+          research_password_file="/tmp/groktocrawl-research-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password"
+          python3 -c 'from secrets import token_urlsafe; print(token_urlsafe(32))' > "$research_password_file"
+          chmod 600 "$research_password_file"
+          echo "DURABLE_RESEARCH_POSTGRES_PASSWORD_FILE=$research_password_file" >> .env
+          echo "DURABLE_RESEARCH_POSTGRES_DSN=postgresql://groktocrawl@research-postgres:5432/groktocrawl" >> .env
           echo "LLM_BASE_URL=http://llm-svc:8011/v1?run_id=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> .env
           echo "LLM_MODEL=fixture-model" >> .env
           echo "CAPTCHA_VISION_BASE_URL=http://llm-svc:8011/v1" >> .env
@@ -187,6 +194,7 @@ jobs:
           --group runtime
           --profile indexing
           --profile fixture
+          --profile durable-research
           --receipt runtime-public-image-prefetch.json
         timeout-minutes: 10
 
@@ -199,7 +207,7 @@ jobs:
           if-no-files-found: warn
 
       - name: Build all application and fixture images from checkout
-        run: docker compose --profile indexing --profile fixture build
+        run: docker compose --profile indexing --profile fixture --profile durable-research build
         timeout-minutes: 25
 
       - name: Build and probe the capture gateway image
@@ -217,7 +225,7 @@ jobs:
           docker exec -i "$container" python - < tests/integration/capture_egress_image_probe.py
 
       - name: Start the Docker stack
-        run: docker compose --profile indexing --profile fixture up -d --no-build --pull never
+        run: docker compose --profile indexing --profile fixture --profile durable-research up -d --no-build --pull never
         timeout-minutes: 10
 
       - name: Wait for services to be healthy
@@ -450,6 +458,10 @@ jobs:
           docker cp .github/workflows/docker.yml "$svc":/app/.github/workflows/docker.yml
           docker cp .github/workflows/. "$svc":/app/.github/workflows/
           docker cp docker-compose.ci.yml "$svc":/app/docker-compose.ci.yml
+          docker cp compose.ci-research-inputs.yml "$svc":/app/compose.ci-research-inputs.yml
+          # Preserve checkout-relative SQL inputs for the seeding contract test.
+          docker compose exec -T agent-svc mkdir -p /app/agent-svc/agent/experimental/migrations
+          docker cp agent-svc/agent/experimental/migrations/. "$svc":/app/agent-svc/agent/experimental/migrations/
           # Include the protected-profile inputs read by service contract tests.
           docker cp compose.experimental-candidate.yml "$svc":/app/compose.experimental-candidate.yml
           docker cp compose.protected-flare-ci.yml "$svc":/app/compose.protected-flare-ci.yml
@@ -592,9 +604,12 @@ jobs:
           set +e
           docker compose exec -T agent-svc env \
             QA_OUTCOME_PATH=/app/qa-outcomes-integration.json \
+            DURABLE_RESEARCH_REDIS_URL=redis://valkey:6379/15 \
+            DURABLE_RESEARCH_POSTGRES_DSN=postgresql://groktocrawl@research-postgres:5432/groktocrawl \
+            PGPASSFILE=/tmp/postgres.pgpass \
             PYTHONPATH=/app/agent-svc:/app/scraper-svc:/app/llm-svc:/app/parse-svc:/app/portal-svc:/app/browser-svc:/app \
             SCRAPER_BASE_URL=http://scraper-svc:8001 \
-            SEARCH_BASE_URL=http://slopsearx:8080 \
+            SEARCH_BASE_URL=http://slopsearx-fixture:8080 \
             LLM_BASE_URL=http://llm-svc:8011/v1?run_id=${{ github.run_id }}-${{ github.run_attempt }} \
             TEST_SITE_BASE_URL=http://test-site:8000 \
             TIER3_FIXTURE_BASE_URL=http://tier3-fixture:8000 \
@@ -816,7 +831,9 @@ jobs:
 
       - name: Tear down stack
         if: always()
-        run: docker compose --profile indexing --profile fixture down --volumes --remove-orphans --timeout 30
+        run: |
+          docker compose --profile indexing --profile fixture --profile durable-research down --volumes --remove-orphans --timeout 30
+          rm -f "/tmp/groktocrawl-research-postgres-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.password"
 
   protected-browser-profile:
     name: Protected Browser Egress Profile

AGENTS.md

@@ -50,6 +50,10 @@ Architecture Decision Records (ADRs) live in `docs/adr/` and capture the context
 
 Jobs are processed with `asyncio.create_task()` inside the API process. Valkey persists job records, not execution: interrupted jobs are not resumed after restart, partial artifacts can remain, and undelivered webhooks are not replayed. `TaskTracker` provides only a short graceful-shutdown window. Restart-safe execution is deferred until it is an explicit product requirement; any future design must define ownership, leases, retries, cancellation, artifact consistency, and webhook idempotency in an ADR before selecting queue technology.
 
+### Durable research execution
+
+The separate `/experimental/research/v1` live run path uses server-configured search, acquisition, and model clients when its four existing feature flags are enabled and the explicit development fixture flag is false. PostgreSQL schema16 retains source bytes and audited report sets; Valkey retains scoped admission, owner generations, operation intents/receipts, and terminal projections. Do not replace a failed or unknown external operation with an unrecorded retry. The worker renews ownership; claim, publication, and cancellation serialize on a scoped PostgreSQL advisory lock, and deletion fences late publication. Recovery checks committed PostgreSQL results before invoking any adapter. Legacy `/v2` jobs keep their separate contract. Read [ADR-0070](docs/adr/0070-execute-live-durable-research.md) before changing these boundaries.
+
 ### Three-tier smart scraper
 
 Tier 1: Check `/llms.txt` at the site root (one GET, whole site in markdown)