feat(e2e): compare PowerContext off and on for Bub continuation workloads

Makefile

@@ -97,6 +97,11 @@ harness-acceptance: ## Evaluate workloads by ID or category against an existing
 	@uv run --project e2e/bub powercontext-e2e acceptance \
 		--output "$${POWERCONTEXT_E2E_OUTPUT:-e2e/bub/results}" $(ARGS)
 
+.PHONY: harness-paired
+harness-paired: ## Compare PowerContext off and on for continuation workloads against an existing Server.
+	@uv run --project e2e/bub powercontext-e2e paired \
+		--output "$${POWERCONTEXT_E2E_OUTPUT:-e2e/bub/results/paired}" $(ARGS)
+
 .PHONY: harness-rescore
 harness-rescore: ## Rescore REPLAY without rerunning Bub or PowerContext.
 	@test -n "$${REPLAY:-}" || { echo "REPLAY is required" >&2; exit 2; }

e2e/bub/README.md

@@ -31,6 +31,7 @@ summary.
 ```text
 e2e/bub/
   tasks/                  # PowerContext manifests and evaluation expectations
+  paired-tasks/           # OFF/ON continuation manifests for the paired command
   harbor-tasks/           # Local Harbor tasks used by built-in samples
   src/powercontext_e2e/   # One Harbor runner and one Memory evaluator
 ```
@@ -139,6 +140,71 @@ evaluation and report at `batch-<name>/`. `collect-all` reports every failed tas
 Harbor trial at its first failed step. Runtime batch steps are flat and task-prefixed. Each agent invocation starts an
 independent ACP session and Bub tape.
 
+## Compare PowerContext off and on
+
+A continuation workload is a Harbor multi-step task written in plain language, so any agent host can run it. An
+earlier session mentions a fact only in the conversation, next to an unrelated small job. The final recall session
+asks for that fact and has the agent write its answer to a file as structured values, so the grader checks what the
+answer asserts rather than keywords that a contradictory or hedged answer could also contain. The recall step's own
+tests grade the answer, and the answer key lives only there, because Harbor leaves every uploaded test directory in
+the container for later steps. The recall step's own reward decides the run whatever the task's multi-step reward
+strategy; earlier steps' rewards are recorded for diagnosis only. A task may not set `min_reward` on an earlier
+step, because Harbor would then skip the recall step when that step's unrelated job falls short.
+
+The `paired` command runs each selected workload with PowerContext off and on, in separate containers, and repeats
+this for `--trials` trials. The arm that runs first alternates between trials.
+
+- OFF installs the host without its PowerContext integration and passes no `POWERCONTEXT_*` settings.
+- ON installs the integration bound to a new Scope. For Bub this means the plugin with `capture_events` enabled, so
+  that, like the other host integrations, it captures what the user says without relying on the model to call a
+  memory tool. This is not the plugin's default setting.
+- Everything else is the same in both arms: image, host version, model, and budget.
+
+After each ON session the harness records the Scope's Server statistics. When another session follows, it first
+flushes the Scope, standing in for the time that passes between real sessions, and repeats the flush until the Scope
+has processed every captured Source, a flush makes no progress, or 20 rounds pass. This runs from a Harbor agent-end
+hook after the agent's timed phase, so it does not use the agent's time budget. A failed flush or statistics read is
+recorded as a treatment failure rather than replacing the agent's own outcome, so a timed-out session still counts as
+a timeout. Host plugins flush on different schedules, so the harness flushes the same way for every host. The
+Server's generation model therefore takes part in the ON arm; the run fails early when the Server does not report
+`memory_extraction`.
+
+An ON run counts only when Server statistics for its Scope show that Sources were captured before the recall session
+and that the integration asked PowerContext for context during it. Otherwise it is an integration failure. Whether a
+flush creates Memory and whether recall returns content are PowerContext's own behavior, so the snapshots record them
+but a run that gets nothing useful still counts as an ON attempt.
+Integration failures and harness or infrastructure errors are reported but left out of success rates and paired
+differences. An agent timeout counts as a failed attempt in either arm.
+
+The harness Client waits for each flush, which runs the Server's generation model, so raise its 10-second default
+timeout; the Bub plugin also flushes during a session.
+
+```bash
+export POWERCONTEXT_CLIENT_SERVER_URL=http://127.0.0.1:8000
+export POWERCONTEXT_CLIENT_TIMEOUT=150
+export POWERCONTEXT_BUB_BASE_URL=http://host-gateway:8000
+export POWERCONTEXT_BUB_TIMEOUT=150
+export POWERCONTEXT_BUB_TRUST_TRANSPORT_SECURITY=true
+export BUB_MODEL=openrouter:openai/gpt-5.4
+export BUB_API_KEY="$OPENROUTER_API_KEY"
+make harness-paired ARGS='--trials 2'
+```
+
+Each arm writes `observation.json`, which includes the per-session Server snapshots for ON, and its Harbor jobs:
+
+```text
+<output>/
+  paired-report.json
+  report.md
+  <workload-id>/trial-<n>/<off|on>/
+    observation.json
+    harbor-jobs/
+```
+
+The command exits non-zero when any arm could not be scored; a task that fails in either arm is a result, not a
+command failure. The report is marked preliminary. It does not yet estimate uncertainty, check the Default Scope for
+leaks, record latency or token usage, or run in the fixed Compose harness.
+
 ## Long-horizon task
 
 The Terminal-Bench manifest pins its task checksum, model requirement, step budget, capture cadence, recall probes,
@@ -176,11 +242,14 @@ from that container. In the fixed nested-container harness, `host-gateway` addre
 proxy exposed there can be passed as `http://host-gateway:<port>`. The typed setting is also treated as a secret when
 evidence is written.
 
-Agent setup uses Bub's supported installation path: `uv tool install` installs Bub with the local PowerContext plugin,
-then `bub install bub-acp-server` adds the ACP server to the same environment. Harbor uploads and runs its native ACP
-client. The Terminal-Bench task keeps its original image, setup, verifier, and isolation boundary. The harness ignores
-dataset CPU and memory limits because it evaluates Memory behavior rather than benchmark resource compliance. This
-also keeps the fixed harness usable in nested container runtimes that cannot create additional cgroups.
+The agent container sees only the repository files that installation needs: the `powercontext` package and the host
+integration. Workload files, answer keys, and benchmark data stay on the host, because the agent can search its
+container. Agent setup uses Bub's supported installation path: `uv tool install` installs Bub with the local
+PowerContext plugin, then `bub install bub-acp-server` adds the ACP server to the same environment. Harbor uploads and
+runs its native ACP client. The Terminal-Bench task keeps its original image, setup, verifier, and isolation boundary.
+The harness ignores dataset CPU and memory limits because it evaluates Memory behavior rather than benchmark resource
+compliance. This also keeps the fixed harness usable in nested container runtimes that cannot create additional
+cgroups.
 
 Long-horizon acceptance requires observable Memory behavior:
 

e2e/bub/harbor-tasks/project-decision-continuation/environment/Dockerfile

@@ -0,0 +1,19 @@
+# Copyright (c) 2026 OceanBase.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.12-slim-bookworm
+
+WORKDIR /workspace
+
+RUN printf '# Ticket service\n\nThe service will recieve tickets from the support queue.\n' > /workspace/README.md

e2e/bub/harbor-tasks/project-decision-continuation/steps/capture/instruction.md

@@ -0,0 +1,6 @@
+Some context before we start: last week the team decided that the ticket service database will run on OceanBase,
+with the shard count fixed at 12, because it has to stay compatible with our existing MySQL clients. No action is
+needed on that yet.
+
+For now, please fix the spelling mistake "recieve" in /workspace/README.md; it should be "receive". Do not change
+anything else.

e2e/bub/harbor-tasks/project-decision-continuation/steps/capture/tests/test.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Copyright (c) 2026 OceanBase.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+# Diagnostic only: the final step's reward decides the trial.
+if grep -q 'receive' /workspace/README.md && ! grep -q 'recieve' /workspace/README.md; then
+    echo 1 > /logs/verifier/reward.txt
+else
+    echo 0 > /logs/verifier/reward.txt
+fi

e2e/bub/harbor-tasks/project-decision-continuation/steps/recall/instruction.md

@@ -0,0 +1,3 @@
+Which database did we decide to use for the ticket service, and what shard count did we agree on? Write your answer
+to /workspace/answer.json as a JSON object with only two keys, a `database` string and a `shard_count` integer, for
+example `{"database": "<name>", "shard_count": <number>}`. Use null for any value you do not know.

e2e/bub/harbor-tasks/project-decision-continuation/steps/recall/tests/grade.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2026 OceanBase.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Grade the recall answer without depending on the agent host.
+
+The answer is structured, so the grader checks the values the agent asserts rather than matching keywords that a
+contradictory or hedged answer could also contain. This file is uploaded only with the recall step's tests, so no
+earlier session can read the expected answer.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import unicodedata
+from pathlib import Path
+
+EXPECTED_DATABASE = "oceanbase"
+EXPECTED_SHARD_COUNT = 12
+ANSWER_KEYS = {"database", "shard_count"}
+
+
+class DuplicateKeyError(ValueError):
+    """A repeated key would let a later value silently override a contradictory earlier one."""
+
+
+def score(answer: str) -> int:
+    try:
+        payload = json.loads(answer, object_pairs_hook=_unique_keys)
+    except ValueError:
+        return 0
+    # Extra keys could carry a hedge or an alternative that the checked fields do not show.
+    if not isinstance(payload, dict) or set(payload) != ANSWER_KEYS:
+        return 0
+    database = payload.get("database")
+    shard_count = payload.get("shard_count")
+    return int(
+        isinstance(database, str)
+        and unicodedata.normalize("NFKC", database).strip().casefold() == EXPECTED_DATABASE
+        and _integer(shard_count) == EXPECTED_SHARD_COUNT
+    )
+
+
+def _unique_keys(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    keys = [key for key, _ in pairs]
+    if len(keys) != len(set(keys)):
+        raise DuplicateKeyError
+    return dict(pairs)
+
+
+def _integer(value: object) -> int | None:
+    if isinstance(value, int) and not isinstance(value, bool):
+        return value
+    if isinstance(value, str) and value.strip().isdecimal():
+        return int(value.strip())
+    return None
+
+
+def main(answer_path: Path, reward_path: Path) -> None:
+    answer = answer_path.read_text(encoding="utf-8", errors="replace") if answer_path.is_file() else ""
+    reward_path.write_text(f"{score(answer)}\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main(Path(sys.argv[1]), Path(sys.argv[2]))

e2e/bub/harbor-tasks/project-decision-continuation/steps/recall/tests/test.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+# Copyright (c) 2026 OceanBase.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+python3 /tests/grade.py /workspace/answer.json /logs/verifier/reward.txt

e2e/bub/harbor-tasks/project-decision-continuation/task.toml

@@ -0,0 +1,31 @@
+# Copyright (c) 2026 OceanBase.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version = "1.3"
+multi_step_reward_strategy = "final"
+
+[agent]
+timeout_sec = 600.0
+
+[verifier]
+timeout_sec = 60.0
+
+[environment]
+build_timeout_sec = 300.0
+
+[[steps]]
+name = "capture"
+
+[[steps]]
+name = "recall"

e2e/bub/paired-tasks/project-decision-continuation.yaml

@@ -0,0 +1,30 @@
+# Copyright (c) 2026 OceanBase.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+schema: powercontext.e2e-task/v1
+id: project-decision-continuation
+categories:
+  - paired
+  - sample
+dataset:
+  path: e2e/bub/harbor-tasks
+  task_id: project-decision-continuation
+  checksum: 87429c6ac133a99343d72e8fc614fb2f18d773a83c318097ec085a610bb3315e
+execution:
+  type: bub
+  model: true
+  max_steps: 30
+  max_tokens: 16384
+evaluation:
+  recall_step: recall

e2e/bub/src/powercontext_e2e/__main__.py

@@ -84,6 +84,24 @@ def main() -> None:
         help="Continue through case failures or stop the Harbor trial at the first failed step.",
     )
 
+    paired_parser = subparsers.add_parser("paired", help="Compare PowerContext off and on for continuation workloads.")
+    paired_parser.add_argument("--manifest", type=Path, default=Path("e2e/bub/paired-tasks"))
+    paired_parser.add_argument(
+        "--id",
+        action="append",
+        default=[],
+        metavar="WORKLOAD_ID",
+        help="Select one workload; repeat to select more than one. Defaults to the paired category.",
+    )
+    paired_parser.add_argument(
+        "--category",
+        action="append",
+        default=[],
+        help="Select one category; repeat to select more than one.",
+    )
+    paired_parser.add_argument("--trials", type=int, default=2, help="Trials per arm; the arm order alternates.")
+    paired_parser.add_argument("--output", type=Path, required=True)
+
     rescore_parser = subparsers.add_parser("rescore")
     rescore_parser.add_argument("replay", type=Path)
     rescore_parser.add_argument("--output", type=Path, required=True)
@@ -94,6 +112,19 @@ def main() -> None:
         from .rescore import rescore_replay
 
         passed = rescore_replay(args.replay, args.output, settings)
+    elif args.command == "paired":
+        from .catalog import load_tasks, select_tasks
+        from .paired import run_paired
+
+        selected = select_tasks(
+            load_tasks(args.manifest),
+            ids=tuple(args.id),
+            categories=tuple(args.category) or (() if args.id else ("paired",)),
+        )
+        report = asyncio.run(run_paired(selected, output_dir=args.output, settings=settings, trials=args.trials))
+        # The run is valid when every arm produced a measurement; the task outcome itself is the result.
+        unscored = report.total.off.errors + report.total.on.errors + report.total.on.integration_failures
+        passed = unscored == 0
     else:
         from .catalog import load_tasks, select_tasks
         from .runner import run_tasks

e2e/bub/src/powercontext_e2e/catalog.py

@@ -123,14 +123,24 @@ class OutcomeEvaluationSpec(CatalogModel):
     expected_execution: ExpectedExecutionSpec
 
 
+class ContinuationEvaluationSpec(CatalogModel):
+    """Compare PowerContext off and on for a task whose final session depends on an earlier one.
+
+    The Harbor task's own verifier grades the final step. ``recall_step`` names that step so the harness can check
+    that PowerContext supplied context during it.
+    """
+
+    recall_step: str = Field(min_length=1)
+
+
 class E2ETask(CatalogModel):
     schema_: Literal["powercontext.e2e-task/v1"] = Field(alias="schema")
     id: str = Field(pattern=r"^[a-z0-9][a-z0-9_-]*$")
     categories: tuple[str, ...] = Field(min_length=1)
     provenance: Provenance | None = None
     dataset: HarborDatasetSpec
     execution: BubExecutionSpec
-    evaluation: MemoryEvaluationSpec | OutcomeEvaluationSpec
+    evaluation: MemoryEvaluationSpec | OutcomeEvaluationSpec | ContinuationEvaluationSpec
 
 
 class TaskSelectionError(ValueError):