Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions loopx/capabilities/explore/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,32 @@ attachments also require the same `input_revision`. Use a new finding id per
evaluated candidate, and reuse an id only to correct that finding. Missing values
are explicit `null`; zero and negative values remain valid measurements.

To reuse verified comparison conditions, save these three fields once in
`conditions.json`:

```json
{"comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1",
"evaluation_ref":"evaluator:v1"}
```

Pass `--measurement-context-json conditions.json` alongside
`--measurements-json observations.json`. Each observation then omits those
three fields, for example:

```json
[{"metric":"quality","value":9,"unit":"points","higher_is_better":true,
"candidate_ref":"candidate:b","evidence_ref":"result:b"}]
```

Ordinary and path-delta attachments support the same optional
`measurement_context` object alongside `measurements`. The writer expands it
into the existing complete records; it stores no mutable context reference.
Observation overrides, even equal ones, and incomplete or extra context fields
reject before recording. The original complete-item format remains supported.
Reusing a declaration does not verify comparability: select a different context
when task, data, evaluator, model or budget changes. Keep unknown conditions
separate. This convenience grants no evaluation, correction or execution authority.

Keep comparison conditions separate from the candidate and its result:

| Field | Meaning across successive findings |
Expand Down
21 changes: 21 additions & 0 deletions loopx/capabilities/explore/README.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,27 @@
每个受评测候选使用新 finding id,仅纠正同一结论时复用 id。未测量显式记为 `null`,
零分和负分仍为有效测量。

若要复用已核实的比较条件,将三个字段一次保存到 `conditions.json`:

```json
{"comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1",
"evaluation_ref":"evaluator:v1"}
```

同时传入 `--measurement-context-json conditions.json` 与
`--measurements-json observations.json`。此时每项观测省略这三个字段,例如:

```json
[{"metric":"quality","value":9,"unit":"points","higher_is_better":true,
"candidate_ref":"candidate:b","evidence_ref":"result:b"}]
```

普通附件及 path-delta 附件也支持在 `measurements` 旁提供可选
`measurement_context` 对象。写入前会展开成原有完整记录,不保存可变上下文引用。
观测重复填写比较字段(即使值相同)、上下文字段缺失或多余都会在记录前拒绝。
原有完整条目格式继续可用。复用声明不认证可比性;任务、数据、评测器、模型或预算
改变时另选上下文,条件未知时保持分开。此便捷输入不授予评测、纠正或执行权限。

比较条件与候选、单次结果分别记录:

| 字段 | 连续 finding 中的含义 |
Expand Down
12 changes: 9 additions & 3 deletions loopx/capabilities/explore/measurements.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,24 @@
from ...control_plane.effect_runtime import EffectRuntimeRejected, effect_runtime_result
from ...control_plane.runtime.public_safety import validate_public_safe_value

_UNSET = object()


@lru_cache(maxsize=512)
def _normalize(encoded: str) -> str:
try:
result = effect_runtime_result("explore.measurements.normalize", {"measurements": json.loads(encoded)})
result = effect_runtime_result("explore.measurements.normalize", json.loads(encoded))
except EffectRuntimeRejected as exc:
# Preserve the result-log reader's invalid-source contract. Runtime
# unavailability still propagates separately from malformed evidence.
raise ValueError(str(exc)) from exc
return json.dumps(result["measurements"], sort_keys=True, allow_nan=False)


def normalize_measurements(value):
def normalize_measurements(value, *, measurement_context=_UNSET):
validate_public_safe_value(value, path="measurements")
return json.loads(_normalize(json.dumps(value, sort_keys=True, allow_nan=False)))
request = {"measurements": value}
if measurement_context is not _UNSET:
validate_public_safe_value(measurement_context, path="measurement_context")
request["measurement_context"] = measurement_context
return json.loads(_normalize(json.dumps(request, sort_keys=True, allow_nan=False)))
13 changes: 11 additions & 2 deletions loopx/cli_commands/explore.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ def register_explore_commands(
finding.add_argument("--summary", help="Optional compact public-safe detail.")
finding.add_argument("--confidence", type=float, help="0..1 confidence.")
finding.add_argument("--measurements-json", help="JSON file containing one to eight sourced measurements for this node.")
finding.add_argument("--measurement-context-json", help="Reuse explicit comparison_scope, input_revision and evaluation_ref from this JSON file; --measurements-json then supplies only metric/result fields.")
_add_common_record_args(finding)

summary = sub.add_parser(
Expand Down Expand Up @@ -568,6 +569,15 @@ def handle_explore_command(
)
payload = _append_event_payload(event, runtime_root=runtime_root, goal_id=args.goal_id)
elif args.explore_command == "finding":
measurements = (json.loads(Path(args.measurements_json).read_text(encoding="utf-8"))
if args.measurements_json else None)
if args.measurement_context_json:
if measurements is None:
raise ValueError("--measurement-context-json requires --measurements-json")
from ..capabilities.explore.measurements import normalize_measurements

measurements = normalize_measurements(measurements, measurement_context=json.loads(
Path(args.measurement_context_json).read_text(encoding="utf-8")))
event = build_explore_finding_event(
goal_id=args.goal_id,
title=args.title,
Expand All @@ -581,8 +591,7 @@ def handle_explore_command(
evidence_refs=args.evidence_ref,
tags=args.tag,
supersedes=args.supersedes,
measurements=(json.loads(Path(args.measurements_json).read_text(encoding="utf-8"))
if args.measurements_json else None),
measurements=measurements,
)
payload = _append_event_payload(event, runtime_root=runtime_root, goal_id=args.goal_id)
elif args.explore_command == "observe":
Expand Down
20 changes: 18 additions & 2 deletions loopx/control_plane/capabilities/explore_measurements.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,26 @@ export interface ExploreMeasurementFeedback {
improvement: number | null; measured_count: number; unknown_count: number;
}

export function normalizeExploreMeasurements(value: unknown): ExploreMeasurement[] {
export function normalizeExploreMeasurements(value: unknown, context?: unknown): ExploreMeasurement[] {
if (!Array.isArray(value) || !value.length || value.length > 8) {
throw new EffectRuntimeRequestError("measurements requires one to eight explicit measurements");
}
if (context !== undefined) {
const shared = requireJsonObject(context, "measurement_context");
const keys = ["comparison_scope", "input_revision", "evaluation_ref"];
if (keys.some(key => !Object.hasOwn(shared, key)) || Object.keys(shared).some(key => !keys.includes(key))) {
throw new EffectRuntimeRequestError("measurement_context requires only comparison_scope, input_revision and evaluation_ref");
}
return normalizeExploreMeasurements(value.map(item => {
const observation = requireJsonObject(item, "measurement");
if (keys.some(key => Object.hasOwn(observation, key))) {
throw new EffectRuntimeRequestError("measurement cannot override its explicit comparison context");
}
// Authoring only: persist the existing complete measurement, never a
// mutable context reference or inferred comparison membership.
return {...shared, ...observation};
}));
}
const seen = new Set<string>();
return value.map(item => {
const row = requireJsonObject(item, "measurement");
Expand Down Expand Up @@ -107,5 +123,5 @@ export function projectExploreMeasurementFeedback(findings: readonly JsonObject[
}

export function normalizeExploreMeasurementsRequest(params: JsonObject): JsonObject {
return {measurements: normalizeExploreMeasurements(params.measurements)};
return {measurements: normalizeExploreMeasurements(params.measurements, params.measurement_context)};
}
11 changes: 8 additions & 3 deletions loopx/control_plane/capabilities/explore_result_writeback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ export function exploreResultWritebackAffordance(): JsonObject {
"Keep raw logs local.",
"Attach measurements when an evaluation returns quantitative evidence: include its metric, unit, explicit direction, candidate and result reference. Use the same comparison_scope only for the same task, evaluator, model/resource budget and evaluation conditions; separate local proxies from native results.",
"input_revision identifies the evaluated task/data revision; evaluation_ref identifies the evaluator/scoring protocol version. Keep both stable across candidates only when those conditions are verified unchanged. Put candidate code/artifact revisions in candidate_ref and individual evaluation/submission receipts in evidence_ref, not in the comparison keys. The measurement input revision and evidence reference must match this finding. If comparability is unknown, keep results separate and explain the uncertainty; never invent a shared version to obtain a delta.",
"To reuse explicitly verified conditions, optional measurement_context supplies comparison_scope, input_revision and evaluation_ref for observations that omit those fields; conflicting fields reject. CLI finding accepts the same context via --measurement-context-json. This copies declared conditions, not proof of comparability.",
"Use null for an unmeasured value. Read measurement_feedback before repeating a measured route: compare the latest recorded candidate with its same-scope best, investigate regressions and retain useful alternatives; a score does not itself close or authorize work.",
"When an experiment has execution evidence, attach execution_observation: preserve its exact candidate and input version, planned versus executed method, change reason and measured resources. Missing measurements are null, not zero. A finding source digest is not an executable digest. Result evidence must name the candidate it actually evaluated, including delayed or regressing results; it does not certify current workspace quality or Goal completion.",
"Observations allow 320 characters, interpretations 1200; the complete scoped summary is preserved within 2000 characters.",
Expand All @@ -83,7 +84,7 @@ export function exploreResultWritebackAffordance(): JsonObject {
}

const FIELDS = ["schema_version", "node_id", "question", "applicability", "input_revision",
"observation", "interpretation", "status", "evidence_refs", "question_update", "relations", "execution_observation", "measurements"];
"observation", "interpretation", "status", "evidence_refs", "question_update", "relations", "execution_observation", "measurements", "measurement_context"];
// Reuse the persisted Explore node/edge vocabulary, distinct from finding status.
const QUESTION_STATUSES = ["open", "exploring", "blocked", "resolved", "dead_end"] as const;
const RELATION_TYPES = ["subtopic_of", "depends_on", "answers", "supports", "refutes", "leads_to"] as const;
Expand Down Expand Up @@ -166,7 +167,11 @@ function normalizeAttachment(value: unknown, visionPacket?: unknown, linkedScope
});
const execution = Object.hasOwn(row, "execution_observation")
? normalizeExploreExecutionObservation({observation: row.execution_observation}) : null;
const measurements = Object.hasOwn(row, "measurements") ? normalizeExploreMeasurements(row.measurements) : null;
if (Object.hasOwn(row, "measurement_context") && !Object.hasOwn(row, "measurements")) {
throw new EffectRuntimeRequestError("measurement_context requires measurements");
}
const measurements = Object.hasOwn(row, "measurements")
? normalizeExploreMeasurements(row.measurements, row.measurement_context) : null;
if (measurements?.some(item => item.input_revision !== text(row.input_revision, "input_revision", 160)
|| !refs.includes(item.evidence_ref))) {
throw new EffectRuntimeRequestError("measurements must retain this finding's input revision and evidence references");
Expand All @@ -189,7 +194,7 @@ function normalizeAttachment(value: unknown, visionPacket?: unknown, linkedScope
}
/** Explicit reference, not inferred evidence. Preserve every route item verbatim. */
function attachmentFromPathDelta(row: JsonObject, visionPacket: unknown, linkedScope: unknown): JsonObject {
const fields = ["schema_version", "node_id", "question", "applicability", "input_revision", "status", "evidence_refs", "question_update", "relations", "execution_observation", "measurements"];
const fields = ["schema_version", "node_id", "question", "applicability", "input_revision", "status", "evidence_refs", "question_update", "relations", "execution_observation", "measurements", "measurement_context"];
if (Object.keys(row).some(key => !fields.includes(key))) {
throw new EffectRuntimeRequestError("Explore path_delta attachment contains unknown fields");
}
Expand Down
87 changes: 87 additions & 0 deletions tests/control_plane/test_explore_measurements.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,93 @@ def context(args):
("registry_path", "runtime_root", "goal_id", "agent_id")})


@pytest.mark.parametrize("provider", ["file", "sqlite"])
def test_cli_reuses_explicit_context_and_preserves_normalized_history(tmp_path, monkeypatch, provider):
from canonical_authority_fixture import isolate_sqlite_runtime, promoted_create_fixture

isolate_sqlite_runtime(tmp_path, monkeypatch)
source, runtime, _state = promoted_create_fixture(tmp_path, provider=provider)
config = json.loads(source.read_text())
config["goals"][0]["explore_graph"] = {"enabled": True}
source.write_text(json.dumps(config))
args = dict(registry_path=source, runtime_root=runtime, goal_id="goal-a", agent_id="agent-a")
command = [sys.executable, "-m", "loopx.cli", "--format", "json", "--registry", str(source),
"--runtime-root", str(runtime), "explore", "finding", "--goal-id", "goal-a",
"--agent-id", "agent-a", "--node", "comparison", "--status", "tentative"]
shared = {key: measurement("first", 8)[key]
for key in ("comparison_scope", "input_revision", "evaluation_ref")}
conditions = tmp_path / "conditions.json"
conditions.write_text(json.dumps(shared))
observations = tmp_path / "observations.json"
log = explore_result_log_path(runtime, "goal-a")
for index, score in enumerate([8, 9, 6, None]):
row = {key: value for key, value in measurement(str(index), score).items() if key not in shared}
observations.write_text(json.dumps([row]))
captured = subprocess.run(command + ["--title", f"Measured candidate {index}",
"--finding-id", f"candidate-{index}", "--evidence-ref", row["evidence_ref"],
"--measurements-json", str(observations), "--measurement-context-json", str(conditions)],
capture_output=True, text=True)
assert captured.returncode == 0, captured.stdout + captured.stderr
events = load_explore_result_events_strict(log, goal_id="goal-a")
assert events[-1]["measurements"] == [measurement(str(index), score)]
assert "measurement_context" not in events[-1]
feedback = context(args)["graph"]["measurement_feedback"][0]
if index == 2:
assert feedback["improvement"] == -3
assert feedback["best"]["candidate_ref"] == "candidate:1"
assert feedback["latest"]["measurement"]["value"] is None
assert feedback["best"]["value"] == 9
assert feedback["measured_count"] == 3
assert feedback["unknown_count"] == 1
assert conditions.read_text() == json.dumps(shared), "recording cannot rewrite the caller's conditions"
healthy = log.read_bytes()
# Even an equal duplicate field must not silently override the shared source.
observations.write_text(json.dumps([{**row, "evaluation_ref": shared["evaluation_ref"]}]))
rejected = subprocess.run(command + ["--title", "Conflicting candidate", "--evidence-ref", row["evidence_ref"],
"--measurements-json", str(observations), "--measurement-context-json", str(conditions)],
capture_output=True, text=True)
assert rejected.returncode != 0
assert log.read_bytes() == healthy
rejected = subprocess.run(command + ["--title", "Missing observations",
"--measurement-context-json", str(conditions)], capture_output=True, text=True)
assert rejected.returncode != 0
assert log.read_bytes() == healthy
conditions.write_text(json.dumps({**shared, "evaluation_ref": "evaluator:changed"}))
observations.write_text(json.dumps([{**row, "value": 99,
"candidate_ref": "candidate:changed", "evidence_ref": "result:changed"}]))
changed = subprocess.run(command + ["--title", "Changed evaluation conditions",
"--finding-id", "changed-condition", "--evidence-ref", "result:changed",
"--measurements-json", str(observations), "--measurement-context-json", str(conditions)],
capture_output=True, text=True)
assert changed.returncode == 0, changed.stdout + changed.stderr
groups = context(args)["graph"]["measurement_feedback"]
assert len(groups) == 2
assert next(group for group in groups if group["evaluation_ref"] == "evaluator:v1")["best"]["value"] == 9
assert next(group for group in groups if group["evaluation_ref"] == "evaluator:changed")["improvement"] is None
assert log.read_bytes().startswith(healthy), "new conditions cannot rewrite earlier evidence"


def test_context_attachment_replays_as_the_equivalent_complete_measurement(tmp_path):
args = fixture(tmp_path)
current = payload()
complete = measurement("one", 8)
conditions = {key: complete[key] for key in ("comparison_scope", "input_revision", "evaluation_ref")}
observation = {key: value for key, value in complete.items() if key not in conditions}
current["explore_result"]["measurement_context"] = conditions
current["explore_result"]["measurements"] = [observation]
prepared = prepare_result_attachment(current["explore_result"], **args)
first = deliver_result_attachment(payload=current, **args)
assert first["ok"], first
log = explore_result_log_path(args["runtime_root"], args["goal_id"])
original = log.read_bytes()
current["explore_result"].pop("measurement_context")
current["explore_result"]["measurements"] = [complete]
repeated = deliver_result_attachment(payload=current, **args)
assert repeated["ok"] and repeated["idempotent_replay"]
assert log.read_bytes() == original
assert prepared["measurements"] == [complete]


@pytest.mark.parametrize("provider", ["file", "sqlite"])
def test_published_example_compares_candidates_but_isolates_changed_conditions(tmp_path, monkeypatch, provider):
from canonical_authority_fixture import isolate_sqlite_runtime, promoted_create_fixture
Expand Down
Loading
Loading