Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 129 additions & 2 deletions autoresearch/prefill/supervisor.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,118 @@ def validate_candidate(candidate: dict) -> None:
raise ValueError("candidate must forbid fallback")


def _select_repair_target(current: dict, ledger: dict) -> str:
leaves = _pending_leaf_ids(ledger)
if not leaves:
raise ValueError("proof ledger has no unresolved leaf")
current_target = str(current.get("target_obligation_id", ""))
if current_target in leaves:
return current_target
parents = {
str(item.get("obligation_id", "")): str(item.get("parent_id", ""))
for item in ledger.get("obligations", [])
}

def distance_from_current(obligation_id: str) -> int:
distance = 0
cursor = obligation_id
visited = set()
while cursor and cursor not in visited:
if cursor == current_target:
return distance
visited.add(cursor)
cursor = parents.get(cursor, "")
distance += 1
return -1

descendants = [
(distance_from_current(obligation_id), obligation_id)
for obligation_id in leaves
if distance_from_current(obligation_id) >= 0
]
if descendants:
return max(descendants)[1]
return leaves[0]


def repair_candidate_schema(
candidate: dict,
*,
current: dict,
ledger: dict,
) -> tuple[dict, list[str]]:
aliases = {
"candidate_id": ("id", "strategy_id"),
"target_obligation_id": (
"target", "target_id", "obligation_id",
),
"hypothesis": ("strategy_hypothesis",),
"generator_directive": (
"generator", "generator_prompt", "generator_strategy",
),
"critic_directive": (
"critic", "critic_prompt", "critic_strategy",
),
"prefill_compute_chunk_tokens": (
"chunk_tokens", "compute_chunk_tokens",
),
}
normalized = {
str(key).strip().lower().replace("-", "_"): value
for key, value in candidate.items()
}
repaired = dict(candidate)
changed: list[str] = []
for field, field_aliases in aliases.items():
if repaired.get(field):
continue
for alias in (field, *field_aliases):
value = normalized.get(alias)
if value not in (None, ""):
repaired[field] = value
changed.append(field)
break
target = str(repaired.get("target_obligation_id", ""))
leaves = _pending_leaf_ids(ledger)
if target not in leaves:
repaired["target_obligation_id"] = _select_repair_target(
current,
ledger,
)
if "target_obligation_id" not in changed:
changed.append("target_obligation_id")
target = repaired["target_obligation_id"]
statement = next(
str(item.get("statement", ""))
for item in ledger.get("obligations", [])
if item.get("obligation_id") == target
)
hypothesis = str(repaired.get("hypothesis", "")).strip()
if not repaired.get("generator_directive") and hypothesis:
repaired["generator_directive"] = (
f"Focus exclusively on {target}: {statement} "
f"Construct and test this hypothesis: {hypothesis}"
)
changed.append("generator_directive")
if not repaired.get("critic_directive") and hypothesis:
repaired["critic_directive"] = (
f"Attempt to falsify the {target} hypothesis: {hypothesis} "
"Identify the first invalid inference and one strictly smaller "
"missing lemma."
)
changed.append("critic_directive")
if not repaired.get("prefill_compute_chunk_tokens"):
repaired["prefill_compute_chunk_tokens"] = int(
current["prefill_compute_chunk_tokens"],
)
changed.append("prefill_compute_chunk_tokens")
else:
repaired["prefill_compute_chunk_tokens"] = int(
repaired["prefill_compute_chunk_tokens"],
)
return repaired, changed


def render_candidate(candidate: dict) -> str:
validate_candidate(candidate)
return (
Expand Down Expand Up @@ -319,9 +431,24 @@ def propose_candidate(
raise RuntimeError(
f"strategy agent did not reach EOS: {session.last_stop_reason}",
)
candidate = _extract_json(
tokenizer.decode(generated, skip_special_tokens=True),
strategy_output = tokenizer.decode(generated, skip_special_tokens=True)
print(
f"[autoresearch] Strategy Output: {strategy_output.strip()}",
flush=True,
)
candidate = _extract_json(strategy_output)
candidate, repaired_fields = repair_candidate_schema(
candidate,
current=current,
ledger=ledger,
)
if repaired_fields:
print(
"[autoresearch] phase=strategy-schema-repair "
f"fields={','.join(sorted(set(repaired_fields)))} "
f"target={candidate.get('target_obligation_id', '')}",
flush=True,
)
candidate.update({
"snapshot_mode": "final_only",
"max_segment_seconds": 300.0,
Expand Down
79 changes: 79 additions & 0 deletions tests/inference_engine/bench/test_autoresearch_supervisor.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
deploy_candidate,
parse_research_verdict,
read_results,
repair_candidate_schema,
render_candidate,
should_keep,
StrategyPrefillHeartbeat,
Expand Down Expand Up @@ -46,6 +47,84 @@ def test_candidate_render_is_executable_and_strict(tmp_path):
raise AssertionError("fallback candidate must be rejected")


def test_strategy_schema_repair_prefers_current_branch_leaf():
current = _candidate()
ledger = {"obligations": [
{
"obligation_id": "RH-C1",
"statement": "Operator construction.",
"status": "UNRESOLVED",
"parent_id": "",
},
{
"obligation_id": "RH-C1-child",
"statement": "Prove the operator is self-adjoint.",
"status": "UNRESOLVED",
"parent_id": "RH-C1",
},
{
"obligation_id": "RH-C2",
"statement": "Zero convergence.",
"status": "UNRESOLVED",
"parent_id": "",
},
]}
repaired, fields = repair_candidate_schema(
{
"candidate_id": "trial-child",
"hypothesis": "The proposed domain yields a symmetric operator.",
},
current=current,
ledger=ledger,
)
assert repaired["target_obligation_id"] == "RH-C1-child"
assert repaired["prefill_compute_chunk_tokens"] == 256
assert "RH-C1-child" in repaired["generator_directive"]
assert "strictly smaller" in repaired["critic_directive"]
assert set(fields) == {
"target_obligation_id",
"generator_directive",
"critic_directive",
"prefill_compute_chunk_tokens",
}


def test_strategy_schema_repair_accepts_uppercase_and_alias_keys():
repaired, fields = repair_candidate_schema(
{
"CANDIDATE_ID": "alias-trial",
"TARGET": "RH-C2",
"HYPOTHESIS": "Test convergence.",
"GENERATOR_PROMPT": "Construct the approximation.",
"CRITIC_PROMPT": "Falsify the approximation.",
"CHUNK_TOKENS": "128",
},
current=_candidate(),
ledger={"obligations": [{
"obligation_id": "RH-C2",
"statement": "Zero convergence.",
"status": "UNRESOLVED",
"parent_id": "",
}]},
)
validate_candidate({
**repaired,
"snapshot_mode": "final_only",
"require_full_context": True,
"allow_fallback": False,
})
assert repaired["candidate_id"] == "alias-trial"
assert repaired["prefill_compute_chunk_tokens"] == 128
assert set(fields) == {
"candidate_id",
"target_obligation_id",
"hypothesis",
"generator_directive",
"critic_directive",
"prefill_compute_chunk_tokens",
}


def test_keep_requires_novel_mathematical_advancement():
baseline = {
"proof_obligations_unresolved": "5",
Expand Down
Loading