diff --git a/benchmarks/factory_harbor_agent.py b/benchmarks/factory_harbor_agent.py index 82ee482e2..637734ff4 100644 --- a/benchmarks/factory_harbor_agent.py +++ b/benchmarks/factory_harbor_agent.py @@ -277,7 +277,14 @@ async def install(self, environment: BaseEnvironment) -> None: "if command -v apk &> /dev/null; then" " apk add --no-cache curl bash nodejs npm procps git;" " elif command -v apt-get &> /dev/null; then" - " apt-get update && apt-get install -y curl procps git;" + " if ! command -v curl &> /dev/null || ! command -v git &> /dev/null; then" + " sed -i 's|http://mirrors.aliyun.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list 2>/dev/null || true;" + " sed -i 's|http://mirrors.cloud.aliyuncs.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list 2>/dev/null || true;" + " rm -f /etc/apt/trusted.gpg.d/* 2>/dev/null || true;" + " apt-get clean; rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* 2>/dev/null || true;" + " apt-get -o Acquire::AllowInsecureRepositories=true -o Acquire::AllowDowngradeToInsecureRepositories=true update;" + " apt-get --allow-unauthenticated --no-install-recommends install -y curl procps git || true;" + " fi;" " elif command -v yum &> /dev/null; then" " yum install -y curl procps-ng git;" " fi" diff --git a/factory/cli/_helpers.py b/factory/cli/_helpers.py index 41bee78d2..b79c41b9d 100644 --- a/factory/cli/_helpers.py +++ b/factory/cli/_helpers.py @@ -39,6 +39,7 @@ "frontend-design-scan", "evolve", "deep-research", + "outer-loop", ] @@ -55,6 +56,7 @@ "study", "swebench", "frontend-design-scan", + "outer-loop", ] diff --git a/factory/cli/_parser_groups.py b/factory/cli/_parser_groups.py index cf1253a4a..09e7b7e88 100644 --- a/factory/cli/_parser_groups.py +++ b/factory/cli/_parser_groups.py @@ -457,6 +457,22 @@ def add_entry_point_parsers(sub: argparse._SubParsersAction) -> None: # type: i "tool (CEO drives via factory workflow tool commands), " "deterministic (headless WorkflowExecutor, no CEO)") + # Outer-loop specific arguments + p.add_argument("--benchmark", default=None, + help="Benchmark adapter for outer-loop fitness evaluation (required for --mode outer-loop)") + p.add_argument("--budget", type=int, default=None, dest="ol_budget", + help="Max evaluations for outer-loop evolution (default: 100)") + p.add_argument("--population", type=int, default=None, + help="Population size per generation for outer-loop (default: 4)") + p.add_argument("--target-score", type=float, default=None, dest="target_score", + help="Optional early stop score for outer-loop evolution") + p.add_argument("--seed", default=None, dest="seed_mode", + help="Existing mode name to seed the outer-loop from (default: benchmark's contributed workflow)") + p.add_argument("--training-instances", default=None, dest="training_instances", + help="Comma-separated problem IDs for evolution (required for --mode outer-loop)") + p.add_argument("--holdout-instances", default=None, dest="holdout_instances", + help="Comma-separated held-out problem IDs for overfit detection (required for --mode outer-loop)") + p = sub.add_parser("run", help="Run factory cycle (delegates to CEO agent)") p.add_argument("path", help="Project path, GitHub URL, idea file path, or prompt") p.add_argument( diff --git a/factory/cli/_task_builder.py b/factory/cli/_task_builder.py index 1f34505d5..d6f343d4e 100644 --- a/factory/cli/_task_builder.py +++ b/factory/cli/_task_builder.py @@ -65,6 +65,13 @@ def _mode_suffix(mode: str, discover_only: bool) -> str: "with structural graph context included. " "Terminal mode — does not chain to other modes." ), + "outer-loop": ( + "\n\nRun Outer Loop mode: evolutionary swarm search for workflow optimization. " + "Seed a population from the specified mode, evolve via mutation + selection + " + "novelty filtering + LLM-guided design, evaluate against benchmark instances. " + "Post-evolution: holdout audit for overfit detection, export best workflow. " + "The full step-by-step playbook is in your system prompt above." + ), } if mode == "discover": if discover_only: diff --git a/factory/outer_loop/__init__.py b/factory/outer_loop/__init__.py new file mode 100644 index 000000000..9dfe12eac --- /dev/null +++ b/factory/outer_loop/__init__.py @@ -0,0 +1,63 @@ +"""Outer loop — evolutionary swarm search for workflow optimization.""" + +from factory.outer_loop.designer import DesignerAgent, extract_telemetry +from factory.outer_loop.engine import BudgetTracker, SwarmEngine +from factory.outer_loop.filesystem import ( + export_best_workflow, + init_filesystem, + load_checkpoint, + load_config, + save_best, + save_checkpoint, + save_generation, + save_map_elites, +) +from factory.outer_loop.direct_evaluator import DirectFeatureBenchEvaluator +from factory.outer_loop.harbor_evaluator import ( + HarborEvaluator, + create_seed_workflow, + workflow_to_harbor_yaml, +) +from factory.outer_loop.models import ( + AuditResult, + EvalResult, + GenerationSummary, + HyperparameterRecord, + Individual, + MutationRecord, + MutationType, + OuterLoopResult, + OuterLoopState, + SwarmConfig, +) +from factory.outer_loop.workflow import outer_loop_workflow + +__all__ = [ + "AuditResult", + "BudgetTracker", + "DirectFeatureBenchEvaluator", + "DesignerAgent", + "EvalResult", + "GenerationSummary", + "HarborEvaluator", + "HyperparameterRecord", + "Individual", + "MutationRecord", + "MutationType", + "OuterLoopResult", + "OuterLoopState", + "SwarmConfig", + "SwarmEngine", + "create_seed_workflow", + "export_best_workflow", + "extract_telemetry", + "init_filesystem", + "load_checkpoint", + "load_config", + "outer_loop_workflow", + "save_best", + "save_checkpoint", + "save_generation", + "save_map_elites", + "workflow_to_harbor_yaml", +] diff --git a/factory/outer_loop/designer.py b/factory/outer_loop/designer.py new file mode 100644 index 000000000..c31af5794 --- /dev/null +++ b/factory/outer_loop/designer.py @@ -0,0 +1,344 @@ +"""Designer Agent — dual-mode workflow designer and informed mutation proposer. + +Design mode: creates from-scratch workflow designs (minimal, thorough, custom). +Mutation mode: proposes targeted mutations based on failure telemetry. + +v1 uses deterministic templates. LLM integration comes when the outer loop +runs against real benchmarks. +""" + +from __future__ import annotations + +import structlog + +from factory.outer_loop.models import EvalResult, MutationRecord, MutationType +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + GateNode, + Workflow, +) + +log = structlog.get_logger() + + +class DesignerAgent: + """LLM-guided workflow designer with design and mutation modes. + + Design mode produces from-scratch workflows for seed diversity. + Mutation mode proposes targeted mutations from failure telemetry. + """ + + def design_minimal(self, benchmark_spec: str) -> Workflow: + """Create a 3-4 node workflow optimized for speed. + + Structure: researcher → builder → gate + """ + nodes: dict[str, AgentNode | FnNode | GateNode] = { + "researcher": AgentNode( + id="researcher", + role=AgentRole.RESEARCHER, + writes={".factory/strategy/research.md"}, + timeout=300, + ), + "builder": AgentNode( + id="builder", + role=AgentRole.BUILDER, + reads={".factory/strategy/research.md"}, + writes={".factory/reviews/builder-latest.md"}, + timeout=600, + ), + "gate_qa": GateNode( + id="gate_qa", + evaluator_type="agent", + evaluator_role=AgentRole.HEALTH_CHECKER, + reads={".factory/reviews/builder-latest.md"}, + ), + } + edges = [ + Edge(source="researcher", target="builder"), + Edge(source="builder", target="gate_qa"), + ] + wf = Workflow( + name=f"minimal_{_slug(benchmark_spec)}", + nodes=nodes, # type: ignore[arg-type] + edges=edges, + start_node="researcher", + ) + log.info("designed_minimal", nodes=len(wf.nodes), benchmark=benchmark_spec[:40]) + return wf + + def design_thorough(self, benchmark_spec: str) -> Workflow: + """Create an 8-10 node workflow optimized for thoroughness. + + Structure: study → researcher → strategist → fork(builder_a, builder_b) + → join → code_reviewer → adversarial_tester → gate + """ + from factory.workflow.primitives import ForkNode, JoinNode + + nodes: dict[str, AgentNode | FnNode | GateNode | ForkNode | JoinNode] = { + "study": FnNode( + id="study", + command="factory study {project_path}", + writes={".factory/strategy/observations.md"}, + ), + "researcher": AgentNode( + id="researcher", + role=AgentRole.RESEARCHER, + reads={".factory/strategy/observations.md"}, + writes={".factory/strategy/research.md"}, + timeout=600, + ), + "strategist": AgentNode( + id="strategist", + role=AgentRole.STRATEGIST, + reads={".factory/strategy/research.md"}, + writes={".factory/strategy/current.md"}, + timeout=600, + ), + "fork_builders": ForkNode( + id="fork_builders", + targets=["builder_a", "builder_b"], + reads={".factory/strategy/current.md"}, + ), + "builder_a": AgentNode( + id="builder_a", + role=AgentRole.BUILDER, + reads={".factory/strategy/current.md"}, + writes={".factory/reviews/builder-a.md"}, + timeout=1200, + ), + "builder_b": AgentNode( + id="builder_b", + role=AgentRole.BUILDER, + reads={".factory/strategy/current.md"}, + writes={".factory/reviews/builder-b.md"}, + timeout=1200, + ), + "join_builders": JoinNode( + id="join_builders", + sources=["builder_a", "builder_b"], + ), + "code_reviewer": AgentNode( + id="code_reviewer", + role=AgentRole.CODE_REVIEWER, + reads={".factory/reviews/builder-a.md", ".factory/reviews/builder-b.md"}, + writes={".factory/reviews/code-review.md"}, + timeout=900, + ), + "adversarial_tester": AgentNode( + id="adversarial_tester", + role=AgentRole.ADVERSARIAL_TESTER, + reads={".factory/reviews/code-review.md"}, + writes={".factory/reviews/adversarial-qa.md"}, + timeout=1800, + ), + "gate_qa": GateNode( + id="gate_qa", + evaluator_type="agent", + evaluator_role=AgentRole.CEO, + reads={".factory/reviews/adversarial-qa.md"}, + ), + } + edges = [ + Edge(source="study", target="researcher"), + Edge(source="researcher", target="strategist"), + Edge(source="strategist", target="fork_builders"), + Edge(source="fork_builders", target="builder_a"), + Edge(source="fork_builders", target="builder_b"), + Edge(source="builder_a", target="join_builders"), + Edge(source="builder_b", target="join_builders"), + Edge(source="join_builders", target="code_reviewer"), + Edge(source="code_reviewer", target="adversarial_tester"), + Edge(source="adversarial_tester", target="gate_qa"), + ] + wf = Workflow( + name=f"thorough_{_slug(benchmark_spec)}", + nodes=nodes, # type: ignore[arg-type] + edges=edges, + start_node="study", + ) + log.info("designed_thorough", nodes=len(wf.nodes), benchmark=benchmark_spec[:40]) + return wf + + def design_custom(self, benchmark_spec: str, constraints: dict[str, object]) -> Workflow: + """Create a custom from-scratch workflow with optional constraints. + + Constraints can specify: + - max_nodes: int — cap on node count + - require_roles: list[str] — roles that must be present + - parallel: bool — whether to include fork/join parallelism + """ + raw_max = constraints.get("max_nodes", 6) + max_nodes = int(raw_max) if isinstance(raw_max, (int, float, str)) else 6 + raw_roles = constraints.get("require_roles", []) + require_roles: list[object] = list(raw_roles) if isinstance(raw_roles, list) else [] + + nodes: dict[str, AgentNode | FnNode | GateNode] = {} + edges: list[Edge] = [] + prev_id: str | None = None + + core_roles: list[tuple[str, AgentRole]] = [ + ("researcher", AgentRole.RESEARCHER), + ("strategist", AgentRole.STRATEGIST), + ("builder", AgentRole.BUILDER), + ] + + for role_str in require_roles: + if isinstance(role_str, str) and not any(r[0] == role_str for r in core_roles): + try: + role_enum = AgentRole(role_str) + core_roles.append((role_str, role_enum)) + except ValueError: + pass + + node_budget = max_nodes - 1 + for node_id, role in core_roles: + if len(nodes) >= node_budget: + break + nodes[node_id] = AgentNode( + id=node_id, + role=role, + timeout=600, + ) + if prev_id is not None: + edges.append(Edge(source=prev_id, target=node_id)) + prev_id = node_id + + if prev_id is not None: + gate_id = "gate_qa" + nodes[gate_id] = GateNode( # type: ignore[assignment] + id=gate_id, + evaluator_type="agent", + evaluator_role=AgentRole.HEALTH_CHECKER, + ) + edges.append(Edge(source=prev_id, target=gate_id)) + + start = core_roles[0][0] if core_roles else "gate_qa" + wf = Workflow( + name=f"custom_{_slug(benchmark_spec)}", + nodes=nodes, # type: ignore[arg-type] + edges=edges, + start_node=start, + ) + log.info("designed_custom", nodes=len(wf.nodes), benchmark=benchmark_spec[:40]) + return wf + + def propose( + self, + parent_workflow: Workflow, + telemetry: dict[str, object], + archive_stats: dict[str, object], + benchmark_spec: str, + ) -> list[MutationRecord]: + """Propose 1-3 targeted mutations based on failure telemetry. + + Heuristics: + - High failure rate on a node → propose removing or replacing it + - Dominant failure is timeout → propose reducing parallelism or increasing timeout + - Low diversity → propose inserting a new agent role not yet present + """ + proposals: list[MutationRecord] = [] + + node_stats = telemetry.get("node_stats", {}) + if isinstance(node_stats, dict): + for node_id, stats in node_stats.items(): + if not isinstance(stats, dict): + continue + failure_rate = stats.get("failure_rate", 0.0) + if isinstance(failure_rate, (int, float)) and failure_rate > 0.5: + proposals.append(MutationRecord( + operator=MutationType.NODE_REMOVE, + target_node=node_id, + before={"failure_rate": failure_rate}, + after={"action": "remove_failing_node"}, + rationale=f"Node {node_id} has {failure_rate:.0%} failure rate", + )) + + dominant_failure = telemetry.get("dominant_failure", "") + if dominant_failure == "timeout": + agent_nodes = [ + nid for nid, node in parent_workflow.nodes.items() + if type(node).__name__ == "AgentNode" + ] + if agent_nodes: + target = agent_nodes[0] + current_timeout = getattr(parent_workflow.nodes[target], "timeout", 600) + new_timeout = min((current_timeout or 600) * 2, 3600) + proposals.append(MutationRecord( + operator=MutationType.PARAM_MUTATE, + target_node=target, + before={"timeout": current_timeout}, + after={"timeout": new_timeout}, + rationale="Dominant failure is timeout — increase timeout", + )) + + diversity = archive_stats.get("diversity", 1.0) + if isinstance(diversity, (int, float)) and diversity < 0.3: + present_roles = { + node.role.value # type: ignore[union-attr] + for node in parent_workflow.nodes.values() + if hasattr(node, "role") + } + missing = set(AgentRole) - {AgentRole(r) for r in present_roles if r in [ar.value for ar in AgentRole]} + if missing: + new_role = next(iter(missing)) + proposals.append(MutationRecord( + operator=MutationType.NODE_INSERT, + target_node=None, + before={"present_roles": sorted(present_roles)}, + after={"new_role": new_role.value}, + rationale=f"Low diversity ({diversity:.2f}) — insert {new_role.value}", + )) + + if not proposals: + proposals.append(MutationRecord( + operator=MutationType.PARAM_MUTATE, + target_node=None, + before={}, + after={"action": "explore"}, + rationale="No specific failure signal — propose parameter exploration", + )) + + return proposals[:3] + + +def extract_telemetry(eval_result: EvalResult) -> dict[str, object]: + """Extract structured diagnostics from an EvalResult. + + Returns a dict with: + - node_stats: per-node success/failure data (from details if available) + - dominant_failure: most common failure category + - benchmark_score: the raw benchmark score + - cost_usd: evaluation cost + - complexity: workflow complexity metric + """ + details = eval_result.details or {} + + node_stats: dict[str, object] = {} + raw_stats = details.get("node_stats", {}) + if isinstance(raw_stats, dict): + node_stats = dict(raw_stats) + + dominant_failure = "" + raw_failure = details.get("dominant_failure", "") + if isinstance(raw_failure, str): + dominant_failure = raw_failure + + return { + "node_stats": node_stats, + "dominant_failure": dominant_failure, + "benchmark_score": eval_result.benchmark_score, + "hygiene_score": eval_result.hygiene_score, + "cost_usd": eval_result.cost_usd, + "complexity": eval_result.complexity, + "score": eval_result.score, + } + + +def _slug(text: str) -> str: + """Convert text to a short slug for workflow naming.""" + clean = text.lower().replace(" ", "_")[:20] + return "".join(c for c in clean if c.isalnum() or c == "_").strip("_") or "default" diff --git a/factory/outer_loop/direct_evaluator.py b/factory/outer_loop/direct_evaluator.py new file mode 100644 index 000000000..d37b5e243 --- /dev/null +++ b/factory/outer_loop/direct_evaluator.py @@ -0,0 +1,443 @@ +"""Direct FeatureBench evaluator — runs agents on the host, verifies in Docker. + +Three-step architecture: +1. Extract /testbed/ from Docker image to a local temp dir +2. Run factory agents DIRECTLY ON THE HOST against the extracted testbed +3. Copy the modified testbed into a fresh container via docker cp + exec + (avoids bind-mount cross-platform issues with amd64 images on arm64 hosts) + +This avoids installing agents inside Docker containers entirely. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import structlog + +from factory.outer_loop.models import EvalResult +from factory.workflow.primitives import AgentNode, ForkNode, GateNode, JoinNode, Workflow + +log = structlog.get_logger() + +_FEATUREBENCH_DIR = Path(__file__).resolve().parents[2] / "featurebench" +_PYTEST_F2P_RE = re.compile(r"pytest\s+(.+?)\s*>\s*/tmp/f2p_output") +_PYTEST_P2P_RE = re.compile(r"pytest\s+(.+?)\s*>\s*/tmp/p2p_output") +_INSTALL_RE = re.compile(r"#\s*Repo-specific install[^\n]*\n(pip install[^\n]+)") + + +def _parse_from_line(dockerfile: Path) -> str: + """Extract the base image from a Dockerfile's FROM line.""" + for line in dockerfile.read_text().splitlines(): + stripped = line.strip() + if stripped.upper().startswith("FROM "): + return stripped.split()[1] + raise ValueError(f"No FROM line found in {dockerfile}") + + +def _parse_deleted_files(patch_path: Path) -> list[str]: + """Parse file paths deleted by a diff (--- a/path lines in deleted-file hunks).""" + deleted: list[str] = [] + if not patch_path.exists(): + return deleted + text = patch_path.read_text() + in_delete_block = False + for line in text.splitlines(): + if line.startswith("deleted file"): + in_delete_block = True + elif line.startswith("diff --git"): + in_delete_block = False + elif in_delete_block and line.startswith("--- a/"): + deleted.append(line[6:]) + return deleted + + +def _parse_test_sh(test_sh: Path) -> tuple[str | None, str | None, str]: + """Extract F2P test args, P2P test args, and install command from test.sh.""" + text = test_sh.read_text() + + f2p_match = _PYTEST_F2P_RE.search(text) + f2p_args = f2p_match.group(1).strip() if f2p_match else None + + p2p_match = _PYTEST_P2P_RE.search(text) + p2p_args = p2p_match.group(1).strip() if p2p_match else None + + install_match = _INSTALL_RE.search(text) + install_cmd = install_match.group(1).strip() if install_match else "pip install -e . || true" + + return f2p_args, p2p_args, install_cmd + + +def _topo_sort_nodes(workflow: Workflow) -> list[str]: + """Topological sort of workflow nodes using Kahn's algorithm.""" + adj: dict[str, list[str]] = {nid: [] for nid in workflow.nodes} + in_degree: dict[str, int] = {nid: 0 for nid in workflow.nodes} + for edge in workflow.edges: + if edge.source in adj and edge.target in in_degree: + adj[edge.source].append(edge.target) + in_degree[edge.target] += 1 + + queue = [nid for nid, deg in in_degree.items() if deg == 0] + order: list[str] = [] + while queue: + queue.sort() + node = queue.pop(0) + order.append(node) + for neighbor in adj[node]: + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + queue.append(neighbor) + return order + + +class DirectFeatureBenchEvaluator: + """Evaluates workflows on FeatureBench without installing agents in containers. + + Implements the ``EvaluatorFn`` protocol:: + + __call__(workflow, project_dir, instances) -> EvalResult + """ + + def __init__( + self, + featurebench_dir: Path | None = None, + agent_timeout: int = 1800, + ) -> None: + self._featurebench_dir = featurebench_dir or _FEATUREBENCH_DIR + self._agent_timeout = agent_timeout + + def __call__( + self, + workflow: Workflow, + project_dir: str, + instances: list[str], + ) -> EvalResult: + resolved = 0 + total = len(instances) + per_instance: dict[str, object] = {} + + for instance_id in instances: + success = self._eval_instance(workflow, instance_id) + per_instance[instance_id] = {"resolved": success} + if success: + resolved += 1 + + score = resolved / max(total, 1) + log.info( + "direct_eval_done", + resolved=resolved, + total=total, + score=score, + ) + return EvalResult( + score=score, + benchmark_score=score, + complexity=float(len(workflow.nodes)), + details={"instances": per_instance}, + ) + + def _eval_instance(self, workflow: Workflow, instance_id: str) -> bool: + """Evaluate a single FeatureBench instance. Returns True if resolved.""" + task_dir = self._featurebench_dir / instance_id + if not task_dir.exists(): + log.error("task_dir_missing", instance=instance_id) + return False + + dockerfile = task_dir / "environment" / "Dockerfile" + if not dockerfile.exists(): + log.error("dockerfile_missing", instance=instance_id) + return False + + image = _parse_from_line(dockerfile) + workdir = Path(tempfile.mkdtemp(prefix=f"fb-{instance_id[:30]}-", dir="/tmp")) + + try: + # 1. Pull image if needed + log.info("pulling_image", image=image, instance=instance_id) + subprocess.run( + ["docker", "pull", "--platform", "linux/amd64", image], + capture_output=True, + text=True, + timeout=600, + ) + + # 2. Extract /testbed/ from Docker image + log.info("extracting_testbed", instance=instance_id) + cid_result = subprocess.run( + ["docker", "create", "--platform", "linux/amd64", image], + capture_output=True, + text=True, + timeout=60, + ) + if cid_result.returncode != 0: + log.error("docker_create_failed", stderr=cid_result.stderr, instance=instance_id) + return False + + cid = cid_result.stdout.strip() + try: + cp_result = subprocess.run( + ["docker", "cp", f"{cid}:/testbed", str(workdir / "testbed")], + capture_output=True, + text=True, + timeout=120, + ) + if cp_result.returncode != 0: + log.error("docker_cp_failed", stderr=cp_result.stderr, instance=instance_id) + return False + finally: + subprocess.run(["docker", "rm", cid], capture_output=True, timeout=30) + + testbed = workdir / "testbed" + + # 3. Initialize git in testbed if not already a repo + if not (testbed / ".git").exists(): + subprocess.run(["git", "init"], cwd=testbed, capture_output=True, timeout=30) + subprocess.run(["git", "add", "."], cwd=testbed, capture_output=True, timeout=60) + subprocess.run( + ["git", "commit", "-m", "initial"], + cwd=testbed, + capture_output=True, + timeout=60, + env={"GIT_AUTHOR_NAME": "test", "GIT_AUTHOR_EMAIL": "test@test", + "GIT_COMMITTER_NAME": "test", "GIT_COMMITTER_EMAIL": "test@test", + "PATH": "/usr/bin:/bin:/usr/local/bin"}, + ) + + # 4. Apply setup_patch (scramble the implementation) + setup_patch = task_dir / "environment" / "setup_patch.diff" + if setup_patch.exists() and setup_patch.stat().st_size > 0: + log.info("applying_setup_patch", instance=instance_id) + subprocess.run( + ["git", "apply", "--whitespace=nowarn", str(setup_patch)], + cwd=testbed, + capture_output=True, + timeout=30, + ) + + # Delete test files listed in test_patch.diff (lv1) + test_patch = task_dir / "environment" / "test_patch.diff" + deleted_files = _parse_deleted_files(test_patch) + for f in deleted_files: + target = testbed / f + if target.exists(): + target.unlink() + log.debug("deleted_test_file", file=f, instance=instance_id) + + # 5. Copy instruction.md to testbed + instruction = task_dir / "instruction.md" + if instruction.exists(): + shutil.copy(instruction, testbed / "task-instruction.md") + + # 6. Create .factory dir for agent output + factory_dir = testbed / ".factory" + factory_dir.mkdir(exist_ok=True) + (factory_dir / "reviews").mkdir(exist_ok=True) + + # 7. Run the workflow's agents on the testbed + log.info("running_agents", instance=instance_id, nodes=len(workflow.nodes)) + self._run_workflow_agents(workflow, testbed) + + # 8. Verify: run test.sh inside Docker with the modified testbed mounted + log.info("verifying_in_docker", instance=instance_id) + success = self._verify_in_docker(task_dir, image, testbed) + log.info( + "instance_result", + instance=instance_id, + resolved=success, + ) + return success + + except subprocess.TimeoutExpired: + log.warning("instance_timeout", instance=instance_id) + return False + except Exception as exc: + log.error("instance_error", instance=instance_id, error=str(exc)) + return False + finally: + shutil.rmtree(workdir, ignore_errors=True) + + def _run_workflow_agents(self, workflow: Workflow, testbed: Path) -> None: + """Run workflow agents in topological order on the testbed.""" + order = _topo_sort_nodes(workflow) + for node_id in order: + node = workflow.nodes[node_id] + if isinstance(node, AgentNode): + timeout = node.timeout or self._agent_timeout + prompt = node.prompt_template + if not prompt: + continue + + log.info("running_agent", node=node_id, role=node.role.value, timeout=timeout) + result = subprocess.run( + [ + "factory", + "agent", + node.role.value, + "--task", + prompt, + "--project", + str(testbed), + "--timeout", + str(timeout), + "--disallowedTools", + "WebSearch,WebFetch", + ], + capture_output=True, + text=True, + timeout=timeout + 120, + ) + log.info( + "agent_finished", + node=node_id, + returncode=result.returncode, + stdout_len=len(result.stdout), + ) + elif isinstance(node, (GateNode, ForkNode, JoinNode)): + pass + + def _verify_in_docker( + self, task_dir: Path, image: str, testbed: Path + ) -> bool: + """Run pytest via docker cp + exec — avoids bind-mount cross-platform issues.""" + test_patch = task_dir / "environment" / "test_patch.diff" + test_sh = task_dir / "tests" / "test.sh" + + f2p_args, p2p_args, install_cmd = (None, None, "pip install -e . || true") + if test_sh.exists(): + f2p_args, p2p_args, install_cmd = _parse_test_sh(test_sh) + + # Restore deleted test files into the host testbed before copying to container + test_files = _parse_deleted_files(test_patch) + if test_files: + if test_patch.exists() and test_patch.stat().st_size > 0: + apply_result = subprocess.run( + ["git", "apply", "--reverse", "--whitespace=nowarn", str(test_patch)], + cwd=testbed, + capture_output=True, + text=True, + timeout=30, + ) + if apply_result.returncode != 0: + log.warning( + "reverse_patch_failed", + stderr=apply_result.stderr, + task_dir=str(task_dir), + ) + f2p_cmd = f"pytest -rA --tb=short --color=no {' '.join(test_files)}" + elif f2p_args: + f2p_cmd = f"pytest -rA --tb=short --color=no {f2p_args}" + else: + log.error("no_test_target", task_dir=str(task_dir)) + return False + + # 1. Create container (kept alive with sleep so we can exec into it) + cid_result = subprocess.run( + [ + "docker", "create", "--platform", "linux/amd64", + image, + "bash", "-c", "sleep 600", + ], + capture_output=True, + text=True, + timeout=60, + ) + if cid_result.returncode != 0: + log.error("docker_create_verify_failed", stderr=cid_result.stderr) + return False + cid = cid_result.stdout.strip() + + try: + # 2. Copy only changed files into the container (avoids symlink conflicts + # where docker cp fails with "cannot overwrite directory with non-directory") + diff_result = subprocess.run( + ["git", "diff", "--name-only", "HEAD"], + cwd=testbed, + capture_output=True, + text=True, + timeout=30, + ) + changed_files: list[str] = [] + if diff_result.returncode == 0: + changed_files.extend(f for f in diff_result.stdout.strip().splitlines() if f) + + untracked_result = subprocess.run( + ["git", "ls-files", "--others", "--exclude-standard"], + cwd=testbed, + capture_output=True, + text=True, + timeout=30, + ) + if untracked_result.returncode == 0: + changed_files.extend(f for f in untracked_result.stdout.strip().splitlines() if f) + + log.info("copying_changed_files", count=len(changed_files), task_dir=str(task_dir)) + + # Start container first so we can mkdir for new files + start_result = subprocess.run( + ["docker", "start", cid], + capture_output=True, + text=True, + timeout=30, + ) + if start_result.returncode != 0: + log.error("docker_start_failed", stderr=start_result.stderr) + return False + + parents_ensured: set[str] = set() + for rel_path in changed_files: + src = testbed / rel_path + if not src.exists() or not src.is_file(): + continue + parent = str(Path(rel_path).parent) + if parent and parent != "." and parent not in parents_ensured: + subprocess.run( + ["docker", "exec", cid, "mkdir", "-p", f"/testbed/{parent}"], + capture_output=True, + timeout=10, + ) + parents_ensured.add(parent) + cp_result = subprocess.run( + ["docker", "cp", str(src), f"{cid}:/testbed/{rel_path}"], + capture_output=True, + text=True, + timeout=30, + ) + if cp_result.returncode != 0: + log.warning("docker_cp_file_failed", file=rel_path, stderr=cp_result.stderr) + + # 3. Exec the test inside the container + script = ( + f"source /opt/miniconda3/bin/activate testbed; " + f"cd /testbed; " + f"{install_cmd} 2>&1 | tail -2; " + f"{f2p_cmd}" + ) + if p2p_args: + script += f"; pytest -rA --tb=short --color=no {p2p_args}" + + result = subprocess.run( + ["docker", "exec", cid, "bash", "-c", script], + capture_output=True, + text=True, + timeout=600, + ) + + log.info( + "docker_verify_done", + returncode=result.returncode, + stdout_tail=result.stdout[-500:] if result.stdout else "", + stderr_tail=result.stderr[-500:] if result.stderr else "", + ) + + return result.returncode == 0 + finally: + # 4. Cleanup: force-remove the container + subprocess.run( + ["docker", "rm", "-f", cid], + capture_output=True, + timeout=30, + ) diff --git a/factory/outer_loop/engine.py b/factory/outer_loop/engine.py new file mode 100644 index 000000000..aac2494ff --- /dev/null +++ b/factory/outer_loop/engine.py @@ -0,0 +1,427 @@ +"""Core evolutionary search controller for workflow optimization.""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING + +import structlog + +from factory.outer_loop.designer import DesignerAgent +from factory.outer_loop.evaluator import SwarmEvaluator +from factory.outer_loop.models import ( + GenerationSummary, + HyperparameterRecord, + MutationRecord, + OuterLoopResult, + SwarmConfig, +) +from factory.outer_loop.mutations import ( + MutationStrategy, + WeightedRandomStrategy, + apply_random_mutation, +) +from factory.outer_loop.overfit import OverfitDetector +from factory.outer_loop.population import MAPElitesArchive, Population +from factory.outer_loop.similarity import NoveltyFilter +from factory.outer_loop.subset import FixedSubsetSelector, SubsetSelector +from factory.workflow.primitives import Workflow + +if TYPE_CHECKING: + pass + +log = structlog.get_logger() + +PLATEAU_WINDOW = 3 + + +class BudgetTracker: + """Tracks evaluation budget consumption, cost, and wall-clock time.""" + + def __init__(self, total_budget: int) -> None: + self._total = total_budget + self._consumed = 0 + self._cost_usd = 0.0 + self._start_time = time.monotonic() + self._warned_80 = False + self._warned_95 = False + + @property + def remaining(self) -> int: + return max(0, self._total - self._consumed) + + @property + def consumed(self) -> int: + return self._consumed + + @property + def total_cost_usd(self) -> float: + return self._cost_usd + + @property + def elapsed_seconds(self) -> float: + return time.monotonic() - self._start_time + + @property + def exhausted(self) -> bool: + return self._consumed >= self._total + + def consume(self, count: int = 1, cost_usd: float = 0.0) -> None: + self._consumed += count + self._cost_usd += cost_usd + pct = self._consumed / self._total if self._total > 0 else 1.0 + if pct >= 0.95 and not self._warned_95: + log.warning("budget_95_percent", consumed=self._consumed, total=self._total) + self._warned_95 = True + elif pct >= 0.80 and not self._warned_80: + log.warning("budget_80_percent", consumed=self._consumed, total=self._total) + self._warned_80 = True + + +class SwarmEngine: + """Orchestrates the evolutionary search loop.""" + + def __init__( + self, + config: SwarmConfig, + evaluator: SwarmEvaluator, + strategy: MutationStrategy | None = None, + subset_selector: SubsetSelector | None = None, + overfit_detector: OverfitDetector | None = None, + novelty_filter: NoveltyFilter | None = None, + designer: DesignerAgent | None = None, + ) -> None: + self._config = config + self._evaluator = evaluator + self._strategy: MutationStrategy = strategy or WeightedRandomStrategy( + mutation_rate=config.mutation_rate, + ) + self._subset: SubsetSelector = subset_selector or FixedSubsetSelector( + config.training_instances, + ) + self._overfit = overfit_detector or OverfitDetector() + self._novelty = novelty_filter or NoveltyFilter(min_edit_distance=3) + self._designer = designer or DesignerAgent() + self._budget = BudgetTracker(config.budget) + self._archive = MAPElitesArchive() + self._score_trajectory: list[float] = [] + + @property + def archive(self) -> MAPElitesArchive: + return self._archive + + @property + def budget(self) -> BudgetTracker: + return self._budget + + def seed( + self, + base_workflow: Workflow, + config: SwarmConfig | None = None, + ) -> Population: + """Create the initial population from a base workflow. + + Slot 0: unmodified seed. + Slots 1..N-designer_count: random mutations of seed. + Last designer_count slots: from-scratch designs via DesignerAgent. + """ + cfg = config or self._config + pop = Population() + + seed_ind = Population.make_individual(base_workflow, generation=0) + pop.add(seed_ind) + self._novelty.add(base_workflow) + + designer_count = cfg.designer_count + mutation_slots = max(0, cfg.population_size - 1 - designer_count) + + attempts = 0 + max_attempts = mutation_slots * 10 + while pop.size < 1 + mutation_slots and attempts < max_attempts: + attempts += 1 + result = apply_random_mutation( + base_workflow, + self._strategy, + generation=0, + frozen_nodes=set(cfg.frozen_node_ids), + ) + if result is None: + continue + mutated_wf, mutation_rec = result + if not self._novelty.is_novel(mutated_wf): + continue + self._novelty.add(mutated_wf) + ind = Population.make_individual( + mutated_wf, + generation=0, + parent_id=seed_ind.id, + mutation_record=mutation_rec, + ) + pop.add(ind) + + if designer_count > 0: + self._add_designer_variants(pop, cfg, designer_count) + + log.info( + "population_seeded", + size=pop.size, + target=cfg.population_size, + designer_variants=min(designer_count, pop.size), + ) + return pop + + def _add_designer_variants( + self, + pop: Population, + cfg: SwarmConfig, + designer_count: int, + ) -> None: + """Add from-scratch designed workflows to the population.""" + benchmark_spec = cfg.benchmark + designs: list[Workflow] = [] + + if designer_count >= 1: + try: + minimal = self._designer.design_minimal(benchmark_spec) + designs.append(minimal) + except Exception: + log.warning("designer_minimal_failed", exc_info=True) + + if designer_count >= 2: + try: + thorough = self._designer.design_thorough(benchmark_spec) + designs.append(thorough) + except Exception: + log.warning("designer_thorough_failed", exc_info=True) + + for i in range(2, designer_count): + try: + custom = self._designer.design_custom( + benchmark_spec, + {"max_nodes": 4 + i, "parallel": i % 2 == 0}, + ) + designs.append(custom) + except Exception: + log.warning("designer_custom_failed", index=i, exc_info=True) + + for wf in designs: + if pop.size >= cfg.population_size: + break + if self._novelty.is_novel(wf): + self._novelty.add(wf) + ind = Population.make_individual(wf, generation=0) + pop.add(ind) + + def evolve_generation( + self, + population: Population, + generation: int, + project_dir: str = "", + ) -> GenerationSummary: + """Run one generation of evolution.""" + instances = self._subset.select( + self._config.training_instances, generation, self._budget.remaining + ) + + # Evaluate current population + for ind in population.individuals: + if self._budget.exhausted: + break + wf = Workflow.from_dict(ind.workflow_data) # type: ignore[arg-type] + ev = self._evaluator.evaluate(wf, project_dir, instances) + self._budget.consume(1, cost_usd=ev.cost_usd) + updated = ind.model_copy(update={"score": ev.score, "cost_usd": ev.cost_usd}) + population.remove(ind.id) + population.add(updated) + self._archive.add(updated) + + # Select parents and create offspring + mutations_applied: list[MutationRecord] = [] + novel_count = 0 + rejected_dupes = 0 + offspring: list[tuple[Workflow, MutationRecord, str]] = [] + + mutation_rate = self._strategy.get_mutation_rate(generation) + for _ in range(self._config.population_size): + parent = self._archive.sample_parent(self._config.tournament_size) + if parent is None: + continue + parent_wf = Workflow.from_dict(parent.workflow_data) # type: ignore[arg-type] + mutation_result = apply_random_mutation( + parent_wf, + self._strategy, + generation, + frozen_nodes=set(self._config.frozen_node_ids), + ) + if mutation_result is None: + continue + child_wf, mutation_rec = mutation_result + if self._novelty.is_novel(child_wf): + self._novelty.add(child_wf) + offspring.append((child_wf, mutation_rec, parent.id)) + mutations_applied.append(mutation_rec) + novel_count += 1 + else: + rejected_dupes += 1 + + # Evaluate offspring and add to population + for child_wf, mutation_rec, parent_id in offspring: + if self._budget.exhausted: + break + eval_result = self._evaluator.evaluate(child_wf, project_dir, instances) + self._budget.consume(1, cost_usd=eval_result.cost_usd) + ind = Population.make_individual( + child_wf, + generation=generation, + parent_id=parent_id, + mutation_record=mutation_rec, + score=eval_result.score, + cost_usd=eval_result.cost_usd, + ) + population.add(ind) + self._archive.add(ind) + + # Track best score + best = population.best() + best_score = best.score if best else 0.0 + mean_score = population.mean_score() + diversity = self._archive.diversity_metric() + self._score_trajectory.append(best_score) + + hp_record = HyperparameterRecord( + generation=generation, + mutation_rate=mutation_rate, + population_size=population.size, + tournament_size=self._config.tournament_size, + designer_ratio=self._strategy.get_designer_ratio(generation), + operator_weights=( + self._strategy.get_operator_weights() + if hasattr(self._strategy, "get_operator_weights") + else {} + ), + best_score=best_score, + mean_score=mean_score, + diversity=diversity, + novel_count=novel_count, + ) + + # Holdout evaluation for best candidate + holdout_score = 0.0 + if best and self._config.holdout_instances: + best_wf = Workflow.from_dict(best.workflow_data) # type: ignore[arg-type] + holdout_result = self._evaluator.evaluate(best_wf, project_dir, self._config.holdout_instances) + holdout_score = holdout_result.score + self._budget.consume(1, cost_usd=holdout_result.cost_usd) + log.info( + "holdout_eval", + generation=generation, + holdout_score=holdout_score, + training_best=best_score, + ) + + return GenerationSummary( + generation=generation, + population_size=population.size, + best_score=best_score, + mean_score=mean_score, + diversity=diversity, + mutations_applied=mutations_applied, + novel_count=novel_count, + rejected_duplicates=rejected_dupes, + holdout_score=holdout_score, + hyperparameters=hp_record, + ) + + def _detect_plateau(self) -> bool: + """Detect plateau: 3 consecutive non-improving generations.""" + if len(self._score_trajectory) < PLATEAU_WINDOW + 1: + return False + recent = self._score_trajectory[-(PLATEAU_WINDOW + 1):] + baseline = recent[0] + return all(s <= baseline for s in recent[1:]) + + def run( + self, + base_workflow: Workflow, + project_dir: str = "", + ) -> OuterLoopResult: + """Run the full evolutionary search loop.""" + population = self.seed(base_workflow) + generation = 0 + summaries: list[GenerationSummary] = [] + hp_history: list[HyperparameterRecord] = [] + + while not self._should_terminate(generation): + log.info("generation_start", generation=generation, budget_remaining=self._budget.remaining) + summary = self.evolve_generation(population, generation, project_dir) + summaries.append(summary) + if summary.hyperparameters: + hp_history.append(summary.hyperparameters) + + # Plateau detection with adaptive response + if self._detect_plateau(): + if hasattr(self._strategy, "on_plateau"): + self._strategy.on_plateau() # type: ignore[union-attr] + log.info("plateau_detected_adapting", generation=generation) + elif len(self._score_trajectory) >= 2 and self._score_trajectory[-1] > self._score_trajectory[-2]: + if hasattr(self._strategy, "on_improvement"): + self._strategy.on_improvement() # type: ignore[union-attr] + + generation += 1 + + convergence_reason = self._get_convergence_reason(generation) + log.info("evolution_complete", reason=convergence_reason, generations=generation) + + # Post-evolution overfit audit + best = self._archive.best() + audit_result = None + if best and self._config.holdout_instances: + best_wf = Workflow.from_dict(best.workflow_data) # type: ignore[arg-type] + audit_result = self._overfit.audit( + best_wf, + self._config.training_instances, + self._config.holdout_instances, + self._evaluator, + project_dir, + ) + + pareto = self._archive.pareto_front() + + return OuterLoopResult( + best_workflow_data=best.workflow_data if best else {}, + best_score=best.score if best else 0.0, + holdout_score=audit_result.holdout_score if audit_result else 0.0, + overfit_flag=audit_result.overfit_flag if audit_result else False, + trajectory=summaries, + total_cost_usd=self._budget.total_cost_usd, + convergence_reason=convergence_reason, + generations_completed=generation, + total_evaluations=self._budget.consumed, + archive_size=self._archive.size, + pareto_front=pareto, + hyperparameter_history=hp_history, + ) + + def _should_terminate(self, generation: int) -> bool: + if self._budget.exhausted: + return True + if self._config.target_score is not None and self._score_trajectory: + if self._score_trajectory[-1] >= self._config.target_score: + return True + if self._detect_plateau(): + # Give one extra generation after plateau adaptation + if len(self._score_trajectory) >= PLATEAU_WINDOW + 2: + recent = self._score_trajectory[-(PLATEAU_WINDOW + 2):] + if all(s <= recent[0] for s in recent[1:]): + return True + return False + + def _get_convergence_reason(self, generation: int) -> str: + if self._budget.exhausted: + return "budget_exhausted" + if self._config.target_score is not None and self._score_trajectory: + if self._score_trajectory[-1] >= self._config.target_score: + return "target_score_reached" + if self._detect_plateau(): + return "plateau" + return "unknown" diff --git a/factory/outer_loop/evaluator.py b/factory/outer_loop/evaluator.py new file mode 100644 index 000000000..0d5473f7f --- /dev/null +++ b/factory/outer_loop/evaluator.py @@ -0,0 +1,139 @@ +"""Fitness evaluation for workflow candidates in the evolutionary search.""" + +from __future__ import annotations + +import time +from typing import Protocol, runtime_checkable + +import structlog + +from factory.outer_loop.models import EvalResult, SwarmConfig +from factory.outer_loop.similarity import structural_hash +from factory.workflow.primitives import Workflow + +log = structlog.get_logger() + + +class FitnessCache: + """Cache evaluation results keyed by (structural_hash, frozenset(instances)).""" + + def __init__(self) -> None: + self._cache: dict[tuple[str, frozenset[str]], tuple[float, float, float]] = {} + + def get( + self, workflow: Workflow, instances: list[str] + ) -> tuple[float, float, float] | None: + key = (structural_hash(workflow), frozenset(instances)) + return self._cache.get(key) + + def put( + self, workflow: Workflow, instances: list[str], score: float, cost: float + ) -> None: + key = (structural_hash(workflow), frozenset(instances)) + self._cache[key] = (score, cost, time.time()) + + @property + def size(self) -> int: + return len(self._cache) + + +@runtime_checkable +class EvaluatorFn(Protocol): + """Protocol for pluggable evaluation functions. + + For v1, this is a simple callable. Phase 4 will wire in InnerLoop. + """ + + def __call__( + self, workflow: Workflow, project_dir: str, instances: list[str] + ) -> EvalResult: ... + + +class SwarmEvaluator: + """Evaluates workflow candidates against benchmark instances.""" + + def __init__( + self, + config: SwarmConfig, + evaluator_fn: EvaluatorFn | None = None, + ) -> None: + self._config = config + self._evaluator_fn = evaluator_fn + self._cache = FitnessCache() + + @property + def cache(self) -> FitnessCache: + return self._cache + + def evaluate( + self, + workflow: Workflow, + project_dir: str, + instances: list[str], + ) -> EvalResult: + """Evaluate a workflow on the given instances, using cache if available.""" + cached = self._cache.get(workflow, instances) + if cached is not None: + score, cost, _ = cached + log.info("fitness_cache_hit", score=score) + return EvalResult(score=score, cost_usd=cost, benchmark_score=score) + + if not self._check_mandatory_components(workflow): + log.warning("mandatory_component_missing", workflow=workflow.name) + return EvalResult(score=0.0, details={"rejected": "mandatory_component_missing"}) + + if not self._check_frozen_nodes(workflow): + log.warning("frozen_node_violated", workflow=workflow.name) + return EvalResult(score=0.0, details={"rejected": "frozen_node_violated"}) + + if self._evaluator_fn is not None: + result = self._evaluator_fn(workflow, project_dir, instances) + else: + result = EvalResult(score=0.0, details={"note": "no_evaluator_fn_configured"}) + + composite = self._compute_composite(result) + result = result.model_copy(update={"score": composite}) + + self._cache.put(workflow, instances, composite, result.cost_usd) + return result + + def evaluate_batch( + self, + workflows: list[Workflow], + project_dir: str, + instances: list[str], + parallelism: int = 1, + ) -> list[EvalResult]: + """Evaluate multiple workflows. Currently sequential; parallelism is reserved.""" + return [self.evaluate(wf, project_dir, instances) for wf in workflows] + + def _compute_composite(self, result: EvalResult) -> float: + """Multi-metric fitness: 0.6*benchmark + 0.2*hygiene + 0.1*(1-cost) + 0.1*(1-complexity).""" + norm_cost = min(result.cost_usd / 10.0, 1.0) if result.cost_usd > 0 else 0.0 + norm_complexity = min(result.complexity / 20.0, 1.0) if result.complexity > 0 else 0.0 + return ( + 0.6 * result.benchmark_score + + 0.2 * result.hygiene_score + + 0.1 * (1.0 - norm_cost) + + 0.1 * (1.0 - norm_complexity) + ) + + def _check_mandatory_components(self, workflow: Workflow) -> bool: + """Verify workflow contains all mandatory node roles.""" + if not self._config.mandatory_node_roles: + return True + present_roles: set[str] = set() + for node in workflow.nodes.values(): + if hasattr(node, "role"): + present_roles.add(node.role.value if hasattr(node.role, "value") else str(node.role)) + for role in self._config.mandatory_node_roles: + if role not in present_roles: + return False + return True + + def _check_frozen_nodes(self, workflow: Workflow) -> bool: + """Verify no frozen node was removed from the workflow.""" + for fid in self._config.frozen_node_ids: + if fid not in workflow.nodes: + return False + return True diff --git a/factory/outer_loop/filesystem.py b/factory/outer_loop/filesystem.py new file mode 100644 index 000000000..ec8d11688 --- /dev/null +++ b/factory/outer_loop/filesystem.py @@ -0,0 +1,229 @@ +"""Experiment filesystem setup and checkpoint/resume for the outer loop.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import structlog + +from factory.outer_loop.models import ( + GenerationSummary, + OuterLoopResult, + OuterLoopState, + SwarmConfig, +) +from factory.outer_loop.population import MAPElitesArchive, Population +from factory.workflow.primitives import Workflow + +log = structlog.get_logger() + + +def init_filesystem(project_path: Path, config: SwarmConfig) -> Path: + """Create the .factory/outer-loop/ directory structure. + + Returns the outer-loop root directory. + """ + root = project_path / ".factory" / "outer-loop" + root.mkdir(parents=True, exist_ok=True) + + (root / "archive").mkdir(exist_ok=True) + (root / "map-elites").mkdir(exist_ok=True) + (root / "best").mkdir(exist_ok=True) + + config_path = root / "config.json" + config_path.write_text( + json.dumps(config.model_dump(mode="json"), indent=2) + ) + + state = OuterLoopState(budget_remaining=config.budget) + state_path = root / "state.json" + state_path.write_text( + json.dumps(state.model_dump(mode="json"), indent=2) + ) + + cache_path = root / "fitness_cache.json" + if not cache_path.exists(): + cache_path.write_text("{}") + + trajectory_path = root / "trajectory.jsonl" + if not trajectory_path.exists(): + trajectory_path.touch() + + log.info("outer_loop_filesystem_initialized", root=str(root)) + return root + + +def save_generation( + project_path: Path, + generation: int, + summary: GenerationSummary, + population: Population, +) -> None: + """Save generation artifacts to .factory/outer-loop/archive/generation-NNN/.""" + root = project_path / ".factory" / "outer-loop" + gen_dir = root / "archive" / f"generation-{generation:03d}" + gen_dir.mkdir(parents=True, exist_ok=True) + + summary_path = gen_dir / "summary.json" + summary_path.write_text( + json.dumps(summary.model_dump(mode="json"), indent=2) + ) + + if summary.hyperparameters: + hp_path = gen_dir / "hyperparameters.json" + hp_path.write_text( + json.dumps(summary.hyperparameters.model_dump(mode="json"), indent=2) + ) + + for i, ind in enumerate(population.individuals): + var_dir = gen_dir / f"variant-{i:02d}" + var_dir.mkdir(exist_ok=True) + (var_dir / "workflow.json").write_text( + json.dumps(ind.workflow_data, indent=2, default=str) + ) + if ind.mutation_record: + (var_dir / "mutation.json").write_text( + json.dumps(ind.mutation_record.model_dump(mode="json"), indent=2) + ) + (var_dir / "scores.json").write_text( + json.dumps({"score": ind.score, "cost_usd": ind.cost_usd}, indent=2) + ) + + traj_path = root / "trajectory.jsonl" + with traj_path.open("a") as f: + entry = { + "generation": generation, + "best_score": summary.best_score, + "mean_score": summary.mean_score, + "diversity": summary.diversity, + "novel_count": summary.novel_count, + } + f.write(json.dumps(entry) + "\n") + + +def save_checkpoint( + project_path: Path, + state: OuterLoopState, +) -> None: + """Write OuterLoopState to .factory/outer-loop/state.json.""" + state_path = project_path / ".factory" / "outer-loop" / "state.json" + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text( + json.dumps(state.model_dump(mode="json"), indent=2) + ) + log.info("outer_loop_checkpoint_saved", generation=state.generation) + + +def load_checkpoint(project_path: Path) -> OuterLoopState | None: + """Load OuterLoopState from .factory/outer-loop/state.json if it exists.""" + state_path = project_path / ".factory" / "outer-loop" / "state.json" + if not state_path.exists(): + return None + try: + data = json.loads(state_path.read_text()) + return OuterLoopState.model_validate(data, strict=False) + except Exception: + log.warning("outer_loop_checkpoint_load_failed", exc_info=True) + return None + + +def load_config(project_path: Path) -> SwarmConfig | None: + """Load SwarmConfig from .factory/outer-loop/config.json if it exists.""" + config_path = project_path / ".factory" / "outer-loop" / "config.json" + if not config_path.exists(): + return None + try: + data = json.loads(config_path.read_text()) + return SwarmConfig.model_validate(data, strict=False) + except Exception: + log.warning("outer_loop_config_load_failed", exc_info=True) + return None + + +def save_map_elites(project_path: Path, archive: MAPElitesArchive) -> None: + """Persist the MAP-Elites grid to .factory/outer-loop/map-elites/grid.json.""" + grid_path = project_path / ".factory" / "outer-loop" / "map-elites" / "grid.json" + grid_path.parent.mkdir(parents=True, exist_ok=True) + + grid_data: dict[str, object] = {} + for key, ind in archive._grid.items(): + grid_data[str(key)] = ind.model_dump(mode="json") + + grid_path.write_text(json.dumps(grid_data, indent=2, default=str)) + + +def save_best( + project_path: Path, + result: OuterLoopResult, +) -> None: + """Write the best workflow and audit results to .factory/outer-loop/best/.""" + best_dir = project_path / ".factory" / "outer-loop" / "best" + best_dir.mkdir(parents=True, exist_ok=True) + + (best_dir / "workflow.json").write_text( + json.dumps(result.best_workflow_data, indent=2, default=str) + ) + + if result.holdout_score > 0 or result.overfit_flag: + audit = { + "holdout_score": result.holdout_score, + "overfit_flag": result.overfit_flag, + "best_score": result.best_score, + } + (best_dir / "holdout_audit.json").write_text( + json.dumps(audit, indent=2) + ) + + +def export_best_workflow( + project_path: Path, + best_workflow_data: dict[str, object], + benchmark_name: str, +) -> Path: + """Export the best workflow as a portable .factory/workflows/-evolved.py. + + Returns the path to the exported file. + """ + workflows_dir = project_path / ".factory" / "workflows" + workflows_dir.mkdir(parents=True, exist_ok=True) + + export_path = workflows_dir / f"{benchmark_name}-evolved.py" + + wf = Workflow.from_dict(best_workflow_data) # type: ignore[arg-type] + + wf_json = json.dumps(wf.to_dict(), indent=4, default=str) + content = ( + f'"""Auto-evolved workflow for {benchmark_name}."""\n' + f"\n" + f"from factory.workflow.primitives import (\n" + f" AgentNode,\n" + f" AgentRole,\n" + f" Edge,\n" + f" FnNode,\n" + f" GateNode,\n" + f" Study,\n" + f" VerdictType,\n" + f" Workflow,\n" + f")\n" + f"\n" + f"\n" + f"meta = {{\n" + f' "name": "{benchmark_name}-evolved",\n' + f' "description": "Evolved workflow for {benchmark_name} benchmark",\n' + f"}}\n" + f"\n" + f"\n" + f"def workflow() -> Workflow:\n" + f' """Evolved workflow for {benchmark_name}."""\n' + f" return Workflow.from_dict({wf_json})\n" + ) + + export_path.write_text(content) + + also_best = project_path / ".factory" / "outer-loop" / "best" / "workflow.py" + also_best.parent.mkdir(parents=True, exist_ok=True) + also_best.write_text(export_path.read_text()) + + log.info("best_workflow_exported", path=str(export_path)) + return export_path diff --git a/factory/outer_loop/harbor_evaluator.py b/factory/outer_loop/harbor_evaluator.py new file mode 100644 index 000000000..6796b2188 --- /dev/null +++ b/factory/outer_loop/harbor_evaluator.py @@ -0,0 +1,249 @@ +"""Harbor evaluator — runs FeatureBench via Harbor to score workflow candidates. + +Implements the ``EvaluatorFn`` protocol so ``SwarmEvaluator`` can use real +benchmark results as the fitness signal for evolutionary search. +""" + +from __future__ import annotations + +import base64 +import os +import re +import subprocess +from pathlib import Path + +import structlog +import yaml + +from factory.outer_loop.models import EvalResult +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + GateNode, + Workflow, +) + +log = structlog.get_logger() + +_BENCHMARKS_DIR = Path(__file__).resolve().parents[2] / "benchmarks" +_RESOLVED_RE = re.compile(r"Result:\s*RESOLVED") +_COST_RE = re.compile(r'"cost_usd":\s*([0-9.]+)') + + +def create_seed_workflow() -> Workflow: + """Build a simple 4-node seed workflow for FeatureBench evolution. + + Structure: researcher → builder → health_checker → gate + + The ``builder`` node ID matches the registered featurebench workflow + so its prompt_template override takes effect during Harbor evaluation. + """ + nodes: dict[str, AgentNode | GateNode] = { + "researcher": AgentNode( + id="researcher", + role=AgentRole.RESEARCHER, + prompt_template=( + "Study the codebase and task. Read /tmp/task-instruction.md. " + "Explore the repository structure and identify files to modify. " + "Write findings to .factory/reviews/study-output.md." + ), + writes={".factory/reviews/study-output.md"}, + timeout=300, + ), + "builder": AgentNode( + id="builder", + role=AgentRole.BUILDER, + prompt_template=( + "You are implementing a new feature in a Python codebase.\n\n" + "1. Read the FULL task description at /tmp/task-instruction.md.\n" + "2. Read .factory/reviews/study-output.md for codebase context.\n" + "3. CRITICAL: Read the actual source code for every function, class, " + "or module you reference. Do NOT guess signatures or imports.\n" + "4. Implement the feature following interface specs EXACTLY.\n" + "5. Ensure all cross-file imports and references resolve correctly.\n" + "6. Run the project's test suite.\n" + "7. Fix any test failures — trace errors to root cause.\n" + "8. Commit changes on the current branch.\n\n" + "Rules:\n" + "- Act AUTONOMOUSLY — do NOT ask for confirmation\n" + "- Follow interface specs EXACTLY\n" + "- Do NOT modify test files\n" + "- Do NOT create branches or PRs — commit on current branch\n" + "- Do NOT run factory commands" + ), + reads={".factory/reviews/study-output.md"}, + writes={".factory/reviews/builder-latest.md"}, + timeout=7200, + ), + "health_checker": AgentNode( + id="health_checker", + role=AgentRole.HEALTH_CHECKER, + prompt_template=( + "Run the project's test suite and verify the implementation. " + "Report test results and any issues found." + ), + reads={".factory/reviews/builder-latest.md"}, + writes={".factory/reviews/health-check.md"}, + timeout=600, + ), + "gate": GateNode( + id="gate", + evaluator_type="fn", + evaluator_command=( + 'cd "$PROJECT_PATH" && ' + 'CHANGES=$(git diff HEAD~1 --stat 2>/dev/null || echo NO_COMMITS) && ' + 'if [ "$CHANGES" = "NO_COMMITS" ] || [ -z "$CHANGES" ]; then ' + 'echo "HALT: no changes committed"; ' + 'else echo "PROCEED"; fi' + ), + reads={".factory/reviews/health-check.md"}, + ), + } + edges = [ + Edge(source="researcher", target="builder"), + Edge(source="builder", target="health_checker"), + Edge(source="health_checker", target="gate"), + ] + return Workflow( + name="featurebench-seed", + nodes=nodes, # type: ignore[arg-type] + edges=edges, + start_node="researcher", + ) + + +def workflow_to_harbor_yaml(wf: Workflow) -> str: + """Convert a Workflow to a YAML annotation surface for Harbor override. + + Generates YAML that ``yaml_to_workflow()`` applies as prompt/timeout + overrides on the registered featurebench workflow. Only node IDs + matching the registered workflow take effect; non-matching IDs are + silently ignored. + """ + surface: dict[str, dict[str, object]] = {} + for node_id, node in wf.nodes.items(): + if isinstance(node, AgentNode) and node.prompt_template: + slots: dict[str, object] = { + f"task_prompt_{node_id}": node.prompt_template, + } + if node.timeout is not None: + slots[f"timeout_{node_id}"] = node.timeout + surface[node_id] = {"type": "AgentNode", "id": node_id, "slots": slots} + elif isinstance(node, GateNode) and node.gate_prompt: + surface[node_id] = { + "type": "GateNode", + "id": node_id, + "slots": {f"gate_prompt_{node_id}": node.gate_prompt}, + } + return yaml.dump(surface, default_flow_style=False) + + +class HarborEvaluator: + """Evaluates workflow candidates by running FeatureBench instances via Harbor. + + Implements the ``EvaluatorFn`` protocol:: + + __call__(workflow, project_dir, instances) -> EvalResult + + For each instance, runs ``benchmarks/run-harbor.sh featurebench --task `` + with the workflow's prompt overrides injected via ``FACTORY_WORKFLOW_YAML_B64``. + Parses stdout for resolved/not-resolved status and aggregates into a score. + """ + + def __init__( + self, + benchmarks_dir: Path | None = None, + timeout: int = 300, + ) -> None: + self._benchmarks_dir = benchmarks_dir or _BENCHMARKS_DIR + self._timeout = timeout + self._script = self._benchmarks_dir / "run-harbor.sh" + + def __call__( + self, + workflow: Workflow, + project_dir: str, + instances: list[str], + ) -> EvalResult: + """Run the workflow on each instance via Harbor and return aggregate score.""" + if not self._script.exists(): + log.error("run_harbor_script_missing", path=str(self._script)) + return EvalResult(score=0.0, details={"error": "run-harbor.sh not found"}) + + yaml_b64 = base64.b64encode( + workflow_to_harbor_yaml(workflow).encode() + ).decode() + + resolved = 0 + total = len(instances) + total_cost = 0.0 + per_instance: dict[str, object] = {} + + for instance_id in instances: + success, cost = self._run_instance(instance_id, yaml_b64) + per_instance[instance_id] = {"resolved": success, "cost_usd": cost} + if success: + resolved += 1 + total_cost += cost + + score = resolved / max(total, 1) + log.info( + "harbor_eval_done", + resolved=resolved, + total=total, + score=score, + cost_usd=total_cost, + ) + return EvalResult( + score=score, + benchmark_score=score, + cost_usd=total_cost, + complexity=float(len(workflow.nodes)), + details={"instances": per_instance}, + ) + + def _run_instance( + self, instance_id: str, yaml_b64: str + ) -> tuple[bool, float]: + """Run a single instance via run-harbor.sh. Returns (resolved, cost_usd).""" + cmd = [ + str(self._script), + "featurebench", + "--task", + instance_id, + "--timeout", + str(self._timeout), + "--preserve", + ] + env = dict(os.environ) + env["FACTORY_WORKFLOW_YAML_B64"] = yaml_b64 + + log.info("harbor_instance_start", instance=instance_id) + try: + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=self._timeout * 3 + 300, + env=env, + ) + except subprocess.TimeoutExpired: + log.warning("harbor_instance_timeout", instance=instance_id) + return False, 0.0 + except (FileNotFoundError, OSError) as exc: + log.error("harbor_instance_error", instance=instance_id, error=str(exc)) + return False, 0.0 + + resolved = bool(_RESOLVED_RE.search(proc.stdout)) + cost_match = _COST_RE.search(proc.stdout) + cost = float(cost_match.group(1)) if cost_match else 0.0 + + log.info( + "harbor_instance_done", + instance=instance_id, + resolved=resolved, + cost_usd=cost, + returncode=proc.returncode, + ) + return resolved, cost diff --git a/factory/outer_loop/models.py b/factory/outer_loop/models.py new file mode 100644 index 000000000..bf0678cd3 --- /dev/null +++ b/factory/outer_loop/models.py @@ -0,0 +1,185 @@ +"""Pydantic v2 strict models for the outer loop evolutionary search.""" + +from __future__ import annotations + +from enum import Enum + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class MutationType(str, Enum): + """Types of graph mutation operators.""" + + NODE_INSERT = "node_insert" + NODE_REMOVE = "node_remove" + EDGE_REDIRECT = "edge_redirect" + PARALLELIZE = "parallelize" + SERIALIZE = "serialize" + PARAM_MUTATE = "param_mutate" + + +class MutationRecord(BaseModel): + """Record of a single mutation applied to a workflow.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + operator: MutationType + target_node: str | None = None + before: dict[str, object] = Field(default_factory=dict) + after: dict[str, object] = Field(default_factory=dict) + rationale: str = "" + + @field_validator("operator", mode="before") + @classmethod + def _coerce_operator(cls, v: object) -> MutationType: + if isinstance(v, str): + return MutationType(v) + return v # type: ignore[return-value] + + +class Individual(BaseModel): + """A single candidate in the evolutionary population.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + id: str + workflow_data: dict[str, object] + score: float = 0.0 + features: tuple[int, ...] = () + generation: int = 0 + parent_id: str | None = None + mutation_record: MutationRecord | None = None + cost_usd: float = 0.0 + + @field_validator("features", mode="before") + @classmethod + def _coerce_features(cls, v: object) -> tuple[int, ...]: + if isinstance(v, list): + return tuple(v) + return v # type: ignore[return-value] + + +class HyperparameterRecord(BaseModel): + """Per-generation evolutionary hyperparameters for Level 3 training data.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + generation: int + mutation_rate: float + population_size: int + tournament_size: int + designer_ratio: float + operator_weights: dict[str, float] = Field(default_factory=dict) + best_score: float = 0.0 + mean_score: float = 0.0 + diversity: float = 0.0 + novel_count: int = 0 + + +class SwarmConfig(BaseModel): + """Configuration for the evolutionary swarm search.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + benchmark: str + budget: int + population_size: int = 4 + tournament_size: int = 3 + mutation_rate: float = 0.3 + target_score: float | None = None + frozen_node_ids: list[str] = Field(default_factory=list) + mandatory_node_roles: list[str] = Field(default_factory=list) + feature_axes: list[str] = Field( + default_factory=lambda: ["depth", "fork_degree", "agent_count", "gate_count"] + ) + mutation_strategy: str = "weighted_random" + designer_count: int = 2 + training_instances: list[str] = Field(default_factory=list) + holdout_instances: list[str] = Field(default_factory=list) + + @field_validator("holdout_instances") + @classmethod + def _no_overlap_with_training(cls, v: list[str], info: object) -> list[str]: + data = getattr(info, "data", {}) + training = data.get("training_instances", []) + overlap = set(v) & set(training) + if overlap: + raise ValueError( + f"holdout_instances must not overlap with training_instances: {overlap}" + ) + return v + + +class OuterLoopState(BaseModel): + """Checkpoint state for the outer loop evolution.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + generation: int = 0 + total_evaluations: int = 0 + best_score: float = 0.0 + budget_remaining: int = 0 + convergence_reason: str | None = None + score_trajectory: list[float] = Field(default_factory=list) + hyperparameter_history: list[HyperparameterRecord] = Field(default_factory=list) + + +class GenerationSummary(BaseModel): + """Summary of a single generation of evolution.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + generation: int + population_size: int + best_score: float + mean_score: float + diversity: float + mutations_applied: list[MutationRecord] = Field(default_factory=list) + novel_count: int = 0 + rejected_duplicates: int = 0 + holdout_score: float = 0.0 + hyperparameters: HyperparameterRecord | None = None + + +class EvalResult(BaseModel): + """Result of evaluating a single workflow candidate.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + score: float + benchmark_score: float = 0.0 + hygiene_score: float = 0.0 + cost_usd: float = 0.0 + complexity: float = 0.0 + details: dict[str, object] = Field(default_factory=dict) + + +class AuditResult(BaseModel): + """Result of overfit detection on the best evolved workflow.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + training_score: float + holdout_score: float + delta: float + overfit_flag: bool + details: str = "" + + +class OuterLoopResult(BaseModel): + """Result of a complete outer loop evolutionary run.""" + + model_config = ConfigDict(strict=True, extra="forbid") + + best_workflow_data: dict[str, object] = Field(default_factory=dict) + best_score: float = 0.0 + holdout_score: float = 0.0 + overfit_flag: bool = False + trajectory: list[GenerationSummary] = Field(default_factory=list) + total_cost_usd: float = 0.0 + convergence_reason: str = "" + generations_completed: int = 0 + total_evaluations: int = 0 + archive_size: int = 0 + pareto_front: list[Individual] = Field(default_factory=list) + hyperparameter_history: list[HyperparameterRecord] = Field(default_factory=list) diff --git a/factory/outer_loop/mutations.py b/factory/outer_loop/mutations.py new file mode 100644 index 000000000..f5f580159 --- /dev/null +++ b/factory/outer_loop/mutations.py @@ -0,0 +1,557 @@ +"""Structured graph mutation operators and strategy protocol for workflow evolution.""" + +from __future__ import annotations + +import random +from typing import Protocol, runtime_checkable + +import networkx as nx +import structlog + +from factory.outer_loop.models import MutationRecord, MutationType +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + ForkNode, + JoinNode, + NodeType, + Workflow, +) + +log = structlog.get_logger() + + +@runtime_checkable +class MutationStrategy(Protocol): + """Protocol for pluggable mutation operator selection.""" + + def select_operator( + self, parent: Workflow, generation: int, archive_stats: dict[str, object] + ) -> MutationType: ... + + def get_mutation_rate(self, generation: int) -> float: ... + + def get_designer_ratio(self, generation: int) -> float: ... + + +class WeightedRandomStrategy: + """Default mutation strategy: select operators by configurable weights.""" + + def __init__( + self, + weights: dict[str, float] | None = None, + mutation_rate: float = 0.3, + designer_ratio: float = 0.3, + ) -> None: + self.weights = weights or { + MutationType.NODE_INSERT.value: 0.2, + MutationType.NODE_REMOVE.value: 0.15, + MutationType.EDGE_REDIRECT.value: 0.2, + MutationType.PARALLELIZE.value: 0.15, + MutationType.SERIALIZE.value: 0.1, + MutationType.PARAM_MUTATE.value: 0.2, + } + self._mutation_rate = mutation_rate + self._designer_ratio = designer_ratio + + def select_operator( + self, parent: Workflow, generation: int, archive_stats: dict[str, object] + ) -> MutationType: + types = list(MutationType) + w = [self.weights.get(t.value, 0.1) for t in types] + return random.choices(types, weights=w, k=1)[0] + + def get_mutation_rate(self, generation: int) -> float: + return self._mutation_rate + + def get_designer_ratio(self, generation: int) -> float: + return self._designer_ratio + + def get_operator_weights(self) -> dict[str, float]: + return dict(self.weights) + + def on_plateau(self) -> None: + """Increase mutation rate when evolution stalls.""" + self._mutation_rate = min(self._mutation_rate + 0.2, 0.8) + + def on_improvement(self) -> None: + """Reset mutation rate after improvement.""" + self._mutation_rate = 0.3 + + +def validate_and_repair(workflow: Workflow) -> Workflow | None: + """Validate a mutated workflow and attempt repair. Returns None if irreparable.""" + g: nx.DiGraph[str] = nx.DiGraph() + for nid in workflow.nodes: + g.add_node(nid) + for edge in workflow.edges: + if edge.source in workflow.nodes and edge.target in workflow.nodes: + g.add_edge(edge.source, edge.target) + + if workflow.start_node not in workflow.nodes: + return None + + # Prune unreachable nodes + reachable = nx.descendants(g, workflow.start_node) | {workflow.start_node} + unreachable = set(workflow.nodes.keys()) - reachable + for nid in unreachable: + del workflow.nodes[nid] + workflow.edges = [ + e for e in workflow.edges + if e.source in workflow.nodes and e.target in workflow.nodes + ] + + # Rebuild graph and check for cycles without gate conditions + g2: nx.DiGraph[str] = nx.DiGraph() + for nid in workflow.nodes: + g2.add_node(nid) + for edge in workflow.edges: + g2.add_edge(edge.source, edge.target) + + for cycle in nx.simple_cycles(g2): + has_gated_edge = False + for i in range(len(cycle)): + src = cycle[i] + tgt = cycle[(i + 1) % len(cycle)] + if type(workflow.nodes.get(src)).__name__ == "GateNode": + for e in workflow.edges: + if e.source == src and e.target == tgt and e.condition is not None: + has_gated_edge = True + break + if has_gated_edge: + break + if not has_gated_edge: + return None + + # Verify reads/writes chain + for nid, node in workflow.nodes.items(): + if node.reads: + ancestors = nx.ancestors(g2, nid) if nid in g2 else set() + available_writes: set[str] = set() + for anc in ancestors: + anc_node = workflow.nodes.get(anc) + if anc_node: + available_writes |= anc_node.writes + broken_reads = node.reads - available_writes + if broken_reads: + node_copy = node.model_copy(update={"reads": node.reads - broken_reads}) + workflow.nodes[nid] = node_copy # type: ignore[assignment] + + return workflow + + +def _is_frozen(node_id: str, frozen_nodes: set[str]) -> bool: + return node_id in frozen_nodes + + +def _deep_copy_workflow(workflow: Workflow) -> Workflow: + """Deep copy a workflow for mutation.""" + nodes: dict[str, NodeType] = {} + for nid, node in workflow.nodes.items(): + nodes[nid] = node.model_copy(deep=True) + edges = [e.model_copy(deep=True) for e in workflow.edges] + return Workflow( + name=workflow.name, + nodes=nodes, + edges=edges, + start_node=workflow.start_node, + terminal=workflow.terminal, + ) + + +def insert_node( + workflow: Workflow, + new_node: NodeType, + after_node_id: str, + *, + frozen_nodes: set[str] | None = None, +) -> tuple[Workflow, MutationRecord] | None: + """Insert a new node after an existing node, reconnecting edges.""" + frozen = frozen_nodes or set() + if _is_frozen(after_node_id, frozen): + return None + + wf = _deep_copy_workflow(workflow) + if after_node_id not in wf.nodes: + return None + + wf.nodes[new_node.id] = new_node + + outgoing = [e for e in wf.edges if e.source == after_node_id] + if not outgoing: + wf.edges.append(Edge(source=after_node_id, target=new_node.id)) + else: + first_edge = outgoing[0] + old_target = first_edge.target + wf.edges = [e for e in wf.edges if not (e.source == after_node_id and e.target == old_target and e.condition is None)] + wf.edges.append(Edge(source=after_node_id, target=new_node.id)) + wf.edges.append(Edge(source=new_node.id, target=old_target)) + + result = validate_and_repair(wf) + if result is None: + return None + + record = MutationRecord( + operator=MutationType.NODE_INSERT, + target_node=new_node.id, + before={}, + after={"inserted_after": after_node_id}, + rationale=f"Inserted {new_node.id} after {after_node_id}", + ) + return result, record + + +def remove_node( + workflow: Workflow, + node_id: str, + *, + frozen_nodes: set[str] | None = None, +) -> tuple[Workflow, MutationRecord] | None: + """Remove a node and short-circuit its edges.""" + frozen = frozen_nodes or set() + if _is_frozen(node_id, frozen): + return None + + wf = _deep_copy_workflow(workflow) + if node_id not in wf.nodes or node_id == wf.start_node: + return None + + incoming_sources = [e.source for e in wf.edges if e.target == node_id] + outgoing_targets = [e.target for e in wf.edges if e.source == node_id] + + wf.edges = [e for e in wf.edges if e.source != node_id and e.target != node_id] + + for src in incoming_sources: + for tgt in outgoing_targets: + if not any(e.source == src and e.target == tgt for e in wf.edges): + wf.edges.append(Edge(source=src, target=tgt)) + + del wf.nodes[node_id] + + result = validate_and_repair(wf) + if result is None: + return None + + record = MutationRecord( + operator=MutationType.NODE_REMOVE, + target_node=node_id, + before={"node_existed": True}, + after={"short_circuited": True}, + rationale=f"Removed {node_id}, short-circuited edges", + ) + return result, record + + +def redirect_edge( + workflow: Workflow, + source_id: str, + old_target_id: str, + new_target_id: str, + *, + frozen_nodes: set[str] | None = None, +) -> tuple[Workflow, MutationRecord] | None: + """Redirect an edge from old_target to new_target.""" + frozen = frozen_nodes or set() + if _is_frozen(source_id, frozen): + return None + + wf = _deep_copy_workflow(workflow) + if new_target_id not in wf.nodes: + return None + + found = False + new_edges: list[Edge] = [] + for e in wf.edges: + if e.source == source_id and e.target == old_target_id and not found: + new_edges.append(Edge(source=source_id, target=new_target_id, condition=e.condition)) + found = True + else: + new_edges.append(e) + + if not found: + return None + + wf.edges = new_edges + result = validate_and_repair(wf) + if result is None: + return None + + record = MutationRecord( + operator=MutationType.EDGE_REDIRECT, + target_node=source_id, + before={"target": old_target_id}, + after={"target": new_target_id}, + rationale=f"Redirected edge from {source_id}: {old_target_id} → {new_target_id}", + ) + return result, record + + +def parallelize( + workflow: Workflow, + node_ids: list[str], + *, + frozen_nodes: set[str] | None = None, +) -> tuple[Workflow, MutationRecord] | None: + """Convert sequential nodes to parallel execution via ForkNode + JoinNode.""" + frozen = frozen_nodes or set() + if any(_is_frozen(nid, frozen) for nid in node_ids): + return None + if len(node_ids) < 2: + return None + + wf = _deep_copy_workflow(workflow) + for nid in node_ids: + if nid not in wf.nodes: + return None + + fork_id = f"fork_{'_'.join(node_ids[:2])}" + join_id = f"join_{'_'.join(node_ids[:2])}" + + first_node = node_ids[0] + last_node = node_ids[-1] + + predecessors = {e.source for e in wf.edges if e.target == first_node} + successors = {e.target for e in wf.edges if e.source == last_node} + + for nid in node_ids: + wf.edges = [e for e in wf.edges if e.source != nid and e.target != nid] + + wf.nodes[fork_id] = ForkNode(id=fork_id, targets=node_ids) + wf.nodes[join_id] = JoinNode(id=join_id, sources=node_ids) + + for pred in predecessors: + wf.edges.append(Edge(source=pred, target=fork_id)) + + for nid in node_ids: + wf.edges.append(Edge(source=fork_id, target=nid)) + wf.edges.append(Edge(source=nid, target=join_id)) + + for succ in successors: + wf.edges.append(Edge(source=join_id, target=succ)) + + if wf.start_node == first_node: + wf.start_node = fork_id + + result = validate_and_repair(wf) + if result is None: + return None + + record = MutationRecord( + operator=MutationType.PARALLELIZE, + target_node=fork_id, + before={"sequential": node_ids}, + after={"parallel": node_ids}, + rationale=f"Parallelized {node_ids}", + ) + return result, record + + +def serialize( + workflow: Workflow, + fork_id: str, + *, + frozen_nodes: set[str] | None = None, +) -> tuple[Workflow, MutationRecord] | None: + """Collapse a fork/join pair back into sequential execution.""" + frozen = frozen_nodes or set() + if _is_frozen(fork_id, frozen): + return None + + wf = _deep_copy_workflow(workflow) + fork_node = wf.nodes.get(fork_id) + if fork_node is None or type(fork_node).__name__ != "ForkNode": + return None + + targets = fork_node.targets # type: ignore[union-attr] + + join_id: str | None = None + for nid, node in wf.nodes.items(): + if type(node).__name__ == "JoinNode": + sources = node.sources # type: ignore[union-attr] + if set(sources) == set(targets): + join_id = nid + break + + if join_id is None: + return None + + predecessors = {e.source for e in wf.edges if e.target == fork_id} + successors = {e.target for e in wf.edges if e.source == join_id} + + wf.edges = [ + e for e in wf.edges + if e.source != fork_id and e.target != fork_id + and e.source != join_id and e.target != join_id + and not (e.source in targets and e.target == join_id) + ] + + del wf.nodes[fork_id] + del wf.nodes[join_id] + + chain = list(targets) + for pred in predecessors: + wf.edges.append(Edge(source=pred, target=chain[0])) + + for i in range(len(chain) - 1): + wf.edges.append(Edge(source=chain[i], target=chain[i + 1])) + + for succ in successors: + wf.edges.append(Edge(source=chain[-1], target=succ)) + + if wf.start_node == fork_id: + wf.start_node = chain[0] + + result = validate_and_repair(wf) + if result is None: + return None + + record = MutationRecord( + operator=MutationType.SERIALIZE, + target_node=fork_id, + before={"parallel": list(targets)}, + after={"sequential": chain}, + rationale=f"Serialized fork {fork_id}", + ) + return result, record + + +def mutate_params( + workflow: Workflow, + node_id: str, + changes: dict[str, object], + *, + frozen_nodes: set[str] | None = None, +) -> tuple[Workflow, MutationRecord] | None: + """Change parameters on a node (timeout, model, max_iterations).""" + frozen = frozen_nodes or set() + if _is_frozen(node_id, frozen): + return None + + wf = _deep_copy_workflow(workflow) + node = wf.nodes.get(node_id) + if node is None: + return None + + allowed_params = {"timeout", "model", "max_iterations", "blocking"} + filtered_changes = {k: v for k, v in changes.items() if k in allowed_params} + if not filtered_changes: + return None + + before: dict[str, object] = {} + for k in filtered_changes: + if hasattr(node, k): + before[k] = getattr(node, k) + + try: + updated_node = node.model_copy(update=filtered_changes) + wf.nodes[node_id] = updated_node # type: ignore[assignment] + except Exception: + return None + + result = validate_and_repair(wf) + if result is None: + return None + + record = MutationRecord( + operator=MutationType.PARAM_MUTATE, + target_node=node_id, + before=before, + after=dict(filtered_changes), + rationale=f"Changed params on {node_id}: {filtered_changes}", + ) + return result, record + + +def apply_random_mutation( + workflow: Workflow, + strategy: MutationStrategy, + generation: int, + *, + frozen_nodes: set[str] | None = None, + archive_stats: dict[str, object] | None = None, + max_attempts: int = 10, +) -> tuple[Workflow, MutationRecord] | None: + """Apply a random mutation using the given strategy. Retries on failure.""" + frozen = frozen_nodes or set() + stats = archive_stats or {} + + for _ in range(max_attempts): + op = strategy.select_operator(workflow, generation, stats) + result = _try_mutation(workflow, op, frozen) + if result is not None: + return result + + return None + + +def _try_mutation( + workflow: Workflow, + op: MutationType, + frozen: set[str], +) -> tuple[Workflow, MutationRecord] | None: + """Attempt a single mutation of the given type.""" + mutable_nodes = [ + nid for nid in workflow.nodes if nid not in frozen and nid != workflow.start_node + ] + if not mutable_nodes and op != MutationType.NODE_INSERT: + return None + + if op == MutationType.NODE_INSERT: + target = random.choice(list(workflow.nodes.keys())) + new_id = f"agent_{random.randint(100, 999)}" + roles = list(AgentRole) + new_node = AgentNode( + id=new_id, + role=random.choice(roles), + ) + return insert_node(workflow, new_node, target, frozen_nodes=frozen) + + elif op == MutationType.NODE_REMOVE: + target = random.choice(mutable_nodes) + return remove_node(workflow, target, frozen_nodes=frozen) + + elif op == MutationType.EDGE_REDIRECT: + edges_from_mutable = [ + e for e in workflow.edges if e.source not in frozen + ] + if not edges_from_mutable: + return None + edge = random.choice(edges_from_mutable) + possible_targets = [nid for nid in workflow.nodes if nid != edge.target] + if not possible_targets: + return None + new_target = random.choice(possible_targets) + return redirect_edge(workflow, edge.source, edge.target, new_target, frozen_nodes=frozen) + + elif op == MutationType.PARALLELIZE: + if len(mutable_nodes) < 2: + return None + pair = random.sample(mutable_nodes, 2) + return parallelize(workflow, pair, frozen_nodes=frozen) + + elif op == MutationType.SERIALIZE: + fork_ids = [ + nid for nid, n in workflow.nodes.items() + if type(n).__name__ == "ForkNode" and nid not in frozen + ] + if not fork_ids: + return None + return serialize(workflow, random.choice(fork_ids), frozen_nodes=frozen) + + elif op == MutationType.PARAM_MUTATE: + agent_nodes = [ + nid for nid in mutable_nodes + if type(workflow.nodes[nid]).__name__ == "AgentNode" + ] + if not agent_nodes: + return None + target = random.choice(agent_nodes) + param = random.choice(["timeout", "model"]) + if param == "timeout": + changes: dict[str, object] = {"timeout": random.choice([300, 600, 900, 1200, 1800])} + else: + changes = {"model": random.choice(["sonnet", "opus", "haiku"])} + return mutate_params(workflow, target, changes, frozen_nodes=frozen) + + return None diff --git a/factory/outer_loop/overfit.py b/factory/outer_loop/overfit.py new file mode 100644 index 000000000..61f12cbbb --- /dev/null +++ b/factory/outer_loop/overfit.py @@ -0,0 +1,78 @@ +"""Overfit / cheating detection for evolved workflows.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import structlog + +from factory.outer_loop.models import AuditResult + +if TYPE_CHECKING: + from factory.outer_loop.evaluator import SwarmEvaluator + from factory.workflow.primitives import Workflow + +log = structlog.get_logger() + +OVERFIT_THRESHOLD = 0.15 + + +class OverfitDetector: + """Detects overfitting by comparing training vs holdout scores.""" + + def __init__(self, threshold: float = OVERFIT_THRESHOLD) -> None: + self._threshold = threshold + + def audit( + self, + best_workflow: Workflow, + training_instances: list[str], + holdout_instances: list[str], + evaluator: SwarmEvaluator, + project_dir: str = "", + ) -> AuditResult: + """Run the best workflow on both training and holdout instances. + + Flags overfit if (training - holdout) / training > threshold. + """ + train_result = evaluator.evaluate(best_workflow, project_dir, training_instances) + holdout_result = evaluator.evaluate(best_workflow, project_dir, holdout_instances) + + training_score = train_result.score + holdout_score = holdout_result.score + + if training_score > 0: + delta = (training_score - holdout_score) / training_score + else: + delta = 0.0 + + overfit_flag = delta > self._threshold + + if overfit_flag: + log.warning( + "overfit_detected", + training_score=training_score, + holdout_score=holdout_score, + delta=delta, + threshold=self._threshold, + ) + else: + log.info( + "overfit_audit_passed", + training_score=training_score, + holdout_score=holdout_score, + delta=delta, + ) + + details = ( + f"training={training_score:.4f} holdout={holdout_score:.4f} " + f"delta={delta:.4f} threshold={self._threshold}" + ) + + return AuditResult( + training_score=training_score, + holdout_score=holdout_score, + delta=delta, + overfit_flag=overfit_flag, + details=details, + ) diff --git a/factory/outer_loop/population.py b/factory/outer_loop/population.py new file mode 100644 index 000000000..cb9b3c03e --- /dev/null +++ b/factory/outer_loop/population.py @@ -0,0 +1,203 @@ +"""Population management and MAP-Elites archive for evolutionary search.""" + +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import TYPE_CHECKING + +import structlog + +from factory.outer_loop.models import Individual +from factory.outer_loop.similarity import compute_features + +if TYPE_CHECKING: + from factory.workflow.primitives import Workflow + +log = structlog.get_logger() + + +class Population: + """Manages a collection of Individual candidates.""" + + def __init__(self) -> None: + self._individuals: dict[str, Individual] = {} + + @property + def size(self) -> int: + return len(self._individuals) + + @property + def individuals(self) -> list[Individual]: + return list(self._individuals.values()) + + def add(self, individual: Individual) -> None: + self._individuals[individual.id] = individual + + def remove(self, individual_id: str) -> Individual | None: + return self._individuals.pop(individual_id, None) + + def get(self, individual_id: str) -> Individual | None: + return self._individuals.get(individual_id) + + def best(self) -> Individual | None: + if not self._individuals: + return None + return max(self._individuals.values(), key=lambda i: i.score) + + def mean_score(self) -> float: + if not self._individuals: + return 0.0 + return sum(i.score for i in self._individuals.values()) / len(self._individuals) + + @staticmethod + def make_individual( + workflow: Workflow, + *, + generation: int = 0, + parent_id: str | None = None, + mutation_record: object = None, + score: float = 0.0, + cost_usd: float = 0.0, + ) -> Individual: + """Create an Individual from a Workflow, computing features automatically.""" + from factory.outer_loop.models import MutationRecord + + features = compute_features(workflow) + return Individual( + id=uuid.uuid4().hex[:12], + workflow_data=workflow.to_dict(), + score=score, + features=features, + generation=generation, + parent_id=parent_id, + mutation_record=mutation_record if isinstance(mutation_record, MutationRecord) else None, + cost_usd=cost_usd, + ) + + def save(self, directory: Path) -> None: + """Serialize the population to a directory.""" + directory.mkdir(parents=True, exist_ok=True) + data = [ind.model_dump(mode="json") for ind in self._individuals.values()] + (directory / "population.json").write_text(json.dumps(data, indent=2)) + + @classmethod + def load(cls, directory: Path) -> Population: + """Deserialize a population from a directory.""" + pop = cls() + path = directory / "population.json" + if path.exists(): + data = json.loads(path.read_text()) + for item in data: + pop.add(Individual.model_validate(item)) + return pop + + +class MAPElitesArchive: + """4D fixed-resolution grid archive for quality-diversity search. + + Axes: (depth, fork_degree, agent_count, gate_count). + Each cell stores the best-scoring Individual for that feature combination. + """ + + def __init__(self) -> None: + self._grid: dict[tuple[int, ...], Individual] = {} + + @property + def size(self) -> int: + return len(self._grid) + + def add(self, individual: Individual) -> bool: + """Add an individual to the archive. Returns True if it was inserted or replaced.""" + key = individual.features + existing = self._grid.get(key) + if existing is None or individual.score > existing.score: + self._grid[key] = individual + return True + return False + + def best(self) -> Individual | None: + if not self._grid: + return None + return max(self._grid.values(), key=lambda i: i.score) + + def all_individuals(self) -> list[Individual]: + return list(self._grid.values()) + + def sample_parent(self, tournament_size: int = 3) -> Individual | None: + """Tournament selection: pick tournament_size random individuals, return the best.""" + import random + + individuals = list(self._grid.values()) + if not individuals: + return None + k = min(tournament_size, len(individuals)) + tournament = random.sample(individuals, k) + return max(tournament, key=lambda i: i.score) + + def pareto_front(self) -> list[Individual]: + """Return the Pareto-optimal individuals (non-dominated on score + features). + + An individual is dominated if another has >= score and dominates on + all feature axes (higher is better for diversity purposes). + """ + individuals = list(self._grid.values()) + if len(individuals) <= 1: + return list(individuals) + + front: list[Individual] = [] + for candidate in individuals: + dominated = False + for other in individuals: + if other is candidate: + continue + if other.score >= candidate.score and all( + o >= c for o, c in zip(other.features, candidate.features) + ) and ( + other.score > candidate.score + or any(o > c for o, c in zip(other.features, candidate.features)) + ): + dominated = True + break + if not dominated: + front.append(candidate) + return front + + def diversity_metric(self) -> float: + """Fraction of occupied cells relative to a reasonable grid size estimate. + + Returns 0.0 for empty archive, approaches 1.0 as more cells are filled. + """ + if not self._grid: + return 0.0 + unique_per_axis: list[set[int]] = [set() for _ in range(4)] + for key in self._grid: + for i, v in enumerate(key): + if i < 4: + unique_per_axis[i].add(v) + total_possible = 1 + for s in unique_per_axis: + total_possible *= max(len(s), 1) + return len(self._grid) / max(total_possible, 1) + + def save(self, directory: Path) -> None: + """Serialize the archive to a directory.""" + directory.mkdir(parents=True, exist_ok=True) + data: dict[str, object] = {} + for key, ind in self._grid.items(): + str_key = ",".join(str(k) for k in key) + data[str_key] = ind.model_dump(mode="json") + (directory / "grid.json").write_text(json.dumps(data, indent=2)) + + @classmethod + def load(cls, directory: Path) -> MAPElitesArchive: + """Deserialize an archive from a directory.""" + archive = cls() + path = directory / "grid.json" + if path.exists(): + data = json.loads(path.read_text()) + for str_key, ind_data in data.items(): + ind = Individual.model_validate(ind_data) + archive._grid[ind.features] = ind + return archive diff --git a/factory/outer_loop/run_evolution.py b/factory/outer_loop/run_evolution.py new file mode 100644 index 000000000..8ad4a398c --- /dev/null +++ b/factory/outer_loop/run_evolution.py @@ -0,0 +1,133 @@ +"""Standalone outer-loop evolution runner for FeatureBench. + +Usage:: + + python -m factory.outer_loop.run_evolution \\ + --training-instances 'id1,id2,id3,id4,id5' \\ + --holdout-instances 'id6,id7' \\ + --generations 3 \\ + --population 4 \\ + --budget 30 +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import structlog + +from factory.outer_loop.direct_evaluator import DirectFeatureBenchEvaluator +from factory.outer_loop.engine import SwarmEngine +from factory.outer_loop.evaluator import SwarmEvaluator +from factory.outer_loop.harbor_evaluator import create_seed_workflow +from factory.outer_loop.models import SwarmConfig + +log = structlog.get_logger() + + +def main(argv: list[str] | None = None) -> int: + """Run the evolutionary search loop and print results.""" + parser = argparse.ArgumentParser( + description="Run outer-loop evolution on FeatureBench", + ) + parser.add_argument( + "--training-instances", + required=True, + help="Comma-separated training instance IDs", + ) + parser.add_argument( + "--holdout-instances", + default="", + help="Comma-separated holdout instance IDs", + ) + parser.add_argument( + "--generations", + type=int, + default=3, + help="Max generations (default: 3)", + ) + parser.add_argument( + "--population", + type=int, + default=4, + help="Population size (default: 4)", + ) + parser.add_argument( + "--budget", + type=int, + default=30, + help="Total evaluation budget (default: 30)", + ) + parser.add_argument( + "--timeout", + type=int, + default=1800, + help="Per-agent timeout in seconds (default: 1800)", + ) + parser.add_argument( + "--output", + default=".factory/outer-loop/best-workflow.json", + help="Path to write best workflow JSON", + ) + + args = parser.parse_args(argv) + + training = [s.strip() for s in args.training_instances.split(",") if s.strip()] + holdout = [s.strip() for s in args.holdout_instances.split(",") if s.strip()] + + if not training: + print( + "ERROR: --training-instances must contain at least one instance ID", + file=sys.stderr, + ) + return 1 + + config = SwarmConfig( + benchmark="featurebench", + budget=args.budget, + population_size=args.population, + training_instances=training, + holdout_instances=holdout, + ) + + direct_eval = DirectFeatureBenchEvaluator(agent_timeout=args.timeout) + evaluator = SwarmEvaluator(config, evaluator_fn=direct_eval) + engine = SwarmEngine(config=config, evaluator=evaluator) + seed = create_seed_workflow() + + print("=== Outer Loop Evolution — FeatureBench (Direct) ===") + print(f"Seed: {seed.name} ({len(seed.nodes)} nodes)") + print(f"Training: {len(training)} instances") + print(f"Holdout: {len(holdout)} instances") + print(f"Budget: {args.budget} evaluations") + print(f"Population: {args.population}") + print() + + result = engine.run(seed) + + print() + print("=" * 50) + print(f"Best score: {result.best_score:.4f}") + print(f"Holdout score: {result.holdout_score:.4f}") + print(f"Overfit: {result.overfit_flag}") + print(f"Convergence: {result.convergence_reason}") + print(f"Generations: {result.generations_completed}") + print(f"Evaluations: {result.total_evaluations}") + print(f"Cost: ${result.total_cost_usd:.2f}") + print(f"Archive size: {result.archive_size}") + print("=" * 50) + + if result.best_workflow_data: + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(result.best_workflow_data, indent=2)) + print(f"\nBest workflow written to {out}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/factory/outer_loop/similarity.py b/factory/outer_loop/similarity.py new file mode 100644 index 000000000..111d8b949 --- /dev/null +++ b/factory/outer_loop/similarity.py @@ -0,0 +1,134 @@ +"""Novelty filtering, deduplication, and feature extraction for workflows.""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING + +import networkx as nx + +if TYPE_CHECKING: + from factory.workflow.primitives import Workflow + + +def structural_hash(workflow: Workflow) -> str: + """SHA-256 of the canonical form of a workflow graph. + + Nodes are sorted by id; edges are sorted by (source, target). + The trigger function is excluded (not serializable). + """ + nodes_canonical: list[dict[str, object]] = [] + for nid in sorted(workflow.nodes): + node = workflow.nodes[nid] + d = node.model_dump(mode="json") + d["_type"] = type(node).__name__ + nodes_canonical.append(d) + + edges_canonical = sorted( + [e.model_dump(mode="json") for e in workflow.edges], + key=lambda e: (e["source"], e["target"]), + ) + + blob = json.dumps( + {"name": workflow.name, "nodes": nodes_canonical, "edges": edges_canonical}, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(blob.encode()).hexdigest() + + +def _build_nx_graph(workflow: Workflow) -> nx.DiGraph[str]: + """Build a NetworkX DiGraph from a workflow for analysis.""" + g: nx.DiGraph[str] = nx.DiGraph() + for nid in workflow.nodes: + g.add_node(nid, node_type=type(workflow.nodes[nid]).__name__) + for edge in workflow.edges: + g.add_edge(edge.source, edge.target) + return g + + +def graph_edit_distance(w1: Workflow, w2: Workflow) -> int: + """Approximate graph edit distance between two workflows. + + Counts: nodes in w1 not in w2, nodes in w2 not in w1, + edges in w1 not in w2, edges in w2 not in w1, + plus attribute diffs on common nodes (different type = 1 edit). + """ + n1 = set(w1.nodes.keys()) + n2 = set(w2.nodes.keys()) + + e1 = {(e.source, e.target) for e in w1.edges} + e2 = {(e.source, e.target) for e in w2.edges} + + dist = len(n1 - n2) + len(n2 - n1) + len(e1 - e2) + len(e2 - e1) + + for nid in n1 & n2: + if type(w1.nodes[nid]).__name__ != type(w2.nodes[nid]).__name__: + dist += 1 + + return dist + + +def compute_features(workflow: Workflow) -> tuple[int, int, int, int]: + """Extract (depth, fork_degree, agent_count, gate_count) from a workflow. + + - depth: longest path in the DAG + - fork_degree: max parallelism (largest ForkNode.targets count) + - agent_count: number of AgentNode instances + - gate_count: number of GateNode instances + """ + g = _build_nx_graph(workflow) + + try: + depth = nx.dag_longest_path_length(g) + except (nx.NetworkXUnfeasible, nx.NetworkXError): + depth = len(workflow.nodes) + + fork_degree = 0 + agent_count = 0 + gate_count = 0 + + for node in workflow.nodes.values(): + tname = type(node).__name__ + if tname == "ForkNode": + fork_degree = max(fork_degree, len(node.targets)) # type: ignore[union-attr] + elif tname == "AgentNode": + agent_count += 1 + elif tname == "GateNode": + gate_count += 1 + + return (depth, fork_degree, agent_count, gate_count) + + +class NoveltyFilter: + """Rejects near-duplicate workflows based on hash and edit distance.""" + + def __init__(self, min_edit_distance: int = 5, max_archive_size: int = 1000) -> None: + self.seen_hashes: set[str] = set() + self.min_edit_distance = min_edit_distance + self.max_archive_size = max_archive_size + self._archived_workflows: list[Workflow] = [] + + def is_novel(self, workflow: Workflow, threshold: int | None = None) -> bool: + """Check if a workflow is novel (not seen before). + + Returns False if the structural hash was seen before OR if the + graph edit distance to any archived workflow is below threshold. + """ + h = structural_hash(workflow) + if h in self.seen_hashes: + return False + + t = threshold if threshold is not None else self.min_edit_distance + for archived in self._archived_workflows: + if graph_edit_distance(workflow, archived) < t: + return False + + return True + + def add(self, workflow: Workflow) -> None: + """Register a workflow as seen.""" + self.seen_hashes.add(structural_hash(workflow)) + if len(self._archived_workflows) < self.max_archive_size: + self._archived_workflows.append(workflow) diff --git a/factory/outer_loop/subset.py b/factory/outer_loop/subset.py new file mode 100644 index 000000000..022b42765 --- /dev/null +++ b/factory/outer_loop/subset.py @@ -0,0 +1,30 @@ +"""Benchmark subset selection for evolutionary search.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +import structlog + +log = structlog.get_logger() + + +@runtime_checkable +class SubsetSelector(Protocol): + """Protocol for selecting which benchmark instances to evaluate per generation.""" + + def select( + self, all_instances: list[str], generation: int, budget_remaining: int + ) -> list[str]: ... + + +class FixedSubsetSelector: + """Always returns the configured training instances.""" + + def __init__(self, training_instances: list[str]) -> None: + self._training_instances = list(training_instances) + + def select( + self, all_instances: list[str], generation: int, budget_remaining: int + ) -> list[str]: + return list(self._training_instances) diff --git a/factory/outer_loop/workflow.py b/factory/outer_loop/workflow.py new file mode 100644 index 000000000..fbde55229 --- /dev/null +++ b/factory/outer_loop/workflow.py @@ -0,0 +1,168 @@ +"""Outer-loop workflow graph definition. + +Defines outer_loop_workflow() returning a Workflow: + study → seed_population → [generation loop: evaluate_batch → select → mutate → + novelty_filter → designer_agent → gate_plateau] → holdout_audit → export_best → archivist +""" + +from __future__ import annotations + +from typing import Any + +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + GateNode, + Study, + VerdictType, + Workflow, +) + + +def outer_loop_workflow() -> Workflow: + """Define the outer-loop evolutionary search workflow graph. + + study → seed_population → evaluate_batch → select → mutate → + novelty_filter → designer_agent → gate_plateau → + holdout_audit → export_best → archivist + """ + nodes: dict[str, Any] = {} + + nodes["study"] = Study( + id="study", + command="factory study {project_path}", + writes={".factory/strategy/observations.md"}, + ) + + nodes["seed_population"] = FnNode( + id="seed_population", + command="factory outer-loop seed {project_path}", + notes=( + "Create the initial population from the seed workflow plus " + "designer-generated variants (minimal + thorough). " + "Reads SwarmConfig from .factory/outer-loop/config.json." + ), + reads={".factory/strategy/observations.md"}, + writes={".factory/outer-loop/state.json"}, + ) + + nodes["evaluate_batch"] = FnNode( + id="evaluate_batch", + command="factory outer-loop evaluate {project_path}", + notes=( + "Parallel evaluation of population via SwarmEvaluator on training instances. " + "Each candidate evaluated in isolated context." + ), + reads={".factory/outer-loop/state.json"}, + writes={".factory/outer-loop/fitness_cache.json"}, + ) + + nodes["select"] = FnNode( + id="select", + command="factory outer-loop select {project_path}", + notes="Tournament selection + MAP-Elites archive update.", + reads={".factory/outer-loop/fitness_cache.json"}, + writes={".factory/outer-loop/map-elites/grid.json"}, + ) + + nodes["mutate"] = FnNode( + id="mutate", + command="factory outer-loop mutate {project_path}", + notes="Apply mutation operators via MutationStrategy to selected parents.", + reads={".factory/outer-loop/map-elites/grid.json"}, + writes={".factory/outer-loop/state.json"}, + ) + + nodes["novelty_filter"] = FnNode( + id="novelty_filter", + command="factory outer-loop filter {project_path}", + notes="Reject near-duplicate candidates before evaluation.", + reads={".factory/outer-loop/state.json"}, + writes={".factory/outer-loop/state.json"}, + ) + + nodes["designer_agent"] = AgentNode( + id="designer_agent", + role=AgentRole.RESEARCHER, + prompt_template=( + "Read the current best workflow and failure telemetry from " + ".factory/outer-loop/. Propose targeted mutations based on " + "execution data. Write mutation proposals to " + ".factory/outer-loop/designer-proposals.json." + ), + reads={".factory/outer-loop/state.json", ".factory/outer-loop/fitness_cache.json"}, + writes={".factory/outer-loop/designer-proposals.json"}, + ) + + nodes["gate_plateau"] = GateNode( + id="gate_plateau", + evaluator_type="fn", + evaluator_command=( + 'python3 -c "' + "import json; from pathlib import Path; " + "state = json.loads(Path('{project_path}/.factory/outer-loop/state.json').read_text()); " + "traj = state.get('score_trajectory', []); " + "budget = state.get('budget_remaining', 0); " + "plateau = len(traj) >= 4 and all(s <= traj[-4] for s in traj[-3:]); " + "done = budget <= 0 or plateau; " + "print('HALT' if done else 'PROCEED')" + '"' + ), + reads={".factory/outer-loop/state.json"}, + ) + + nodes["holdout_audit"] = FnNode( + id="holdout_audit", + command="factory outer-loop audit {project_path}", + notes=( + "Run best workflow on held-out instances via OverfitDetector. " + "Flags if >15% score drop from training to holdout." + ), + reads={".factory/outer-loop/state.json"}, + writes={".factory/outer-loop/best/holdout_audit.json"}, + ) + + nodes["export_best"] = FnNode( + id="export_best", + command="factory outer-loop export {project_path}", + notes="Write best workflow as a portable .factory/workflows/-evolved.py.", + reads={".factory/outer-loop/best/holdout_audit.json"}, + writes={".factory/outer-loop/best/workflow.py"}, + ) + + nodes["archivist"] = AgentNode( + id="archivist", + role=AgentRole.ARCHIVIST, + prompt_template=( + "Archive the outer-loop evolutionary run results. " + "Read the final state and best workflow from .factory/outer-loop/. " + "Write a summary of the evolution to .factory/archive/outer-loop.md." + ), + reads={".factory/outer-loop/best/workflow.py", ".factory/outer-loop/state.json"}, + writes={".factory/archive/outer-loop.md"}, + blocking=False, + ) + + edges = [ + Edge(source="study", target="seed_population"), + Edge(source="seed_population", target="evaluate_batch"), + Edge(source="evaluate_batch", target="select"), + Edge(source="select", target="mutate"), + Edge(source="mutate", target="novelty_filter"), + Edge(source="novelty_filter", target="designer_agent"), + Edge(source="designer_agent", target="gate_plateau"), + # Generation loop: continue or exit + Edge(source="gate_plateau", target="evaluate_batch", condition=VerdictType.PROCEED), + Edge(source="gate_plateau", target="holdout_audit", condition=VerdictType.HALT), + Edge(source="holdout_audit", target="export_best"), + Edge(source="export_best", target="archivist"), + ] + + return Workflow( + name="outer-loop", + nodes=nodes, + edges=edges, + start_node="study", + ) diff --git a/factory/workflow/definitions.py b/factory/workflow/definitions.py index 5e3c607f1..cde86480f 100644 --- a/factory/workflow/definitions.py +++ b/factory/workflow/definitions.py @@ -4104,6 +4104,9 @@ def _get_builtin_registry() -> dict[str, Any]: "parallel-improve": parallel_improve_workflow, "plan": lambda: design_workflow(just_plan=True), "evolve": evolve_workflow, + "outer-loop": lambda: __import__( + "factory.outer_loop.workflow", fromlist=["outer_loop_workflow"] + ).outer_loop_workflow(), "deep-research": lambda: __import__( "factory.workflow.deep_research", fromlist=["workflow"] ).workflow(), diff --git a/factory/workflow/primitives.py b/factory/workflow/primitives.py index dbf916b2e..dfabe7233 100644 --- a/factory/workflow/primitives.py +++ b/factory/workflow/primitives.py @@ -308,6 +308,71 @@ def subgraph( ] return Workflow(name=name, nodes=nodes, edges=edges, start_node=start_node) + def to_dict(self) -> dict[str, Any]: + """Serialize the workflow to a JSON-safe dict. + + Includes a ``_type`` discriminator on each node so ``from_dict`` + can reconstruct the correct node subclass. The ``trigger`` field + is excluded (not serializable). + """ + nodes_out: dict[str, Any] = {} + for nid, node in self.nodes.items(): + d = node.model_dump(mode="json") + d["_type"] = type(node).__name__ + nodes_out[nid] = d + + edges_out = [e.model_dump(mode="json") for e in self.edges] + + return { + "name": self.name, + "nodes": nodes_out, + "edges": edges_out, + "start_node": self.start_node, + "terminal": self.terminal, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Workflow: + """Reconstruct a Workflow from a dict produced by ``to_dict``.""" + _NODE_TYPE_MAP: dict[str, type[Node]] = { + "AgentNode": AgentNode, + "FnNode": FnNode, + "GateNode": GateNode, + "ForkNode": ForkNode, + "JoinNode": JoinNode, + "SubgraphForkNode": SubgraphForkNode, + "SelectionNode": SelectionNode, + "Study": Study, + "LLMNode": LLMNode, + } + + # JSON serializes sets as lists; convert back for strict Pydantic models + _SET_FIELDS = {"reads", "writes"} + + nodes: dict[str, NodeType] = {} + for nid, node_data in data["nodes"].items(): + node_data = dict(node_data) + type_name = node_data.pop("_type", "FnNode") + node_cls = _NODE_TYPE_MAP.get(type_name) + if node_cls is None: + raise ValueError(f"Unknown node type: {type_name}") + for field in _SET_FIELDS: + if field in node_data and isinstance(node_data[field], list): + node_data[field] = set(node_data[field]) + nodes[nid] = node_cls.model_validate( # type: ignore[assignment] + node_data, strict=False, + ) + + edges = [Edge.model_validate(e, strict=False) for e in data["edges"]] + + return cls( + name=data["name"], + nodes=nodes, + edges=edges, + start_node=data["start_node"], + terminal=data.get("terminal", False), + ) + # ── factory ────────────────────────────────────────────────────── diff --git a/tests/test_outer_loop/__init__.py b/tests/test_outer_loop/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_outer_loop/conftest.py b/tests/test_outer_loop/conftest.py new file mode 100644 index 000000000..e6ca197c5 --- /dev/null +++ b/tests/test_outer_loop/conftest.py @@ -0,0 +1,67 @@ +"""Shared fixtures for outer loop tests.""" + +from __future__ import annotations + +import pytest + +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + GateNode, + VerdictType, + Workflow, +) + + +@pytest.fixture() +def simple_workflow() -> Workflow: + """A simple 5-node workflow for mutation testing. + + study → researcher → strategist → builder → gate_qa + """ + nodes = { + "study": FnNode( + id="study", + command="factory study {project_path}", + writes={".factory/strategy/observations.md"}, + ), + "researcher": AgentNode( + id="researcher", + role=AgentRole.RESEARCHER, + reads={".factory/strategy/observations.md"}, + writes={".factory/strategy/research.md"}, + ), + "strategist": AgentNode( + id="strategist", + role=AgentRole.STRATEGIST, + reads={".factory/strategy/research.md"}, + writes={".factory/strategy/current.md"}, + ), + "builder": AgentNode( + id="builder", + role=AgentRole.BUILDER, + reads={".factory/strategy/current.md"}, + writes={".factory/reviews/builder-latest.md"}, + ), + "gate_qa": GateNode( + id="gate_qa", + evaluator_type="agent", + evaluator_role=AgentRole.CEO, + reads={".factory/reviews/builder-latest.md"}, + ), + } + edges = [ + Edge(source="study", target="researcher"), + Edge(source="researcher", target="strategist"), + Edge(source="strategist", target="builder"), + Edge(source="builder", target="gate_qa"), + Edge(source="gate_qa", target="builder", condition=VerdictType.RELOOP), + ] + return Workflow( + name="test_simple", + nodes=nodes, + edges=edges, + start_node="study", + ) diff --git a/tests/test_outer_loop/test_cli.py b/tests/test_outer_loop/test_cli.py new file mode 100644 index 000000000..6f89d7aa9 --- /dev/null +++ b/tests/test_outer_loop/test_cli.py @@ -0,0 +1,195 @@ +"""Tests for outer-loop CLI argument parsing and mode registration.""" + +from __future__ import annotations + +import pytest + + +class TestOuterLoopModeRegistration: + def test_outer_loop_in_ceo_modes(self) -> None: + from factory.cli._helpers import CEO_MODES + + assert "outer-loop" in CEO_MODES + + def test_outer_loop_in_run_modes(self) -> None: + from factory.cli._helpers import RUN_MODES + + assert "outer-loop" in RUN_MODES + + def test_outer_loop_workflow_registered(self) -> None: + from factory.workflow.definitions import _get_builtin_registry + + registry = _get_builtin_registry() + assert "outer-loop" in registry + + +class TestOuterLoopCLIParsing: + def _parse_ceo(self, *args: str) -> object: + from factory.cli._main import build_parser + + parser = build_parser() + return parser.parse_args(["ceo", *args]) + + def test_mode_outer_loop_accepted(self) -> None: + ns = self._parse_ceo("/tmp/project", "--mode", "outer-loop") + assert ns.mode == "outer-loop" + + def test_benchmark_parsed(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--benchmark", "featurebench", + ) + assert ns.benchmark == "featurebench" + + def test_budget_parsed(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--budget", "50", + ) + assert ns.ol_budget == 50 + + def test_population_parsed(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--population", "8", + ) + assert ns.population == 8 + + def test_target_score_parsed(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--target-score", "0.85", + ) + assert ns.target_score == pytest.approx(0.85) + + def test_seed_mode_parsed(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--seed", "improve", + ) + assert ns.seed_mode == "improve" + + def test_training_instances_parsed(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--training-instances", "fb-1,fb-2,fb-3", + ) + assert ns.training_instances == "fb-1,fb-2,fb-3" + + def test_holdout_instances_parsed(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--holdout-instances", "fb-4,fb-5", + ) + assert ns.holdout_instances == "fb-4,fb-5" + + def test_training_instances_as_list(self) -> None: + """Verify comma-separated strings can be split into lists.""" + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--training-instances", "a,b,c", + ) + instances = ns.training_instances.split(",") + assert instances == ["a", "b", "c"] + + def test_defaults_when_not_specified(self) -> None: + ns = self._parse_ceo("/tmp/project", "--mode", "outer-loop") + assert ns.benchmark is None + assert ns.ol_budget is None + assert ns.population is None + assert ns.target_score is None + assert ns.seed_mode is None + assert ns.training_instances is None + assert ns.holdout_instances is None + + def test_all_args_together(self) -> None: + ns = self._parse_ceo( + "/tmp/project", "--mode", "outer-loop", + "--benchmark", "terminalbench", + "--budget", "100", + "--population", "6", + "--target-score", "0.9", + "--seed", "evolve", + "--training-instances", "t1,t2,t3,t4,t5", + "--holdout-instances", "h1,h2", + ) + assert ns.mode == "outer-loop" + assert ns.benchmark == "terminalbench" + assert ns.ol_budget == 100 + assert ns.population == 6 + assert ns.target_score == pytest.approx(0.9) + assert ns.seed_mode == "evolve" + assert ns.training_instances.split(",") == ["t1", "t2", "t3", "t4", "t5"] + assert ns.holdout_instances.split(",") == ["h1", "h2"] + + +class TestOuterLoopWorkflowGraph: + def test_workflow_validates(self) -> None: + from factory.outer_loop.workflow import outer_loop_workflow + + wf = outer_loop_workflow() + issues = wf.validate_graph() + assert issues == [], f"Workflow validation issues: {issues}" + + def test_workflow_name(self) -> None: + from factory.outer_loop.workflow import outer_loop_workflow + + wf = outer_loop_workflow() + assert wf.name == "outer-loop" + + def test_workflow_start_node(self) -> None: + from factory.outer_loop.workflow import outer_loop_workflow + + wf = outer_loop_workflow() + assert wf.start_node == "study" + + def test_workflow_has_expected_nodes(self) -> None: + from factory.outer_loop.workflow import outer_loop_workflow + + wf = outer_loop_workflow() + expected = { + "study", "seed_population", "evaluate_batch", "select", + "mutate", "novelty_filter", "designer_agent", "gate_plateau", + "holdout_audit", "export_best", "archivist", + } + assert set(wf.nodes.keys()) == expected + + def test_workflow_generation_loop(self) -> None: + """gate_plateau has a PROCEED edge back to evaluate_batch (loop).""" + from factory.outer_loop.workflow import outer_loop_workflow + from factory.workflow.primitives import VerdictType + + wf = outer_loop_workflow() + loop_edge = [ + e for e in wf.edges + if e.source == "gate_plateau" + and e.target == "evaluate_batch" + and e.condition == VerdictType.PROCEED + ] + assert len(loop_edge) == 1 + + def test_workflow_exit_to_holdout(self) -> None: + """gate_plateau HALT goes to holdout_audit.""" + from factory.outer_loop.workflow import outer_loop_workflow + from factory.workflow.primitives import VerdictType + + wf = outer_loop_workflow() + exit_edge = [ + e for e in wf.edges + if e.source == "gate_plateau" + and e.target == "holdout_audit" + and e.condition == VerdictType.HALT + ] + assert len(exit_edge) == 1 + + def test_workflow_serialization_round_trip(self) -> None: + from factory.outer_loop.workflow import outer_loop_workflow + from factory.workflow.primitives import Workflow + + wf = outer_loop_workflow() + data = wf.to_dict() + restored = Workflow.from_dict(data) + + assert restored.name == wf.name + assert set(restored.nodes.keys()) == set(wf.nodes.keys()) + assert len(restored.edges) == len(wf.edges) diff --git a/tests/test_outer_loop/test_designer.py b/tests/test_outer_loop/test_designer.py new file mode 100644 index 000000000..ef0e8a97a --- /dev/null +++ b/tests/test_outer_loop/test_designer.py @@ -0,0 +1,223 @@ +"""Tests for DesignerAgent — design mode and mutation mode.""" + +from __future__ import annotations + +from factory.outer_loop.designer import DesignerAgent +from factory.outer_loop.models import MutationType + + +class TestDesignMinimal: + def test_produces_3_to_4_nodes(self) -> None: + designer = DesignerAgent() + wf = designer.design_minimal("test benchmark") + assert 3 <= len(wf.nodes) <= 4 + + def test_valid_workflow(self) -> None: + designer = DesignerAgent() + wf = designer.design_minimal("test benchmark") + issues = wf.validate_graph() + assert issues == [], f"Validation issues: {issues}" + + def test_has_builder(self) -> None: + designer = DesignerAgent() + wf = designer.design_minimal("test benchmark") + roles = { + node.role.value + for node in wf.nodes.values() + if hasattr(node, "role") + } + assert "builder" in roles + + def test_has_gate(self) -> None: + designer = DesignerAgent() + wf = designer.design_minimal("test benchmark") + gate_nodes = [ + n for n in wf.nodes.values() + if type(n).__name__ == "GateNode" + ] + assert len(gate_nodes) >= 1 + + def test_name_includes_benchmark(self) -> None: + designer = DesignerAgent() + wf = designer.design_minimal("feature_bench") + assert "minimal" in wf.name + assert "feature_bench" in wf.name + + def test_serialization_roundtrip(self) -> None: + from factory.workflow.primitives import Workflow + + designer = DesignerAgent() + wf = designer.design_minimal("test benchmark") + data = wf.to_dict() + restored = Workflow.from_dict(data) + assert len(restored.nodes) == len(wf.nodes) + assert restored.start_node == wf.start_node + + +class TestDesignThorough: + def test_produces_8_to_10_nodes(self) -> None: + designer = DesignerAgent() + wf = designer.design_thorough("test benchmark") + assert 8 <= len(wf.nodes) <= 10 + + def test_valid_workflow(self) -> None: + designer = DesignerAgent() + wf = designer.design_thorough("test benchmark") + issues = wf.validate_graph() + assert issues == [], f"Validation issues: {issues}" + + def test_has_parallel_builders(self) -> None: + designer = DesignerAgent() + wf = designer.design_thorough("test benchmark") + fork_nodes = [ + n for n in wf.nodes.values() + if type(n).__name__ == "ForkNode" + ] + assert len(fork_nodes) >= 1 + + def test_has_code_reviewer(self) -> None: + designer = DesignerAgent() + wf = designer.design_thorough("test benchmark") + roles = { + node.role.value + for node in wf.nodes.values() + if hasattr(node, "role") + } + assert "code_reviewer" in roles + + def test_has_adversarial_tester(self) -> None: + designer = DesignerAgent() + wf = designer.design_thorough("test benchmark") + roles = { + node.role.value + for node in wf.nodes.values() + if hasattr(node, "role") + } + assert "adversarial_tester" in roles + + def test_has_study_node(self) -> None: + designer = DesignerAgent() + wf = designer.design_thorough("test benchmark") + assert "study" in wf.nodes + + def test_serialization_roundtrip(self) -> None: + from factory.workflow.primitives import Workflow + + designer = DesignerAgent() + wf = designer.design_thorough("test benchmark") + data = wf.to_dict() + restored = Workflow.from_dict(data) + assert len(restored.nodes) == len(wf.nodes) + assert restored.start_node == wf.start_node + + +class TestDesignCustom: + def test_respects_max_nodes(self) -> None: + designer = DesignerAgent() + wf = designer.design_custom("bench", {"max_nodes": 5}) + assert len(wf.nodes) <= 5 + + def test_valid_workflow(self) -> None: + designer = DesignerAgent() + wf = designer.design_custom("bench", {"max_nodes": 6}) + issues = wf.validate_graph() + assert issues == [], f"Validation issues: {issues}" + + def test_includes_required_roles(self) -> None: + designer = DesignerAgent() + wf = designer.design_custom( + "bench", {"max_nodes": 8, "require_roles": ["health_checker"]} + ) + roles = { + node.role.value + for node in wf.nodes.values() + if hasattr(node, "role") + } + assert "health_checker" in roles + + +class TestPropose: + def test_returns_mutation_records(self, simple_workflow) -> None: # type: ignore[no-untyped-def] + designer = DesignerAgent() + proposals = designer.propose( + simple_workflow, + telemetry={"node_stats": {}, "dominant_failure": ""}, + archive_stats={"diversity": 0.5}, + benchmark_spec="test", + ) + assert len(proposals) >= 1 + assert len(proposals) <= 3 + + def test_high_failure_rate_proposes_removal(self, simple_workflow) -> None: # type: ignore[no-untyped-def] + designer = DesignerAgent() + proposals = designer.propose( + simple_workflow, + telemetry={ + "node_stats": {"researcher": {"failure_rate": 0.8}}, + "dominant_failure": "", + }, + archive_stats={"diversity": 0.5}, + benchmark_spec="test", + ) + remove_proposals = [ + p for p in proposals if p.operator == MutationType.NODE_REMOVE + ] + assert len(remove_proposals) >= 1 + assert remove_proposals[0].target_node == "researcher" + + def test_timeout_failure_proposes_param_mutate(self, simple_workflow) -> None: # type: ignore[no-untyped-def] + designer = DesignerAgent() + proposals = designer.propose( + simple_workflow, + telemetry={ + "node_stats": {}, + "dominant_failure": "timeout", + }, + archive_stats={"diversity": 0.5}, + benchmark_spec="test", + ) + timeout_proposals = [ + p for p in proposals if p.operator == MutationType.PARAM_MUTATE + ] + assert len(timeout_proposals) >= 1 + + def test_low_diversity_proposes_insertion(self, simple_workflow) -> None: # type: ignore[no-untyped-def] + designer = DesignerAgent() + proposals = designer.propose( + simple_workflow, + telemetry={"node_stats": {}, "dominant_failure": ""}, + archive_stats={"diversity": 0.1}, + benchmark_spec="test", + ) + insert_proposals = [ + p for p in proposals if p.operator == MutationType.NODE_INSERT + ] + assert len(insert_proposals) >= 1 + + def test_no_signal_still_returns_proposal(self, simple_workflow) -> None: # type: ignore[no-untyped-def] + designer = DesignerAgent() + proposals = designer.propose( + simple_workflow, + telemetry={}, + archive_stats={}, + benchmark_spec="test", + ) + assert len(proposals) >= 1 + + def test_max_3_proposals(self, simple_workflow) -> None: # type: ignore[no-untyped-def] + designer = DesignerAgent() + proposals = designer.propose( + simple_workflow, + telemetry={ + "node_stats": { + "researcher": {"failure_rate": 0.9}, + "strategist": {"failure_rate": 0.9}, + "builder": {"failure_rate": 0.9}, + "gate_qa": {"failure_rate": 0.9}, + }, + "dominant_failure": "timeout", + }, + archive_stats={"diversity": 0.1}, + benchmark_spec="test", + ) + assert len(proposals) <= 3 diff --git a/tests/test_outer_loop/test_e2e.py b/tests/test_outer_loop/test_e2e.py new file mode 100644 index 000000000..dd49e74ab --- /dev/null +++ b/tests/test_outer_loop/test_e2e.py @@ -0,0 +1,439 @@ +"""End-to-end integration test for the outer loop evolutionary search. + +Creates a simple seed workflow, uses a mock evaluator that rewards more agent +nodes (so evolution discovers this), runs 3 generations with population=4, +and verifies the evolutionary loop actually improves over the seed. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from factory.outer_loop.engine import SwarmEngine +from factory.outer_loop.evaluator import SwarmEvaluator +from factory.outer_loop.filesystem import ( + export_best_workflow, + init_filesystem, + load_checkpoint, + save_best, + save_checkpoint, + save_generation, + save_map_elites, +) +from factory.outer_loop.models import ( + EvalResult, + OuterLoopState, + SwarmConfig, +) +from factory.outer_loop.mutations import WeightedRandomStrategy +from factory.outer_loop.population import Population +from factory.outer_loop.similarity import NoveltyFilter +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + GateNode, + VerdictType, + Workflow, +) + + +def _seed_workflow() -> Workflow: + """A simple 3-node seed workflow.""" + return Workflow( + name="seed", + nodes={ + "study": FnNode( + id="study", + command="factory study {project_path}", + writes={".factory/obs.md"}, + ), + "builder": AgentNode( + id="builder", + role=AgentRole.BUILDER, + reads={".factory/obs.md"}, + writes={".factory/build.md"}, + ), + "gate": GateNode( + id="gate", + evaluator_type="fn", + reads={".factory/build.md"}, + ), + }, + edges=[ + Edge(source="study", target="builder"), + Edge(source="builder", target="gate"), + Edge(source="gate", target="builder", condition=VerdictType.RELOOP), + ], + start_node="study", + ) + + +def _make_feature_evaluator() -> SwarmEvaluator: + """Evaluator that rewards more agent nodes — evolution should discover this.""" + def eval_fn( + wf: Workflow, project_dir: str, instances: list[str], + ) -> EvalResult: + agent_count = sum( + 1 for n in wf.nodes.values() if isinstance(n, AgentNode) + ) + node_count = len(wf.nodes) + score = min(0.3 + agent_count * 0.1 + node_count * 0.02, 0.95) + return EvalResult( + score=0.0, + benchmark_score=score, + hygiene_score=0.6, + cost_usd=0.01, + complexity=float(node_count), + ) + + config = SwarmConfig( + benchmark="test-e2e", + budget=60, + population_size=4, + tournament_size=2, + mutation_rate=0.5, + training_instances=["t1", "t2", "t3"], + holdout_instances=["h1"], + ) + return SwarmEvaluator(config, evaluator_fn=eval_fn) + + +def _make_holdout_evaluator(training_score: float = 0.8) -> SwarmEvaluator: + """Evaluator with distinct training vs holdout behavior for overfit testing.""" + def eval_fn( + wf: Workflow, project_dir: str, instances: list[str], + ) -> EvalResult: + if any(i.startswith("h") for i in instances): + score = training_score * 0.7 + else: + score = training_score + return EvalResult( + score=0.0, + benchmark_score=score, + hygiene_score=0.6, + cost_usd=0.01, + complexity=float(len(wf.nodes)), + ) + + config = SwarmConfig( + benchmark="test-overfit", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + return SwarmEvaluator(config, evaluator_fn=eval_fn) + + +class TestE2EEvolution: + def test_evolution_improves_over_seed(self) -> None: + """The best evolved workflow should score higher than the seed.""" + seed_wf = _seed_workflow() + evaluator = _make_feature_evaluator() + config = SwarmConfig( + benchmark="test-e2e", + budget=60, + population_size=4, + tournament_size=2, + mutation_rate=0.5, + training_instances=["t1", "t2", "t3"], + holdout_instances=["h1"], + ) + + seed_score = evaluator.evaluate(seed_wf, "", ["t1", "t2", "t3"]).score + + strategy = WeightedRandomStrategy(mutation_rate=0.5) + novelty = NoveltyFilter(min_edit_distance=1) + engine = SwarmEngine( + config, evaluator, + strategy=strategy, + novelty_filter=novelty, + ) + + result = engine.run(seed_wf) + + assert result.best_score > seed_score, ( + f"Best evolved score {result.best_score} should exceed " + f"seed score {seed_score}" + ) + + def test_archive_populated(self) -> None: + """MAP-Elites archive should have entries after evolution.""" + seed_wf = _seed_workflow() + evaluator = _make_feature_evaluator() + config = SwarmConfig( + benchmark="test-e2e", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + engine = SwarmEngine(config, evaluator) + result = engine.run(seed_wf) + + assert result.archive_size > 0 + + def test_trajectory_recorded(self) -> None: + """Generation trajectory should be recorded.""" + seed_wf = _seed_workflow() + evaluator = _make_feature_evaluator() + config = SwarmConfig( + benchmark="test-e2e", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + engine = SwarmEngine(config, evaluator) + result = engine.run(seed_wf) + + assert len(result.trajectory) >= 1 + assert result.generations_completed >= 1 + + def test_hyperparameter_history_complete(self) -> None: + """Every generation should have a HyperparameterRecord.""" + seed_wf = _seed_workflow() + evaluator = _make_feature_evaluator() + config = SwarmConfig( + benchmark="test-e2e", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + engine = SwarmEngine(config, evaluator) + result = engine.run(seed_wf) + + assert len(result.hyperparameter_history) == result.generations_completed + for hp in result.hyperparameter_history: + assert hp.mutation_rate > 0 + assert hp.population_size > 0 + + def test_best_workflow_is_valid(self) -> None: + """The best workflow should be a valid Workflow.""" + seed_wf = _seed_workflow() + evaluator = _make_feature_evaluator() + config = SwarmConfig( + benchmark="test-e2e", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + engine = SwarmEngine(config, evaluator) + result = engine.run(seed_wf) + + assert result.best_workflow_data != {} + reconstructed = Workflow.from_dict(result.best_workflow_data) # type: ignore[arg-type] + assert len(reconstructed.nodes) > 0 + assert len(reconstructed.edges) > 0 + + def test_pareto_front_non_empty(self) -> None: + """Pareto front should contain at least one individual.""" + seed_wf = _seed_workflow() + evaluator = _make_feature_evaluator() + config = SwarmConfig( + benchmark="test-e2e", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + engine = SwarmEngine(config, evaluator) + result = engine.run(seed_wf) + + assert len(result.pareto_front) > 0 + + +class TestE2EOverfitDetection: + def test_overfit_flagged(self) -> None: + """When holdout score drops >15%, overfit should be flagged.""" + seed_wf = _seed_workflow() + evaluator = _make_holdout_evaluator(training_score=0.8) + config = SwarmConfig( + benchmark="test-overfit", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + engine = SwarmEngine(config, evaluator) + result = engine.run(seed_wf) + + assert result.overfit_flag is True + assert result.holdout_score > 0 + + +class TestE2EFilesystem: + def test_init_and_checkpoint(self, tmp_path: Path) -> None: + """Filesystem init creates directories and checkpoint round-trips.""" + config = SwarmConfig( + benchmark="test-fs", + budget=10, + training_instances=["t1"], + holdout_instances=["h1"], + ) + root = init_filesystem(tmp_path, config) + + assert (root / "config.json").exists() + assert (root / "state.json").exists() + assert (root / "fitness_cache.json").exists() + assert (root / "trajectory.jsonl").exists() + assert (root / "archive").is_dir() + assert (root / "map-elites").is_dir() + assert (root / "best").is_dir() + + loaded_state = load_checkpoint(tmp_path) + assert loaded_state is not None + assert loaded_state.budget_remaining == 10 + + def test_save_and_load_checkpoint(self, tmp_path: Path) -> None: + """Checkpoint save/load round-trip preserves state.""" + config = SwarmConfig( + benchmark="test-ckpt", + budget=50, + training_instances=["t1"], + holdout_instances=["h1"], + ) + init_filesystem(tmp_path, config) + + state = OuterLoopState( + generation=3, + total_evaluations=25, + best_score=0.72, + budget_remaining=25, + score_trajectory=[0.5, 0.6, 0.65, 0.72], + ) + save_checkpoint(tmp_path, state) + + loaded = load_checkpoint(tmp_path) + assert loaded is not None + assert loaded.generation == 3 + assert loaded.total_evaluations == 25 + assert loaded.best_score == 0.72 + assert loaded.budget_remaining == 25 + assert len(loaded.score_trajectory) == 4 + + def test_export_best_workflow(self, tmp_path: Path) -> None: + """Export produces a portable Python file.""" + seed_wf = _seed_workflow() + wf_data = seed_wf.to_dict() + + path = export_best_workflow(tmp_path, wf_data, "test-bench") + + assert path.exists() + content = path.read_text() + assert "meta" in content + assert "test-bench-evolved" in content + assert "def workflow()" in content + + def test_save_generation_creates_artifacts(self, tmp_path: Path) -> None: + """save_generation creates generation directory with artifacts.""" + from factory.outer_loop.models import GenerationSummary, HyperparameterRecord + from factory.outer_loop.population import Population + + config = SwarmConfig( + benchmark="test-gen", + budget=10, + training_instances=["t1"], + holdout_instances=["h1"], + ) + init_filesystem(tmp_path, config) + + seed_wf = _seed_workflow() + pop = Population() + ind = Population.make_individual(seed_wf, generation=0) + ind = ind.model_copy(update={"score": 0.5}) + pop.add(ind) + + hp = HyperparameterRecord( + generation=0, + mutation_rate=0.3, + population_size=1, + tournament_size=2, + designer_ratio=0.3, + best_score=0.5, + mean_score=0.5, + ) + summary = GenerationSummary( + generation=0, + population_size=1, + best_score=0.5, + mean_score=0.5, + diversity=0.0, + hyperparameters=hp, + ) + save_generation(tmp_path, 0, summary, pop) + + gen_dir = tmp_path / ".factory" / "outer-loop" / "archive" / "generation-000" + assert gen_dir.exists() + assert (gen_dir / "summary.json").exists() + assert (gen_dir / "hyperparameters.json").exists() + assert (gen_dir / "variant-00" / "workflow.json").exists() + assert (gen_dir / "variant-00" / "scores.json").exists() + + traj = tmp_path / ".factory" / "outer-loop" / "trajectory.jsonl" + lines = traj.read_text().strip().splitlines() + assert len(lines) == 1 + entry = json.loads(lines[0]) + assert entry["generation"] == 0 + assert entry["best_score"] == 0.5 + + +class TestE2EFullPipeline: + def test_full_pipeline_with_filesystem(self, tmp_path: Path) -> None: + """Full pipeline: init → evolve → save → export.""" + seed_wf = _seed_workflow() + config = SwarmConfig( + benchmark="test-full", + budget=30, + population_size=4, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + evaluator = _make_feature_evaluator() + + init_filesystem(tmp_path, config) + + engine = SwarmEngine( + config, evaluator, + novelty_filter=NoveltyFilter(min_edit_distance=1), + ) + result = engine.run(seed_wf) + + state = OuterLoopState( + generation=result.generations_completed, + total_evaluations=result.total_evaluations, + best_score=result.best_score, + budget_remaining=config.budget - result.total_evaluations, + convergence_reason=result.convergence_reason, + score_trajectory=[s.best_score for s in result.trajectory], + hyperparameter_history=result.hyperparameter_history, + ) + save_checkpoint(tmp_path, state) + save_best(tmp_path, result) + save_map_elites(tmp_path, engine.archive) + + for i, summary in enumerate(result.trajectory): + pop = Population() + ind = Population.make_individual(seed_wf, generation=i) + ind = ind.model_copy(update={"score": summary.best_score}) + pop.add(ind) + save_generation(tmp_path, i, summary, pop) + + export_path = export_best_workflow( + tmp_path, result.best_workflow_data, "test-full", + ) + + assert export_path.exists() + assert (tmp_path / ".factory" / "outer-loop" / "state.json").exists() + assert (tmp_path / ".factory" / "outer-loop" / "best" / "workflow.json").exists() + assert (tmp_path / ".factory" / "outer-loop" / "map-elites" / "grid.json").exists() + + loaded = load_checkpoint(tmp_path) + assert loaded is not None + assert loaded.generation == result.generations_completed + assert loaded.best_score == result.best_score diff --git a/tests/test_outer_loop/test_engine.py b/tests/test_outer_loop/test_engine.py new file mode 100644 index 000000000..a7c5eeaca --- /dev/null +++ b/tests/test_outer_loop/test_engine.py @@ -0,0 +1,339 @@ +"""Tests for SwarmEngine and BudgetTracker.""" + +from __future__ import annotations + +import pytest + +from factory.outer_loop.engine import BudgetTracker, SwarmEngine +from factory.outer_loop.evaluator import SwarmEvaluator +from factory.outer_loop.models import EvalResult, SwarmConfig +from factory.outer_loop.mutations import WeightedRandomStrategy +from factory.outer_loop.similarity import NoveltyFilter +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + GateNode, + VerdictType, + Workflow, +) + + +def _make_config(**overrides: object) -> SwarmConfig: + defaults: dict[str, object] = { + "benchmark": "test", + "budget": 30, + "population_size": 4, + "tournament_size": 2, + "mutation_rate": 0.3, + "training_instances": ["t1", "t2"], + "holdout_instances": ["h1"], + } + defaults.update(overrides) + return SwarmConfig(**defaults) # type: ignore[arg-type] + + +def _make_workflow() -> Workflow: + return Workflow( + name="test_evo", + nodes={ + "study": FnNode( + id="study", command="factory study", writes={".factory/obs.md"}, + ), + "researcher": AgentNode( + id="researcher", role=AgentRole.RESEARCHER, + reads={".factory/obs.md"}, writes={".factory/research.md"}, + ), + "strategist": AgentNode( + id="strategist", role=AgentRole.STRATEGIST, + reads={".factory/research.md"}, writes={".factory/current.md"}, + ), + "builder": AgentNode( + id="builder", role=AgentRole.BUILDER, + reads={".factory/current.md"}, writes={".factory/build.md"}, + ), + "gate": GateNode( + id="gate", evaluator_type="fn", + reads={".factory/build.md"}, + ), + }, + edges=[ + Edge(source="study", target="researcher"), + Edge(source="researcher", target="strategist"), + Edge(source="strategist", target="builder"), + Edge(source="builder", target="gate"), + Edge(source="gate", target="builder", condition=VerdictType.RELOOP), + ], + start_node="study", + ) + + +def _make_deterministic_evaluator( + base_score: float = 0.5, increment: float = 0.02, +) -> SwarmEvaluator: + """Returns an evaluator that gives incrementally higher scores to different workflows.""" + counter: dict[str, int] = {"n": 0} + + def eval_fn(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + counter["n"] += 1 + score = min(base_score + counter["n"] * increment, 1.0) + return EvalResult( + score=0.0, benchmark_score=score, hygiene_score=0.7, + cost_usd=0.1, complexity=len(wf.nodes), + ) + + config = _make_config() + return SwarmEvaluator(config, evaluator_fn=eval_fn) + + +class TestBudgetTracker: + def test_initial_state(self) -> None: + bt = BudgetTracker(100) + assert bt.remaining == 100 + assert bt.consumed == 0 + assert not bt.exhausted + assert bt.total_cost_usd == 0.0 + + def test_consume(self) -> None: + bt = BudgetTracker(10) + bt.consume(3, cost_usd=1.5) + assert bt.consumed == 3 + assert bt.remaining == 7 + assert bt.total_cost_usd == 1.5 + + def test_exhausted(self) -> None: + bt = BudgetTracker(5) + bt.consume(5) + assert bt.exhausted + assert bt.remaining == 0 + + def test_over_consume(self) -> None: + bt = BudgetTracker(3) + bt.consume(5) + assert bt.exhausted + assert bt.remaining == 0 + + def test_elapsed(self) -> None: + bt = BudgetTracker(10) + assert bt.elapsed_seconds >= 0 + + +class TestSwarmEngineSeed: + def test_seed_creates_population(self) -> None: + config = _make_config(population_size=4) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + pop = engine.seed(wf) + assert pop.size >= 1 + assert pop.size <= 4 + + def test_seed_slot_zero_is_original(self) -> None: + config = _make_config(population_size=3, designer_count=0) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + pop = engine.seed(wf) + individuals = pop.individuals + original = [i for i in individuals if i.parent_id is None] + assert len(original) == 1 + + def test_seed_diversity(self) -> None: + config = _make_config(population_size=4) + evaluator = _make_deterministic_evaluator() + novelty = NoveltyFilter(min_edit_distance=1) + engine = SwarmEngine(config, evaluator, novelty_filter=novelty) + wf = _make_workflow() + + pop = engine.seed(wf) + ids = {i.id for i in pop.individuals} + assert len(ids) == pop.size + + +class TestSwarmEngineEvolve: + def test_evolve_generation_returns_summary(self) -> None: + config = _make_config(budget=50, population_size=3) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + pop = engine.seed(wf) + + summary = engine.evolve_generation(pop, generation=1) + + assert summary.generation == 1 + assert summary.population_size > 0 + assert summary.best_score >= 0 + assert summary.hyperparameters is not None + assert summary.hyperparameters.generation == 1 + + def test_evolve_updates_archive(self) -> None: + config = _make_config(budget=50, population_size=3) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + pop = engine.seed(wf) + + engine.evolve_generation(pop, generation=1) + assert engine.archive.size > 0 + + def test_hyperparameter_record_logged(self) -> None: + config = _make_config(budget=50, population_size=3) + evaluator = _make_deterministic_evaluator() + strategy = WeightedRandomStrategy(mutation_rate=0.4, designer_ratio=0.2) + engine = SwarmEngine(config, evaluator, strategy=strategy) + wf = _make_workflow() + pop = engine.seed(wf) + + summary = engine.evolve_generation(pop, generation=0) + + assert summary.hyperparameters is not None + hp = summary.hyperparameters + assert hp.mutation_rate == 0.4 + assert hp.designer_ratio == 0.2 + assert hp.population_size > 0 + + +class TestSwarmEngineRun: + def test_run_terminates_on_budget(self) -> None: + config = _make_config(budget=10, population_size=2) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + result = engine.run(wf) + + assert result.convergence_reason == "budget_exhausted" + assert result.total_evaluations <= 10 + assert result.generations_completed >= 1 + assert len(result.trajectory) > 0 + + def test_run_terminates_on_target_score(self) -> None: + config = _make_config(budget=100, population_size=2, target_score=0.6) + + def high_score_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult( + score=0.0, benchmark_score=0.9, hygiene_score=0.9, + cost_usd=0.01, complexity=3.0, + ) + + evaluator = SwarmEvaluator(config, evaluator_fn=high_score_eval) + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + result = engine.run(wf) + assert result.convergence_reason == "target_score_reached" + assert result.best_score >= 0.6 + + def test_run_holdout_audit(self) -> None: + config = _make_config(budget=15, population_size=2) + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + if "h1" in instances: + return EvalResult(score=0.0, benchmark_score=0.6, hygiene_score=0.6) + return EvalResult(score=0.0, benchmark_score=0.7, hygiene_score=0.7) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + result = engine.run(wf) + assert result.holdout_score > 0 + assert isinstance(result.overfit_flag, bool) + + def test_run_hyperparameter_history(self) -> None: + config = _make_config(budget=15, population_size=2) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + result = engine.run(wf) + assert len(result.hyperparameter_history) == result.generations_completed + + def test_run_pareto_front(self) -> None: + config = _make_config(budget=15, population_size=2) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + result = engine.run(wf) + assert result.archive_size > 0 + assert len(result.pareto_front) > 0 + + def test_run_result_fields(self) -> None: + config = _make_config(budget=10, population_size=2) + evaluator = _make_deterministic_evaluator() + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + result = engine.run(wf) + assert result.best_workflow_data != {} + assert result.total_cost_usd >= 0 + assert result.convergence_reason != "" + + +class TestSwarmEnginePlateau: + def test_plateau_detection(self) -> None: + config = _make_config(budget=100, population_size=2) + + def flat_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult( + score=0.0, benchmark_score=0.5, hygiene_score=0.5, + cost_usd=0.01, complexity=3.0, + ) + + evaluator = SwarmEvaluator(config, evaluator_fn=flat_eval) + strategy = WeightedRandomStrategy(mutation_rate=0.3) + engine = SwarmEngine(config, evaluator, strategy=strategy) + wf = _make_workflow() + + result = engine.run(wf) + # With flat scores, should eventually plateau + assert result.convergence_reason in ("plateau", "budget_exhausted") + + def test_plateau_increases_mutation_rate(self) -> None: + strategy = WeightedRandomStrategy(mutation_rate=0.3) + assert strategy.get_mutation_rate(0) == 0.3 + strategy.on_plateau() + assert strategy.get_mutation_rate(0) == pytest.approx(0.5) + + def test_improvement_resets_mutation_rate(self) -> None: + strategy = WeightedRandomStrategy(mutation_rate=0.3) + strategy.on_plateau() + assert strategy.get_mutation_rate(0) == pytest.approx(0.5) + strategy.on_improvement() + assert strategy.get_mutation_rate(0) == 0.3 + + +class TestSwarmEngineIntegration: + def test_3_generations_with_mock(self) -> None: + """Integration test: 3 generations, pop=4, mock fitness, verify trajectory.""" + config = _make_config(budget=50, population_size=4, target_score=None) + + eval_counter: dict[str, int] = {"n": 0} + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + eval_counter["n"] += 1 + score = min(0.3 + eval_counter["n"] * 0.01, 1.0) + return EvalResult( + score=0.0, benchmark_score=score, hygiene_score=0.6, + cost_usd=0.05, complexity=float(len(wf.nodes)), + ) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + engine = SwarmEngine(config, evaluator) + wf = _make_workflow() + + result = engine.run(wf) + + assert result.generations_completed >= 1 + assert result.total_evaluations > 0 + assert len(result.trajectory) >= 1 + assert result.best_score > 0 + assert len(result.hyperparameter_history) == result.generations_completed + + for hp in result.hyperparameter_history: + assert hp.mutation_rate > 0 + assert hp.population_size > 0 diff --git a/tests/test_outer_loop/test_evaluator.py b/tests/test_outer_loop/test_evaluator.py new file mode 100644 index 000000000..cffb8f7f2 --- /dev/null +++ b/tests/test_outer_loop/test_evaluator.py @@ -0,0 +1,185 @@ +"""Tests for SwarmEvaluator and FitnessCache.""" + +from __future__ import annotations + +from factory.outer_loop.evaluator import FitnessCache, SwarmEvaluator +from factory.outer_loop.models import EvalResult, SwarmConfig +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + GateNode, + VerdictType, + Workflow, +) + + +def _make_config(**overrides: object) -> SwarmConfig: + defaults: dict[str, object] = { + "benchmark": "test", + "budget": 50, + "training_instances": ["t1", "t2"], + "holdout_instances": ["h1"], + } + defaults.update(overrides) + return SwarmConfig(**defaults) # type: ignore[arg-type] + + +def _make_simple_workflow(name: str = "test_wf") -> Workflow: + return Workflow( + name=name, + nodes={ + "study": FnNode(id="study", command="echo study", writes={".factory/obs.md"}), + "builder": AgentNode( + id="builder", role=AgentRole.BUILDER, reads={".factory/obs.md"}, + ), + "gate": GateNode(id="gate", evaluator_type="fn"), + }, + edges=[ + Edge(source="study", target="builder"), + Edge(source="builder", target="gate"), + ], + start_node="study", + ) + + +class TestFitnessCache: + def test_miss_then_hit(self) -> None: + cache = FitnessCache() + wf = _make_simple_workflow() + instances = ["t1", "t2"] + + assert cache.get(wf, instances) is None + cache.put(wf, instances, 0.85, 1.5) + result = cache.get(wf, instances) + assert result is not None + score, cost, ts = result + assert score == 0.85 + assert cost == 1.5 + assert ts > 0 + + def test_different_instances_separate_keys(self) -> None: + cache = FitnessCache() + wf = _make_simple_workflow() + cache.put(wf, ["t1"], 0.7, 1.0) + cache.put(wf, ["t1", "t2"], 0.85, 2.0) + + r1 = cache.get(wf, ["t1"]) + r2 = cache.get(wf, ["t1", "t2"]) + assert r1 is not None and r2 is not None + assert r1[0] == 0.7 + assert r2[0] == 0.85 + + def test_size(self) -> None: + cache = FitnessCache() + wf = _make_simple_workflow() + assert cache.size == 0 + cache.put(wf, ["t1"], 0.5, 0.0) + assert cache.size == 1 + + +class TestSwarmEvaluator: + def test_evaluate_with_fn(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult( + score=0.0, benchmark_score=0.8, hygiene_score=0.9, + cost_usd=1.0, complexity=5.0, + ) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + wf = _make_simple_workflow() + result = evaluator.evaluate(wf, "/tmp/test", ["t1"]) + + assert result.score > 0 + assert result.benchmark_score == 0.8 + + def test_evaluate_uses_cache(self) -> None: + config = _make_config() + call_count = 0 + + def counting_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + nonlocal call_count + call_count += 1 + return EvalResult(score=0.0, benchmark_score=0.7, hygiene_score=0.8) + + evaluator = SwarmEvaluator(config, evaluator_fn=counting_eval) + wf = _make_simple_workflow() + evaluator.evaluate(wf, "/tmp/test", ["t1"]) + evaluator.evaluate(wf, "/tmp/test", ["t1"]) + assert call_count == 1 + + def test_mandatory_component_rejection(self) -> None: + config = _make_config(mandatory_node_roles=["health_checker"]) + evaluator = SwarmEvaluator(config) + wf = _make_simple_workflow() + result = evaluator.evaluate(wf, "/tmp/test", ["t1"]) + assert result.score == 0.0 + assert result.details.get("rejected") == "mandatory_component_missing" + + def test_mandatory_component_passes(self) -> None: + config = _make_config(mandatory_node_roles=["builder"]) + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult(score=0.0, benchmark_score=0.5, hygiene_score=0.5) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + wf = _make_simple_workflow() + result = evaluator.evaluate(wf, "/tmp/test", ["t1"]) + assert result.score > 0 + + def test_frozen_node_rejection(self) -> None: + config = _make_config(frozen_node_ids=["missing_node"]) + evaluator = SwarmEvaluator(config) + wf = _make_simple_workflow() + result = evaluator.evaluate(wf, "/tmp/test", ["t1"]) + assert result.score == 0.0 + assert result.details.get("rejected") == "frozen_node_violated" + + def test_frozen_node_passes(self) -> None: + config = _make_config(frozen_node_ids=["study"]) + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult(score=0.0, benchmark_score=0.6, hygiene_score=0.7) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + wf = _make_simple_workflow() + result = evaluator.evaluate(wf, "/tmp/test", ["t1"]) + assert result.score > 0 + + def test_evaluate_batch(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult(score=0.0, benchmark_score=0.5, hygiene_score=0.5) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + wf1 = _make_simple_workflow("wf1") + wf2 = _make_simple_workflow("wf2") + results = evaluator.evaluate_batch([wf1, wf2], "/tmp/test", ["t1"]) + assert len(results) == 2 + assert all(r.score > 0 for r in results) + + def test_multi_metric_composition(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult( + score=0.0, benchmark_score=1.0, hygiene_score=1.0, + cost_usd=0.0, complexity=0.0, + ) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + wf = _make_simple_workflow() + result = evaluator.evaluate(wf, "/tmp/test", ["t1"]) + # 0.6*1.0 + 0.2*1.0 + 0.1*(1-0) + 0.1*(1-0) = 1.0 + assert result.score == 1.0 + + def test_no_evaluator_fn(self) -> None: + config = _make_config() + evaluator = SwarmEvaluator(config) + wf = _make_simple_workflow() + result = evaluator.evaluate(wf, "/tmp/test", ["t1"]) + assert result.details.get("note") == "no_evaluator_fn_configured" diff --git a/tests/test_outer_loop/test_harbor_evaluator.py b/tests/test_outer_loop/test_harbor_evaluator.py new file mode 100644 index 000000000..1ed4fab9d --- /dev/null +++ b/tests/test_outer_loop/test_harbor_evaluator.py @@ -0,0 +1,258 @@ +"""Tests for HarborEvaluator, create_seed_workflow, and workflow_to_harbor_yaml.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from unittest.mock import patch + +import yaml + +from factory.outer_loop.harbor_evaluator import ( + HarborEvaluator, + create_seed_workflow, + workflow_to_harbor_yaml, +) +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + GateNode, + Workflow, +) + + +class TestCreateSeedWorkflow: + def test_returns_valid_workflow(self) -> None: + wf = create_seed_workflow() + assert isinstance(wf, Workflow) + assert wf.name == "featurebench-seed" + + def test_has_four_nodes(self) -> None: + wf = create_seed_workflow() + assert len(wf.nodes) == 4 + assert set(wf.nodes.keys()) == {"researcher", "builder", "health_checker", "gate"} + + def test_has_correct_edges(self) -> None: + wf = create_seed_workflow() + edge_pairs = [(e.source, e.target) for e in wf.edges] + assert ("researcher", "builder") in edge_pairs + assert ("builder", "health_checker") in edge_pairs + assert ("health_checker", "gate") in edge_pairs + + def test_start_node_is_researcher(self) -> None: + wf = create_seed_workflow() + assert wf.start_node == "researcher" + + def test_builder_has_prompt(self) -> None: + wf = create_seed_workflow() + builder = wf.nodes["builder"] + assert isinstance(builder, AgentNode) + assert builder.prompt_template + assert "task-instruction" in builder.prompt_template + + def test_roundtrip_serialization(self) -> None: + wf = create_seed_workflow() + d = wf.to_dict() + restored = Workflow.from_dict(d) + assert restored.name == wf.name + assert set(restored.nodes.keys()) == set(wf.nodes.keys()) + assert len(restored.edges) == len(wf.edges) + + def test_node_roles(self) -> None: + wf = create_seed_workflow() + assert wf.nodes["researcher"].role == AgentRole.RESEARCHER # type: ignore[union-attr] + assert wf.nodes["builder"].role == AgentRole.BUILDER # type: ignore[union-attr] + assert wf.nodes["health_checker"].role == AgentRole.HEALTH_CHECKER # type: ignore[union-attr] + assert isinstance(wf.nodes["gate"], GateNode) + + +class TestWorkflowToHarborYaml: + def test_produces_valid_yaml(self) -> None: + wf = create_seed_workflow() + result = workflow_to_harbor_yaml(wf) + parsed = yaml.safe_load(result) + assert isinstance(parsed, dict) + + def test_includes_agent_nodes_with_prompts(self) -> None: + wf = create_seed_workflow() + result = workflow_to_harbor_yaml(wf) + parsed = yaml.safe_load(result) + assert "builder" in parsed + assert "task_prompt_builder" in parsed["builder"]["slots"] + + def test_includes_timeout(self) -> None: + wf = create_seed_workflow() + result = workflow_to_harbor_yaml(wf) + parsed = yaml.safe_load(result) + builder_slots = parsed["builder"]["slots"] + assert "timeout_builder" in builder_slots + assert builder_slots["timeout_builder"] == 7200 + + def test_gate_without_gate_prompt_excluded(self) -> None: + wf = create_seed_workflow() + result = workflow_to_harbor_yaml(wf) + parsed = yaml.safe_load(result) + assert "gate" not in parsed + + def test_gate_with_prompt_included(self) -> None: + wf = Workflow( + name="test", + nodes={ + "g": GateNode( + id="g", + evaluator_type="fn", + gate_prompt="Check if tests pass", + ), + }, + edges=[], + start_node="g", + ) + result = workflow_to_harbor_yaml(wf) + parsed = yaml.safe_load(result) + assert "g" in parsed + assert "gate_prompt_g" in parsed["g"]["slots"] + + def test_empty_prompt_excluded(self) -> None: + wf = Workflow( + name="test", + nodes={ + "b": AgentNode(id="b", role=AgentRole.BUILDER, prompt_template=""), + }, + edges=[], + start_node="b", + ) + result = workflow_to_harbor_yaml(wf) + parsed = yaml.safe_load(result) + assert parsed is None or "b" not in (parsed or {}) + + +class TestHarborEvaluator: + def test_missing_script_returns_zero(self, tmp_path: Path) -> None: + evaluator = HarborEvaluator(benchmarks_dir=tmp_path, timeout=60) + wf = create_seed_workflow() + result = evaluator(wf, "/tmp/test", ["instance1"]) + assert result.score == 0.0 + assert "error" in result.details + + def test_all_resolved(self, tmp_path: Path) -> None: + script = tmp_path / "run-harbor.sh" + script.write_text("#!/bin/bash\necho 'Result: RESOLVED'\necho '\"cost_usd\": 1.5'") + script.chmod(0o755) + + evaluator = HarborEvaluator(benchmarks_dir=tmp_path, timeout=60) + wf = create_seed_workflow() + result = evaluator(wf, "/tmp/test", ["i1", "i2"]) + assert result.score == 1.0 + assert result.benchmark_score == 1.0 + assert result.cost_usd == 3.0 + + def test_partial_resolve(self, tmp_path: Path) -> None: + call_count = 0 + + def mock_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: + nonlocal call_count + call_count += 1 + if call_count == 1: + stdout = "Result: RESOLVED\n\"cost_usd\": 1.0" + else: + stdout = "Result: NOT RESOLVED\n\"cost_usd\": 0.5" + return subprocess.CompletedProcess( + args=[], returncode=0, stdout=stdout, stderr="" + ) + + script = tmp_path / "run-harbor.sh" + script.write_text("#!/bin/bash\necho test") + script.chmod(0o755) + + evaluator = HarborEvaluator(benchmarks_dir=tmp_path, timeout=60) + wf = create_seed_workflow() + + with patch("subprocess.run", side_effect=mock_run): + result = evaluator(wf, "/tmp/test", ["i1", "i2"]) + + assert result.score == 0.5 + assert result.cost_usd == 1.5 + + def test_timeout_scores_zero(self, tmp_path: Path) -> None: + def mock_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: + raise subprocess.TimeoutExpired(cmd="test", timeout=60) + + script = tmp_path / "run-harbor.sh" + script.write_text("#!/bin/bash\necho test") + script.chmod(0o755) + + evaluator = HarborEvaluator(benchmarks_dir=tmp_path, timeout=60) + wf = create_seed_workflow() + + with patch("subprocess.run", side_effect=mock_run): + result = evaluator(wf, "/tmp/test", ["i1"]) + + assert result.score == 0.0 + + def test_complexity_from_node_count(self, tmp_path: Path) -> None: + script = tmp_path / "run-harbor.sh" + script.write_text("#!/bin/bash\necho 'Result: RESOLVED'") + script.chmod(0o755) + + evaluator = HarborEvaluator(benchmarks_dir=tmp_path, timeout=60) + wf = create_seed_workflow() + result = evaluator(wf, "/tmp/test", ["i1"]) + assert result.complexity == 4.0 + + def test_passes_yaml_b64_to_env(self, tmp_path: Path) -> None: + captured_env: dict[str, str] = {} + + def mock_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: + env = kwargs.get("env", {}) + assert isinstance(env, dict) + captured_env.update(env) + return subprocess.CompletedProcess( + args=[], returncode=0, stdout="Result: NOT RESOLVED", stderr="" + ) + + script = tmp_path / "run-harbor.sh" + script.write_text("#!/bin/bash\necho test") + script.chmod(0o755) + + evaluator = HarborEvaluator(benchmarks_dir=tmp_path, timeout=60) + wf = create_seed_workflow() + + with patch("subprocess.run", side_effect=mock_run): + evaluator(wf, "/tmp/test", ["i1"]) + + assert "FACTORY_WORKFLOW_YAML_B64" in captured_env + + def test_implements_evaluator_fn_protocol(self) -> None: + from factory.outer_loop.evaluator import EvaluatorFn + + evaluator = HarborEvaluator(timeout=60) + assert isinstance(evaluator, EvaluatorFn) + + +class TestRunEvolution: + def test_main_missing_training_instances(self) -> None: + from factory.outer_loop.run_evolution import main + + result = main(["--training-instances", ""]) + assert result == 1 + + def test_main_parses_instances(self) -> None: + from factory.outer_loop.run_evolution import main + + with patch( + "factory.outer_loop.run_evolution.SwarmEngine" + ) as mock_engine_cls: + mock_engine = mock_engine_cls.return_value + from factory.outer_loop.models import OuterLoopResult + + mock_engine.run.return_value = OuterLoopResult( + convergence_reason="budget_exhausted", + ) + result = main([ + "--training-instances", "a,b,c", + "--holdout-instances", "d,e", + "--budget", "1", + "--population", "2", + ]) + assert result == 0 + mock_engine.run.assert_called_once() diff --git a/tests/test_outer_loop/test_models.py b/tests/test_outer_loop/test_models.py new file mode 100644 index 000000000..554eb30d0 --- /dev/null +++ b/tests/test_outer_loop/test_models.py @@ -0,0 +1,208 @@ +"""Tests for outer loop Pydantic models.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from factory.outer_loop.models import ( + GenerationSummary, + HyperparameterRecord, + Individual, + MutationRecord, + MutationType, + OuterLoopState, + SwarmConfig, +) + + +class TestMutationType: + def test_all_variants(self) -> None: + assert len(MutationType) == 6 + assert MutationType.NODE_INSERT.value == "node_insert" + assert MutationType.PARAM_MUTATE.value == "param_mutate" + + +class TestMutationRecord: + def test_basic(self) -> None: + rec = MutationRecord( + operator=MutationType.NODE_INSERT, + target_node="agent_1", + rationale="test", + ) + assert rec.operator == MutationType.NODE_INSERT + assert rec.before == {} + assert rec.after == {} + + def test_round_trip(self) -> None: + rec = MutationRecord( + operator=MutationType.EDGE_REDIRECT, + target_node="gate_1", + before={"target": "a"}, + after={"target": "b"}, + rationale="redirect", + ) + dumped = rec.model_dump(mode="json") + restored = MutationRecord.model_validate(dumped) + assert restored == rec + + def test_extra_forbid(self) -> None: + with pytest.raises(ValidationError): + MutationRecord( + operator=MutationType.NODE_INSERT, + target_node="x", + rationale="test", + unknown_field="bad", # type: ignore[call-arg] + ) + + +class TestIndividual: + def test_basic(self) -> None: + ind = Individual( + id="abc123", + workflow_data={"name": "test"}, + score=0.85, + features=(3, 2, 5, 1), + generation=1, + ) + assert ind.score == 0.85 + assert ind.features == (3, 2, 5, 1) + assert ind.parent_id is None + + def test_round_trip(self) -> None: + ind = Individual( + id="xyz", + workflow_data={"name": "w"}, + score=0.5, + features=(1, 0, 2, 1), + generation=0, + parent_id="abc", + mutation_record=MutationRecord( + operator=MutationType.NODE_REMOVE, + target_node="n1", + rationale="r", + ), + cost_usd=1.5, + ) + dumped = ind.model_dump(mode="json") + restored = Individual.model_validate(dumped) + assert restored.parent_id == "abc" + assert restored.mutation_record is not None + assert restored.mutation_record.operator == MutationType.NODE_REMOVE + + +class TestHyperparameterRecord: + def test_basic(self) -> None: + rec = HyperparameterRecord( + generation=0, + mutation_rate=0.3, + population_size=4, + tournament_size=3, + designer_ratio=0.3, + operator_weights={"node_insert": 0.2, "node_remove": 0.15}, + best_score=0.8, + mean_score=0.6, + diversity=0.4, + novel_count=3, + ) + assert rec.generation == 0 + assert rec.operator_weights["node_insert"] == 0.2 + + def test_round_trip(self) -> None: + rec = HyperparameterRecord( + generation=5, + mutation_rate=0.5, + population_size=8, + tournament_size=5, + designer_ratio=0.4, + ) + dumped = rec.model_dump(mode="json") + restored = HyperparameterRecord.model_validate(dumped) + assert restored == rec + + +class TestSwarmConfig: + def test_defaults(self) -> None: + cfg = SwarmConfig(benchmark="featurebench", budget=100) + assert cfg.population_size == 4 + assert cfg.tournament_size == 3 + assert cfg.mutation_rate == 0.3 + assert cfg.designer_count == 2 + assert cfg.mutation_strategy == "weighted_random" + + def test_no_overlap(self) -> None: + with pytest.raises(ValidationError, match="overlap"): + SwarmConfig( + benchmark="test", + budget=50, + training_instances=["p1", "p2", "p3"], + holdout_instances=["p3", "p4"], + ) + + def test_disjoint_ok(self) -> None: + cfg = SwarmConfig( + benchmark="test", + budget=50, + training_instances=["p1", "p2", "p3"], + holdout_instances=["p4", "p5"], + ) + assert len(cfg.training_instances) == 3 + assert len(cfg.holdout_instances) == 2 + + +class TestOuterLoopState: + def test_defaults(self) -> None: + state = OuterLoopState() + assert state.generation == 0 + assert state.convergence_reason is None + assert state.hyperparameter_history == [] + + def test_with_history(self) -> None: + rec = HyperparameterRecord( + generation=0, + mutation_rate=0.3, + population_size=4, + tournament_size=3, + designer_ratio=0.3, + ) + state = OuterLoopState( + generation=1, + total_evaluations=8, + best_score=0.85, + budget_remaining=92, + score_trajectory=[0.7, 0.85], + hyperparameter_history=[rec], + ) + dumped = state.model_dump(mode="json") + restored = OuterLoopState.model_validate(dumped) + assert len(restored.hyperparameter_history) == 1 + + +class TestGenerationSummary: + def test_basic(self) -> None: + summary = GenerationSummary( + generation=0, + population_size=4, + best_score=0.8, + mean_score=0.6, + diversity=0.4, + novel_count=3, + rejected_duplicates=1, + ) + assert summary.hyperparameters is None + assert summary.mutations_applied == [] + + def test_with_mutations(self) -> None: + rec = MutationRecord( + operator=MutationType.PARALLELIZE, + rationale="speed up", + ) + summary = GenerationSummary( + generation=1, + population_size=4, + best_score=0.9, + mean_score=0.75, + diversity=0.5, + mutations_applied=[rec], + ) + assert len(summary.mutations_applied) == 1 diff --git a/tests/test_outer_loop/test_mutations.py b/tests/test_outer_loop/test_mutations.py new file mode 100644 index 000000000..9d0357d6e --- /dev/null +++ b/tests/test_outer_loop/test_mutations.py @@ -0,0 +1,250 @@ +"""Tests for mutation operators and MutationStrategy.""" + +from __future__ import annotations + + +from factory.outer_loop.models import MutationType +from factory.outer_loop.mutations import ( + MutationStrategy, + WeightedRandomStrategy, + apply_random_mutation, + insert_node, + mutate_params, + parallelize, + redirect_edge, + remove_node, + serialize, + validate_and_repair, +) +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + Workflow, +) + + +class TestInsertNode: + def test_insert_between_nodes(self, simple_workflow: Workflow) -> None: + new_node = AgentNode(id="reviewer", role=AgentRole.CODE_REVIEWER) + result = insert_node(simple_workflow, new_node, "strategist") + assert result is not None + wf, rec = result + assert "reviewer" in wf.nodes + assert rec.operator == MutationType.NODE_INSERT + + def test_insert_respects_frozen(self, simple_workflow: Workflow) -> None: + new_node = AgentNode(id="new", role=AgentRole.RESEARCHER) + result = insert_node( + simple_workflow, new_node, "researcher", frozen_nodes={"researcher"} + ) + assert result is None + + def test_insert_after_nonexistent(self, simple_workflow: Workflow) -> None: + new_node = AgentNode(id="new", role=AgentRole.RESEARCHER) + result = insert_node(simple_workflow, new_node, "nonexistent") + assert result is None + + +class TestRemoveNode: + def test_remove_middle_node(self, simple_workflow: Workflow) -> None: + result = remove_node(simple_workflow, "strategist") + assert result is not None + wf, rec = result + assert "strategist" not in wf.nodes + assert rec.operator == MutationType.NODE_REMOVE + has_edge = any( + e.source == "researcher" and e.target == "builder" for e in wf.edges + ) + assert has_edge + + def test_remove_start_node_fails(self, simple_workflow: Workflow) -> None: + result = remove_node(simple_workflow, "study") + assert result is None + + def test_remove_frozen_fails(self, simple_workflow: Workflow) -> None: + result = remove_node(simple_workflow, "builder", frozen_nodes={"builder"}) + assert result is None + + +class TestRedirectEdge: + def test_redirect_edge(self, simple_workflow: Workflow) -> None: + result = redirect_edge(simple_workflow, "researcher", "strategist", "builder") + assert result is not None + wf, rec = result + assert rec.operator == MutationType.EDGE_REDIRECT + has_new = any( + e.source == "researcher" and e.target == "builder" for e in wf.edges + ) + assert has_new + + def test_redirect_nonexistent_target(self, simple_workflow: Workflow) -> None: + result = redirect_edge(simple_workflow, "researcher", "strategist", "nonexistent") + assert result is None + + def test_redirect_frozen_source(self, simple_workflow: Workflow) -> None: + result = redirect_edge( + simple_workflow, "researcher", "strategist", "builder", + frozen_nodes={"researcher"}, + ) + assert result is None + + +class TestParallelize: + def test_parallelize_two_nodes(self, simple_workflow: Workflow) -> None: + result = parallelize(simple_workflow, ["researcher", "strategist"]) + assert result is not None + wf, rec = result + assert rec.operator == MutationType.PARALLELIZE + fork_nodes = [nid for nid, n in wf.nodes.items() if type(n).__name__ == "ForkNode"] + join_nodes = [nid for nid, n in wf.nodes.items() if type(n).__name__ == "JoinNode"] + assert len(fork_nodes) >= 1 + assert len(join_nodes) >= 1 + + def test_parallelize_single_node_fails(self, simple_workflow: Workflow) -> None: + result = parallelize(simple_workflow, ["researcher"]) + assert result is None + + def test_parallelize_frozen_fails(self, simple_workflow: Workflow) -> None: + result = parallelize( + simple_workflow, ["researcher", "strategist"], + frozen_nodes={"researcher"}, + ) + assert result is None + + +class TestSerialize: + def test_serialize_reverses_parallelize(self, simple_workflow: Workflow) -> None: + par_result = parallelize(simple_workflow, ["researcher", "strategist"]) + assert par_result is not None + wf_par, _ = par_result + + fork_ids = [nid for nid, n in wf_par.nodes.items() if type(n).__name__ == "ForkNode"] + assert len(fork_ids) >= 1 + + ser_result = serialize(wf_par, fork_ids[0]) + assert ser_result is not None + wf_ser, rec = ser_result + assert rec.operator == MutationType.SERIALIZE + assert not any(type(n).__name__ == "ForkNode" for n in wf_ser.nodes.values()) + + def test_serialize_nonexistent_fails(self, simple_workflow: Workflow) -> None: + result = serialize(simple_workflow, "nonexistent") + assert result is None + + def test_serialize_non_fork_fails(self, simple_workflow: Workflow) -> None: + result = serialize(simple_workflow, "researcher") + assert result is None + + +class TestMutateParams: + def test_change_timeout(self, simple_workflow: Workflow) -> None: + result = mutate_params(simple_workflow, "researcher", {"timeout": 1200}) + assert result is not None + wf, rec = result + assert rec.operator == MutationType.PARAM_MUTATE + node = wf.nodes["researcher"] + assert hasattr(node, "timeout") + assert node.timeout == 1200 # type: ignore[union-attr] + + def test_change_model(self, simple_workflow: Workflow) -> None: + result = mutate_params(simple_workflow, "researcher", {"model": "opus"}) + assert result is not None + wf, _ = result + assert wf.nodes["researcher"].model == "opus" # type: ignore[union-attr] + + def test_disallowed_param_ignored(self, simple_workflow: Workflow) -> None: + result = mutate_params(simple_workflow, "researcher", {"role": "builder"}) + assert result is None + + def test_frozen_fails(self, simple_workflow: Workflow) -> None: + result = mutate_params( + simple_workflow, "researcher", {"timeout": 900}, + frozen_nodes={"researcher"}, + ) + assert result is None + + +class TestValidateAndRepair: + def test_valid_workflow_passes(self, simple_workflow: Workflow) -> None: + result = validate_and_repair(simple_workflow) + assert result is not None + + def test_prunes_unreachable(self) -> None: + nodes = { + "start": FnNode(id="start", command="echo start"), + "reachable": FnNode(id="reachable", command="echo r"), + "orphan": FnNode(id="orphan", command="echo orphan"), + } + edges = [Edge(source="start", target="reachable")] + wf = Workflow(name="test", nodes=nodes, edges=edges, start_node="start") + result = validate_and_repair(wf) + assert result is not None + assert "orphan" not in result.nodes + + def test_cycle_without_gate_returns_none(self) -> None: + nodes = { + "a": FnNode(id="a", command="echo a"), + "b": FnNode(id="b", command="echo b"), + } + edges = [ + Edge(source="a", target="b"), + Edge(source="b", target="a"), + ] + wf = Workflow(name="test", nodes=nodes, edges=edges, start_node="a") + result = validate_and_repair(wf) + assert result is None + + +class TestWeightedRandomStrategy: + def test_implements_protocol(self) -> None: + strategy = WeightedRandomStrategy() + assert isinstance(strategy, MutationStrategy) + + def test_select_operator_returns_valid(self, simple_workflow: Workflow) -> None: + strategy = WeightedRandomStrategy() + op = strategy.select_operator(simple_workflow, 0, {}) + assert isinstance(op, MutationType) + + def test_mutation_rate(self) -> None: + strategy = WeightedRandomStrategy(mutation_rate=0.5) + assert strategy.get_mutation_rate(0) == 0.5 + assert strategy.get_mutation_rate(10) == 0.5 + + def test_designer_ratio(self) -> None: + strategy = WeightedRandomStrategy(designer_ratio=0.4) + assert strategy.get_designer_ratio(0) == 0.4 + + def test_operator_weights(self) -> None: + weights = {t.value: (1.0 if t == MutationType.NODE_INSERT else 0.0) for t in MutationType} + strategy = WeightedRandomStrategy(weights=weights) + ops = [strategy.select_operator(Workflow( + name="dummy", + nodes={"a": FnNode(id="a", command="x")}, + edges=[], + start_node="a", + ), 0, {}) for _ in range(20)] + assert all(op == MutationType.NODE_INSERT for op in ops) + + +class TestApplyRandomMutation: + def test_produces_valid_result(self, simple_workflow: Workflow) -> None: + strategy = WeightedRandomStrategy() + result = apply_random_mutation( + simple_workflow, strategy, generation=0, max_attempts=20, + ) + if result is not None: + wf, rec = result + assert isinstance(rec.operator, MutationType) + assert wf.start_node in wf.nodes + + def test_with_frozen_nodes(self, simple_workflow: Workflow) -> None: + strategy = WeightedRandomStrategy() + all_nodes = set(simple_workflow.nodes.keys()) + result = apply_random_mutation( + simple_workflow, strategy, generation=0, + frozen_nodes=all_nodes, + max_attempts=5, + ) + assert result is None diff --git a/tests/test_outer_loop/test_overfit.py b/tests/test_outer_loop/test_overfit.py new file mode 100644 index 000000000..d13cc66a4 --- /dev/null +++ b/tests/test_outer_loop/test_overfit.py @@ -0,0 +1,140 @@ +"""Tests for OverfitDetector.""" + +from __future__ import annotations + +from factory.outer_loop.evaluator import SwarmEvaluator +from factory.outer_loop.models import EvalResult, SwarmConfig +from factory.outer_loop.overfit import OverfitDetector +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + Workflow, +) + + +def _make_config() -> SwarmConfig: + return SwarmConfig( + benchmark="test", + budget=50, + training_instances=["t1", "t2"], + holdout_instances=["h1"], + ) + + +def _make_workflow() -> Workflow: + return Workflow( + name="test", + nodes={ + "a": FnNode(id="a", command="echo a"), + "b": AgentNode(id="b", role=AgentRole.BUILDER), + }, + edges=[Edge(source="a", target="b")], + start_node="a", + ) + + +class TestOverfitDetector: + def test_no_overfit(self) -> None: + config = _make_config() + scores = {"t1": 0.8, "t2": 0.8, "h1": 0.75} + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + avg = sum(scores.get(i, 0.0) for i in instances) / max(len(instances), 1) + return EvalResult(score=avg, benchmark_score=avg, hygiene_score=0.8) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + detector = OverfitDetector(threshold=0.15) + + wf = _make_workflow() + result = detector.audit(wf, ["t1", "t2"], ["h1"], evaluator, "/tmp") + + assert not result.overfit_flag + assert result.training_score > 0 + assert result.holdout_score > 0 + assert result.delta < 0.15 + + def test_overfit_detected(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + if "h1" in instances: + return EvalResult(score=0.5, benchmark_score=0.5, hygiene_score=0.5) + return EvalResult(score=0.9, benchmark_score=0.9, hygiene_score=0.9) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + detector = OverfitDetector(threshold=0.15) + + wf = _make_workflow() + result = detector.audit(wf, ["t1", "t2"], ["h1"], evaluator, "/tmp") + + assert result.overfit_flag + assert result.delta > 0.15 + + def test_equal_scores(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult(score=0.7, benchmark_score=0.7, hygiene_score=0.7) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + detector = OverfitDetector() + + wf = _make_workflow() + result = detector.audit(wf, ["t1"], ["h1"], evaluator, "/tmp") + + assert not result.overfit_flag + assert result.delta == 0.0 + + def test_zero_training_score(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult(score=0.0, benchmark_score=0.0) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + detector = OverfitDetector() + + wf = _make_workflow() + result = detector.audit(wf, ["t1"], ["h1"], evaluator, "/tmp") + + assert not result.overfit_flag + assert result.delta == 0.0 + + def test_custom_threshold(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + if "h1" in instances: + # Composite: 0.6*0.9 + 0.2*1.0 + 0.1 + 0.1 = 0.94 + return EvalResult(score=0.0, benchmark_score=0.9, hygiene_score=1.0) + # Composite: 0.6*1.0 + 0.2*1.0 + 0.1 + 0.1 = 1.0 + return EvalResult(score=0.0, benchmark_score=1.0, hygiene_score=1.0) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + # Delta = (1.0 - 0.94) / 1.0 = 0.06 → passes at 0.15, fails at 0.05 + detector_strict = OverfitDetector(threshold=0.05) + detector_loose = OverfitDetector(threshold=0.15) + + wf = _make_workflow() + strict_result = detector_strict.audit(wf, ["t1"], ["h1"], evaluator, "/tmp") + loose_result = detector_loose.audit(wf, ["t1"], ["h1"], evaluator, "/tmp") + + assert strict_result.overfit_flag + assert not loose_result.overfit_flag + + def test_details_populated(self) -> None: + config = _make_config() + + def mock_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult(score=0.8, benchmark_score=0.8) + + evaluator = SwarmEvaluator(config, evaluator_fn=mock_eval) + detector = OverfitDetector() + wf = _make_workflow() + result = detector.audit(wf, ["t1"], ["h1"], evaluator, "/tmp") + + assert "training=" in result.details + assert "holdout=" in result.details + assert "delta=" in result.details diff --git a/tests/test_outer_loop/test_population.py b/tests/test_outer_loop/test_population.py new file mode 100644 index 000000000..f0e36ffb9 --- /dev/null +++ b/tests/test_outer_loop/test_population.py @@ -0,0 +1,177 @@ +"""Tests for Population and MAPElitesArchive.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from factory.outer_loop.models import Individual +from factory.outer_loop.population import MAPElitesArchive, Population +from factory.workflow.primitives import Workflow + + +class TestPopulation: + def test_add_and_size(self) -> None: + pop = Population() + assert pop.size == 0 + ind = Individual(id="a", workflow_data={"name": "w"}, score=0.5, features=(1, 0, 2, 1)) + pop.add(ind) + assert pop.size == 1 + + def test_remove(self) -> None: + pop = Population() + ind = Individual(id="a", workflow_data={"name": "w"}, score=0.5, features=(1, 0, 2, 1)) + pop.add(ind) + removed = pop.remove("a") + assert removed is not None + assert pop.size == 0 + assert pop.remove("nonexistent") is None + + def test_get(self) -> None: + pop = Population() + ind = Individual(id="a", workflow_data={"name": "w"}, score=0.5, features=(1, 0, 2, 1)) + pop.add(ind) + assert pop.get("a") is not None + assert pop.get("b") is None + + def test_best(self) -> None: + pop = Population() + assert pop.best() is None + pop.add(Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1))) + pop.add(Individual(id="b", workflow_data={}, score=0.9, features=(2, 1, 3, 2))) + pop.add(Individual(id="c", workflow_data={}, score=0.7, features=(1, 1, 2, 1))) + best = pop.best() + assert best is not None + assert best.id == "b" + + def test_mean_score(self) -> None: + pop = Population() + assert pop.mean_score() == 0.0 + pop.add(Individual(id="a", workflow_data={}, score=0.4, features=())) + pop.add(Individual(id="b", workflow_data={}, score=0.8, features=())) + assert pop.mean_score() == pytest.approx(0.6) + + def test_individuals_list(self) -> None: + pop = Population() + pop.add(Individual(id="a", workflow_data={}, score=0.5, features=())) + pop.add(Individual(id="b", workflow_data={}, score=0.7, features=())) + assert len(pop.individuals) == 2 + + def test_make_individual(self, simple_workflow: Workflow) -> None: + ind = Population.make_individual(simple_workflow, generation=1, score=0.8) + assert ind.generation == 1 + assert ind.score == 0.8 + assert len(ind.features) == 4 + assert ind.parent_id is None + + def test_serialization_round_trip(self, simple_workflow: Workflow, tmp_path: Path) -> None: + pop = Population() + ind = Population.make_individual(simple_workflow, generation=0, score=0.7) + pop.add(ind) + + pop.save(tmp_path / "pop") + loaded = Population.load(tmp_path / "pop") + + assert loaded.size == 1 + loaded_ind = loaded.individuals[0] + assert loaded_ind.id == ind.id + assert loaded_ind.score == ind.score + + +class TestMAPElitesArchive: + def test_add_and_size(self) -> None: + archive = MAPElitesArchive() + assert archive.size == 0 + ind = Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1)) + assert archive.add(ind) is True + assert archive.size == 1 + + def test_add_replaces_lower_score(self) -> None: + archive = MAPElitesArchive() + ind1 = Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1)) + ind2 = Individual(id="b", workflow_data={}, score=0.9, features=(1, 0, 2, 1)) + archive.add(ind1) + assert archive.add(ind2) is True + assert archive.size == 1 + assert archive.best().id == "b" # type: ignore[union-attr] + + def test_add_keeps_higher_score(self) -> None: + archive = MAPElitesArchive() + ind1 = Individual(id="a", workflow_data={}, score=0.9, features=(1, 0, 2, 1)) + ind2 = Individual(id="b", workflow_data={}, score=0.5, features=(1, 0, 2, 1)) + archive.add(ind1) + assert archive.add(ind2) is False + assert archive.best().id == "a" # type: ignore[union-attr] + + def test_best_empty(self) -> None: + assert MAPElitesArchive().best() is None + + def test_best(self) -> None: + archive = MAPElitesArchive() + archive.add(Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1))) + archive.add(Individual(id="b", workflow_data={}, score=0.9, features=(2, 1, 3, 2))) + best = archive.best() + assert best is not None + assert best.id == "b" + + def test_sample_parent_returns_something(self) -> None: + archive = MAPElitesArchive() + assert archive.sample_parent() is None + archive.add(Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1))) + result = archive.sample_parent(tournament_size=1) + assert result is not None + assert result.id == "a" + + def test_tournament_selection(self) -> None: + archive = MAPElitesArchive() + for i in range(10): + archive.add( + Individual(id=f"i{i}", workflow_data={}, score=i * 0.1, features=(i, 0, i, 0)) + ) + results = [archive.sample_parent(tournament_size=3) for _ in range(20)] + scores = [r.score for r in results if r is not None] + assert all(s >= 0.0 for s in scores) + + def test_pareto_front_single(self) -> None: + archive = MAPElitesArchive() + archive.add(Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1))) + front = archive.pareto_front() + assert len(front) == 1 + + def test_pareto_front_dominated(self) -> None: + archive = MAPElitesArchive() + archive.add(Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1))) + archive.add(Individual(id="b", workflow_data={}, score=0.9, features=(2, 1, 3, 2))) + front = archive.pareto_front() + assert len(front) == 1 + assert front[0].id == "b" + + def test_pareto_front_non_dominated(self) -> None: + archive = MAPElitesArchive() + archive.add(Individual(id="a", workflow_data={}, score=0.9, features=(1, 0, 5, 0))) + archive.add(Individual(id="b", workflow_data={}, score=0.5, features=(5, 3, 1, 3))) + front = archive.pareto_front() + assert len(front) == 2 + + def test_diversity_metric_empty(self) -> None: + assert MAPElitesArchive().diversity_metric() == 0.0 + + def test_diversity_metric_nonzero(self) -> None: + archive = MAPElitesArchive() + archive.add(Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1))) + archive.add(Individual(id="b", workflow_data={}, score=0.7, features=(2, 1, 3, 2))) + d = archive.diversity_metric() + assert 0.0 < d <= 1.0 + + def test_serialization_round_trip(self, tmp_path: Path) -> None: + archive = MAPElitesArchive() + archive.add(Individual(id="a", workflow_data={}, score=0.5, features=(1, 0, 2, 1))) + archive.add(Individual(id="b", workflow_data={}, score=0.9, features=(2, 1, 3, 2))) + + archive.save(tmp_path / "archive") + loaded = MAPElitesArchive.load(tmp_path / "archive") + + assert loaded.size == 2 + assert loaded.best() is not None + assert loaded.best().id == "b" # type: ignore[union-attr] diff --git a/tests/test_outer_loop/test_seed_diversity.py b/tests/test_outer_loop/test_seed_diversity.py new file mode 100644 index 000000000..6a7fa69ae --- /dev/null +++ b/tests/test_outer_loop/test_seed_diversity.py @@ -0,0 +1,146 @@ +"""Tests for seed population diversity with designer-created variants.""" + +from __future__ import annotations + +from factory.outer_loop.designer import DesignerAgent +from factory.outer_loop.engine import SwarmEngine +from factory.outer_loop.evaluator import SwarmEvaluator +from factory.outer_loop.models import EvalResult, SwarmConfig +from factory.outer_loop.similarity import NoveltyFilter, compute_features +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + GateNode, + VerdictType, + Workflow, +) + + +def _make_config(**overrides: object) -> SwarmConfig: + defaults: dict[str, object] = { + "benchmark": "test_bench", + "budget": 50, + "population_size": 6, + "tournament_size": 2, + "mutation_rate": 0.3, + "training_instances": ["t1", "t2"], + "holdout_instances": ["h1"], + "designer_count": 2, + } + defaults.update(overrides) + return SwarmConfig(**defaults) # type: ignore[arg-type] + + +def _make_base_workflow() -> Workflow: + return Workflow( + name="seed_base", + nodes={ + "study": FnNode( + id="study", command="factory study", writes={".factory/obs.md"}, + ), + "researcher": AgentNode( + id="researcher", role=AgentRole.RESEARCHER, + reads={".factory/obs.md"}, writes={".factory/research.md"}, + ), + "strategist": AgentNode( + id="strategist", role=AgentRole.STRATEGIST, + reads={".factory/research.md"}, writes={".factory/current.md"}, + ), + "builder": AgentNode( + id="builder", role=AgentRole.BUILDER, + reads={".factory/current.md"}, writes={".factory/build.md"}, + ), + "gate": GateNode( + id="gate", evaluator_type="fn", + reads={".factory/build.md"}, + ), + }, + edges=[ + Edge(source="study", target="researcher"), + Edge(source="researcher", target="strategist"), + Edge(source="strategist", target="builder"), + Edge(source="builder", target="gate"), + Edge(source="gate", target="builder", condition=VerdictType.RELOOP), + ], + start_node="study", + ) + + +def _make_noop_evaluator(config: SwarmConfig) -> SwarmEvaluator: + def noop_eval(wf: Workflow, project_dir: str, instances: list[str]) -> EvalResult: + return EvalResult( + score=0.5, benchmark_score=0.5, hygiene_score=0.5, + cost_usd=0.01, complexity=float(len(wf.nodes)), + ) + return SwarmEvaluator(config, evaluator_fn=noop_eval) + + +class TestSeedWithDesigner: + def test_seed_includes_designer_variants(self) -> None: + config = _make_config(population_size=6, designer_count=2) + evaluator = _make_noop_evaluator(config) + novelty = NoveltyFilter(min_edit_distance=1) + engine = SwarmEngine(config, evaluator, novelty_filter=novelty) + wf = _make_base_workflow() + + pop = engine.seed(wf) + + assert pop.size >= 3 + originals = [i for i in pop.individuals if i.parent_id is None] + assert len(originals) >= 2 + + def test_feature_vectors_differ(self) -> None: + designer = DesignerAgent() + minimal = designer.design_minimal("test") + thorough = designer.design_thorough("test") + + min_features = compute_features(minimal) + thor_features = compute_features(thorough) + + assert min_features != thor_features + assert min_features[2] < thor_features[2] + + def test_designer_count_zero_skips_designs(self) -> None: + config = _make_config(population_size=4, designer_count=0) + evaluator = _make_noop_evaluator(config) + engine = SwarmEngine(config, evaluator) + wf = _make_base_workflow() + + pop = engine.seed(wf) + + originals = [i for i in pop.individuals if i.parent_id is None] + assert len(originals) == 1 + + def test_designer_count_3_includes_custom(self) -> None: + config = _make_config(population_size=8, designer_count=3) + evaluator = _make_noop_evaluator(config) + novelty = NoveltyFilter(min_edit_distance=1) + engine = SwarmEngine(config, evaluator, novelty_filter=novelty) + wf = _make_base_workflow() + + pop = engine.seed(wf) + + originals = [i for i in pop.individuals if i.parent_id is None] + assert len(originals) >= 3 + + def test_minimal_has_fewer_nodes_than_thorough(self) -> None: + designer = DesignerAgent() + minimal = designer.design_minimal("test") + thorough = designer.design_thorough("test") + + assert len(minimal.nodes) < len(thorough.nodes) + + def test_minimal_has_fewer_agents_than_thorough(self) -> None: + designer = DesignerAgent() + minimal = designer.design_minimal("test") + thorough = designer.design_thorough("test") + + min_agents = sum( + 1 for n in minimal.nodes.values() if type(n).__name__ == "AgentNode" + ) + thor_agents = sum( + 1 for n in thorough.nodes.values() if type(n).__name__ == "AgentNode" + ) + assert min_agents < thor_agents diff --git a/tests/test_outer_loop/test_similarity.py b/tests/test_outer_loop/test_similarity.py new file mode 100644 index 000000000..8463befa6 --- /dev/null +++ b/tests/test_outer_loop/test_similarity.py @@ -0,0 +1,183 @@ +"""Tests for structural hashing, GED, feature extraction, and novelty filtering.""" + +from __future__ import annotations + +from factory.outer_loop.similarity import ( + NoveltyFilter, + compute_features, + graph_edit_distance, + structural_hash, +) +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + Edge, + FnNode, + ForkNode, + GateNode, + JoinNode, + Workflow, +) + + +class TestStructuralHash: + def test_deterministic(self, simple_workflow: Workflow) -> None: + h1 = structural_hash(simple_workflow) + h2 = structural_hash(simple_workflow) + assert h1 == h2 + + def test_different_workflows_different_hash(self, simple_workflow: Workflow) -> None: + other = Workflow( + name="other", + nodes={"a": FnNode(id="a", command="echo a")}, + edges=[], + start_node="a", + ) + assert structural_hash(simple_workflow) != structural_hash(other) + + def test_same_structure_same_hash(self) -> None: + nodes1 = { + "a": FnNode(id="a", command="echo a"), + "b": FnNode(id="b", command="echo b"), + } + edges1 = [Edge(source="a", target="b")] + wf1 = Workflow(name="w", nodes=nodes1, edges=edges1, start_node="a") + + nodes2 = { + "a": FnNode(id="a", command="echo a"), + "b": FnNode(id="b", command="echo b"), + } + edges2 = [Edge(source="a", target="b")] + wf2 = Workflow(name="w", nodes=nodes2, edges=edges2, start_node="a") + + assert structural_hash(wf1) == structural_hash(wf2) + + +class TestGraphEditDistance: + def test_identical_workflows(self, simple_workflow: Workflow) -> None: + assert graph_edit_distance(simple_workflow, simple_workflow) == 0 + + def test_different_node_sets(self) -> None: + wf1 = Workflow( + name="w1", + nodes={ + "a": FnNode(id="a", command="x"), + "b": FnNode(id="b", command="x"), + }, + edges=[Edge(source="a", target="b")], + start_node="a", + ) + wf2 = Workflow( + name="w2", + nodes={ + "a": FnNode(id="a", command="x"), + "c": FnNode(id="c", command="x"), + }, + edges=[Edge(source="a", target="c")], + start_node="a", + ) + dist = graph_edit_distance(wf1, wf2) + assert dist >= 2 + + def test_type_change_adds_distance(self) -> None: + wf1 = Workflow( + name="w", + nodes={"a": FnNode(id="a", command="x")}, + edges=[], + start_node="a", + ) + wf2 = Workflow( + name="w", + nodes={"a": AgentNode(id="a", role=AgentRole.RESEARCHER)}, + edges=[], + start_node="a", + ) + assert graph_edit_distance(wf1, wf2) == 1 + + +class TestComputeFeatures: + def test_simple_workflow(self, simple_workflow: Workflow) -> None: + depth, fork_degree, agent_count, gate_count = compute_features(simple_workflow) + assert depth >= 4 + assert fork_degree == 0 + assert agent_count == 3 + assert gate_count == 1 + + def test_workflow_with_fork(self) -> None: + nodes = { + "start": FnNode(id="start", command="x"), + "fork": ForkNode(id="fork", targets=["a", "b", "c"]), + "a": AgentNode(id="a", role=AgentRole.RESEARCHER), + "b": AgentNode(id="b", role=AgentRole.BUILDER), + "c": AgentNode(id="c", role=AgentRole.STRATEGIST), + "join": JoinNode(id="join", sources=["a", "b", "c"]), + "gate": GateNode(id="gate", evaluator_type="fn"), + } + edges = [ + Edge(source="start", target="fork"), + Edge(source="fork", target="a"), + Edge(source="fork", target="b"), + Edge(source="fork", target="c"), + Edge(source="a", target="join"), + Edge(source="b", target="join"), + Edge(source="c", target="join"), + Edge(source="join", target="gate"), + ] + wf = Workflow(name="forked", nodes=nodes, edges=edges, start_node="start") + depth, fork_degree, agent_count, gate_count = compute_features(wf) + assert fork_degree == 3 + assert agent_count == 3 + assert gate_count == 1 + + +class TestNoveltyFilter: + def test_first_workflow_is_novel(self, simple_workflow: Workflow) -> None: + nf = NoveltyFilter() + assert nf.is_novel(simple_workflow) is True + + def test_duplicate_is_not_novel(self, simple_workflow: Workflow) -> None: + nf = NoveltyFilter() + nf.add(simple_workflow) + assert nf.is_novel(simple_workflow) is False + + def test_similar_workflow_rejected_by_ged(self, simple_workflow: Workflow) -> None: + nf = NoveltyFilter(min_edit_distance=2) + nf.add(simple_workflow) + + other = Workflow( + name=simple_workflow.name, + nodes=dict(simple_workflow.nodes), + edges=list(simple_workflow.edges), + start_node=simple_workflow.start_node, + ) + assert nf.is_novel(other) is False + + def test_very_different_workflow_is_novel(self, simple_workflow: Workflow) -> None: + nf = NoveltyFilter(min_edit_distance=2) + nf.add(simple_workflow) + + other = Workflow( + name="totally_different", + nodes={ + "x": FnNode(id="x", command="echo x"), + "y": FnNode(id="y", command="echo y"), + "z": FnNode(id="z", command="echo z"), + }, + edges=[ + Edge(source="x", target="y"), + Edge(source="y", target="z"), + ], + start_node="x", + ) + assert nf.is_novel(other) is True + + def test_custom_threshold(self, simple_workflow: Workflow) -> None: + nf = NoveltyFilter(min_edit_distance=100) + nf.add(simple_workflow) + other = Workflow( + name="other", + nodes={"a": FnNode(id="a", command="x")}, + edges=[], + start_node="a", + ) + assert nf.is_novel(other, threshold=1) is True diff --git a/tests/test_outer_loop/test_subset.py b/tests/test_outer_loop/test_subset.py new file mode 100644 index 000000000..9f944b14e --- /dev/null +++ b/tests/test_outer_loop/test_subset.py @@ -0,0 +1,33 @@ +"""Tests for SubsetSelector and FixedSubsetSelector.""" + +from __future__ import annotations + +from factory.outer_loop.subset import FixedSubsetSelector, SubsetSelector + + +class TestFixedSubsetSelector: + def test_returns_configured_instances(self) -> None: + selector = FixedSubsetSelector(["t1", "t2", "t3"]) + result = selector.select(["t1", "t2", "t3", "t4", "t5"], generation=0, budget_remaining=100) + assert result == ["t1", "t2", "t3"] + + def test_ignores_generation_and_budget(self) -> None: + selector = FixedSubsetSelector(["a", "b"]) + r1 = selector.select(["a", "b", "c"], generation=0, budget_remaining=100) + r2 = selector.select(["a", "b", "c"], generation=5, budget_remaining=10) + assert r1 == r2 + + def test_returns_copy(self) -> None: + instances = ["x", "y"] + selector = FixedSubsetSelector(instances) + result = selector.select([], generation=0, budget_remaining=50) + result.append("z") + assert selector.select([], generation=0, budget_remaining=50) == ["x", "y"] + + def test_protocol_conformance(self) -> None: + selector = FixedSubsetSelector(["t1"]) + assert isinstance(selector, SubsetSelector) + + def test_empty_instances(self) -> None: + selector = FixedSubsetSelector([]) + assert selector.select(["a", "b"], generation=0, budget_remaining=10) == [] diff --git a/tests/test_outer_loop/test_telemetry.py b/tests/test_outer_loop/test_telemetry.py new file mode 100644 index 000000000..7cc69d0bf --- /dev/null +++ b/tests/test_outer_loop/test_telemetry.py @@ -0,0 +1,87 @@ +"""Tests for telemetry extraction from EvalResult.""" + +from __future__ import annotations + +from factory.outer_loop.designer import extract_telemetry +from factory.outer_loop.models import EvalResult + + +class TestExtractTelemetry: + def test_basic_fields(self) -> None: + result = EvalResult( + score=0.75, + benchmark_score=0.8, + hygiene_score=0.7, + cost_usd=1.5, + complexity=5.0, + ) + telemetry = extract_telemetry(result) + + assert telemetry["benchmark_score"] == 0.8 + assert telemetry["hygiene_score"] == 0.7 + assert telemetry["cost_usd"] == 1.5 + assert telemetry["complexity"] == 5.0 + assert telemetry["score"] == 0.75 + + def test_node_stats_from_details(self) -> None: + result = EvalResult( + score=0.5, + details={ + "node_stats": { + "builder": {"failure_rate": 0.3, "tokens": 5000}, + "researcher": {"failure_rate": 0.0, "tokens": 2000}, + }, + }, + ) + telemetry = extract_telemetry(result) + + node_stats = telemetry["node_stats"] + assert isinstance(node_stats, dict) + assert "builder" in node_stats + assert "researcher" in node_stats + + def test_dominant_failure_from_details(self) -> None: + result = EvalResult( + score=0.3, + details={"dominant_failure": "timeout"}, + ) + telemetry = extract_telemetry(result) + + assert telemetry["dominant_failure"] == "timeout" + + def test_empty_details(self) -> None: + result = EvalResult(score=0.5) + telemetry = extract_telemetry(result) + + assert telemetry["node_stats"] == {} + assert telemetry["dominant_failure"] == "" + + def test_missing_node_stats(self) -> None: + result = EvalResult( + score=0.5, + details={"some_other_key": "value"}, + ) + telemetry = extract_telemetry(result) + + assert telemetry["node_stats"] == {} + assert telemetry["dominant_failure"] == "" + + def test_all_fields_present(self) -> None: + result = EvalResult( + score=0.6, + benchmark_score=0.7, + hygiene_score=0.5, + cost_usd=2.0, + complexity=8.0, + details={ + "node_stats": {"gate": {"failure_rate": 0.1}}, + "dominant_failure": "crash", + }, + ) + telemetry = extract_telemetry(result) + + expected_keys = { + "node_stats", "dominant_failure", "benchmark_score", + "hygiene_score", "cost_usd", "complexity", "score", + } + assert set(telemetry.keys()) == expected_keys diff --git a/tests/test_outer_loop/test_workflow_serialization.py b/tests/test_outer_loop/test_workflow_serialization.py new file mode 100644 index 000000000..55970e4f3 --- /dev/null +++ b/tests/test_outer_loop/test_workflow_serialization.py @@ -0,0 +1,99 @@ +"""Tests for Workflow.to_dict() / from_dict() round-trip serialization.""" + +from __future__ import annotations + +import pytest + +from factory.workflow.definitions import register_all +from factory.workflow.primitives import Workflow + + +class TestWorkflowRoundTrip: + """Verify that to_dict → from_dict preserves structural identity.""" + + def test_simple_workflow(self, simple_workflow: Workflow) -> None: + data = simple_workflow.to_dict() + restored = Workflow.from_dict(data) + + assert restored.name == simple_workflow.name + assert restored.start_node == simple_workflow.start_node + assert restored.terminal == simple_workflow.terminal + assert set(restored.nodes.keys()) == set(simple_workflow.nodes.keys()) + assert len(restored.edges) == len(simple_workflow.edges) + + for nid in simple_workflow.nodes: + orig = simple_workflow.nodes[nid] + rest = restored.nodes[nid] + assert type(orig).__name__ == type(rest).__name__ + assert orig.id == rest.id + + def test_round_trip_preserves_node_types(self, simple_workflow: Workflow) -> None: + data = simple_workflow.to_dict() + restored = Workflow.from_dict(data) + + for nid, node_data in data["nodes"].items(): + assert "_type" in node_data + restored_node = restored.nodes[nid] + assert type(restored_node).__name__ == node_data["_type"] + + def test_round_trip_preserves_edges(self, simple_workflow: Workflow) -> None: + data = simple_workflow.to_dict() + restored = Workflow.from_dict(data) + + orig_edges = {(e.source, e.target, e.condition) for e in simple_workflow.edges} + rest_edges = {(e.source, e.target, e.condition) for e in restored.edges} + assert orig_edges == rest_edges + + def test_validates_after_round_trip(self, simple_workflow: Workflow) -> None: + data = simple_workflow.to_dict() + restored = Workflow.from_dict(data) + issues = restored.validate_graph() + assert issues == [] + + def test_unknown_node_type_raises(self) -> None: + data = { + "name": "bad", + "nodes": {"n1": {"_type": "UnknownNode", "id": "n1"}}, + "edges": [], + "start_node": "n1", + } + with pytest.raises(ValueError, match="Unknown node type"): + Workflow.from_dict(data) + + +class TestBuiltinWorkflowRoundTrips: + """Round-trip all builtin workflows through to_dict/from_dict.""" + + @pytest.fixture(scope="class") + def all_workflows(self) -> dict[str, Workflow]: + return register_all() + + def test_all_workflows_round_trip(self, all_workflows: dict[str, Workflow]) -> None: + assert len(all_workflows) > 0 + for name, wf in all_workflows.items(): + data = wf.to_dict() + restored = Workflow.from_dict(data) + + assert restored.name == wf.name, f"{name}: name mismatch" + assert restored.start_node == wf.start_node, f"{name}: start_node mismatch" + assert set(restored.nodes.keys()) == set(wf.nodes.keys()), ( + f"{name}: node set mismatch" + ) + assert len(restored.edges) == len(wf.edges), f"{name}: edge count mismatch" + + for nid in wf.nodes: + assert type(restored.nodes[nid]).__name__ == type(wf.nodes[nid]).__name__, ( + f"{name}: node {nid} type mismatch" + ) + + def test_all_workflows_validate_after_round_trip( + self, all_workflows: dict[str, Workflow] + ) -> None: + for name, wf in all_workflows.items(): + data = wf.to_dict() + restored = Workflow.from_dict(data) + issues = restored.validate_graph() + orig_issues = wf.validate_graph() + assert issues == orig_issues, ( + f"{name}: validation issues differ after round-trip: {issues} vs {orig_issues}" + ) diff --git a/uv.lock b/uv.lock index dea8a6212..cf4772604 100644 --- a/uv.lock +++ b/uv.lock @@ -165,6 +165,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "anthropic" +version = "0.122.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/23/9987d70b74e3481d5bc5d2021d3e10fd5f60c1f7b54088ea86506d9b7f2b/anthropic-0.122.0.tar.gz", hash = "sha256:ffec56ae96657c8d19fa575ec96f140f380c353a07ab7d61b92eb18ee6536601", size = 1021535, upload-time = "2026-08-13T18:36:00.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/5b/f5c87e71097a9f89f1b414d1ef7ae8439051fae57d5e4ee90946082982b8/anthropic-0.122.0-py3-none-any.whl", hash = "sha256:45ec906452ffae6b5f7f0c53d01f50bfb7e4ce878d7ae8e4309d13171e557e67", size = 1041853, upload-time = "2026-08-13T18:36:01.831Z" }, +] + +[package.optional-dependencies] +vertex = [ + { name = "google-auth", extra = ["requests"] }, +] + [[package]] name = "anyio" version = "4.13.0" @@ -697,6 +721,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/2a/1b016902351a523aa2bd446b50a5bc1175d7a7d1cf90fe2ef904f9b84ebc/cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4", size = 3412829, upload-time = "2026-04-08T01:57:48.874Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + [[package]] name = "durationpy" version = "0.10" @@ -865,6 +907,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] +[[package]] +name = "google-auth" +version = "2.56.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/4c/fa42116a48bab3f7a143cf5042ecff7df9c8b73f8a376203cd534d1dc966/google_auth-2.56.3.tar.gz", hash = "sha256:40e229fc901f0a305b553050e5fce562d509bee0435be053abfa91582b51b90c", size = 367110, upload-time = "2026-08-06T06:24:01.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/b3/6117b2f24065cd7e2c4f140e9a193e215f089ca8ba314cf91eb9d0b7fe0a/google_auth-2.56.3-py3-none-any.whl", hash = "sha256:8ec438808f813ad034535000261eed1067475d229d05bbf4216e78c3f2362e53", size = 259116, upload-time = "2026-08-06T06:22:51.788Z" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + [[package]] name = "googleapis-common-protos" version = "1.75.0" @@ -1133,6 +1193,92 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/3f/fae6cc967d120ec89e31c5418a51176d8278b3087fbb384a9176754f353c/jiter-0.16.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3", size = 309289, upload-time = "2026-06-29T13:02:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/97c6c3562c077f6247d6e6ce5c82562500b6316c0d928e97e106b7a1321a/jiter-0.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e", size = 315181, upload-time = "2026-06-29T13:02:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/7b/89/d8d073f8aa2667e46c6c0873f86fe4a512bba4293cc730f626a076211a62/jiter-0.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037", size = 340939, upload-time = "2026-06-29T13:02:55.412Z" }, + { url = "https://files.pythonhosted.org/packages/87/c9/db4fda3ed73fb864139305e935e5b8b38a5a24692a5a9dd356c22f1b9c8d/jiter-0.16.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e", size = 364932, upload-time = "2026-06-29T13:02:57.28Z" }, + { url = "https://files.pythonhosted.org/packages/a2/74/52b5e86241057f52ddd7c9a580f90effb51f9d06239f6fc612279b91a838/jiter-0.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b", size = 461132, upload-time = "2026-06-29T13:02:58.994Z" }, + { url = "https://files.pythonhosted.org/packages/a9/87/544a700f7447c1f31c5d7833821a4daa5683165c2d5a094fbf5b5800c3dc/jiter-0.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c", size = 374857, upload-time = "2026-06-29T13:03:00.455Z" }, + { url = "https://files.pythonhosted.org/packages/40/cd/0fcc3f7d39183674d5bfa9ec640faaeb506c60be7c8f94625dfba366e37c/jiter-0.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee", size = 347053, upload-time = "2026-06-29T13:03:02.045Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/c7e64e7932ad597fa395b61440b249ada6366716e25c6e08dd2afbd021e6/jiter-0.16.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03", size = 356153, upload-time = "2026-06-29T13:03:03.706Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/1c719044f14da814e1a060191ab19b96f3e99207bc5b4bfc6d6be34b3f80/jiter-0.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea", size = 393956, upload-time = "2026-06-29T13:03:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/3b/dc/7b2f303a2847207e265503853a2d964a55354cffd62a5f2936c155486798/jiter-0.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91", size = 521081, upload-time = "2026-06-29T13:03:06.886Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/501cf6e1e09caeb420195179ffc6f62aca603f1220ec53fd80d0d70b3e56/jiter-0.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3", size = 552085, upload-time = "2026-06-29T13:03:08.339Z" }, + { url = "https://files.pythonhosted.org/packages/79/54/aa5be86520113b79455c3877f3d1f07a348098df4083ba3688e9537e52dd/jiter-0.16.0-cp311-cp311-win32.whl", hash = "sha256:560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7", size = 206755, upload-time = "2026-06-29T13:03:09.653Z" }, + { url = "https://files.pythonhosted.org/packages/64/ec/2feb893eb330bd69b413866f4d5daada33c3962f1c6f270c91ca2d87fdf9/jiter-0.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1", size = 199155, upload-time = "2026-06-29T13:03:10.979Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9c/ca040d94415048a3666fc237774df8151c96f8d2b661cbe3b184acc95876/jiter-0.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056", size = 194403, upload-time = "2026-06-29T13:03:12.341Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/06/d3/8e278946d43eeca2585b4dd0834a887cd71136329b837f3a16ed86a8b4b0/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad", size = 304518, upload-time = "2026-06-29T13:05:00.172Z" }, + { url = "https://files.pythonhosted.org/packages/72/43/28d4ef495028bf0506a413d4db3f4eb3e7288a382e0f065f306a17bbeb5e/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f", size = 310207, upload-time = "2026-06-29T13:05:02.123Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ca/c366b1012da1d640de975d9683acd44e4d150d9068845d0ca2610435253f/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5", size = 342771, upload-time = "2026-06-29T13:05:03.55Z" }, + { url = "https://files.pythonhosted.org/packages/16/52/50cc4056fc1ae02e7154704e7ecc89df0afb8300222cfe8a52d3f67e4730/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85", size = 346468, upload-time = "2026-06-29T13:05:05.452Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -2252,6 +2398,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pybase64" version = "1.4.3" @@ -2845,6 +3012,7 @@ wheels = [ name = "remote-factory" source = { editable = "." } dependencies = [ + { name = "anthropic", extra = ["vertex"] }, { name = "fastapi" }, { name = "filelock" }, { name = "graphifyy" }, @@ -2883,6 +3051,7 @@ docs = [ [package.metadata] requires-dist = [ + { name = "anthropic", extras = ["vertex"], specifier = ">=0.52" }, { name = "fastapi", specifier = ">=0.115" }, { name = "filelock", specifier = ">=3.0" }, { name = "graphifyy", specifier = ">=0.9" }, @@ -3104,6 +3273,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sse-starlette" version = "3.3.4"