From 1e967633ab869f4695973f32b1f7c89ff49c0afd Mon Sep 17 00:00:00 2001 From: Paul Tune Date: Tue, 28 Oct 2025 23:00:38 +1100 Subject: [PATCH] Fix heavy hitter config loading --- configs/heavy_hitters_workload.yaml | 6 +++- heavy_hitters_evaluator.py | 26 ++++++++++++-- run_heavy_hitters.py | 4 +-- .../workflow/configuration.py | 36 ++++++++++++++++--- src/randomize_evolve/workflow/workflow.py | 2 +- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/configs/heavy_hitters_workload.yaml b/configs/heavy_hitters_workload.yaml index df061ea..cb71d2e 100644 --- a/configs/heavy_hitters_workload.yaml +++ b/configs/heavy_hitters_workload.yaml @@ -25,6 +25,8 @@ prompt: - Implement all data structures yourself; no external packages. - Keep memory bounded — prefer sketches, compact heaps, or sparse maps. - Ensure deterministic behaviour for identical inputs. + - Modify only the code inside the EVOLVE-BLOCK markers so the harness can + safely integrate your design. Inspiration: - Count-Min Sketch variations with bias reduction or conservative updates. @@ -39,12 +41,14 @@ prompt: - "Maximise heavy-hitter recall (>0.9 is excellent)." - "Keep mean relative error below 15%." - "Minimise zero-frequency overestimation (avoid ghost heavy hitters)." - - "Keep per-observation memory under 32 bits when possible." + - "Keep per-observation memory under 32 bits when possible and improve on the baseline's ~25 bits." conversation_starters: - "Combine a Count-Min sketch with a bounded dictionary of suspects and decay their counts." - "Use multiple reservoirs to capture bursty heavy hitters while retaining estimates via sketches." - "Implement a layered SpaceSaving structure with compressed fingerprints." + - "Evolve a reversible hash sketch that tracks residual counts and refreshes stale entries with decay." + - "Blend top-k maintenance with stratified sampling so rare-but-growing keys can displace stale heavy hitters." database: population_size: 120 diff --git a/heavy_hitters_evaluator.py b/heavy_hitters_evaluator.py index 858b816..4ca8eba 100644 --- a/heavy_hitters_evaluator.py +++ b/heavy_hitters_evaluator.py @@ -3,6 +3,7 @@ import concurrent.futures import importlib.util import math +import sys import traceback from pathlib import Path from types import ModuleType @@ -10,10 +11,15 @@ from openevolve.evaluation_result import EvaluationResult -from randomize_evolve.evaluators.heavy_hitters import ( +_REPO_ROOT = Path(__file__).resolve().parent +_SRC_PATH = _REPO_ROOT / "src" +if str(_SRC_PATH) not in sys.path: + sys.path.insert(0, str(_SRC_PATH)) + +from randomize_evolve.evaluators.heavy_hitters import ( # noqa: E402 (import after path tweak) EvaluationResult as HeavyEvaluationResult, ) -from randomize_evolve.evaluators.heavy_hitters import Evaluator, EvaluatorConfig +from randomize_evolve.evaluators.heavy_hitters import Evaluator, EvaluatorConfig # noqa: E402 EVALUATION_TIMEOUT_S = 75 @@ -175,4 +181,18 @@ def _error_result(message: str, artifacts: dict) -> EvaluationResult: def _score_to_reward(score: float) -> float: - return 0.0 if not math.isfinite(score) else 1.0 / (1.0 + max(score, 0.0)) + """Map the evaluator's raw score into a smooth reward in ``[0, 1]``. + + The heavy hitter evaluator accumulates large penalty values to emphasise + accuracy, latency, and memory usage. The previous transformation + ``1 / (1 + score)`` compressed the dynamic range of rewards so aggressively + that evolutionary search treated most candidates as equally poor. By using + an exponential falloff we keep the ordering identical while ensuring that + meaningful improvements translate into noticeably higher rewards. + """ + + if not math.isfinite(score): + return 0.0 + + scaled = max(score, 0.0) / 4000.0 + return math.exp(-scaled) diff --git a/run_heavy_hitters.py b/run_heavy_hitters.py index 82aea65..827616f 100644 --- a/run_heavy_hitters.py +++ b/run_heavy_hitters.py @@ -18,7 +18,7 @@ _INITIAL_PROGRAM_PATH = pathlib.Path(__file__).parent / "initial_program_heavy_hitters.py" INITIAL_PROGRAM_SOURCE = ProgramSource(_INITIAL_PROGRAM_PATH.read_text(encoding="utf-8")) -_EVALUATOR_PATH = Path(__file__).parent / "heavy_hitters_evaluator.py" +_EVALUATOR_PATH = pathlib.Path(__file__).parent / "heavy_hitters_evaluator.py" _CONFIG_LOADER = ConfigLoader() @@ -27,7 +27,7 @@ def _build_runner() -> OpenEvolveRunner: def _build_workflow(provider) -> "EvolutionWorkflow": - from randomize_evolve.workflow.workflow import EvolutionWorkflow + from src.randomize_evolve.workflow.workflow import EvolutionWorkflow runner = _build_runner() reporter = EvolutionReporter() diff --git a/src/randomize_evolve/workflow/configuration.py b/src/randomize_evolve/workflow/configuration.py index 9792a18..380ebec 100644 --- a/src/randomize_evolve/workflow/configuration.py +++ b/src/randomize_evolve/workflow/configuration.py @@ -1,10 +1,17 @@ import os -from dataclasses import dataclass +from dataclasses import dataclass, fields from pathlib import Path from typing import Any, Callable, Optional, Protocol import yaml -from openevolve.config import Config, LLMModelConfig +from openevolve.config import ( + Config, + DatabaseConfig, + EvaluatorConfig, + LLMConfig, + LLMModelConfig, + PromptConfig, +) class APIKeyProvider: @@ -140,10 +147,29 @@ def _load_with_openevolve(self, path: Path) -> Optional[Config]: return None def _construct_from_dict(self, data: dict[str, Any]) -> Optional[Config]: + filtered: dict[str, Any] = {} + allowed_top = {field.name for field in fields(Config)} + for key, value in data.items(): + if key in allowed_top: + filtered[key] = value + + for section, schema in ( + ("llm", LLMConfig), + ("prompt", PromptConfig), + ("database", DatabaseConfig), + ("evaluator", EvaluatorConfig), + ): + section_data = filtered.get(section) + if isinstance(section_data, dict): + allowed_fields = {field.name for field in fields(schema)} + filtered[section] = { + name: section_data[name] + for name in allowed_fields + if name in section_data + } + try: - if hasattr(Config, "model_validate"): - return Config.model_validate(data) # type: ignore[attr-defined,return-value] - return Config(**data) # type: ignore[arg-type] + return Config.from_dict(filtered) except Exception: return None diff --git a/src/randomize_evolve/workflow/workflow.py b/src/randomize_evolve/workflow/workflow.py index 438c9cf..3598934 100644 --- a/src/randomize_evolve/workflow/workflow.py +++ b/src/randomize_evolve/workflow/workflow.py @@ -1,6 +1,6 @@ from typing import Any -from randomize_evolve.workflow.program import ProgramSource, TemporaryProgramFile +from .program import ProgramSource, TemporaryProgramFile class EvolutionWorkflow: