From 591cac5232dfaf5aebd826efa8234a253d13da4e Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Wed, 19 Aug 2026 15:09:50 -0400 Subject: [PATCH] Integrate CutoffEnvironment into eval pipeline for temporal framing Apply CutoffEnvironment.prepare_question() to each question before forecasting, using freeze_datetime (or forecast_due_date as fallback) to add temporal context framing to question backgrounds. Co-Authored-By: Claude Opus 4.6 (1M context) --- eval.py | 11 ++ tests/test_cutoff_integration.py | 217 +++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 tests/test_cutoff_integration.py diff --git a/eval.py b/eval.py index 975d99e..b8e744e 100644 --- a/eval.py +++ b/eval.py @@ -18,6 +18,7 @@ litellm.suppress_debug_info = True +from cutoff import CutoffEnvironment # noqa: E402 from fetch_data import MARKET_SOURCES, Question, QuestionSet, Resolution, ResolvedQuestion, load_data, join_resolved_questions, fetch_question_set, fetch_all_resolutions, list_question_set_files, fetch_leaderboard, refresh_cache # noqa: E402 from logging_config import configure_logging, generate_run_id, get_logger # noqa: E402 from score import ScoringResult, brier_skill_score, score_forecasts # noqa: E402 @@ -284,6 +285,14 @@ def _build_question(q: Question | ResolvedQuestion, forecast_due_date: str | Non ) +def _apply_cutoff(question: Question) -> Question: + freeze = question.freeze_datetime or getattr(question, "forecast_due_date", None) + if not freeze: + return question + env = CutoffEnvironment(str(freeze)) + return env.prepare_question(question) + + async def run_eval( forecaster: Forecaster, n_held_out: int = 2, @@ -345,6 +354,8 @@ async def run_eval( logger.info("forecasting_questions", n_base=len(questions), n_resolved=len(iteration_resolved)) model_slug = _model_slug(agent_name, run_label=run_label, prompt_variant=prompt_variant) + questions = [_apply_cutoff(q) for q in questions] + if is_async_forecaster(forecaster): forecasts = await _run_async( forecaster, questions, model_slug, # type: ignore[arg-type] diff --git a/tests/test_cutoff_integration.py b/tests/test_cutoff_integration.py new file mode 100644 index 0000000..93a2c0d --- /dev/null +++ b/tests/test_cutoff_integration.py @@ -0,0 +1,217 @@ +"""Tests for CutoffEnvironment integration into the eval pipeline.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +from fetch_data import Question, QuestionSet, ResolvedQuestion +from eval import run_eval, _apply_cutoff + + +def _dummy_forecaster(question: Question, resolution_date: str | None = None, **kwargs: Any) -> float: + return 0.5 + + +class TestApplyCutoff: + def test_question_with_freeze_datetime_gets_temporal_framing(self) -> None: + q = Question( + id="q1", source="acled", question="Will X happen?", + background="Original background", + freeze_datetime="2024-06-15", + ) + result = _apply_cutoff(q) + assert "forecast based on information available as of this date" in result.background.lower() + assert "2024-06-15" in result.background + assert "Original background" in result.background + + def test_question_without_freeze_datetime_unchanged(self) -> None: + q = Question( + id="q2", source="metaculus", question="Will Y happen?", + background="Some background", + ) + result = _apply_cutoff(q) + assert result.background == "Some background" + + def test_question_with_none_freeze_datetime_unchanged(self) -> None: + q = Question( + id="q3", source="metaculus", question="Will Z happen?", + background="Background text", + freeze_datetime=None, + ) + result = _apply_cutoff(q) + assert result.background == "Background text" + + def test_question_with_empty_background_gets_context(self) -> None: + q = Question( + id="q4", source="acled", question="Will A happen?", + background="", + freeze_datetime="2024-03-01", + ) + result = _apply_cutoff(q) + assert "2024-03-01" in result.background + assert "forecast based on information available" in result.background.lower() + + def test_preserves_other_fields(self) -> None: + q = Question( + id="q5", source="fred", question="GDP growth?", + background="Econ data", + resolution_criteria="Resolves YES if...", + freeze_datetime="2024-09-01", + resolution_dates=["2024-12-31"], + ) + result = _apply_cutoff(q) + assert result.id == "q5" + assert result.source == "fred" + assert result.question == "GDP growth?" + assert result.resolution_criteria == "Resolves YES if..." + assert result.resolution_dates == ["2024-12-31"] + + def test_original_question_not_mutated(self) -> None: + q = Question( + id="q6", source="acled", question="Test?", + background="Original", + freeze_datetime="2024-01-01", + ) + _apply_cutoff(q) + assert q.background == "Original" + + def test_falls_back_to_forecast_due_date(self) -> None: + q = Question( + id="q7", source="acled", question="Test?", + background="BG", + freeze_datetime=None, + forecast_due_date="2024-07-15", + ) + result = _apply_cutoff(q) + assert "2024-07-15" in result.background + assert "forecast based on information available" in result.background.lower() + + +class TestCutoffInRunEval: + def test_cutoff_applied_in_pipeline(self, tmp_path: Path, monkeypatch: Any) -> None: + """run_eval should apply temporal framing to questions with freeze_datetime.""" + import eval as eval_mod + + resolved = [ + ResolvedQuestion( + id="q0", source="acled", question="Q0", + outcome=1, forecast_due_date="2024-01-01", + freeze_datetime="2024-06-15", + resolution_date="2024-06-29", + ), + ResolvedQuestion( + id="q1", source="metaculus", question="Q1", + outcome=0, forecast_due_date="2024-01-01", + freeze_datetime="2024-06-15", + ), + ] + question_sets = [ + QuestionSet( + forecast_due_date="2024-01-01", + question_set="set_0", + questions=[ + Question(id="q0", source="acled", question="Q0", + freeze_datetime="2024-06-15", + resolution_dates=["2024-06-29"]), + Question(id="q1", source="metaculus", question="Q1", + freeze_datetime="2024-06-15"), + ], + ), + QuestionSet(forecast_due_date="2024-02-01", question_set="set_1", questions=[]), + QuestionSet(forecast_due_date="2024-03-01", question_set="set_2", questions=[]), + ] + + results_dir = tmp_path / "results" + results_dir.mkdir() + + seen_questions: dict[str, str] = {} + + def _capturing_forecaster(question: Question, **kwargs: Any) -> float: + seen_questions[question.id] = question.background or "" + return 0.5 + + monkeypatch.setattr(eval_mod, "RESULTS_DIR", results_dir) + monkeypatch.setattr(eval_mod, "load_data", lambda: (question_sets, resolved)) + monkeypatch.setattr(eval_mod, "CACHE_DIR", tmp_path / "cache") + + asyncio.run(run_eval(_capturing_forecaster, n_held_out=2, raw=True)) + + assert "q0" in seen_questions + assert "2024-06-15" in seen_questions["q0"] + assert "forecast based on information available" in seen_questions["q0"].lower() + + def test_cutoff_skipped_without_any_date(self, tmp_path: Path, monkeypatch: Any) -> None: + """Questions without freeze_datetime or forecast_due_date are unchanged.""" + import eval as eval_mod + + resolved = [ + ResolvedQuestion( + id="q0", source="metaculus", question="Q0", + outcome=1, forecast_due_date="2024-01-01", + ), + ] + question_sets = [ + QuestionSet( + forecast_due_date="2024-01-01", + question_set="set_0", + questions=[ + Question(id="q0", source="metaculus", question="Q0"), + ], + ), + QuestionSet(forecast_due_date="2024-02-01", question_set="set_1", questions=[]), + QuestionSet(forecast_due_date="2024-03-01", question_set="set_2", questions=[]), + ] + + results_dir = tmp_path / "results" + results_dir.mkdir() + + monkeypatch.setattr(eval_mod, "RESULTS_DIR", results_dir) + monkeypatch.setattr(eval_mod, "load_data", lambda: (question_sets, resolved)) + monkeypatch.setattr(eval_mod, "CACHE_DIR", tmp_path / "cache") + + q_no_date = Question(id="bare", source="metaculus", question="Bare?", background="Just text") + result = _apply_cutoff(q_no_date) + assert result.background == "Just text" + + def test_cutoff_uses_forecast_due_date_fallback(self, tmp_path: Path, monkeypatch: Any) -> None: + """When freeze_datetime is absent but forecast_due_date is set, cutoff uses it.""" + import eval as eval_mod + + resolved = [ + ResolvedQuestion( + id="q0", source="metaculus", question="Q0", + outcome=1, forecast_due_date="2024-01-01", + ), + ] + question_sets = [ + QuestionSet( + forecast_due_date="2024-01-01", + question_set="set_0", + questions=[ + Question(id="q0", source="metaculus", question="Q0"), + ], + ), + QuestionSet(forecast_due_date="2024-02-01", question_set="set_1", questions=[]), + QuestionSet(forecast_due_date="2024-03-01", question_set="set_2", questions=[]), + ] + + results_dir = tmp_path / "results" + results_dir.mkdir() + + seen_questions: dict[str, str] = {} + + def _capturing_forecaster(question: Question, **kwargs: Any) -> float: + seen_questions[question.id] = question.background or "" + return 0.5 + + monkeypatch.setattr(eval_mod, "RESULTS_DIR", results_dir) + monkeypatch.setattr(eval_mod, "load_data", lambda: (question_sets, resolved)) + monkeypatch.setattr(eval_mod, "CACHE_DIR", tmp_path / "cache") + + asyncio.run(run_eval(_capturing_forecaster, n_held_out=2, raw=True)) + + assert "q0" in seen_questions + assert "2024-01-01" in seen_questions["q0"] + assert "forecast based on information available" in seen_questions["q0"].lower()