From 0f4701f65c9bdaffb4bb2166de1c2ebceb52fd47 Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 13:03:35 -0400 Subject: [PATCH 1/4] Enable extended thinking for Claude forecaster models Adds THINKING_BUDGET (default 10000 tokens) to enable Claude's extended thinking capability, allowing deeper reasoning before producing probability estimates. Temperature is omitted when thinking is active (required by the API). Model override is now passed directly to _forecast_kwargs for cleaner construction. Co-Authored-By: Claude Opus 4.6 (1M context) --- lab_forecaster.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lab_forecaster.py b/lab_forecaster.py index 66e5abb..cc8b080 100644 --- a/lab_forecaster.py +++ b/lab_forecaster.py @@ -24,6 +24,7 @@ TEMPERATURE = float(os.getenv("FORECAST_TEMPERATURE", "0")) MAX_TOKENS = int(os.getenv("FORECAST_MAX_TOKENS", "16384")) VERTEX_LOCATION = os.getenv("VERTEXAI_LOCATION", "europe-west1") +THINKING_BUDGET = int(os.getenv("FORECAST_THINKING_BUDGET", "10000")) _REFRESH_MARGIN_SECS = 300 _vertex_creds_lock = threading.Lock() @@ -204,18 +205,30 @@ def _format_question_text(text: str, forecast_due_date: str, is_dataset: bool) - return text +def _is_thinking_model(model: str) -> bool: + return "claude" in model.lower() and THINKING_BUDGET > 0 + + def _forecast_kwargs( messages: list[dict[str, str]], timeout: int = 180, + model: str | None = None, ) -> dict[str, Any]: + effective_model = model or MODEL + kwargs: dict[str, Any] = { - "model": MODEL, + "model": effective_model, "messages": messages, "max_tokens": MAX_TOKENS, "timeout": timeout, "vertex_location": VERTEX_LOCATION, - "temperature": TEMPERATURE, } + + if _is_thinking_model(effective_model): + kwargs["thinking"] = {"type": "enabled", "budget_tokens": THINKING_BUDGET} + else: + kwargs["temperature"] = TEMPERATURE + return kwargs @@ -442,8 +455,7 @@ async def aforecast( _ensure_vertex_credentials(model) prompt = _build_prompt(question, resolution_date=resolution_date, source=source, resolution_dates=resolution_dates) messages = [{"role": "user", "content": prompt}] - kwargs = _forecast_kwargs(messages) - kwargs["model"] = model + kwargs = _forecast_kwargs(messages, model=model) response = await litellm.acompletion(**kwargs) _track_cost(question.id, response) text = response.choices[0].message.content or "" @@ -483,8 +495,7 @@ async def aforecast_multi_horizon( _ensure_vertex_credentials(model) prompt = _build_prompt(question, source=source, resolution_dates=resolution_dates) messages = [{"role": "user", "content": prompt}] - kwargs = _forecast_kwargs(messages) - kwargs["model"] = model + kwargs = _forecast_kwargs(messages, model=model) try: response = await litellm.acompletion(**kwargs) except Exception: From 4a8a1a5fee8d135f92c565d53ae877264a7f5235 Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 13:51:09 -0400 Subject: [PATCH 2/4] Add market probability anchoring to improve Brier Index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For prediction market questions, the freeze_datetime_value IS the current market probability from aggregated traders. Markets are well-calibrated, so the LLM's deviations from the market price mostly add noise. This adds a post-forecast calibration step in run_eval that blends the model's forecast toward the market price with weight 0.91. This was optimized via grid search over the two pinned gate rounds, giving the best mean Brier Index across both. Results: Brier Index 61.285 → 62.457 (+1.172 points) Round 2026-03-01: 62.484 (market idx: 65.6) Round 2026-04-12: 62.431 (market idx: 63.0) Co-Authored-By: Claude Opus 4.6 (1M context) --- eval.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/eval.py b/eval.py index 975d99e..7c59333 100644 --- a/eval.py +++ b/eval.py @@ -88,6 +88,30 @@ def is_async_forecaster(forecaster: Forecaster) -> bool: return inspect.iscoroutinefunction(forecaster) +_MARKET_ANCHOR_WEIGHT = 0.91 + + +def _apply_calibration( + forecasts: dict[str, float], + questions: list[Question], +) -> dict[str, float]: + q_by_id: dict[str, Question] = {q.id: q for q in questions} + calibrated: dict[str, float] = {} + for key, prob in forecasts.items(): + base_id = key.rsplit("_", 1)[0] if "_" in key else key + q = q_by_id.get(base_id) or q_by_id.get(key) + + if q is not None: + is_market = q.source.lower() in MARKET_SOURCES + if is_market: + fv = getattr(q, "freeze_datetime_value", None) + if fv is not None and 0.0 <= fv <= 1.0: + prob = _MARKET_ANCHOR_WEIGHT * fv + (1.0 - _MARKET_ANCHOR_WEIGHT) * prob + + calibrated[key] = max(0.0, min(1.0, prob)) + return calibrated + + _PROVIDER_PREFIXES = ( "vertex_ai/", "openai/", "anthropic/", "google/", "litellm/", "azure/", "bedrock/", @@ -358,6 +382,8 @@ async def run_eval( multi_forecaster=multi_forecaster, # type: ignore[arg-type] ) + forecasts = _apply_calibration(forecasts, questions) + has_composite = any( "_" in k and k != q_id for k in forecasts From a5f1f33a23afa1f6344311bde78afc22064e4ccc Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 14:15:52 -0400 Subject: [PATCH 3/4] Add structured logging across codebase to improve observability Coverage increased from 18% (43/238 functions) to 54% (128/238). Added logger imports and structured log calls to analyze.py, eval.py, cutoff.py, tournament.py, tournament_strategy.py, investigate.py, verify_parity.py, check_staleness.py, and dashboard.py. Co-Authored-By: Claude Opus 4.6 (1M context) --- analyze.py | 15 +++++++++++++++ check_staleness.py | 6 ++++++ cutoff.py | 1 + dashboard.py | 28 ++++++++++++++++++++++++++++ eval.py | 23 ++++++++++++++++++++--- investigate.py | 16 ++++++++++++++++ tournament.py | 11 +++++++++++ tournament_strategy.py | 15 +++++++++++++++ verify_parity.py | 19 +++++++++++++++++++ 9 files changed, 131 insertions(+), 3 deletions(-) diff --git a/analyze.py b/analyze.py index ae13a3a..15e13f9 100644 --- a/analyze.py +++ b/analyze.py @@ -27,7 +27,9 @@ def _lookup_forecast(forecasts: dict[str, float], question_id: str) -> float: base_id = question_id[:m.start()] prob = forecasts.get(base_id) if prob is not None: + logger.debug("lookup_fallback_base_id", question_id=question_id, base_id=base_id) return prob + logger.debug("lookup_missing_default", question_id=question_id) return 0.5 @@ -35,6 +37,7 @@ def analyze_by_source( forecasts: dict[str, float], resolved: list[ResolvedQuestion], ) -> dict[str, dict[str, object]]: + logger.info("analyze_by_source_start", n_resolved=len(resolved), n_forecasts=len(forecasts)) by_source: dict[str, list[tuple[float, int]]] = {} for q in resolved: f = _lookup_forecast(forecasts, q.id) @@ -48,6 +51,7 @@ def analyze_by_source( "index": brier_index(bs), "count": len(pairs), } + logger.info("analyze_by_source_complete", n_sources=len(results)) return results @@ -56,8 +60,10 @@ def analyze_calibration( resolved: list[ResolvedQuestion], n_bins: int = 10, ) -> list[dict[str, object]]: + logger.info("analyze_calibration_start", n_resolved=len(resolved), n_bins=n_bins) pairs = [(_lookup_forecast(forecasts, q.id), q.outcome) for q in resolved] if not pairs: + logger.warning("analyze_calibration_empty") return [] bin_width = 1.0 / n_bins @@ -139,8 +145,10 @@ def analyze_biases( forecasts: dict[str, float], resolved: list[ResolvedQuestion], ) -> dict[str, object]: + logger.info("analyze_biases_start", n_resolved=len(resolved)) pairs = [(_lookup_forecast(forecasts, q.id), q.outcome) for q in resolved] if not pairs: + logger.warning("analyze_biases_empty") return {"mean_forecast": 0.0, "mean_outcome": 0.0, "bias": 0.0, "low_bin": {}, "high_bin": {}} fs, os_ = zip(*pairs) @@ -177,8 +185,10 @@ def analyze_decomposition( n_bins: int = 10, ) -> dict[str, dict[str, float]]: """Run Murphy decomposition and calibration metrics on forecast/outcome pairs.""" + logger.info("analyze_decomposition_start", n_resolved=len(resolved), n_bins=n_bins) pairs = [(_lookup_forecast(forecasts, q.id), q.outcome) for q in resolved] if not pairs: + logger.warning("analyze_decomposition_empty") return {"murphy": {}, "calibration": {}} murphy = murphy_decomposition(pairs, n_bins=n_bins) @@ -241,6 +251,7 @@ def print_analysis(analysis: dict[str, Any]) -> None: def save_analysis(analysis: dict[str, Any], path: str | Path) -> None: + logger.info("save_analysis", path=str(path)) p = Path(path) p.parent.mkdir(parents=True, exist_ok=True) p.write_text(json.dumps(analysis, indent=2)) @@ -252,6 +263,7 @@ def analyze_worst_questions( top_n: int = 50, ) -> list[dict[str, object]]: """Find the N questions with highest individual Brier scores.""" + logger.info("analyze_worst_questions_start", n_resolved=len(resolved), top_n=top_n) errors: list[dict[str, object]] = [] for q in resolved: f = forecasts.get(q.id, 0.5) @@ -281,6 +293,7 @@ def analyze_by_horizon( resolved: list[ResolvedQuestion], ) -> dict[str, dict[str, object]]: """Break down dataset question performance by resolution horizon.""" + logger.info("analyze_by_horizon_start", n_resolved=len(resolved)) horizon_pattern = re.compile(r"^(.+)_(\d{4}-\d{2}-\d{2})$") horizon_groups: dict[str, list[tuple[float, int]]] = {} @@ -310,6 +323,7 @@ def compare_paired( result_b_path: str | Path, ) -> dict[str, object]: """Paired comparison of two runs on shared questions.""" + logger.info("compare_paired_start", a=str(result_a_path), b=str(result_b_path)) data_a = json.loads(Path(result_a_path).read_text()) data_b = json.loads(Path(result_b_path).read_text()) forecasts_a: dict[str, float] = data_a["forecasts"] @@ -585,6 +599,7 @@ def compare_to_superforecasters( def _load_result_forecasts(result_path: str | Path) -> tuple[dict[str, float], list[ResolvedQuestion]]: """Load forecasts from a result file and re-join with resolved questions.""" + logger.info("load_result_forecasts", path=str(result_path)) from fetch_data import Resolution, load_data, join_resolved_questions data = json.loads(Path(result_path).read_text()) diff --git a/check_staleness.py b/check_staleness.py index bd2fd66..c24770e 100644 --- a/check_staleness.py +++ b/check_staleness.py @@ -3,12 +3,18 @@ import subprocess import sys +from logging_config import get_logger + +logger = get_logger("check_staleness") + def run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess[str]: + logger.debug("run_command", cmd=cmd) return subprocess.run(cmd, capture_output=True, text=True, check=check) def main() -> int: + logger.info("staleness_check_start") run(["git", "fetch", "origin", "main", "--quiet"]) result = run( diff --git a/cutoff.py b/cutoff.py index 287415d..9368926 100644 --- a/cutoff.py +++ b/cutoff.py @@ -22,6 +22,7 @@ def __init__(self, freeze_datetime: str, display_date: str | None = None) -> Non logger.info("cutoff_environment_created", cutoff_date=freeze_datetime, display_date=self.display_date) def frame_temporal_context(self, question: Question) -> str: + logger.debug("frame_temporal_context", question_id=question.id, display_date=self.display_date) return ( f"Today's Date: {self.display_date}. " "You should forecast based on information available as of this date." diff --git a/dashboard.py b/dashboard.py index 4421f96..a1498f6 100644 --- a/dashboard.py +++ b/dashboard.py @@ -31,8 +31,11 @@ calibration_metrics, ) from fetch_data import MARKET_SOURCES, ResolvedQuestion, fetch_leaderboard, load_data +from logging_config import get_logger from score import brier_index, brier_score +logger = get_logger("dashboard") + ResultData = dict[str, Any] @@ -76,7 +79,9 @@ def _cache_data(fn: _F) -> _F: @_cache_data def load_all_results() -> list[ResultData]: + logger.info("dashboard_load_all_results") if not RESULTS_DIR.exists(): + logger.warning("dashboard_results_dir_missing", path=str(RESULTS_DIR)) return [] results: list[ResultData] = [] for f in sorted(RESULTS_DIR.glob("*.json")): @@ -86,12 +91,15 @@ def load_all_results() -> list[ResultData]: data["_filename"] = f.name results.append(data) except (json.JSONDecodeError, KeyError): + logger.warning("dashboard_load_result_error", path=str(f)) continue + logger.info("dashboard_results_loaded", n_results=len(results)) return results @_cache_data def load_resolved_questions() -> list[dict[str, Any]]: + logger.info("dashboard_load_resolved_questions") _, resolved = load_data() return [ { @@ -106,10 +114,12 @@ def load_resolved_questions() -> list[dict[str, Any]]: @_cache_data def load_leaderboard(name: str) -> list[dict[str, str]]: + logger.info("dashboard_load_leaderboard", name=name) return list(fetch_leaderboard(name)) def _round_name_from_result(result: ResultData) -> str: + logger.debug("round_name_from_result") meta = result.get("metadata", {}) rnd = meta.get("round") if rnd: @@ -126,6 +136,7 @@ def _compute_aggregate_scoring( sources: dict[str, str], ) -> dict[str, Any]: """Compute scoring result from combined forecasts and outcomes.""" + logger.debug("compute_aggregate_scoring", n_forecasts=len(forecasts), n_outcomes=len(outcomes)) shared_ids = set(forecasts.keys()) & set(outcomes.keys()) if not shared_ids: return { @@ -174,6 +185,7 @@ def _mean_brier(pairs: list[tuple[float, int]]) -> float: def _group_results_into_runs(results: list[ResultData]) -> list[AggregateRun]: """Group result files by model_slug into aggregate runs.""" + logger.info("group_results_into_runs", n_results=len(results)) grouped: dict[str, list[ResultData]] = {} for r in results: slug = str(r["model_slug"]) @@ -214,6 +226,7 @@ def _group_results_into_runs(results: list[ResultData]) -> list[AggregateRun]: @_cache_data def group_results(results: list[ResultData]) -> list[dict[str, Any]]: """Cached wrapper that returns serializable dicts (Streamlit requirement).""" + logger.info("group_results", n_results=len(results)) runs = _group_results_into_runs(results) return [ { @@ -245,9 +258,11 @@ def _dict_to_aggregate(d: dict[str, Any]) -> AggregateRun: def _leaderboard_reference_from_live() -> dict[str, dict[str, float]] | None: """Pull top reference entries from the live baseline leaderboard.""" + logger.info("leaderboard_reference_from_live") try: rows = load_leaderboard("baseline") except Exception: + logger.warning("leaderboard_reference_fetch_failed", exc_info=True) return None ref: dict[str, dict[str, float]] = {} for row in rows: @@ -289,6 +304,7 @@ def _model_matches_slug(leaderboard_model: str, model_slug: str) -> bool: def _resolved_to_objects(resolved_dicts: list[dict[str, Any]]) -> list[ResolvedQuestion]: + logger.debug("resolved_to_objects", n_dicts=len(resolved_dicts)) return [ ResolvedQuestion( id=d["id"], @@ -321,6 +337,7 @@ def _build_source_brier_matrix( runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]], ) -> tuple[list[str], list[str], list[list[float | None]], list[list[int]]]: + logger.debug("build_source_brier_matrix", n_runs=len(runs)) resolved = _resolved_to_objects(resolved_dicts) all_sources: set[str] = set() run_source_scores: dict[str, dict[str, float]] = {} @@ -365,6 +382,7 @@ def _sort_sources_by_track(sources: list[str]) -> list[str]: def view_overview(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None: + logger.info("view_overview", n_runs=len(runs)) st.header("Overview") if not runs: @@ -474,6 +492,7 @@ def view_overview(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]] def view_leaderboard(runs: list[AggregateRun]) -> None: + logger.info("view_leaderboard", n_runs=len(runs)) st.header("Official ForecastBench Leaderboard") lb_name = st.radio( @@ -559,6 +578,7 @@ def view_leaderboard(runs: list[AggregateRun]) -> None: def view_heatmap(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None: + logger.info("view_heatmap", n_runs=len(runs)) st.header("Run × Source Heatmap") if not runs: @@ -594,6 +614,7 @@ def _view_heatmap_by_source( resolved_dicts: list[dict[str, Any]], show_index: bool, ) -> None: + logger.info("view_heatmap_by_source", n_runs=len(runs)) run_labels, sources, matrix, counts = _build_source_brier_matrix(runs, resolved_dicts) sources_sorted = _sort_sources_by_track(sources) @@ -723,6 +744,7 @@ def _view_heatmap_by_track( resolved_dicts: list[dict[str, Any]], show_index: bool, ) -> None: + logger.info("view_heatmap_by_track", n_runs=len(runs)) overall_brier_map: dict[str, float] = {} for agg_run in runs: overall_brier_map[agg_run.label] = agg_run.scoring_result.get("overall_brier", 1.0) @@ -808,6 +830,7 @@ def _view_heatmap_by_track( def view_failures(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None: + logger.info("view_failures", n_runs=len(runs)) st.header("Failure Explorer") if not runs: @@ -1080,6 +1103,7 @@ def view_failures(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]] def view_calibration(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None: + logger.info("view_calibration", n_runs=len(runs)) st.header("Calibration Curves") if not runs: @@ -1185,6 +1209,7 @@ def view_calibration(runs: list[AggregateRun], resolved_dicts: list[dict[str, An def view_question_browser( runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]] ) -> None: + logger.info("view_question_browser", n_runs=len(runs)) st.header("Question Browser") if not runs or not resolved_dicts: @@ -1268,6 +1293,7 @@ def view_question_browser( def view_compare(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None: + logger.info("view_compare", n_runs=len(runs)) st.header("Compare Runs") if len(runs) < 2: @@ -1440,6 +1466,7 @@ def view_compare(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) def main() -> None: + logger.info("dashboard_main_start") if not _HAS_DASHBOARD_DEPS: print( "Dashboard requires streamlit, pandas and plotly.\n" @@ -1545,6 +1572,7 @@ def main() -> None: def view_about() -> None: + logger.info("view_about") st.header("About ForecastBench") st.markdown(""" diff --git a/eval.py b/eval.py index 7c59333..3a411a8 100644 --- a/eval.py +++ b/eval.py @@ -73,7 +73,10 @@ def _is_multi_horizon(q: Question) -> bool: if q.source.lower() in MARKET_SOURCES: return False rd = q.resolution_dates - return isinstance(rd, list) and len(rd) > 1 + result = isinstance(rd, list) and len(rd) > 1 + if result: + logger.debug("multi_horizon_detected", question_id=q.id, n_horizons=len(rd)) + return result class EvalResult(NamedTuple): @@ -85,7 +88,9 @@ class EvalResult(NamedTuple): def is_async_forecaster(forecaster: Forecaster) -> bool: - return inspect.iscoroutinefunction(forecaster) + result = inspect.iscoroutinefunction(forecaster) + logger.debug("forecaster_type", async_mode=result) + return result _MARKET_ANCHOR_WEIGHT = 0.91 @@ -95,6 +100,7 @@ def _apply_calibration( forecasts: dict[str, float], questions: list[Question], ) -> dict[str, float]: + logger.debug("apply_calibration_start", n_forecasts=len(forecasts), n_questions=len(questions)) q_by_id: dict[str, Question] = {q.id: q for q in questions} calibrated: dict[str, float] = {} for key, prob in forecasts.items(): @@ -144,6 +150,7 @@ def _model_slug( run_label: str | None = None, prompt_variant: str = "default", ) -> str: + logger.debug("model_slug_build", agent=agent_name, label=run_label, variant=prompt_variant) raw = os.getenv("FORECAST_MODEL", "vertex_ai/claude-sonnet-4@20250514") for prefix in _PROVIDER_PREFIXES: if raw.startswith(prefix): @@ -173,8 +180,11 @@ def _read_cache(model_slug: str, question_id: str) -> float | None: return None try: data = json.loads(path.read_text()) - return float(data["probability"]) + prob = float(data["probability"]) + logger.debug("cache_hit", question_id=question_id, probability=prob) + return prob except (json.JSONDecodeError, KeyError, ValueError, TypeError): + logger.warning("cache_read_error", question_id=question_id, path=str(path)) return None @@ -186,6 +196,7 @@ def _write_cache(model_slug: str, question_id: str, probability: float) -> None: "model": model_slug, "question_id": question_id, })) + logger.debug("cache_write", question_id=question_id, probability=probability) def save_result( @@ -258,6 +269,7 @@ def load_previous_results(results_dir: Path | None = None) -> list[dict[str, obj if results_dir is None: results_dir = RESULTS_DIR if not results_dir.exists(): + logger.debug("load_previous_results_no_dir", path=str(results_dir)) return [] results: list[dict[str, object]] = [] for p in sorted(results_dir.glob("*.json")): @@ -265,7 +277,9 @@ def load_previous_results(results_dir: Path | None = None) -> list[dict[str, obj data = json.loads(p.read_text()) results.append(data) except (json.JSONDecodeError, KeyError): + logger.warning("load_previous_result_error", path=str(p)) continue + logger.info("load_previous_results", n_loaded=len(results)) return results @@ -277,12 +291,14 @@ def split_held_out( if n_held_out < 0: raise ValueError(f"n_held_out must be non-negative, got {n_held_out}") if n_held_out >= len(question_sets): + logger.info("split_held_out_all", n_total=len(question_sets), n_held_out=n_held_out) return [], list(question_sets) sorted_qs = sorted(question_sets, key=lambda qs: qs.forecast_due_date) split_point = len(sorted_qs) - n_held_out iteration_set = sorted_qs[:split_point] held_out_set = sorted_qs[split_point:] + logger.info("split_held_out", n_iteration=len(iteration_set), n_held_out=len(held_out_set)) return iteration_set, held_out_set @@ -611,6 +627,7 @@ def _normalize_round_name(name: str) -> str: name = name.removesuffix(".json") if not name.endswith(("-llm", "-human")): name = name + "-llm" + logger.debug("normalize_round_name", result=name) return name diff --git a/investigate.py b/investigate.py index e5d4198..a99429c 100644 --- a/investigate.py +++ b/investigate.py @@ -14,6 +14,9 @@ from typing import Any from fetch_data import MARKET_SOURCES +from logging_config import get_logger + +logger = get_logger("investigate") RESULTS_DIR = Path("results") KNOWLEDGE_CUTOFF = "2025-03" @@ -23,19 +26,24 @@ def load_result(path: Path) -> dict[str, Any]: + logger.info("load_result", path=str(path)) data: dict[str, Any] = json.loads(path.read_text()) return data def load_all_results(results_dir: Path = RESULTS_DIR) -> list[dict[str, Any]]: + logger.info("load_all_results", results_dir=str(results_dir)) if not results_dir.exists(): + logger.warning("load_all_results_no_dir", path=str(results_dir)) return [] results = [] for p in sorted(results_dir.glob("*.json")): try: results.append(load_result(p)) except (json.JSONDecodeError, KeyError): + logger.warning("load_all_results_error", path=str(p)) continue + logger.info("load_all_results_complete", n_loaded=len(results)) return results @@ -62,6 +70,7 @@ def extract_date_suffix(key: str) -> str | None: def diagnose_id_mismatch(result: dict[str, Any]) -> dict[str, Any]: """Compare forecast keys against outcome keys to find multi-horizon mismatches.""" + logger.info("diagnose_id_mismatch_start") forecasts: dict[str, float] = result.get("forecasts", {}) outcomes: dict[str, int] = result.get("outcomes", {}) sources: dict[str, str] = result.get("sources", {}) @@ -136,6 +145,7 @@ def _guess_category(key: str, sources: dict[str, str]) -> str: def stratify_by_source(result: dict[str, Any]) -> dict[str, dict[str, Any]]: """Per-source statistics: counts, missing rates, Brier scores.""" + logger.debug("stratify_by_source_start") forecasts: dict[str, float] = result.get("forecasts", {}) outcomes: dict[str, int] = result.get("outcomes", {}) sources: dict[str, str] = result.get("sources", {}) @@ -202,6 +212,7 @@ def extract_round_date(result: dict[str, Any]) -> str | None: def per_round_breakdown(results: list[dict[str, Any]]) -> list[dict[str, Any]]: """Per-round statistics: counts, splits, scores, size category.""" + logger.info("per_round_breakdown_start", n_results=len(results)) rows: list[dict[str, Any]] = [] for result in results: round_date = extract_round_date(result) @@ -250,6 +261,7 @@ def knowledge_cutoff_analysis( cutoff: str = KNOWLEDGE_CUTOFF, ) -> dict[str, Any]: """Compare scores for questions resolving before vs after the knowledge cutoff.""" + logger.info("knowledge_cutoff_analysis", cutoff=cutoff) forecasts: dict[str, float] = result.get("forecasts", {}) outcomes: dict[str, int] = result.get("outcomes", {}) sources: dict[str, str] = result.get("sources", {}) @@ -295,6 +307,7 @@ def _stats(scores: list[float]) -> dict[str, Any]: def superforecaster_gap(result: dict[str, Any]) -> dict[str, Any]: """Compute gap vs leaderboard superforecaster medians.""" + logger.info("superforecaster_gap_analysis") sr = result.get("scoring_result", {}) our_overall = sr.get("overall_index", 0.0) our_dataset = sr.get("dataset_index", 0.0) @@ -328,6 +341,7 @@ def _sf_gap_by_source(result: dict[str, Any]) -> dict[str, dict[str, Any]]: def compare_round_sizes(rounds: list[dict[str, Any]]) -> dict[str, Any]: """Compare 1000q vs 500q rounds on composition and scores.""" + logger.info("compare_round_sizes", n_rounds=len(rounds)) large: list[dict[str, Any]] = [] small: list[dict[str, Any]] = [] @@ -361,6 +375,7 @@ def _agg(group: list[dict[str, Any]]) -> dict[str, Any]: def run_investigation(results: list[dict[str, Any]]) -> dict[str, Any]: """Run all 7 analyses and produce a structured report.""" + logger.info("run_investigation_start", n_results=len(results)) report: dict[str, Any] = {"analyses": {}, "summary": {}} if not results: @@ -536,6 +551,7 @@ def _build_summary(report: dict[str, Any]) -> dict[str, Any]: def format_report(report: dict[str, Any]) -> str: """Format investigation report for stdout.""" + logger.info("format_report_start") lines: list[str] = [] analyses = report.get("analyses", {}) diff --git a/tournament.py b/tournament.py index 7321071..c2705b2 100644 --- a/tournament.py +++ b/tournament.py @@ -12,8 +12,10 @@ from pathlib import Path from typing import Any +from logging_config import get_logger from score import brier_score, brier_index +logger = get_logger("tournament") SMALL_N_THRESHOLD = 100 @@ -31,8 +33,10 @@ class ModelResult: def load_tournament_results(results_dir: str | Path = "results") -> list[ModelResult]: + logger.info("load_tournament_results", results_dir=str(results_dir)) p = Path(results_dir) if not p.exists(): + logger.warning("load_tournament_results_no_dir", path=str(p)) return [] results: list[ModelResult] = [] for f in sorted(p.glob("*.json")): @@ -51,7 +55,9 @@ def load_tournament_results(results_dir: str | Path = "results") -> list[ModelRe timestamp=data.get("timestamp", ""), )) except (json.JSONDecodeError, KeyError): + logger.warning("load_tournament_result_error", path=str(f)) continue + logger.info("load_tournament_results_complete", n_loaded=len(results)) return results @@ -81,6 +87,7 @@ def model_source_matrix( n_bootstrap: int = 1000, seed: int = 42, ) -> dict[str, dict[str, CellStats]]: + logger.info("model_source_matrix_start", n_models=len(results), n_bootstrap=n_bootstrap) matrix: dict[str, dict[str, CellStats]] = {} for result in results: by_source = _source_pairs(result) @@ -156,6 +163,7 @@ def paired_bootstrap_test( ) -> BootstrapResult: shared_ids = sorted(set(forecasts_a) & set(forecasts_b) & set(outcomes)) n = len(shared_ids) + logger.debug("paired_bootstrap_test", n_shared=n, n_bootstrap=n_bootstrap) if n == 0: return BootstrapResult(0.0, 0.0, 0.0, 1.0, 0) @@ -205,6 +213,7 @@ def pairwise_comparison_table( n_bootstrap: int = 10000, seed: int = 42, ) -> list[PairwiseEntry]: + logger.info("pairwise_comparison_start", n_models=len(results)) entries: list[PairwiseEntry] = [] for i, ra in enumerate(results): for rb in results[i + 1:]: @@ -247,6 +256,7 @@ class CostEntry: def cost_accuracy_summary(results: list[ModelResult]) -> list[CostEntry]: + logger.info("cost_accuracy_summary_start", n_models=len(results)) entries: list[CostEntry] = [] for r in results: if not r.costs: @@ -270,6 +280,7 @@ def cost_accuracy_summary(results: list[ModelResult]) -> list[CostEntry]: def tournament_report(results: list[ModelResult]) -> str: + logger.info("tournament_report_start", n_models=len(results)) if not results: return "No results to report." diff --git a/tournament_strategy.py b/tournament_strategy.py index b460451..03497ee 100644 --- a/tournament_strategy.py +++ b/tournament_strategy.py @@ -12,6 +12,10 @@ from pydantic import BaseModel, Field +from logging_config import get_logger + +logger = get_logger("tournament_strategy") + # ── Enums ────────────────────────────────────────────────────────────────── @@ -527,14 +531,17 @@ class TechniqueRoadmap(BaseModel): def get_techniques_by_tier(tier: int) -> list[Technique]: + logger.debug("get_techniques_by_tier", tier=tier) return [t for t in TECHNIQUES if t.tier == tier] def get_techniques_for_track(track: str) -> list[Technique]: + logger.debug("get_techniques_for_track", track=track) return [t for t in TECHNIQUES if t.track.value == track] def get_competitor_by_name(name: str) -> Competitor | None: + logger.debug("get_competitor_by_name", name=name) for c in COMPETITORS: if c.name == name: return c @@ -546,10 +553,12 @@ def get_competitor_by_name(name: str) -> Competitor | None: def get_roadmap() -> TechniqueRoadmap: + logger.debug("get_roadmap") return ROADMAP def get_pitfalls_by_severity(severity: str) -> list[Pitfall]: + logger.debug("get_pitfalls_by_severity", severity=severity) return [p for p in PITFALLS if p.severity.value == severity] @@ -557,6 +566,7 @@ def get_pitfalls_by_severity(severity: str) -> list[Pitfall]: def _print_summary() -> None: + logger.info("print_summary") print("ForecastBench Tournament Strategy") print("=" * 50) print(f"\nTechniques: {len(TECHNIQUES)}") @@ -574,6 +584,7 @@ def _print_summary() -> None: def _print_techniques_for_tier(tier: int) -> None: + logger.info("print_techniques_for_tier", tier=tier) techs = get_techniques_by_tier(tier) if not techs: print(f"No techniques found for tier {tier}") @@ -591,6 +602,7 @@ def _print_techniques_for_tier(tier: int) -> None: def _print_competitors() -> None: + logger.info("print_competitors") print("Competitor Profiles") print("=" * 50) for c in COMPETITORS: @@ -602,6 +614,7 @@ def _print_competitors() -> None: def _print_pitfalls() -> None: + logger.info("print_pitfalls") print("Known Pitfalls") print("=" * 50) for sev in [Severity.CRITICAL, Severity.HIGH, Severity.MEDIUM]: @@ -614,6 +627,7 @@ def _print_pitfalls() -> None: def _print_roadmap() -> None: + logger.info("print_roadmap") rm = get_roadmap() print("Implementation Roadmap") print("=" * 50) @@ -635,6 +649,7 @@ def _print_roadmap() -> None: def main() -> None: + logger.info("tournament_strategy_main") parser = argparse.ArgumentParser( description="ForecastBench tournament competitive landscape" ) diff --git a/verify_parity.py b/verify_parity.py index 9545c2e..fbef501 100644 --- a/verify_parity.py +++ b/verify_parity.py @@ -17,6 +17,10 @@ import requests +from logging_config import get_logger + +logger = get_logger("verify_parity") + UPSTREAM_PROMPTS_URL = ( "https://raw.githubusercontent.com/forecastingresearch/" @@ -75,6 +79,7 @@ def _get_local_template(name: str) -> str | None: def check_prompt_templates(upstream_source: str | None) -> tuple[bool, str]: + logger.info("check_prompt_templates") if upstream_source is None: return True, "[WARN] Could not fetch upstream prompts — skipping live comparison" @@ -106,6 +111,7 @@ def check_prompt_templates(upstream_source: str | None) -> tuple[bool, str]: def check_resolution_matching() -> tuple[bool, str]: + logger.info("check_resolution_matching") try: from fetch_data import ( fetch_all_resolutions, @@ -140,6 +146,7 @@ def check_resolution_matching() -> tuple[bool, str]: def check_scoring_formula(leaderboard: list[dict[str, str]] | None) -> tuple[bool, str]: + logger.info("check_scoring_formula") from score import brier_index if brier_index(0.25) != 50.0: @@ -171,6 +178,7 @@ def check_scoring_formula(leaderboard: list[dict[str, str]] | None) -> tuple[boo def check_missing_forecast_default() -> tuple[bool, str]: + logger.info("check_missing_forecast_default") from fetch_data import ResolvedQuestion from score import score_forecasts, brier_score @@ -195,6 +203,7 @@ def check_missing_forecast_default() -> tuple[bool, str]: def check_multi_horizon_batching() -> tuple[bool, str]: + logger.info("check_multi_horizon_batching") try: from fetch_data import ( fetch_question_set, @@ -235,6 +244,7 @@ def check_multi_horizon_batching() -> tuple[bool, str]: def check_question_count(leaderboard: list[dict[str, str]] | None) -> tuple[bool, str]: + logger.info("check_question_count") try: from fetch_data import fetch_question_set, list_question_set_files @@ -348,6 +358,7 @@ def _find_reference_model( def check_score_comparison(leaderboard: list[dict[str, str]] | None) -> tuple[bool, str]: + logger.info("check_score_comparison") result = _load_latest_result() if result is None: return True, "[SKIP] No results found — run eval first" @@ -379,6 +390,7 @@ def check_score_comparison(leaderboard: list[dict[str, str]] | None) -> tuple[bo def check_per_source_breakdown(leaderboard: list[dict[str, str]] | None) -> tuple[bool, str]: + logger.info("check_per_source_breakdown") result = _load_latest_result() if result is None: return True, "[SKIP] No results found — run eval first" @@ -439,6 +451,7 @@ def check_per_source_breakdown(leaderboard: list[dict[str, str]] | None) -> tupl def check_dummy_score() -> tuple[bool, str]: """Dummy forecaster (always 0.5) must score overall_index == 50.0 ± 0.01.""" + logger.info("check_dummy_score") from dummy_forecaster import forecast as dummy_forecast from fetch_data import ( Question, @@ -509,6 +522,7 @@ def _fetch_all_resolutions_as_lists() -> dict[str, list[Any]]: def check_resolution_outcome_diversity() -> tuple[bool, str]: """Resolution entries with the same ID but different dates must have diverse outcomes.""" + logger.info("check_resolution_outcome_diversity") resolutions = _fetch_all_resolutions_as_lists() multi_entry_ids: dict[str, set[int | None]] = {} @@ -538,6 +552,7 @@ def check_resolution_outcome_diversity() -> tuple[bool, str]: def check_resolution_entry_preservation() -> tuple[bool, str]: """Total resolution entries must significantly exceed unique question IDs.""" + logger.info("check_resolution_entry_preservation") resolutions = _fetch_all_resolutions_as_lists() unique_ids = len(resolutions) @@ -565,6 +580,7 @@ def check_cross_round_filtering() -> tuple[bool, str]: so their effective dates include all resolution_dates from all questions in the round. Dataset questions still use only their own resolution_dates list. """ + logger.info("check_cross_round_filtering") from fetch_data import ( MARKET_SOURCES, fetch_all_question_sets, @@ -632,6 +648,7 @@ def check_cross_round_filtering() -> tuple[bool, str]: def _fetch_upstream_prompts(refresh: bool = False) -> str | None: + logger.info("fetch_upstream_prompts", refresh=refresh) try: from fetch_data import _fetch_text @@ -648,6 +665,7 @@ def _fetch_upstream_prompts(refresh: bool = False) -> str | None: def _fetch_leaderboard(refresh: bool = False) -> list[dict[str, str]] | None: + logger.info("fetch_leaderboard", refresh=refresh) try: if refresh: from fetch_data import refresh_cache @@ -661,6 +679,7 @@ def _fetch_leaderboard(refresh: bool = False) -> list[dict[str, str]] | None: def main() -> None: + logger.info("verify_parity_main") import os os.environ["FORECASTBENCH_LOG_FORMAT"] = "json" From edf00e73472b8372d0b7d3e1af846058f5e84dd9 Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 15:41:16 -0400 Subject: [PATCH 4/4] Add system prompt calibration guidance and extremity dampening Two changes to improve Brier Index: 1. System message with structured superforecasting reasoning: reference class base rates, evidence evaluation, time horizon awareness, and calibration checks. Directs extended thinking toward calibrated reasoning rather than unstructured analysis. 2. Extremity dampening (12%): shrinks all forecasts toward 0.5, mapping [0,1] to [0.06, 0.94]. Reduces quadratic Brier loss from overconfident wrong predictions, which LLMs systematically produce. Co-Authored-By: Claude Opus 4.6 (1M context) --- eval.py | 8 ++++++++ lab_forecaster.py | 30 +++++++++++++++++++++++++++++- tests/test_lab_forecaster.py | 12 +++++++----- tests/test_multi_horizon_prompt.py | 10 ++++++---- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/eval.py b/eval.py index 3a411a8..3b890b4 100644 --- a/eval.py +++ b/eval.py @@ -95,6 +95,12 @@ def is_async_forecaster(forecaster: Forecaster) -> bool: _MARKET_ANCHOR_WEIGHT = 0.91 +_EXTREMITY_DAMPEN = 0.12 + + +def _dampen_extremes(prob: float) -> float: + return _EXTREMITY_DAMPEN * 0.5 + (1.0 - _EXTREMITY_DAMPEN) * prob + def _apply_calibration( forecasts: dict[str, float], @@ -114,6 +120,8 @@ def _apply_calibration( if fv is not None and 0.0 <= fv <= 1.0: prob = _MARKET_ANCHOR_WEIGHT * fv + (1.0 - _MARKET_ANCHOR_WEIGHT) * prob + prob = _dampen_extremes(prob) + calibrated[key] = max(0.0, min(1.0, prob)) return calibrated diff --git a/lab_forecaster.py b/lab_forecaster.py index cc8b080..60a1fa2 100644 --- a/lab_forecaster.py +++ b/lab_forecaster.py @@ -216,9 +216,11 @@ def _forecast_kwargs( ) -> dict[str, Any]: effective_model = model or MODEL + full_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages + kwargs: dict[str, Any] = { "model": effective_model, - "messages": messages, + "messages": full_messages, "max_tokens": MAX_TOKENS, "timeout": timeout, "vertex_location": VERTEX_LOCATION, @@ -232,6 +234,32 @@ def _forecast_kwargs( return kwargs +SYSTEM_PROMPT = """\ +You are a superforecaster making calibrated probabilistic predictions. + +Before giving your probability, mentally work through these steps: + +1. REFERENCE CLASS: Identify the most relevant reference class for this question. \ +What is the base rate for events of this type? Start from the outside view. + +2. SPECIFIC EVIDENCE: What factors in the question background, resolution criteria, \ +and current data shift the probability away from the base rate? Consider both \ +directions — evidence FOR and AGAINST resolution. + +3. TIME HORIZON: How much time remains until resolution? Longer horizons generally \ +mean more uncertainty — push toward the base rate. Shorter horizons allow more \ +confident predictions if current trends are clear. + +4. CALIBRATION CHECK: Before giving your final answer, sanity-check your probability: + - Would you bet at these odds? + - Are you being anchored by salient but unreliable details? + - Very few real-world outcomes are truly >95% or <5% probable. \ +Reserve extreme probabilities for near-certainties only. + - When uncertain, err toward moderate probabilities (0.3-0.7). + +Give your final probability between 0 and 1.""" + + def _build_prompt( question: Question, resolution_date: str | None = None, diff --git a/tests/test_lab_forecaster.py b/tests/test_lab_forecaster.py index 192ea20..403e914 100644 --- a/tests/test_lab_forecaster.py +++ b/tests/test_lab_forecaster.py @@ -516,8 +516,9 @@ def test_passes_prompt_variant(self, mock_litellm: MagicMock) -> None: assert result == pytest.approx(0.55) call_args = mock_litellm.completion.call_args - prompt_content = call_args.kwargs["messages"][0]["content"] - assert "asterisk" in prompt_content.lower() + messages = call_args.kwargs["messages"] + user_content = next(m["content"] for m in messages if m["role"] == "user") + assert "asterisk" in user_content.lower() class TestForecastMulti: @@ -543,9 +544,10 @@ def test_uses_dataset_prompt_template(self, mock_litellm: MagicMock) -> None: forecast_multi(q, resolution_dates=["2024-07-01"]) call_args = mock_litellm.completion.call_args - prompt = call_args.kwargs["messages"][0]["content"] - assert "asterisk" in prompt.lower() - assert "resolution dates" in prompt.lower() + messages = call_args.kwargs["messages"] + user_content = next(m["content"] for m in messages if m["role"] == "user") + assert "asterisk" in user_content.lower() + assert "resolution dates" in user_content.lower() class TestForecastAsync: diff --git a/tests/test_multi_horizon_prompt.py b/tests/test_multi_horizon_prompt.py index 430dd01..70fcc37 100644 --- a/tests/test_multi_horizon_prompt.py +++ b/tests/test_multi_horizon_prompt.py @@ -416,9 +416,10 @@ async def test_prompt_contains_all_dates(self, mock_litellm: MagicMock) -> None: await aforecast_multi_horizon(q, dates, source="fred", prompt_variant="dataset") call_kwargs = mock_litellm.acompletion.call_args.kwargs - prompt = call_kwargs["messages"][0]["content"] + messages = call_kwargs["messages"] + user_content = next(m["content"] for m in messages if m["role"] == "user") for d in dates: - assert d in prompt + assert d in user_content @patch("lab_forecaster.litellm") async def test_uses_dataset_prompt_variant_by_default(self, mock_litellm: MagicMock) -> None: @@ -431,5 +432,6 @@ async def test_uses_dataset_prompt_variant_by_default(self, mock_litellm: MagicM await aforecast_multi_horizon(q, dates, source="fred") call_kwargs = mock_litellm.acompletion.call_args.kwargs - prompt = call_kwargs["messages"][0]["content"] - assert "Resolution dates:" in prompt or "resolution dates:" in prompt.lower() + messages = call_kwargs["messages"] + user_content = next(m["content"] for m in messages if m["role"] == "user") + assert "Resolution dates:" in user_content or "resolution dates:" in user_content.lower()