Add calibration-focused system prompt to improve Brier Index

analyze.py

@@ -27,14 +27,17 @@ def _lookup_forecast(forecasts: dict[str, float], question_id: str) -> float:
         base_id = question_id[:m.start()]
         prob = forecasts.get(base_id)
         if prob is not None:
+            logger.debug("lookup_fallback_base_id", question_id=question_id, base_id=base_id)
             return prob
+    logger.debug("lookup_missing_default", question_id=question_id)
     return 0.5
 
 
 def analyze_by_source(
     forecasts: dict[str, float],
     resolved: list[ResolvedQuestion],
 ) -> dict[str, dict[str, object]]:
+    logger.info("analyze_by_source_start", n_resolved=len(resolved), n_forecasts=len(forecasts))
     by_source: dict[str, list[tuple[float, int]]] = {}
     for q in resolved:
         f = _lookup_forecast(forecasts, q.id)
@@ -48,6 +51,7 @@ def analyze_by_source(
             "index": brier_index(bs),
             "count": len(pairs),
         }
+    logger.info("analyze_by_source_complete", n_sources=len(results))
     return results
 
 
@@ -56,8 +60,10 @@ def analyze_calibration(
     resolved: list[ResolvedQuestion],
     n_bins: int = 10,
 ) -> list[dict[str, object]]:
+    logger.info("analyze_calibration_start", n_resolved=len(resolved), n_bins=n_bins)
     pairs = [(_lookup_forecast(forecasts, q.id), q.outcome) for q in resolved]
     if not pairs:
+        logger.warning("analyze_calibration_empty")
         return []
 
     bin_width = 1.0 / n_bins
@@ -139,8 +145,10 @@ def analyze_biases(
     forecasts: dict[str, float],
     resolved: list[ResolvedQuestion],
 ) -> dict[str, object]:
+    logger.info("analyze_biases_start", n_resolved=len(resolved))
     pairs = [(_lookup_forecast(forecasts, q.id), q.outcome) for q in resolved]
     if not pairs:
+        logger.warning("analyze_biases_empty")
         return {"mean_forecast": 0.0, "mean_outcome": 0.0, "bias": 0.0, "low_bin": {}, "high_bin": {}}
 
     fs, os_ = zip(*pairs)
@@ -177,8 +185,10 @@ def analyze_decomposition(
     n_bins: int = 10,
 ) -> dict[str, dict[str, float]]:
     """Run Murphy decomposition and calibration metrics on forecast/outcome pairs."""
+    logger.info("analyze_decomposition_start", n_resolved=len(resolved), n_bins=n_bins)
     pairs = [(_lookup_forecast(forecasts, q.id), q.outcome) for q in resolved]
     if not pairs:
+        logger.warning("analyze_decomposition_empty")
         return {"murphy": {}, "calibration": {}}
 
     murphy = murphy_decomposition(pairs, n_bins=n_bins)
@@ -241,6 +251,7 @@ def print_analysis(analysis: dict[str, Any]) -> None:
 
 
 def save_analysis(analysis: dict[str, Any], path: str | Path) -> None:
+    logger.info("save_analysis", path=str(path))
     p = Path(path)
     p.parent.mkdir(parents=True, exist_ok=True)
     p.write_text(json.dumps(analysis, indent=2))
@@ -252,6 +263,7 @@ def analyze_worst_questions(
     top_n: int = 50,
 ) -> list[dict[str, object]]:
     """Find the N questions with highest individual Brier scores."""
+    logger.info("analyze_worst_questions_start", n_resolved=len(resolved), top_n=top_n)
     errors: list[dict[str, object]] = []
     for q in resolved:
         f = forecasts.get(q.id, 0.5)
@@ -281,6 +293,7 @@ def analyze_by_horizon(
     resolved: list[ResolvedQuestion],
 ) -> dict[str, dict[str, object]]:
     """Break down dataset question performance by resolution horizon."""
+    logger.info("analyze_by_horizon_start", n_resolved=len(resolved))
     horizon_pattern = re.compile(r"^(.+)_(\d{4}-\d{2}-\d{2})$")
     horizon_groups: dict[str, list[tuple[float, int]]] = {}
 
@@ -310,6 +323,7 @@ def compare_paired(
     result_b_path: str | Path,
 ) -> dict[str, object]:
     """Paired comparison of two runs on shared questions."""
+    logger.info("compare_paired_start", a=str(result_a_path), b=str(result_b_path))
     data_a = json.loads(Path(result_a_path).read_text())
     data_b = json.loads(Path(result_b_path).read_text())
     forecasts_a: dict[str, float] = data_a["forecasts"]
@@ -585,6 +599,7 @@ def compare_to_superforecasters(
 
 def _load_result_forecasts(result_path: str | Path) -> tuple[dict[str, float], list[ResolvedQuestion]]:
     """Load forecasts from a result file and re-join with resolved questions."""
+    logger.info("load_result_forecasts", path=str(result_path))
     from fetch_data import Resolution, load_data, join_resolved_questions
 
     data = json.loads(Path(result_path).read_text())

check_staleness.py

@@ -3,12 +3,18 @@
 import subprocess
 import sys
 
+from logging_config import get_logger
+
+logger = get_logger("check_staleness")
+
 
 def run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess[str]:
+    logger.debug("run_command", cmd=cmd)
     return subprocess.run(cmd, capture_output=True, text=True, check=check)
 
 
 def main() -> int:
+    logger.info("staleness_check_start")
     run(["git", "fetch", "origin", "main", "--quiet"])
 
     result = run(

cutoff.py

@@ -22,6 +22,7 @@ def __init__(self, freeze_datetime: str, display_date: str | None = None) -> Non
         logger.info("cutoff_environment_created", cutoff_date=freeze_datetime, display_date=self.display_date)
 
     def frame_temporal_context(self, question: Question) -> str:
+        logger.debug("frame_temporal_context", question_id=question.id, display_date=self.display_date)
         return (
             f"Today's Date: {self.display_date}. "
             "You should forecast based on information available as of this date."

dashboard.py

@@ -31,8 +31,11 @@
     calibration_metrics,
 )
 from fetch_data import MARKET_SOURCES, ResolvedQuestion, fetch_leaderboard, load_data
+from logging_config import get_logger
 from score import brier_index, brier_score
 
+logger = get_logger("dashboard")
+
 
 ResultData = dict[str, Any]
 
@@ -76,7 +79,9 @@ def _cache_data(fn: _F) -> _F:
 
 @_cache_data
 def load_all_results() -> list[ResultData]:
+    logger.info("dashboard_load_all_results")
     if not RESULTS_DIR.exists():
+        logger.warning("dashboard_results_dir_missing", path=str(RESULTS_DIR))
         return []
     results: list[ResultData] = []
     for f in sorted(RESULTS_DIR.glob("*.json")):
@@ -86,12 +91,15 @@ def load_all_results() -> list[ResultData]:
                 data["_filename"] = f.name
                 results.append(data)
         except (json.JSONDecodeError, KeyError):
+            logger.warning("dashboard_load_result_error", path=str(f))
             continue
+    logger.info("dashboard_results_loaded", n_results=len(results))
     return results
 
 
 @_cache_data
 def load_resolved_questions() -> list[dict[str, Any]]:
+    logger.info("dashboard_load_resolved_questions")
     _, resolved = load_data()
     return [
         {
@@ -106,10 +114,12 @@ def load_resolved_questions() -> list[dict[str, Any]]:
 
 @_cache_data
 def load_leaderboard(name: str) -> list[dict[str, str]]:
+    logger.info("dashboard_load_leaderboard", name=name)
     return list(fetch_leaderboard(name))
 
 
 def _round_name_from_result(result: ResultData) -> str:
+    logger.debug("round_name_from_result")
     meta = result.get("metadata", {})
     rnd = meta.get("round")
     if rnd:
@@ -126,6 +136,7 @@ def _compute_aggregate_scoring(
     sources: dict[str, str],
 ) -> dict[str, Any]:
     """Compute scoring result from combined forecasts and outcomes."""
+    logger.debug("compute_aggregate_scoring", n_forecasts=len(forecasts), n_outcomes=len(outcomes))
     shared_ids = set(forecasts.keys()) & set(outcomes.keys())
     if not shared_ids:
         return {
@@ -174,6 +185,7 @@ def _mean_brier(pairs: list[tuple[float, int]]) -> float:
 
 def _group_results_into_runs(results: list[ResultData]) -> list[AggregateRun]:
     """Group result files by model_slug into aggregate runs."""
+    logger.info("group_results_into_runs", n_results=len(results))
     grouped: dict[str, list[ResultData]] = {}
     for r in results:
         slug = str(r["model_slug"])
@@ -214,6 +226,7 @@ def _group_results_into_runs(results: list[ResultData]) -> list[AggregateRun]:
 @_cache_data
 def group_results(results: list[ResultData]) -> list[dict[str, Any]]:
     """Cached wrapper that returns serializable dicts (Streamlit requirement)."""
+    logger.info("group_results", n_results=len(results))
     runs = _group_results_into_runs(results)
     return [
         {
@@ -245,9 +258,11 @@ def _dict_to_aggregate(d: dict[str, Any]) -> AggregateRun:
 
 def _leaderboard_reference_from_live() -> dict[str, dict[str, float]] | None:
     """Pull top reference entries from the live baseline leaderboard."""
+    logger.info("leaderboard_reference_from_live")
     try:
         rows = load_leaderboard("baseline")
     except Exception:
+        logger.warning("leaderboard_reference_fetch_failed", exc_info=True)
         return None
     ref: dict[str, dict[str, float]] = {}
     for row in rows:
@@ -289,6 +304,7 @@ def _model_matches_slug(leaderboard_model: str, model_slug: str) -> bool:
 
 
 def _resolved_to_objects(resolved_dicts: list[dict[str, Any]]) -> list[ResolvedQuestion]:
+    logger.debug("resolved_to_objects", n_dicts=len(resolved_dicts))
     return [
         ResolvedQuestion(
             id=d["id"],
@@ -321,6 +337,7 @@ def _build_source_brier_matrix(
     runs: list[AggregateRun],
     resolved_dicts: list[dict[str, Any]],
 ) -> tuple[list[str], list[str], list[list[float | None]], list[list[int]]]:
+    logger.debug("build_source_brier_matrix", n_runs=len(runs))
     resolved = _resolved_to_objects(resolved_dicts)
     all_sources: set[str] = set()
     run_source_scores: dict[str, dict[str, float]] = {}
@@ -365,6 +382,7 @@ def _sort_sources_by_track(sources: list[str]) -> list[str]:
 
 
 def view_overview(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None:
+    logger.info("view_overview", n_runs=len(runs))
     st.header("Overview")
 
     if not runs:
@@ -474,6 +492,7 @@ def view_overview(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]
 
 
 def view_leaderboard(runs: list[AggregateRun]) -> None:
+    logger.info("view_leaderboard", n_runs=len(runs))
     st.header("Official ForecastBench Leaderboard")
 
     lb_name = st.radio(
@@ -559,6 +578,7 @@ def view_leaderboard(runs: list[AggregateRun]) -> None:
 
 
 def view_heatmap(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None:
+    logger.info("view_heatmap", n_runs=len(runs))
     st.header("Run × Source Heatmap")
 
     if not runs:
@@ -594,6 +614,7 @@ def _view_heatmap_by_source(
     resolved_dicts: list[dict[str, Any]],
     show_index: bool,
 ) -> None:
+    logger.info("view_heatmap_by_source", n_runs=len(runs))
     run_labels, sources, matrix, counts = _build_source_brier_matrix(runs, resolved_dicts)
 
     sources_sorted = _sort_sources_by_track(sources)
@@ -723,6 +744,7 @@ def _view_heatmap_by_track(
     resolved_dicts: list[dict[str, Any]],
     show_index: bool,
 ) -> None:
+    logger.info("view_heatmap_by_track", n_runs=len(runs))
     overall_brier_map: dict[str, float] = {}
     for agg_run in runs:
         overall_brier_map[agg_run.label] = agg_run.scoring_result.get("overall_brier", 1.0)
@@ -808,6 +830,7 @@ def _view_heatmap_by_track(
 
 
 def view_failures(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None:
+    logger.info("view_failures", n_runs=len(runs))
     st.header("Failure Explorer")
 
     if not runs:
@@ -1080,6 +1103,7 @@ def view_failures(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]
 
 
 def view_calibration(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None:
+    logger.info("view_calibration", n_runs=len(runs))
     st.header("Calibration Curves")
 
     if not runs:
@@ -1185,6 +1209,7 @@ def view_calibration(runs: list[AggregateRun], resolved_dicts: list[dict[str, An
 def view_question_browser(
     runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]
 ) -> None:
+    logger.info("view_question_browser", n_runs=len(runs))
     st.header("Question Browser")
 
     if not runs or not resolved_dicts:
@@ -1268,6 +1293,7 @@ def view_question_browser(
 
 
 def view_compare(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]]) -> None:
+    logger.info("view_compare", n_runs=len(runs))
     st.header("Compare Runs")
 
     if len(runs) < 2:
@@ -1440,6 +1466,7 @@ def view_compare(runs: list[AggregateRun], resolved_dicts: list[dict[str, Any]])
 
 
 def main() -> None:
+    logger.info("dashboard_main_start")
     if not _HAS_DASHBOARD_DEPS:
         print(
             "Dashboard requires streamlit, pandas and plotly.\n"
@@ -1545,6 +1572,7 @@ def main() -> None:
 
 
 def view_about() -> None:
+    logger.info("view_about")
     st.header("About ForecastBench")
 
     st.markdown("""