TenSolver Stochastic Benchmark example

examples/TenSolver/README.md

@@ -0,0 +1,18 @@
+# TenSolver Benchmark Example
+
+Benchmarking [TenSolver](https://github.com/SECQUOIA/TenSolver.jl) on 23 QUBO instances from the QpLib subset of QUBOLib (instances 308–330).
+
+## What's included
+
+- **Energy distribution visualizations** — heatmap showing how the sampled energy distribution evolves across DMRG iterations for a representative instance.
+- **Performance ratio** — computed against Gurobi's best-known objectives as the reference minimum and random bitstring sampling as the baseline.
+- **Bootstrap analysis** — performance ratio inferred for 1, 10, 100, and 1000 samples per iteration using the stochastic-benchmark framework.
+- **Resource analysis** — `iterations × samples` used as a proxy for compute; interpolation, train/test split, virtual best baseline, and projection experiments.
+
+## Data files
+
+| File | Description |
+|---|---|
+| `results/energy_history.json` | 1000 energy samples per DMRG iteration for all 23 instances |
+| `results/random_baseline.json` | Mean and min energy from 1000 random bitstrings per instance |
+| `results/gurobi_best.json` | Best objective found by Gurobi (900 s limit) per instance |

examples/TenSolver/results/gurobi_best.json

@@ -0,0 +1,117 @@
+{
+  "308": {
+    "best_objective": -806,
+    "status": 2,
+    "optimal": true
+  },
+  "309": {
+    "best_objective": -440,
+    "status": 2,
+    "optimal": true
+  },
+  "310": {
+    "best_objective": -1686,
+    "status": 9,
+    "optimal": false
+  },
+  "311": {
+    "best_objective": -1542,
+    "status": 9,
+    "optimal": false
+  },
+  "312": {
+    "best_objective": -1844,
+    "status": 9,
+    "optimal": false
+  },
+  "313": {
+    "best_objective": -614,
+    "status": 2,
+    "optimal": true
+  },
+  "314": {
+    "best_objective": -1142,
+    "status": 2,
+    "optimal": true
+  },
+  "315": {
+    "best_objective": -688,
+    "status": 2,
+    "optimal": true
+  },
+  "316": {
+    "best_objective": -514,
+    "status": 2,
+    "optimal": true
+  },
+  "317": {
+    "best_objective": -1402,
+    "status": 2,
+    "optimal": true
+  },
+  "318": {
+    "best_objective": -900,
+    "status": 2,
+    "optimal": true
+  },
+  "319": {
+    "best_objective": -1298,
+    "status": 2,
+    "optimal": true
+  },
+  "320": {
+    "best_objective": -2022,
+    "status": 9,
+    "optimal": false
+  },
+  "321": {
+    "best_objective": -362,
+    "status": 2,
+    "optimal": true
+  },
+  "322": {
+    "best_objective": -1040,
+    "status": 2,
+    "optimal": true
+  },
+  "323": {
+    "best_objective": -23964950,
+    "status": 9,
+    "optimal": false
+  },
+  "324": {
+    "best_objective": -74347446,
+    "status": 2,
+    "optimal": true
+  },
+  "325": {
+    "best_objective": -68476481,
+    "status": 2,
+    "optimal": true
+  },
+  "326": {
+    "best_objective": -86899,
+    "status": 9,
+    "optimal": false
+  },
+  "327": {
+    "best_objective": -25305,
+    "status": 2,
+    "optimal": true
+  },
+  "328": {
+    "best_objective": -50321,
+    "status": 9,
+    "optimal": false
+  },
+  "329": {
+    "best_objective": -71422,
+    "status": 9,
+    "optimal": false
+  },
+  "330": {
+    "best_objective": -256359,
+    "status": 9,
+    "optimal": false
+  }
+}

src/stochastic_benchmark.py

@@ -251,6 +251,11 @@ def initAll(
             self.populate_interp_results()
 
     def get_experiment_parameters(self) -> ExperimentParameters:
+        def baseline_recalibrate(df):
+            baseline = getattr(self, "baseline", None)
+            if baseline is not None:
+                baseline.recalibrate(df)
+
         return ExperimentParameters(
             parameter_names=self.parameter_names,
             instance_cols=self.instance_cols,
@@ -263,7 +268,7 @@ def get_experiment_parameters(self) -> ExperimentParameters:
             training_stats=self.training_stats,
             testing_stats=self.testing_stats,
             evaluate_without_bootstrap=self.evaluate_without_bootstrap,
-            baseline_recalibrate=self.baseline.recalibrate,
+            baseline_recalibrate=baseline_recalibrate,
         )
 
     def run_Bootstrap(self, bsParams_iter, group_name_fcn=None):

tests/test_experiments.py

@@ -167,6 +167,93 @@ def test_run_Interpolate_raises_runtime_error_on_none(self, mock_interp):
             with pytest.raises(RuntimeError, match="Interpolation failed"):
                 sb.run_Interpolate(iParams)
 
+    def test_run_baseline_does_not_require_existing_baseline(self):
+        """run_baseline should construct the first baseline without a placeholder."""
+        import stochastic_benchmark as sb_module
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            sb = sb_module.stochastic_benchmark(
+                here=temp_dir,
+                response_key="PerfRatio",
+                response_dir=1,
+                parameter_names=["iteration", "samples"],
+                instance_cols=["instance"],
+                reduce_mem=False,
+                smooth=False,
+            )
+            sb.interp_results = pd.DataFrame(
+                {
+                    "instance": [1, 1, 2, 2],
+                    "train": [0, 0, 0, 0],
+                    "resource": [1, 2, 1, 2],
+                    "iteration": [1, 2, 1, 2],
+                    "samples": [1, 1, 1, 1],
+                    "Key=PerfRatio": [0.5, 0.8, 0.6, 0.9],
+                    "ConfInt=lower_Key=PerfRatio": [0.4, 0.7, 0.5, 0.8],
+                    "ConfInt=upper_Key=PerfRatio": [0.6, 0.9, 0.7, 1.0],
+                }
+            )
+            sb.training_stats = pd.DataFrame()
+            sb.testing_stats = pd.DataFrame()
+            sb.stat_params = stats.StatsParameters(
+                metrics=["PerfRatio"],
+                stats_measures=[stats.Mean()],
+                lower_bounds={},
+                upper_bounds={},
+            )
+
+            assert not hasattr(sb, "baseline")
+
+            sb.run_baseline()
+
+            assert sb.baseline.name == "VirtualBest"
+
+    def test_experiment_parameters_recalibrate_late_bound_baseline(self):
+        """Experiment parameters should recalibrate a baseline added after creation."""
+        import stochastic_benchmark as sb_module
+        import tempfile
+
+        class RecordingBaseline:
+            def __init__(self):
+                self.calls = []
+
+            def recalibrate(self, df):
+                self.calls.append(df)
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            sb = sb_module.stochastic_benchmark(
+                here=temp_dir,
+                response_key="PerfRatio",
+                response_dir=1,
+                parameter_names=["iteration", "samples"],
+                instance_cols=["instance"],
+                reduce_mem=False,
+                smooth=False,
+            )
+            sb.interp_results = pd.DataFrame()
+            sb.training_stats = pd.DataFrame()
+            sb.testing_stats = pd.DataFrame()
+            sb.stat_params = stats.StatsParameters(
+                metrics=["PerfRatio"],
+                stats_measures=[stats.Mean()],
+                lower_bounds={},
+                upper_bounds={},
+            )
+
+            params = sb.get_experiment_parameters()
+            baseline = RecordingBaseline()
+            sb.baseline = baseline
+            experiment = experiments.StaticRecommendationExperiment(
+                params,
+                pd.DataFrame({"resource": [1], "iteration": [1], "samples": [1]}),
+            )
+            eval_df = pd.DataFrame({"resource": [1], "response": [0.5]})
+
+            experiment.attach_runs(eval_df, process=False)
+
+            assert baseline.calls == [eval_df]
+
 
 class TestStochasticBenchmarkUsesLogger:
     """Tests that stochastic_benchmark uses logger, not print()."""

tests/test_tensolver_notebook.py

@@ -0,0 +1,52 @@
+"""Smoke checks for the TenSolver example notebook."""
+import json
+import os
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use("Agg", force=True)
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+NOTEBOOK = REPO_ROOT / "examples" / "TenSolver" / "TenSolver.ipynb"
+
+
+def _notebook():
+    return json.loads(NOTEBOOK.read_text())
+
+
+def _code_cell_sources():
+    return [
+        "".join(cell["source"])
+        for cell in _notebook()["cells"]
+        if cell["cell_type"] == "code"
+    ]
+
+
+def test_first_tensolver_code_cell_runs_from_repo_root(monkeypatch):
+    """The notebook should locate its data when launched from the repo root."""
+    monkeypatch.chdir(REPO_ROOT)
+    os.environ.setdefault("MPLBACKEND", "Agg")
+
+    namespace = {"__name__": "__main__"}
+    exec(compile(_code_cell_sources()[0], str(NOTEBOOK), "exec"), namespace)
+
+    assert namespace["HERE"] == REPO_ROOT / "examples" / "TenSolver"
+    assert "323" in namespace["all_data"]
+
+
+def test_tensolver_notebook_seeds_stochastic_steps():
+    """Bootstrap and train/test split output should be reproducible."""
+    source = "\n".join(_code_cell_sources())
+
+    assert "RANDOM_SEED = " in source
+    assert source.count("np.random.seed(RANDOM_SEED)") >= 2
+
+
+def test_tensolver_notebook_uses_example_dir_for_outputs():
+    """Checkpoint and plot output should stay in the example directory."""
+    source = "\n".join(_code_cell_sources())
+
+    assert "here=str(HERE)" in source
+    assert "fig.savefig(HERE / \"performance.png\")" in source