per_proportion_benchmark.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

BENCHMARKS="deltablue,chaos,nqueens,richards,go,nbody,fannkuch"
LEVEL="advanced"
SAMPLES_PER_POINT="8"
POINTS="11"
MAX_ATTEMPTS_PER_POINT="400"
ALPHA="0.05"
BOOTSTRAP_RESAMPLES="5000"
OUT_DIR="$SCRIPT_DIR/per_proportion_reports"
SEED=""
START_SKIP_BUILD="${START_SKIP_BUILD:-1}"

usage() {
  cat <<'USAGE'
Usage: ./per_proportion_benchmark.sh [options]

Runs per-proportion runtime benchmarks for de_typer_boxunbox and writes one
Markdown report per benchmark.

Defaults:
  benchmarks: deltablue,chaos,nqueens,richards,go,nbody,fannkuch
  level: advanced
  samples-per-point: 8
  points: 11 (0%..100% detyped)
  max-attempts-per-point: 400
  alpha: 0.05
  bootstrap-resamples: 5000
  out-dir: ./per_proportion_reports

Options:
  --benchmarks CSV              Benchmark list
  --level LEVEL                 advanced|shallow|untyped
  --samples-per-point N         Runtime samples per proportion point
  --points N                    Number of proportion points between 0 and 1
  --max-attempts-per-point N    Max random attempts per point to gather samples
  --alpha FLOAT                 CI alpha (e.g. 0.05)
  --bootstrap-resamples N       Bootstrap resample count
  --out-dir PATH                Output directory for markdown files
  --seed N                      Base RNG seed
  -h, --help                    Show this help
USAGE
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --benchmarks)
      BENCHMARKS="$2"
      shift 2
      ;;
    --level)
      LEVEL="$2"
      shift 2
      ;;
    --samples-per-point)
      SAMPLES_PER_POINT="$2"
      shift 2
      ;;
    --points)
      POINTS="$2"
      shift 2
      ;;
    --max-attempts-per-point)
      MAX_ATTEMPTS_PER_POINT="$2"
      shift 2
      ;;
    --alpha)
      ALPHA="$2"
      shift 2
      ;;
    --bootstrap-resamples)
      BOOTSTRAP_RESAMPLES="$2"
      shift 2
      ;;
    --out-dir)
      OUT_DIR="$2"
      shift 2
      ;;
    --seed)
      SEED="$2"
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown arg: $1" >&2
      usage >&2
      exit 2
      ;;
  esac
done

if [[ -z "$SEED" ]]; then
  SEED="$(date +%s)"
fi

mkdir -p "$OUT_DIR"
rm -f "$OUT_DIR"/*.md

INDEX_MD="$OUT_DIR/README.md"
{
  echo "# Per-Proportion Performance Reports"
  echo
  echo "- generated by \`per_proportion_benchmark.sh\`"
  echo "- level: \`$LEVEL\`"
  echo "- samples per point: \`$SAMPLES_PER_POINT\`"
  echo "- points: \`$POINTS\`"
  echo "- max attempts per point: \`$MAX_ATTEMPTS_PER_POINT\`"
  echo "- alpha: \`$ALPHA\`"
  echo "- bootstrap resamples: \`$BOOTSTRAP_RESAMPLES\`"
  echo "- seed: \`$SEED\`"
  echo
  echo "## Reports"
  echo
} >"$INDEX_MD"

IFS=',' read -r -a BENCH_ARRAY <<<"$BENCHMARKS"

bench_index=0
for bench in "${BENCH_ARRAY[@]}"; do
  bench="$(echo "$bench" | xargs)"
  if [[ -z "$bench" ]]; then
    continue
  fi
  bench_index=$((bench_index + 1))
  bench_seed=$((SEED + bench_index))
  out_file="$OUT_DIR/${bench}.${LEVEL}.md"

  echo "[$bench] running per-proportion benchmark..."

  set +e
  markdown_out="$(
    START_SKIP_BUILD="$START_SKIP_BUILD" \
      bash "$SCRIPT_DIR/start.sh" \
      env \
      PP_BENCH="$bench" \
      PP_LEVEL="$LEVEL" \
      PP_SAMPLES_PER_POINT="$SAMPLES_PER_POINT" \
      PP_POINTS="$POINTS" \
      PP_MAX_ATTEMPTS_PER_POINT="$MAX_ATTEMPTS_PER_POINT" \
      PP_ALPHA="$ALPHA" \
      PP_BOOTSTRAP_RESAMPLES="$BOOTSTRAP_RESAMPLES" \
      PP_SEED="$bench_seed" \
      /bin/bash -lc '
set -euo pipefail
PYTHONPATH=/cinder/Tools/benchmarks /cinder/python - <<'"'"'PY'"'"'
from __future__ import annotations

import math
import os
import random
import re
import time
from dataclasses import dataclass
from pathlib import Path
from statistics import mean, stdev

from de_typer_boxunbox import CinderDetyperBoxUnbox


def quantile(sorted_values: list[float], q: float) -> float:
    assert 0.0 <= q <= 1.0, "q out of range"
    if len(sorted_values) == 0:
        raise ValueError("quantile on empty data")
    if len(sorted_values) == 1:
        return sorted_values[0]
    pos = q * (len(sorted_values) - 1)
    lo = int(math.floor(pos))
    hi = int(math.ceil(pos))
    if lo == hi:
        return sorted_values[lo]
    frac = pos - lo
    return sorted_values[lo] * (1.0 - frac) + sorted_values[hi] * frac


def bootstrap_confidence_interval(
    data: list[float], alpha: float, num_resamples: int, rng: random.Random
) -> tuple[float, float]:
    assert len(data) > 0, "bootstrap requires non-empty data"
    sample_means: list[float] = []
    n = len(data)
    for _ in range(num_resamples):
        resample = [data[rng.randrange(n)] for _ in range(n)]
        sample_means.append(sum(resample) / n)
    sample_means.sort()
    lower = quantile(sample_means, alpha / 2.0)
    upper = quantile(sample_means, 1.0 - alpha / 2.0)
    return lower, upper


def signed_rank_confidence_interval(data: list[float], alpha: float, rng: random.Random) -> tuple[float, float]:
    assert len(data) > 0, "signed rank interval requires non-empty data"
    ordered = sorted(data)
    n = len(ordered)
    ranks = list(range(1, n + 1))
    med = quantile(ordered, 0.5)
    signed_ranks = [rank if value > med else -rank for value, rank in zip(ordered, ranks)]
    sample_mean = sum(ordered) / n
    _sum_ranks = sum(signed_ranks)
    se = math.sqrt((n * (n + 1) * (2 * n + 1)) / 6)
    normals = sorted(rng.gauss(0.0, 1.0) for _ in range(10000))
    z_alpha = abs(quantile(normals, 1.0 - alpha / 2.0))
    lower = sample_mean - (z_alpha * se / math.sqrt(24.0))
    upper = sample_mean + (z_alpha * se / math.sqrt(24.0))
    return lower, upper


def parse_runtime_seconds(stdout: str) -> float | None:
    lines = [ln.strip() for ln in stdout.splitlines() if ln.strip()]
    if len(lines) == 0:
        return None
    float_pattern = re.compile(r"[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?")
    for line in reversed(lines):
        matches = float_pattern.findall(line)
        if len(matches) == 0:
            continue
        try:
            return float(matches[-1])
        except ValueError:
            continue
    return None


def make_perm(fun_count: int, detyped_count: int, rng: random.Random) -> tuple[bool, ...]:
    assert 0 <= detyped_count <= fun_count, "detyped_count out of range"
    if detyped_count == 0:
        return tuple(False for _ in range(fun_count))
    if detyped_count == fun_count:
        return tuple(True for _ in range(fun_count))
    idxs = set(rng.sample(range(fun_count), detyped_count))
    return tuple(i in idxs for i in range(fun_count))


@dataclass
class PointResult:
    detyped_count: int
    detyped_ratio: float
    target_ratio: float
    requested_samples: int
    collected_samples: int
    attempts: int
    typecheck_failures: int
    run_failures: int
    parse_failures: int
    runtimes: list[float]


bench = os.environ["PP_BENCH"]
level = os.environ["PP_LEVEL"]
samples_per_point = int(os.environ["PP_SAMPLES_PER_POINT"])
points = int(os.environ["PP_POINTS"])
max_attempts_per_point = int(os.environ["PP_MAX_ATTEMPTS_PER_POINT"])
alpha = float(os.environ["PP_ALPHA"])
bootstrap_resamples = int(os.environ["PP_BOOTSTRAP_RESAMPLES"])
seed = int(os.environ["PP_SEED"])

benchmark_path = f"/root/static-python-perf/Benchmark/{bench}/{level}/main.py"
now_utc = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())

md: list[str] = []
md.append(f"# Per-Proportion Performance: {bench} ({level})")
md.append("")
md.append(f"- generated: `{now_utc}`")
md.append(f"- seed: `{seed}`")
md.append(f"- samples per point: `{samples_per_point}`")
md.append(f"- points requested: `{points}`")
md.append(f"- max attempts per point: `{max_attempts_per_point}`")
md.append(f"- alpha: `{alpha}`")
md.append(f"- bootstrap resamples: `{bootstrap_resamples}`")
md.append("")

if not Path(benchmark_path).exists():
    md.append("## Status")
    md.append("")
    md.append(f"- missing benchmark file: `{benchmark_path}`")
    print("\n".join(md))
    raise SystemExit(0)

rng = random.Random(seed)
detyper = CinderDetyperBoxUnbox(
    benchmark_file_name=benchmark_path,
    python="/cinder/python",
    scratch_dir="/tmp/detype_boxunbox",
    params=(),
)
fun_count = detyper.fun_count()
md.append(f"- function count: `{fun_count}`")
md.append("")

# Baseline sanity gate.
typed_perm = detyper.get_fully_typed_perm()
detyper.write_permutation(typed_perm)
typed_tc = detyper.execute_typecheck_permutation(typed_perm)
if typed_tc.returncode != 0:
    tail = [ln.strip() for ln in typed_tc.stderr.splitlines() if ln.strip()]
    error_tail = tail[-1] if len(tail) > 0 else "<no stderr>"
    md.append("## Status")
    md.append("")
    md.append("- baseline typed typecheck failed")
    md.append(f"- error: `{error_tail}`")
    print("\n".join(md))
    raise SystemExit(0)

# Build proportion points and dedupe on concrete detyped counts.
assert points >= 2, "points must be >= 2"
targets = [i / (points - 1) for i in range(points)]
detyped_points: list[tuple[int, float]] = []
seen_counts: set[int] = set()
for target_ratio in targets:
    detyped_count = int(round(target_ratio * fun_count))
    if detyped_count in seen_counts:
        continue
    seen_counts.add(detyped_count)
    detyped_points.append((detyped_count, target_ratio))

results: list[PointResult] = []
for detyped_count, target_ratio in detyped_points:
    point_rng = random.Random(rng.randrange(1 << 62))
    attempts = 0
    tc_fail = 0
    run_fail = 0
    parse_fail = 0
    runtimes: list[float] = []
    seen_perm_names: set[str] = set()

    while len(runtimes) < samples_per_point and attempts < max_attempts_per_point:
        attempts += 1
        perm = make_perm(fun_count, detyped_count, point_rng)
        if not any(perm) and detyped_count != 0:
            continue
        perm_name = CinderDetyperBoxUnbox._perm_name(perm)
        if perm_name in seen_perm_names and detyped_count not in (0, fun_count):
            continue
        seen_perm_names.add(perm_name)

        detyper.write_permutation(perm)
        tc_res = detyper.execute_typecheck_permutation(perm)
        if tc_res.returncode != 0:
            tc_fail += 1
            continue

        run_res = detyper.execute_permutation(perm)
        if run_res.returncode != 0:
            run_fail += 1
            continue

        runtime = parse_runtime_seconds(run_res.stdout)
        if runtime is None:
            parse_fail += 1
            continue
        runtimes.append(runtime)

    results.append(
        PointResult(
            detyped_count=detyped_count,
            detyped_ratio=detyped_count / fun_count if fun_count > 0 else 0.0,
            target_ratio=target_ratio,
            requested_samples=samples_per_point,
            collected_samples=len(runtimes),
            attempts=attempts,
            typecheck_failures=tc_fail,
            run_failures=run_fail,
            parse_failures=parse_fail,
            runtimes=runtimes,
        )
    )

typed_point = next((row for row in results if row.detyped_count == 0 and row.collected_samples > 0), None)
typed_mean = mean(typed_point.runtimes) if typed_point is not None else None

md.append("## Results")
md.append("")
md.append("| Detyped Fn | Detyped % | Samples | Mean (s) | StdDev (s) | Bootstrap CI | Signed-Rank CI | Speedup vs Typed |")
md.append("|---:|---:|---:|---:|---:|---|---|---:|")

for row in results:
    if row.collected_samples == 0:
        md.append(
            f"| {row.detyped_count} | {row.detyped_ratio * 100:.1f}% | 0/{row.requested_samples} | N/A | N/A | N/A | N/A | N/A |"
        )
        continue

    row_mean = mean(row.runtimes)
    row_std = stdev(row.runtimes) if row.collected_samples >= 2 else 0.0
    ci_rng = random.Random(seed + row.detyped_count * 101 + 17)
    b_lo, b_hi = bootstrap_confidence_interval(row.runtimes, alpha=alpha, num_resamples=bootstrap_resamples, rng=ci_rng)
    s_lo, s_hi = signed_rank_confidence_interval(row.runtimes, alpha=alpha, rng=ci_rng)

    if typed_mean is not None and row_mean > 0:
        speedup = typed_mean / row_mean
        speedup_txt = f"{speedup:.3f}x"
    else:
        speedup_txt = "N/A"

    md.append(
        "| "
        f"{row.detyped_count} | {row.detyped_ratio * 100:.1f}% | {row.collected_samples}/{row.requested_samples} | "
        f"{row_mean:.6f} | {row_std:.6f} | "
        f"[{b_lo:.6f}, {b_hi:.6f}] | [{s_lo:.6f}, {s_hi:.6f}] | {speedup_txt} |"
    )

md.append("")
md.append("## Diagnostics")
md.append("")
md.append("| Detyped Fn | Attempts | Typecheck Fails | Run Fails | Parse Fails |")
md.append("|---:|---:|---:|---:|---:|")
for row in results:
    md.append(
        f"| {row.detyped_count} | {row.attempts} | {row.typecheck_failures} | {row.run_failures} | {row.parse_failures} |"
    )
md.append("")
md.append("Notes:")
md.append("- Detyped proportion is sampled by selecting exactly K detyped functions at each point.")
md.append("- `Speedup vs Typed` uses the 0-detyped mean runtime as baseline (`typed_mean / point_mean`).")
md.append("- Bootstrap CI is empirical resampling of the sample means.")
md.append("- Signed-rank CI is included as a secondary sanity signal following the provided style.")

print("\n".join(md))
PY
'
  )"
  rc=$?
  set -e

  if [[ $rc -ne 0 ]]; then
    {
      echo "# Per-Proportion Performance: ${bench} (${LEVEL})"
      echo
      echo "- status: failed to run benchmark harness"
      echo "- command return code: \`$rc\`"
    } >"$out_file"
    echo "- [${bench}](${bench}.${LEVEL}.md) (failed)" >>"$INDEX_MD"
    continue
  fi

  printf "%s\n" "$markdown_out" >"$out_file"
  echo "- [${bench}](${bench}.${LEVEL}.md)" >>"$INDEX_MD"
done

echo
echo "wrote reports to: $OUT_DIR"
echo "index: $INDEX_MD"