From 1478611b522143810034235287a1bc9850f47ee5 Mon Sep 17 00:00:00 2001 From: Kai Piper Date: Wed, 22 Jul 2026 09:09:09 +0100 Subject: [PATCH 1/9] Add progressive accepted-probe auto compute --- runtime_python/chat_app.py | 302 +++++- runtime_python/chat_web_app.py | 53 +- .../requirements_runtime_interface.txt | 2 +- source/benchmark_progressive_auto_compute.py | 981 ++++++++++++++++++ source/benchmark_v51_prediction_stability.py | 98 +- source/chat_app.py | 302 +++++- source/chat_web_app.py | 42 +- source/generate_studio_runtime_manifest.py | 7 + source/multimodel_runtime.py | 1 + source/requirements_runtime_interface.txt | 2 +- source/run_v51_prediction_stability_gate.py | 533 ++++++++++ source/studio_runtime_manifest.json | 31 +- source/supermix_multimodel_web_app.py | 27 + test_multimodel_runtime.py | 2 + test_progressive_auto_compute_benchmark.py | 416 ++++++++ test_runtime_auto_compute_controls.py | 364 ++++++- test_studio_runtime_manifest.py | 7 + test_supermix_multimodel_web_app.py | 5 + test_v51_prediction_stability_gate.py | 317 ++++++ 19 files changed, 3375 insertions(+), 117 deletions(-) create mode 100644 source/benchmark_progressive_auto_compute.py create mode 100644 source/run_v51_prediction_stability_gate.py create mode 100644 test_progressive_auto_compute_benchmark.py create mode 100644 test_v51_prediction_stability_gate.py diff --git a/runtime_python/chat_app.py b/runtime_python/chat_app.py index d5e431438..f089d7585 100644 --- a/runtime_python/chat_app.py +++ b/runtime_python/chat_app.py @@ -1,6 +1,7 @@ import argparse import inspect import json +import math import re import time from pathlib import Path @@ -74,6 +75,9 @@ def _int_or_default(value, default: int) -> int: DEFAULT_ADAPTIVE_EXIT_TOL = 1e-3 DEFAULT_AUTO_COMPUTE_CONFIDENCE = 0.55 DEFAULT_AUTO_COMPUTE_ENTROPY = 1.85 +AUTO_COMPUTE_PLAN_SCHEMA_VERSION = "runtime-auto-compute-plan-v2" +AUTO_COMPUTE_STRATEGY = "progressive_accepted_probe" +DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K = 5 def resolve_runtime_compute_cycles(cycles, default: Optional[List[int]] = None, limit: int = 8) -> List[int]: @@ -383,6 +387,8 @@ def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[fl parsed = float(value) except (TypeError, ValueError): parsed = float(default) + if not math.isfinite(parsed): + parsed = float(default) parsed = max(0.0, parsed) if max_value is not None: parsed = min(parsed, float(max_value)) @@ -585,6 +591,261 @@ def forward_with_runtime_compute( return output, diagnostics +def _topk_distribution_js_divergence( + previous: torch.Tensor, + current: torch.Tensor, + top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> float: + """Return midpoint-support JSD while preserving tail probability mass.""" + + left = previous.detach().to(dtype=torch.float64).reshape(-1) + right = current.detach().to(dtype=torch.float64).reshape(-1) + if left.numel() != right.numel() or left.numel() == 0: + return 0.0 + k = min(max(1, int(top_k)), int(left.numel())) + midpoint = 0.5 * (left + right) + indices = torch.topk(midpoint, k=k).indices + left_top = left.index_select(0, indices) + right_top = right.index_select(0, indices) + left_compact = torch.cat( + [left_top, (1.0 - left_top.sum()).clamp_min(0.0).reshape(1)] + ) + right_compact = torch.cat( + [right_top, (1.0 - right_top.sum()).clamp_min(0.0).reshape(1)] + ) + compact_midpoint = 0.5 * (left_compact + right_compact) + epsilon = torch.finfo(left_compact.dtype).tiny + + def _kl(source: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return ( + source + * ( + torch.log(source.clamp_min(epsilon)) + - torch.log(target.clamp_min(epsilon)) + ) + ).sum() + + value = 0.5 * ( + _kl(left_compact, compact_midpoint) + + _kl(right_compact, compact_midpoint) + ) + return round(max(0.0, float(value.item())), 10) + + +def progressive_auto_compute_forward( + model: Any, + x: torch.Tensor, + available_labels: List[int], + *, + cycles: Any = None, + confidence_target: Any = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: Any = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: Any = False, + exit_tol: Any = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: Any = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: Any = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, + auto_reasoning_context: str = "", + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + """Evaluate auto-compute budgets progressively and reuse the accepted probe. + + This preserves the v1 selection policy and row order exactly: the first + confidence/entropy target wins, otherwise the best-confidence row wins + after the full ladder. Probes use the complete current runtime-control set, + and the accepted probe output is returned directly, removing the old N+1 + rerun. Cross-budget persistence and JSD are telemetry only and never select + or reject an output. + """ + + if bool(getattr(model, "training", False)): + raise RuntimeError("progressive auto compute requires model.eval()") + labels = list(available_labels) or list(range(MODEL_CLASSES)) + resolved_cycles = resolve_runtime_compute_cycles(cycles) + confidence_target = min( + 1.0, + _coerce_nonnegative_float( + confidence_target, + DEFAULT_AUTO_COMPUTE_CONFIDENCE, + ), + ) + entropy_target = _coerce_nonnegative_float( + entropy_target, + DEFAULT_AUTO_COMPUTE_ENTROPY, + ) + adaptive = _coerce_bool(adaptive_compute) + idx = torch.tensor(labels, dtype=torch.long, device=x.device) + records: List[Dict[str, Any]] = [] + cached_outputs: List[torch.Tensor] = [] + previous_probs: Optional[torch.Tensor] = None + previous_label: Optional[int] = None + previous_confidence: Optional[float] = None + accepted_index: Optional[int] = None + reason = "best_confidence" + + for requested_cycles in resolved_cycles: + started = time.perf_counter() + with torch.no_grad(): + model_out, compute = forward_with_runtime_compute( + model, + x, + reasoning_cycles=requested_cycles, + adaptive_compute=adaptive, + exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + auto_reasoning_context=auto_reasoning_context, + ) + logits = model_out[0, 0] + available_logits = logits.index_select(0, idx.to(logits.device)) + probs = torch.softmax(available_logits, dim=0) + confidence_tensor, predicted_position = torch.max(probs, dim=0) + entropy_tensor = -(probs * torch.log(probs.clamp_min(1e-8))).sum() + # v1 serialized these values to six decimals before the selector read + # them. Keep that boundary exact so values such as 0.5499996 still + # satisfy a 0.55 target just as they did before. + confidence = round(float(confidence_tensor.item()), 6) + entropy = round(float(entropy_tensor.item()), 6) + predicted_label = int(labels[int(predicted_position.item())]) + # Freeze selection latency before shadow-only diagnostics. Otherwise a + # costly JSD calculation could alter the legacy fallback tie-break. + probe_latency_ms = round((time.perf_counter() - started) * 1000.0, 1) + has_previous = previous_probs is not None + shadow = { + "role": "shadow_diagnostic_only", + "metric": "midpoint_support_top_k_jensen_shannon_divergence_nats", + "top_k": min(max(1, int(distribution_top_k)), len(labels)), + "has_previous_probe": has_previous, + "top1_persistent": ( + bool(previous_label == predicted_label) if has_previous else None + ), + "confidence_delta": ( + round(abs(confidence - float(previous_confidence)), 8) + if previous_confidence is not None + else None + ), + "js_divergence": ( + _topk_distribution_js_divergence( + previous_probs, + probs, + distribution_top_k, + ) + if previous_probs is not None + else None + ), + "selection_enabled": False, + } + row = { + "requested_cycles": int(requested_cycles), + "latency_ms": probe_latency_ms, + "cycles_used": compute.get("cycles_used"), + "predicted_label": predicted_label, + "confidence": confidence, + "entropy": entropy, + "confidence_target_met": bool(confidence >= confidence_target), + "entropy_target_met": bool(entropy <= entropy_target), + "mutual_stability_shadow": shadow, + "compute": compute, + } + records.append(row) + cached_outputs.append(model_out) + + if confidence >= confidence_target: + accepted_index = len(records) - 1 + reason = "confidence_target" + break + if entropy <= entropy_target: + accepted_index = len(records) - 1 + reason = "entropy_target" + break + previous_probs = probs.detach() + previous_label = predicted_label + previous_confidence = confidence + + if accepted_index is None: + accepted_index = max( + range(len(records)), + key=lambda index: ( + float(records[index].get("confidence", 0.0)), + -float(records[index].get("entropy", 999.0)), + -float(records[index].get("latency_ms", 0.0)), + ), + ) + + selected_row = records[accepted_index] + selected_output = cached_outputs[accepted_index] + selected_compute = dict(selected_row["compute"]) + evaluated_cycles = [int(row["requested_cycles"]) for row in records] + skipped_cycles = [ + int(value) for value in resolved_cycles[len(evaluated_cycles) :] + ] + legacy_forward_evaluations = len(resolved_cycles) + 1 + forward_evaluations = len(records) + plan = { + "schema_version": AUTO_COMPUTE_PLAN_SCHEMA_VERSION, + "enabled": True, + "strategy": AUTO_COMPUTE_STRATEGY, + "selected_reasoning_cycles": int(selected_row["requested_cycles"]), + "selected_index": int(accepted_index), + "accepted_probe_index": int(accepted_index), + "reason": reason, + "confidence_target": confidence_target, + "entropy_target": entropy_target, + "candidate_cycles": [int(value) for value in resolved_cycles], + "evaluated_cycles": evaluated_cycles, + "skipped_cycles": skipped_cycles, + "forward_evaluations": forward_evaluations, + "legacy_forward_evaluations": legacy_forward_evaluations, + "forward_reduction_percent": round( + 100.0 + * (legacy_forward_evaluations - forward_evaluations) + / max(1, legacy_forward_evaluations), + 3, + ), + "reused_probe_output": True, + "selection_semantics": "legacy_v1_selection_policy", + "probe_control_scope": "full_runtime_controls_v2", + "mutual_stability_role": "shadow_diagnostic_only", + "rows": records, + } + selected_compute["auto_compute_plan"] = plan + selected_compute["inference_reused"] = True + return selected_output, selected_compute, plan + + +def compact_auto_compute_plan_metrics(plan: Any) -> Dict[str, Any]: + """Return terminal-safe metrics for the probe that supplied the output.""" + + if not isinstance(plan, dict): + return {} + rows = plan.get("rows") if isinstance(plan.get("rows"), list) else [] + try: + selected_index = int(plan.get("selected_index")) + except (TypeError, ValueError): + selected_index = -1 + if not 0 <= selected_index < len(rows): + selected_index = len(rows) - 1 + selected_row = rows[selected_index] if selected_index >= 0 else {} + shadow = ( + selected_row.get("mutual_stability_shadow", {}) + if isinstance(selected_row, dict) + else {} + ) + metrics: Dict[str, Any] = { + "auto_selected": plan.get("selected_reasoning_cycles"), + "auto_reason": plan.get("reason"), + "auto_forwards": ( + f"{plan.get('forward_evaluations')}/" + f"{plan.get('legacy_forward_evaluations')}" + ), + "auto_reused": bool(plan.get("reused_probe_output")), + } + if isinstance(shadow, dict) and shadow.get("js_divergence") is not None: + metrics["auto_shadow_js"] = shadow.get("js_divergence") + return metrics + + def _default_expansion_dim_for_model_size(model_size: str) -> int: if model_size in THIRD_EXPANSION_DIM_MODEL_SIZES: return 1024 @@ -1028,8 +1289,7 @@ def main(): compact_compute = dict(last_compute_metrics) plan = compact_compute.pop("auto_compute_plan", None) if isinstance(plan, dict): - compact_compute["auto_selected"] = plan.get("selected_reasoning_cycles") - compact_compute["auto_reason"] = plan.get("reason") + compact_compute.update(compact_auto_compute_plan_metrics(plan)) print( " last_compute=" + ", ".join(f"{k}={v}" for k, v in compact_compute.items()) @@ -1244,25 +1504,13 @@ def main(): effective_reasoning_cycles = args.reasoning_cycles auto_compute_plan = None if args.auto_compute and model_supports_runtime_compute(model): - auto_rows = evaluate_runtime_compute_budgets( + model_out, last_compute_metrics, auto_compute_plan = progressive_auto_compute_forward( model, x, available_labels, cycles=runtime_auto_compute_cycles(args.reasoning_cycles), - adaptive_compute=args.adaptive_compute, - exit_tol=args.adaptive_exit_tol, - ) - auto_compute_plan = select_auto_runtime_compute_budget( - auto_rows, confidence_target=args.auto_compute_confidence, entropy_target=args.auto_compute_entropy, - ) - effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") - with torch.no_grad(): - model_out, last_compute_metrics = forward_with_runtime_compute( - model, - x, - reasoning_cycles=effective_reasoning_cycles, adaptive_compute=args.adaptive_compute, exit_tol=args.adaptive_exit_tol, exit_entropy_threshold=args.adaptive_exit_entropy, @@ -1270,9 +1518,21 @@ def main(): prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, ) - logits = model_out[0, 0] # (10,) - if auto_compute_plan is not None: - last_compute_metrics["auto_compute_plan"] = auto_compute_plan + effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") + else: + with torch.no_grad(): + model_out, last_compute_metrics = forward_with_runtime_compute( + model, + x, + reasoning_cycles=effective_reasoning_cycles, + adaptive_compute=args.adaptive_compute, + exit_tol=args.adaptive_exit_tol, + exit_entropy_threshold=args.adaptive_exit_entropy, + prediction_stability_patience=args.prediction_stability_patience, + prediction_stability_tol=args.prediction_stability_tol, + auto_reasoning_context=context, + ) + logits = model_out[0, 0] # (10,) t_infer += max(0.0, time.perf_counter() - _t) idx = torch.tensor(available_labels, dtype=torch.long, device=logits.device) @@ -1424,9 +1684,9 @@ def main(): plan = last_compute_metrics.get("auto_compute_plan") if isinstance(last_compute_metrics, dict) else None plan_suffix = "" if isinstance(plan, dict): - plan_suffix = ( - f" auto_selected={plan.get('selected_reasoning_cycles')} " - f"reason={plan.get('reason')}" + plan_metrics = compact_auto_compute_plan_metrics(plan) + plan_suffix = " " + " ".join( + f"{key}={value}" for key, value in plan_metrics.items() ) print( f"{TerminalColors.SYSTEM}Compute:{TerminalColors.RESET} " diff --git a/runtime_python/chat_web_app.py b/runtime_python/chat_web_app.py index 3104db5d9..2e94dc6a6 100644 --- a/runtime_python/chat_web_app.py +++ b/runtime_python/chat_web_app.py @@ -220,7 +220,7 @@
-
+
@@ -301,7 +301,18 @@ const entropy=fmtNum(compute.gating_entropy); if(entropy) parts.push(`gate entropy ${entropy}`); const exitEntropy=fmtNum(compute.exit_entropy_threshold); if(exitEntropy) parts.push(`exit entropy ${exitEntropy}`); if(compute.auto_reasoning_policy&&Array.isArray(compute.auto_reasoning_policy.reasons)) parts.push(`policy ${compute.auto_reasoning_policy.reasons.slice(0,3).join(',')}`); - if(compute.auto_compute_plan) parts.push(`budget ${compute.auto_compute_plan.selected_reasoning_cycles} (${compute.auto_compute_plan.reason})`); + if(compute.auto_compute_plan){ + const plan=compute.auto_compute_plan; + parts.push(`budget ${plan.selected_reasoning_cycles} (${plan.reason})`); + parts.push(`${plan.forward_evaluations}/${plan.legacy_forward_evaluations} forwards`); + if(plan.reused_probe_output) parts.push('probe reused'); + const rows=Array.isArray(plan.rows)?plan.rows:[]; + const selectedIndex=Number(plan.selected_index); + const selectedRow=Number.isInteger(selectedIndex)&&selectedIndex>=0&&selectedIndex str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _git_text(*args: str) -> str | None: + try: + completed = subprocess.run( + ["git", *args], + cwd=PROJECT_ROOT, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + return completed.stdout.strip() + + +def collect_provenance( + *, + device: Any, + device_info: Mapping[str, Any], +) -> Dict[str, Any]: + """Capture the controller, model, data, runtime, and repository identity.""" + + source_paths = ( + Path(__file__).resolve(), + PROJECT_ROOT / "source" / "chat_app.py", + PROJECT_ROOT / "source" / "benchmark_cognitive_leap_ultra_v51.py", + PROJECT_ROOT / "source" / "model_variants.py", + ) + source_hashes = { + path.relative_to(PROJECT_ROOT).as_posix(): _sha256_file(path) + for path in source_paths + } + status = _git_text("status", "--porcelain", "--untracked-files=all") + resolved_device = str(device_info.get("resolved", device)) + device_metadata: Dict[str, Any] = { + "requested": str(device_info.get("requested", "unknown")), + "resolved": resolved_device, + "torch_device": str(device), + } + if resolved_device.startswith("cuda") and torch.cuda.is_available(): + device_metadata["name"] = torch.cuda.get_device_name(device) + device_metadata["capability"] = list(torch.cuda.get_device_capability(device)) + + return { + "git": { + "commit": _git_text("rev-parse", "HEAD"), + "branch": _git_text("rev-parse", "--abbrev-ref", "HEAD"), + "worktree_dirty": bool(status), + }, + "source_sha256": source_hashes, + "runtime": { + "python": { + "version": platform.python_version(), + "implementation": platform.python_implementation(), + "executable": sys.executable, + "platform": platform.platform(), + }, + "torch": { + "version": str(torch.__version__), + "cuda_version": str(torch.version.cuda) if torch.version.cuda else None, + "default_dtype": str(torch.get_default_dtype()), + "deterministic_algorithms": bool( + torch.are_deterministic_algorithms_enabled() + ), + }, + "device": device_metadata, + "threads": { + "torch_num_threads": int(torch.get_num_threads()), + "torch_num_interop_threads": int(torch.get_num_interop_threads()), + "logical_cpu_count": int(os.cpu_count() or 1), + }, + }, + } + + +def _finite_float(value: Any, *, label: str) -> float: + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{label} must be finite, got {value!r}") + return number + + +def _synchronize_for_timing(device: torch.device) -> None: + device_type = str(device).split(":", 1)[0].lower() + if device_type == "cuda" and torch.cuda.is_available(): + torch.cuda.synchronize(device) + elif device_type == "xpu" and hasattr(torch, "xpu"): + torch.xpu.synchronize() # type: ignore[attr-defined] + elif device_type == "mps" and hasattr(torch, "mps"): + torch.mps.synchronize() + + +def _latency_summary(values: Sequence[float]) -> Dict[str, float]: + if not values: + raise ValueError("At least one latency observation is required") + tensor = torch.tensor(list(values), dtype=torch.float64) + return { + "total_ms": round(float(tensor.sum().item()), 6), + "mean_ms": round(float(tensor.mean().item()), 6), + "p50_ms": round(float(torch.quantile(tensor, 0.50).item()), 6), + "p95_ms": round(float(torch.quantile(tensor, 0.95).item()), 6), + } + + +@torch.no_grad() +def legacy_v1_auto_compute_forward( + model: Any, + x: torch.Tensor, + available_labels: List[int], + *, + cycles: Any = None, + confidence_target: Any = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: Any = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: Any = False, + exit_tol: Any = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: Any = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: Any = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, + auto_reasoning_context: str = "", + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + """Execute the retired v1 probe-all/select/rerun controller literally.""" + + del distribution_top_k # The v1 selector had no distribution-stability input. + rows = evaluate_runtime_compute_budgets( + model, + x, + available_labels, + cycles=cycles, + adaptive_compute=bool(adaptive_compute), + exit_tol=exit_tol, + ) + selection = select_auto_runtime_compute_budget( + rows, + confidence_target=float(confidence_target), + entropy_target=float(entropy_target), + ) + selected_cycles = int(selection["selected_reasoning_cycles"]) + output, compute = forward_with_runtime_compute( + model, + x, + reasoning_cycles=selected_cycles, + adaptive_compute=adaptive_compute, + exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + auto_reasoning_context=auto_reasoning_context, + ) + plan = { + **selection, + "schema_version": "runtime-auto-compute-plan-v1-benchmark-reference", + "strategy": "probe_all_select_and_rerun", + "candidate_cycles": [ + int(value) for value in resolve_runtime_compute_cycles(cycles) + ], + "evaluated_cycles": [int(row["requested_cycles"]) for row in rows], + "forward_evaluations": len(rows) + 1, + "reused_probe_output": False, + "mutual_stability_role": "not_present_in_v1", + } + compute = dict(compute) + compute["auto_compute_plan"] = plan + compute["inference_reused"] = False + return output, compute, plan + + +def _prediction(output: torch.Tensor, labels: Sequence[int]) -> int: + logits = output[0, 0] + indices = torch.tensor(list(labels), dtype=torch.long, device=logits.device) + position = int(logits.index_select(0, indices).argmax(dim=0).item()) + return int(labels[position]) + + +def _shadow_selection_is_disabled(plan: Mapping[str, Any]) -> bool: + if plan.get("mutual_stability_role") != "shadow_diagnostic_only": + return False + for row in plan.get("rows", []): + if not isinstance(row, Mapping): + return False + shadow = row.get("mutual_stability_shadow") + if not isinstance(shadow, Mapping): + return False + if shadow.get("role") != "shadow_diagnostic_only": + return False + if shadow.get("selection_enabled") is not False: + return False + return True + + +def _max_abs_difference( + left: torch.Tensor, right: torch.Tensor +) -> float | None: + if tuple(left.shape) != tuple(right.shape): + return None + return float( + (left.detach().to(dtype=torch.float64, device="cpu") + - right.detach().to(dtype=torch.float64, device="cpu")) + .abs() + .max() + .item() + ) + + +@torch.no_grad() +def benchmark_requests( + model: Any, + x: torch.Tensor, + y: torch.Tensor, + *, + available_labels: Sequence[int] = tuple(range(10)), + cycles: Any = DEFAULT_CYCLES, + confidence_target: float = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: float = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: bool = False, + exit_tol: float = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: float = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: int = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: float = DEFAULT_PREDICTION_STABILITY_TOL, + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + order_offset: int = 0, + legacy_fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]] + | None = None, + progressive_fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]] + | None = None, +) -> Dict[str, Any]: + """Run a paired, request-level, counterbalanced controller comparison.""" + + if bool(getattr(model, "training", False)): + raise RuntimeError("controller benchmark requires model.eval()") + request_count = int(len(y)) + if request_count <= 0 or int(len(x)) != request_count: + raise ValueError("x and y must contain the same positive request count") + labels = [int(label) for label in available_labels] + if not labels: + raise ValueError("available_labels must not be empty") + resolved_cycles = resolve_runtime_compute_cycles(cycles) + settings = { + "cycles": resolved_cycles, + "confidence_target": _finite_float( + confidence_target, label="confidence target" + ), + "entropy_target": _finite_float(entropy_target, label="entropy target"), + "adaptive_compute": bool(adaptive_compute), + "exit_tol": _finite_float(exit_tol, label="exit tolerance"), + "exit_entropy_threshold": _finite_float( + exit_entropy_threshold, label="exit entropy threshold" + ), + "prediction_stability_patience": int(prediction_stability_patience), + "prediction_stability_tol": _finite_float( + prediction_stability_tol, label="prediction stability tolerance" + ), + "distribution_top_k": int(distribution_top_k), + } + if settings["prediction_stability_patience"] < 0: + raise ValueError("prediction_stability_patience must be nonnegative") + if settings["distribution_top_k"] <= 0: + raise ValueError("distribution_top_k must be positive") + + legacy = legacy_fn or legacy_v1_auto_compute_forward + progressive = progressive_fn or progressive_auto_compute_forward + first_sample = x[:1] + + # Match the transition immediately before each seed's first measured mode. + if int(order_offset) % 2 == 0: + legacy(model, first_sample, labels, **settings) + progressive(model, first_sample, labels, **settings) + else: + progressive(model, first_sample, labels, **settings) + legacy(model, first_sample, labels, **settings) + + legacy_latencies: List[float] = [] + progressive_latencies: List[float] = [] + measurement_orders: Counter[str] = Counter() + selected_cycle_pairs: Counter[str] = Counter() + selected_cycle_disagreements = 0 + prediction_disagreements = 0 + output_disagreements = 0 + legacy_correct = 0 + progressive_correct = 0 + legacy_forward_evaluations = 0 + progressive_forward_evaluations = 0 + progressive_not_better_count = 0 + shadow_selection_violations = 0 + maximum_abs_logit_difference = 0.0 + mismatch_details: List[Dict[str, Any]] = [] + + def timed_call( + fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]], + sample: torch.Tensor, + values: List[float], + ) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + _synchronize_for_timing(sample.device) + started = time.perf_counter() + result = fn(model, sample, labels, **settings) + _synchronize_for_timing(sample.device) + values.append((time.perf_counter() - started) * 1000.0) + return result + + for index in range(request_count): + sample = x[index : index + 1] + legacy_first = (index + int(order_offset)) % 2 == 0 + if legacy_first: + measurement_orders["legacy_then_progressive"] += 1 + legacy_result = timed_call(legacy, sample, legacy_latencies) + progressive_result = timed_call( + progressive, sample, progressive_latencies + ) + else: + measurement_orders["progressive_then_legacy"] += 1 + progressive_result = timed_call( + progressive, sample, progressive_latencies + ) + legacy_result = timed_call(legacy, sample, legacy_latencies) + + legacy_output, _legacy_compute, legacy_plan = legacy_result + progressive_output, _progressive_compute, progressive_plan = ( + progressive_result + ) + legacy_cycles = int(legacy_plan["selected_reasoning_cycles"]) + progressive_cycles = int(progressive_plan["selected_reasoning_cycles"]) + legacy_prediction = _prediction(legacy_output, labels) + progressive_prediction = _prediction(progressive_output, labels) + truth = int(y[index].item()) + shapes_match = tuple(legacy_output.shape) == tuple(progressive_output.shape) + outputs_equal = bool( + shapes_match and torch.equal(legacy_output, progressive_output) + ) + max_abs_difference = _max_abs_difference( + legacy_output, progressive_output + ) + if max_abs_difference is not None: + maximum_abs_logit_difference = max( + maximum_abs_logit_difference, max_abs_difference + ) + + legacy_evaluations = int(legacy_plan["forward_evaluations"]) + progressive_evaluations = int(progressive_plan["forward_evaluations"]) + legacy_forward_evaluations += legacy_evaluations + progressive_forward_evaluations += progressive_evaluations + progressive_not_better_count += int( + progressive_evaluations >= legacy_evaluations + ) + shadow_ok = _shadow_selection_is_disabled(progressive_plan) + shadow_selection_violations += int(not shadow_ok) + selected_cycle_pairs[f"{legacy_cycles}->{progressive_cycles}"] += 1 + selected_cycle_disagreements += int(legacy_cycles != progressive_cycles) + prediction_disagreements += int( + legacy_prediction != progressive_prediction + ) + output_disagreements += int(not outputs_equal) + legacy_correct += int(legacy_prediction == truth) + progressive_correct += int(progressive_prediction == truth) + + if ( + legacy_cycles != progressive_cycles + or legacy_prediction != progressive_prediction + or not outputs_equal + or not shadow_ok + ) and len(mismatch_details) < MAX_MISMATCH_DETAILS: + mismatch_details.append( + { + "request_index": index, + "truth": truth, + "legacy_selected_cycles": legacy_cycles, + "progressive_selected_cycles": progressive_cycles, + "legacy_prediction": legacy_prediction, + "progressive_prediction": progressive_prediction, + "output_shapes_match": shapes_match, + "outputs_exactly_equal": outputs_equal, + "max_abs_logit_difference": max_abs_difference, + "shadow_selection_disabled": shadow_ok, + } + ) + + legacy_latency = _latency_summary(legacy_latencies) + progressive_latency = _latency_summary(progressive_latencies) + forward_reduction = ( + 100.0 + * (legacy_forward_evaluations - progressive_forward_evaluations) + / max(1, legacy_forward_evaluations) + ) + latency_reduction = ( + 100.0 + * (legacy_latency["total_ms"] - progressive_latency["total_ms"]) + / max(1e-12, legacy_latency["total_ms"]) + ) + checks = { + "exact_selected_cycle_agreement": { + "passed": selected_cycle_disagreements == 0, + "observed_disagreements": selected_cycle_disagreements, + "required": 0, + }, + "exact_prediction_agreement": { + "passed": prediction_disagreements == 0, + "observed_disagreements": prediction_disagreements, + "required": 0, + }, + "exact_output_tensor_agreement": { + "passed": output_disagreements == 0, + "observed_disagreements": output_disagreements, + "required": 0, + }, + "positive_forward_evaluation_reduction": { + "passed": progressive_forward_evaluations < legacy_forward_evaluations, + "legacy_forward_evaluations": legacy_forward_evaluations, + "progressive_forward_evaluations": progressive_forward_evaluations, + }, + "per_request_forward_evaluation_reduction": { + "passed": progressive_not_better_count == 0, + "non_reducing_request_count": progressive_not_better_count, + "required": 0, + }, + "mutual_stability_is_shadow_only": { + "passed": shadow_selection_violations == 0, + "selection_policy_violation_count": shadow_selection_violations, + "required": 0, + }, + } + return { + "request_count": request_count, + "measurement_order": { + "offset": int(order_offset), + "legacy_then_progressive": measurement_orders[ + "legacy_then_progressive" + ], + "progressive_then_legacy": measurement_orders[ + "progressive_then_legacy" + ], + }, + "legacy_v1": { + "strategy": "probe_all_select_and_rerun", + "correct_predictions": legacy_correct, + "forward_evaluations": legacy_forward_evaluations, + "latency": legacy_latency, + }, + "progressive": { + "strategy": "progressive_accepted_probe", + "correct_predictions": progressive_correct, + "forward_evaluations": progressive_forward_evaluations, + "latency": progressive_latency, + }, + "comparison": { + "selected_cycle_pairs": dict(sorted(selected_cycle_pairs.items())), + "exact_selected_cycle_disagreement_count": ( + selected_cycle_disagreements + ), + "exact_prediction_disagreement_count": prediction_disagreements, + "exact_output_tensor_disagreement_count": output_disagreements, + "maximum_absolute_logit_difference": round( + maximum_abs_logit_difference, 12 + ), + "forward_evaluation_reduction_percent": round( + forward_reduction, 6 + ), + "mean_latency_reduction_percent": round(latency_reduction, 6), + "mismatch_details": mismatch_details, + }, + "gates": { + "passed": all(bool(check["passed"]) for check in checks.values()), + "checks": checks, + }, + } + + +def aggregate_results(seed_results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: + if not seed_results: + raise ValueError("At least one seed result is required") + total_requests = 0 + total_selected_cycle_disagreements = 0 + total_prediction_disagreements = 0 + total_output_disagreements = 0 + total_legacy_correct = 0 + total_progressive_correct = 0 + total_legacy_forward_evaluations = 0 + total_progressive_forward_evaluations = 0 + total_legacy_latency_ms = 0.0 + total_progressive_latency_ms = 0.0 + maximum_abs_logit_difference = 0.0 + shadow_selection_violations = 0 + non_reducing_requests = 0 + selected_cycle_pairs: Counter[str] = Counter() + per_seed_latency_reductions: List[float] = [] + per_seed: List[Dict[str, Any]] = [] + seen_seeds: set[int] = set() + + for seed_result in seed_results: + seed = int(seed_result["seed"]) + if seed in seen_seeds: + raise ValueError(f"Duplicate seed result: {seed}") + seen_seeds.add(seed) + metrics = seed_result["metrics"] + request_count = int(metrics["request_count"]) + if request_count <= 0: + raise ValueError(f"Seed {seed} has no requests") + order = metrics["measurement_order"] + legacy_first = int(order["legacy_then_progressive"]) + progressive_first = int(order["progressive_then_legacy"]) + offset = int(order["offset"]) + expected_legacy_first = ( + (request_count + 1) // 2 if offset % 2 == 0 else request_count // 2 + ) + if ( + legacy_first != expected_legacy_first + or progressive_first != request_count - expected_legacy_first + ): + raise ValueError(f"Seed {seed} measurement order is not counterbalanced") + + legacy = metrics["legacy_v1"] + progressive = metrics["progressive"] + comparison = metrics["comparison"] + legacy_latency = _finite_float( + legacy["latency"]["total_ms"], label="legacy latency" + ) + progressive_latency = _finite_float( + progressive["latency"]["total_ms"], label="progressive latency" + ) + if legacy_latency <= 0.0 or progressive_latency < 0.0: + raise ValueError(f"Seed {seed} latency totals are invalid") + seed_latency_reduction = ( + 100.0 * (legacy_latency - progressive_latency) / legacy_latency + ) + per_seed_latency_reductions.append(seed_latency_reduction) + total_requests += request_count + total_selected_cycle_disagreements += int( + comparison["exact_selected_cycle_disagreement_count"] + ) + total_prediction_disagreements += int( + comparison["exact_prediction_disagreement_count"] + ) + total_output_disagreements += int( + comparison["exact_output_tensor_disagreement_count"] + ) + total_legacy_correct += int(legacy["correct_predictions"]) + total_progressive_correct += int(progressive["correct_predictions"]) + total_legacy_forward_evaluations += int(legacy["forward_evaluations"]) + total_progressive_forward_evaluations += int( + progressive["forward_evaluations"] + ) + total_legacy_latency_ms += legacy_latency + total_progressive_latency_ms += progressive_latency + maximum_abs_logit_difference = max( + maximum_abs_logit_difference, + _finite_float( + comparison["maximum_absolute_logit_difference"], + label="maximum absolute logit difference", + ), + ) + for pair, count in comparison["selected_cycle_pairs"].items(): + selected_cycle_pairs[str(pair)] += int(count) + checks = metrics["gates"]["checks"] + shadow_selection_violations += int( + checks["mutual_stability_is_shadow_only"][ + "selection_policy_violation_count" + ] + ) + non_reducing_requests += int( + checks["per_request_forward_evaluation_reduction"][ + "non_reducing_request_count" + ] + ) + per_seed.append( + { + "seed": seed, + "samples": request_count, + "selected_cycle_disagreements": int( + comparison["exact_selected_cycle_disagreement_count"] + ), + "prediction_disagreements": int( + comparison["exact_prediction_disagreement_count"] + ), + "output_tensor_disagreements": int( + comparison["exact_output_tensor_disagreement_count"] + ), + "forward_evaluation_reduction_percent": float( + comparison["forward_evaluation_reduction_percent"] + ), + "mean_latency_reduction_percent": round( + seed_latency_reduction, 6 + ), + "measurement_order": dict(order), + } + ) + + forward_reduction = ( + 100.0 + * (total_legacy_forward_evaluations - total_progressive_forward_evaluations) + / max(1, total_legacy_forward_evaluations) + ) + weighted_latency_reduction = ( + 100.0 + * (total_legacy_latency_ms - total_progressive_latency_ms) + / max(1e-12, total_legacy_latency_ms) + ) + checks = { + "exact_selected_cycle_agreement": { + "passed": total_selected_cycle_disagreements == 0, + "observed_disagreements": total_selected_cycle_disagreements, + "required": 0, + }, + "exact_prediction_agreement": { + "passed": total_prediction_disagreements == 0, + "observed_disagreements": total_prediction_disagreements, + "required": 0, + }, + "exact_output_tensor_agreement": { + "passed": total_output_disagreements == 0, + "observed_disagreements": total_output_disagreements, + "required": 0, + }, + "positive_forward_evaluation_reduction": { + "passed": ( + total_progressive_forward_evaluations + < total_legacy_forward_evaluations + ), + "legacy_forward_evaluations": total_legacy_forward_evaluations, + "progressive_forward_evaluations": total_progressive_forward_evaluations, + }, + "per_request_forward_evaluation_reduction": { + "passed": non_reducing_requests == 0, + "non_reducing_request_count": non_reducing_requests, + "required": 0, + }, + "mutual_stability_is_shadow_only": { + "passed": shadow_selection_violations == 0, + "selection_policy_violation_count": shadow_selection_violations, + "required": 0, + }, + } + return { + "summary": { + "seed_count": len(seed_results), + "total_samples": total_requests, + "exact_selected_cycle_disagreement_count": ( + total_selected_cycle_disagreements + ), + "exact_prediction_disagreement_count": total_prediction_disagreements, + "exact_output_tensor_disagreement_count": total_output_disagreements, + "maximum_absolute_logit_difference": round( + maximum_abs_logit_difference, 12 + ), + "legacy_correct_predictions": total_legacy_correct, + "progressive_correct_predictions": total_progressive_correct, + "legacy_forward_evaluations": total_legacy_forward_evaluations, + "progressive_forward_evaluations": total_progressive_forward_evaluations, + "forward_evaluation_reduction_percent": round(forward_reduction, 6), + "weighted_mean_latency_reduction_percent": round( + weighted_latency_reduction, 6 + ), + "median_per_seed_latency_reduction_percent": round( + float(statistics.median(per_seed_latency_reductions)), 6 + ), + "weighted_legacy_mean_latency_ms": round( + total_legacy_latency_ms / total_requests, 6 + ), + "weighted_progressive_mean_latency_ms": round( + total_progressive_latency_ms / total_requests, 6 + ), + "selected_cycle_pairs": dict(sorted(selected_cycle_pairs.items())), + }, + "per_seed_summary": per_seed, + "gates": { + "passed": all(bool(check["passed"]) for check in checks.values()), + "checks": checks, + }, + } + + +def run_benchmark( + *, + weights: Path, + seeds: Sequence[int], + samples_per_seed: int, + device: Any, + device_info: Mapping[str, Any], + cycles: Sequence[int] = DEFAULT_CYCLES, + confidence_target: float = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: float = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: bool = False, + exit_tol: float = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: float = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: int = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: float = DEFAULT_PREDICTION_STABILITY_TOL, + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + model_loader: Callable[[Path, Any], Any] | None = None, + task_factory: Callable[..., Tuple[torch.Tensor, torch.Tensor]] | None = None, + benchmark_fn: Callable[..., Dict[str, Any]] | None = None, + provenance: Mapping[str, Any] | None = None, +) -> Dict[str, Any]: + weights = Path(weights).resolve() + if not weights.is_file(): + raise FileNotFoundError(f"Missing v51 checkpoint: {weights}") + normalized_seeds = [int(seed) for seed in seeds] + if not normalized_seeds: + raise ValueError("At least one held-out seed is required") + if len(set(normalized_seeds)) != len(normalized_seeds): + raise ValueError("Seeds must be unique") + overlapping_seeds = sorted(TRAINING_AND_ORIGINAL_TEST_SEEDS.intersection(normalized_seeds)) + if overlapping_seeds: + raise ValueError( + "Held-out seeds overlap v51 training/original-test seeds: " + + ", ".join(str(seed) for seed in overlapping_seeds) + ) + if int(samples_per_seed) <= 0: + raise ValueError("samples_per_seed must be positive") + resolved_cycles = resolve_runtime_compute_cycles(cycles) + if len(resolved_cycles) < 2: + raise ValueError("At least two distinct compute-cycle candidates are required") + normalized_confidence_target = _finite_float( + confidence_target, label="confidence target" + ) + normalized_entropy_target = _finite_float( + entropy_target, label="entropy target" + ) + if not 0.0 <= normalized_confidence_target <= 1.0: + raise ValueError("confidence_target must be between 0 and 1") + if normalized_entropy_target < 0.0: + raise ValueError("entropy_target must be nonnegative") + + loader = model_loader or _load_model + task = task_factory or make_chained_task + benchmark = benchmark_fn or benchmark_requests + checkpoint_sha256 = _sha256_file(weights) + model = loader(weights, device) + if hasattr(model, "eval"): + model.eval() + seed_results: List[Dict[str, Any]] = [] + for seed_index, seed in enumerate(normalized_seeds): + x, y = task(int(samples_per_seed), seed=seed) + metrics = benchmark( + model, + x.to(device), + y.to(device), + cycles=resolved_cycles, + confidence_target=normalized_confidence_target, + entropy_target=normalized_entropy_target, + adaptive_compute=bool(adaptive_compute), + exit_tol=float(exit_tol), + exit_entropy_threshold=float(exit_entropy_threshold), + prediction_stability_patience=int(prediction_stability_patience), + prediction_stability_tol=float(prediction_stability_tol), + distribution_top_k=int(distribution_top_k), + order_offset=seed_index % 2, + ) + seed_results.append({"seed": seed, "metrics": metrics}) + + aggregate = aggregate_results(seed_results) + return { + "schema_version": BENCHMARK_SCHEMA_VERSION, + "created_at": datetime.now(timezone.utc).isoformat(), + "checkpoint": { + "path": str(weights), + "sha256": checkpoint_sha256, + }, + "configuration": { + "seeds": normalized_seeds, + "samples_per_seed": int(samples_per_seed), + "total_samples": int(samples_per_seed) * len(normalized_seeds), + "data": { + "generator": "make_chained_task", + "kind": "deterministic_synthetic_chained_arithmetic", + "split": "held_out_rng_seeds", + "excluded_training_and_original_test_seeds": sorted( + TRAINING_AND_ORIGINAL_TEST_SEEDS + ), + }, + "cycles": resolved_cycles, + "confidence_target": normalized_confidence_target, + "entropy_target": normalized_entropy_target, + "adaptive_compute": bool(adaptive_compute), + "exit_tol": float(exit_tol), + "exit_entropy_threshold": float(exit_entropy_threshold), + "prediction_stability_patience": int( + prediction_stability_patience + ), + "prediction_stability_tol": float(prediction_stability_tol), + "distribution_top_k": int(distribution_top_k), + "counterbalance": ( + "alternate_per_request_with_alternating_seed_offset" + ), + "latency_is_release_gate": False, + "mutual_stability_role": "shadow_diagnostic_only", + }, + "provenance": dict(provenance) + if provenance is not None + else collect_provenance(device=device, device_info=device_info), + **aggregate, + "seed_results": seed_results, + } + + +def _strict_json(payload: Mapping[str, Any]) -> str: + return json.dumps( + payload, + indent=2, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Compare progressive accepted-probe auto compute with legacy v1 " + "on deterministic held-out v51 requests." + ) + ) + parser.add_argument("--weights", default=str(DEFAULT_WEIGHTS)) + parser.add_argument("--output", default=str(DEFAULT_OUTPUT)) + parser.add_argument("--seeds", nargs="+", type=int, default=list(DEFAULT_SEEDS)) + parser.add_argument( + "--samples-per-seed", + "--samples_per_seed", + dest="samples_per_seed", + type=int, + default=DEFAULT_SAMPLES_PER_SEED, + ) + parser.add_argument("--cycles", nargs="+", type=int, default=list(DEFAULT_CYCLES)) + parser.add_argument( + "--confidence-target", + "--confidence_target", + type=float, + default=DEFAULT_AUTO_COMPUTE_CONFIDENCE, + ) + parser.add_argument( + "--entropy-target", + "--entropy_target", + type=float, + default=DEFAULT_AUTO_COMPUTE_ENTROPY, + ) + parser.add_argument("--adaptive-compute", action="store_true") + parser.add_argument( + "--exit-tol", type=float, default=DEFAULT_ADAPTIVE_EXIT_TOL + ) + parser.add_argument( + "--exit-entropy-threshold", + type=float, + default=DEFAULT_ADAPTIVE_EXIT_ENTROPY, + ) + parser.add_argument( + "--prediction-stability-patience", + type=int, + default=DEFAULT_PREDICTION_STABILITY_PATIENCE, + ) + parser.add_argument( + "--prediction-stability-tol", + type=float, + default=DEFAULT_PREDICTION_STABILITY_TOL, + ) + parser.add_argument( + "--distribution-top-k", + type=int, + default=DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + ) + parser.add_argument("--device", default="auto") + parser.add_argument( + "--device-preference", default="cuda,npu,xpu,dml,mps,cpu" + ) + parser.add_argument("--torch-num-threads", type=int, default=0) + parser.add_argument("--torch-interop-threads", type=int, default=0) + parser.add_argument("--strict-determinism", action="store_true") + parser.add_argument( + "--enforce-gates", + action="store_true", + help="Exit with status 2 unless every exact-agreement gate passes.", + ) + return parser + + +def main(argv: Iterable[str] | None = None) -> int: + args = build_parser().parse_args(argv) + configure_torch_runtime( + torch_num_threads=int(args.torch_num_threads), + torch_interop_threads=int(args.torch_interop_threads), + strict_determinism=bool(args.strict_determinism), + ) + device, device_info = resolve_device( + args.device, preference=args.device_preference + ) + payload = run_benchmark( + weights=Path(args.weights), + seeds=args.seeds, + samples_per_seed=int(args.samples_per_seed), + device=device, + device_info=device_info, + cycles=args.cycles, + confidence_target=float(args.confidence_target), + entropy_target=float(args.entropy_target), + adaptive_compute=bool(args.adaptive_compute), + exit_tol=float(args.exit_tol), + exit_entropy_threshold=float(args.exit_entropy_threshold), + prediction_stability_patience=int(args.prediction_stability_patience), + prediction_stability_tol=float(args.prediction_stability_tol), + distribution_top_k=int(args.distribution_top_k), + ) + encoded = _strict_json(payload) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(encoded + "\n", encoding="utf-8") + sys.stdout.write(encoded + "\n") + if bool(args.enforce_gates) and not bool(payload["gates"]["passed"]): + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source/benchmark_v51_prediction_stability.py b/source/benchmark_v51_prediction_stability.py index d3887fcd4..fea1fab11 100644 --- a/source/benchmark_v51_prediction_stability.py +++ b/source/benchmark_v51_prediction_stability.py @@ -39,6 +39,7 @@ def _load_model(weights: Path, device: torch.device) -> ChampionNetCognitiveLeap def _latency_summary(values: List[float]) -> Dict[str, float]: tensor = torch.tensor(values, dtype=torch.float64) return { + "total_ms": round(float(tensor.sum().item()), 6), "mean_ms": round(float(tensor.mean().item()), 3), "p50_ms": round(float(torch.quantile(tensor, 0.50).item()), 3), "p95_ms": round(float(torch.quantile(tensor, 0.95).item()), 3), @@ -55,6 +56,18 @@ def _scalar_summary(values: List[float]) -> Dict[str, float]: } +def _synchronize_for_timing(device: torch.device) -> None: + """Finish queued accelerator work so request latency is measured honestly.""" + + device_type = str(device).split(":", 1)[0].lower() + if device_type == "cuda" and torch.cuda.is_available(): + torch.cuda.synchronize(device) + elif device_type == "xpu" and hasattr(torch, "xpu"): + torch.xpu.synchronize() # type: ignore[attr-defined] + elif device_type == "mps" and hasattr(torch, "mps"): + torch.mps.synchronize() + + @torch.no_grad() def benchmark_serving_requests( model: ChampionNetCognitiveLeapUltraExpert, @@ -66,6 +79,7 @@ def benchmark_serving_requests( stability_patience: int, stability_tol: float, distribution_top_k: int, + order_offset: int = 0, ) -> Dict[str, Any]: head = model.layers[10] fixed_predictions: List[int] = [] @@ -76,30 +90,42 @@ def benchmark_serving_requests( adaptive_latest_js_drift: List[float] = [] adaptive_max_js_drift: List[float] = [] exit_reasons: Counter[str] = Counter() + measurement_orders: Counter[str] = Counter() # Keep initialization and one-time kernel setup outside measured requests. - model(x[:1], reasoning_cycles=fixed_cycles) - model( - x[:1], - reasoning_cycles=max_cycles, - adaptive_compute=True, - exit_tol=0.0, - exit_entropy_threshold=0.0, - prediction_stability_patience=stability_patience, - prediction_stability_tol=stability_tol, - prediction_stability_top_k=distribution_top_k, - ) + # End the warmup on the opposite mode from the first measured request so + # both seed offsets receive the same cross-mode transition. + def warm_adaptive() -> None: + model( + x[:1], + reasoning_cycles=max_cycles, + adaptive_compute=True, + exit_tol=0.0, + exit_entropy_threshold=0.0, + prediction_stability_patience=stability_patience, + prediction_stability_tol=stability_tol, + prediction_stability_top_k=distribution_top_k, + ) - for index in range(len(y)): - sample = x[index : index + 1] + if int(order_offset) % 2 == 0: + model(x[:1], reasoning_cycles=fixed_cycles) + warm_adaptive() + else: + warm_adaptive() + model(x[:1], reasoning_cycles=fixed_cycles) + def run_fixed(sample: torch.Tensor) -> torch.Tensor: + _synchronize_for_timing(sample.device) started = time.perf_counter() - fixed_output = model(sample, reasoning_cycles=fixed_cycles) + output = model(sample, reasoning_cycles=fixed_cycles) + _synchronize_for_timing(sample.device) fixed_latencies.append((time.perf_counter() - started) * 1000.0) - fixed_predictions.append(int(fixed_output.argmax(dim=-1).item())) + return output + def run_adaptive(sample: torch.Tensor) -> torch.Tensor: + _synchronize_for_timing(sample.device) started = time.perf_counter() - adaptive_output = model( + output = model( sample, reasoning_cycles=max_cycles, adaptive_compute=True, @@ -109,8 +135,8 @@ def benchmark_serving_requests( prediction_stability_tol=stability_tol, prediction_stability_top_k=distribution_top_k, ) + _synchronize_for_timing(sample.device) adaptive_latencies.append((time.perf_counter() - started) * 1000.0) - adaptive_predictions.append(int(adaptive_output.argmax(dim=-1).item())) adaptive_cycles.append(float(head.last_cycles_used.item())) adaptive_latest_js_drift.append( float(head.last_prediction_topk_js_divergence.item()) @@ -119,13 +145,33 @@ def benchmark_serving_requests( float(head.last_prediction_topk_js_divergence_max.item()) ) exit_reasons[str(head.last_exit_reason)] += 1 + return output + + for index in range(len(y)): + sample = x[index : index + 1] + fixed_first = (index + int(order_offset)) % 2 == 0 + if fixed_first: + measurement_orders["fixed_then_adaptive"] += 1 + fixed_output = run_fixed(sample) + adaptive_output = run_adaptive(sample) + else: + measurement_orders["adaptive_then_fixed"] += 1 + adaptive_output = run_adaptive(sample) + fixed_output = run_fixed(sample) + + fixed_predictions.append(int(fixed_output.argmax(dim=-1).item())) + adaptive_predictions.append(int(adaptive_output.argmax(dim=-1).item())) truth = y.detach().cpu() fixed_tensor = torch.tensor(fixed_predictions) adaptive_tensor = torch.tensor(adaptive_predictions) - fixed_accuracy = float((fixed_tensor == truth).float().mean().item()) - adaptive_accuracy = float((adaptive_tensor == truth).float().mean().item()) - agreement = float((fixed_tensor == adaptive_tensor).float().mean().item()) + fixed_correct = int((fixed_tensor == truth).sum().item()) + adaptive_correct = int((adaptive_tensor == truth).sum().item()) + disagreement_count = int((fixed_tensor != adaptive_tensor).sum().item()) + request_count = int(len(y)) + fixed_accuracy = fixed_correct / max(1, request_count) + adaptive_accuracy = adaptive_correct / max(1, request_count) + agreement = 1.0 - (disagreement_count / max(1, request_count)) fixed_latency = _latency_summary(fixed_latencies) adaptive_latency = _latency_summary(adaptive_latencies) mean_cycles = sum(adaptive_cycles) / max(1, len(adaptive_cycles)) @@ -134,6 +180,7 @@ def benchmark_serving_requests( "fixed": { "cycles": int(fixed_cycles), "accuracy": round(fixed_accuracy, 6), + "correct_predictions": fixed_correct, "latency": fixed_latency, }, "prediction_stability": { @@ -141,7 +188,9 @@ def benchmark_serving_requests( "patience": int(stability_patience), "confidence_tolerance": float(stability_tol), "accuracy": round(adaptive_accuracy, 6), + "correct_predictions": adaptive_correct, "mean_cycles_used": round(mean_cycles, 4), + "total_cycles_used": round(sum(adaptive_cycles), 6), "cycle_counts": { str(cycle): count for cycle, count in sorted(Counter(adaptive_cycles).items()) @@ -159,6 +208,13 @@ def benchmark_serving_requests( "comparison": { "accuracy_delta": round(adaptive_accuracy - fixed_accuracy, 6), "prediction_agreement": round(agreement, 6), + "exact_disagreement_count": disagreement_count, + "request_count": request_count, + "measurement_order": { + "offset": int(order_offset), + "fixed_then_adaptive": measurement_orders["fixed_then_adaptive"], + "adaptive_then_fixed": measurement_orders["adaptive_then_fixed"], + }, "cycle_reduction_percent": round( 100.0 * (float(fixed_cycles) - mean_cycles) / max(1.0, float(fixed_cycles)), 3, @@ -184,6 +240,7 @@ def main() -> None: parser.add_argument("--stability_patience", type=int, default=2) parser.add_argument("--stability_tol", type=float, default=0.005) parser.add_argument("--distribution_top_k", type=int, default=5) + parser.add_argument("--order_offset", type=int, default=0) parser.add_argument("--device", default="auto") parser.add_argument("--device_preference", default="cuda,npu,xpu,dml,mps,cpu") parser.add_argument("--torch_num_threads", type=int, default=0) @@ -209,6 +266,7 @@ def main() -> None: stability_patience=max(0, int(args.stability_patience)), stability_tol=max(0.0, float(args.stability_tol)), distribution_top_k=max(1, int(args.distribution_top_k)), + order_offset=int(args.order_offset), ) payload = { "created_at": datetime.now(timezone.utc).isoformat(), diff --git a/source/chat_app.py b/source/chat_app.py index d5e431438..f089d7585 100644 --- a/source/chat_app.py +++ b/source/chat_app.py @@ -1,6 +1,7 @@ import argparse import inspect import json +import math import re import time from pathlib import Path @@ -74,6 +75,9 @@ def _int_or_default(value, default: int) -> int: DEFAULT_ADAPTIVE_EXIT_TOL = 1e-3 DEFAULT_AUTO_COMPUTE_CONFIDENCE = 0.55 DEFAULT_AUTO_COMPUTE_ENTROPY = 1.85 +AUTO_COMPUTE_PLAN_SCHEMA_VERSION = "runtime-auto-compute-plan-v2" +AUTO_COMPUTE_STRATEGY = "progressive_accepted_probe" +DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K = 5 def resolve_runtime_compute_cycles(cycles, default: Optional[List[int]] = None, limit: int = 8) -> List[int]: @@ -383,6 +387,8 @@ def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[fl parsed = float(value) except (TypeError, ValueError): parsed = float(default) + if not math.isfinite(parsed): + parsed = float(default) parsed = max(0.0, parsed) if max_value is not None: parsed = min(parsed, float(max_value)) @@ -585,6 +591,261 @@ def forward_with_runtime_compute( return output, diagnostics +def _topk_distribution_js_divergence( + previous: torch.Tensor, + current: torch.Tensor, + top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> float: + """Return midpoint-support JSD while preserving tail probability mass.""" + + left = previous.detach().to(dtype=torch.float64).reshape(-1) + right = current.detach().to(dtype=torch.float64).reshape(-1) + if left.numel() != right.numel() or left.numel() == 0: + return 0.0 + k = min(max(1, int(top_k)), int(left.numel())) + midpoint = 0.5 * (left + right) + indices = torch.topk(midpoint, k=k).indices + left_top = left.index_select(0, indices) + right_top = right.index_select(0, indices) + left_compact = torch.cat( + [left_top, (1.0 - left_top.sum()).clamp_min(0.0).reshape(1)] + ) + right_compact = torch.cat( + [right_top, (1.0 - right_top.sum()).clamp_min(0.0).reshape(1)] + ) + compact_midpoint = 0.5 * (left_compact + right_compact) + epsilon = torch.finfo(left_compact.dtype).tiny + + def _kl(source: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return ( + source + * ( + torch.log(source.clamp_min(epsilon)) + - torch.log(target.clamp_min(epsilon)) + ) + ).sum() + + value = 0.5 * ( + _kl(left_compact, compact_midpoint) + + _kl(right_compact, compact_midpoint) + ) + return round(max(0.0, float(value.item())), 10) + + +def progressive_auto_compute_forward( + model: Any, + x: torch.Tensor, + available_labels: List[int], + *, + cycles: Any = None, + confidence_target: Any = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: Any = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: Any = False, + exit_tol: Any = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: Any = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: Any = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, + auto_reasoning_context: str = "", + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + """Evaluate auto-compute budgets progressively and reuse the accepted probe. + + This preserves the v1 selection policy and row order exactly: the first + confidence/entropy target wins, otherwise the best-confidence row wins + after the full ladder. Probes use the complete current runtime-control set, + and the accepted probe output is returned directly, removing the old N+1 + rerun. Cross-budget persistence and JSD are telemetry only and never select + or reject an output. + """ + + if bool(getattr(model, "training", False)): + raise RuntimeError("progressive auto compute requires model.eval()") + labels = list(available_labels) or list(range(MODEL_CLASSES)) + resolved_cycles = resolve_runtime_compute_cycles(cycles) + confidence_target = min( + 1.0, + _coerce_nonnegative_float( + confidence_target, + DEFAULT_AUTO_COMPUTE_CONFIDENCE, + ), + ) + entropy_target = _coerce_nonnegative_float( + entropy_target, + DEFAULT_AUTO_COMPUTE_ENTROPY, + ) + adaptive = _coerce_bool(adaptive_compute) + idx = torch.tensor(labels, dtype=torch.long, device=x.device) + records: List[Dict[str, Any]] = [] + cached_outputs: List[torch.Tensor] = [] + previous_probs: Optional[torch.Tensor] = None + previous_label: Optional[int] = None + previous_confidence: Optional[float] = None + accepted_index: Optional[int] = None + reason = "best_confidence" + + for requested_cycles in resolved_cycles: + started = time.perf_counter() + with torch.no_grad(): + model_out, compute = forward_with_runtime_compute( + model, + x, + reasoning_cycles=requested_cycles, + adaptive_compute=adaptive, + exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + auto_reasoning_context=auto_reasoning_context, + ) + logits = model_out[0, 0] + available_logits = logits.index_select(0, idx.to(logits.device)) + probs = torch.softmax(available_logits, dim=0) + confidence_tensor, predicted_position = torch.max(probs, dim=0) + entropy_tensor = -(probs * torch.log(probs.clamp_min(1e-8))).sum() + # v1 serialized these values to six decimals before the selector read + # them. Keep that boundary exact so values such as 0.5499996 still + # satisfy a 0.55 target just as they did before. + confidence = round(float(confidence_tensor.item()), 6) + entropy = round(float(entropy_tensor.item()), 6) + predicted_label = int(labels[int(predicted_position.item())]) + # Freeze selection latency before shadow-only diagnostics. Otherwise a + # costly JSD calculation could alter the legacy fallback tie-break. + probe_latency_ms = round((time.perf_counter() - started) * 1000.0, 1) + has_previous = previous_probs is not None + shadow = { + "role": "shadow_diagnostic_only", + "metric": "midpoint_support_top_k_jensen_shannon_divergence_nats", + "top_k": min(max(1, int(distribution_top_k)), len(labels)), + "has_previous_probe": has_previous, + "top1_persistent": ( + bool(previous_label == predicted_label) if has_previous else None + ), + "confidence_delta": ( + round(abs(confidence - float(previous_confidence)), 8) + if previous_confidence is not None + else None + ), + "js_divergence": ( + _topk_distribution_js_divergence( + previous_probs, + probs, + distribution_top_k, + ) + if previous_probs is not None + else None + ), + "selection_enabled": False, + } + row = { + "requested_cycles": int(requested_cycles), + "latency_ms": probe_latency_ms, + "cycles_used": compute.get("cycles_used"), + "predicted_label": predicted_label, + "confidence": confidence, + "entropy": entropy, + "confidence_target_met": bool(confidence >= confidence_target), + "entropy_target_met": bool(entropy <= entropy_target), + "mutual_stability_shadow": shadow, + "compute": compute, + } + records.append(row) + cached_outputs.append(model_out) + + if confidence >= confidence_target: + accepted_index = len(records) - 1 + reason = "confidence_target" + break + if entropy <= entropy_target: + accepted_index = len(records) - 1 + reason = "entropy_target" + break + previous_probs = probs.detach() + previous_label = predicted_label + previous_confidence = confidence + + if accepted_index is None: + accepted_index = max( + range(len(records)), + key=lambda index: ( + float(records[index].get("confidence", 0.0)), + -float(records[index].get("entropy", 999.0)), + -float(records[index].get("latency_ms", 0.0)), + ), + ) + + selected_row = records[accepted_index] + selected_output = cached_outputs[accepted_index] + selected_compute = dict(selected_row["compute"]) + evaluated_cycles = [int(row["requested_cycles"]) for row in records] + skipped_cycles = [ + int(value) for value in resolved_cycles[len(evaluated_cycles) :] + ] + legacy_forward_evaluations = len(resolved_cycles) + 1 + forward_evaluations = len(records) + plan = { + "schema_version": AUTO_COMPUTE_PLAN_SCHEMA_VERSION, + "enabled": True, + "strategy": AUTO_COMPUTE_STRATEGY, + "selected_reasoning_cycles": int(selected_row["requested_cycles"]), + "selected_index": int(accepted_index), + "accepted_probe_index": int(accepted_index), + "reason": reason, + "confidence_target": confidence_target, + "entropy_target": entropy_target, + "candidate_cycles": [int(value) for value in resolved_cycles], + "evaluated_cycles": evaluated_cycles, + "skipped_cycles": skipped_cycles, + "forward_evaluations": forward_evaluations, + "legacy_forward_evaluations": legacy_forward_evaluations, + "forward_reduction_percent": round( + 100.0 + * (legacy_forward_evaluations - forward_evaluations) + / max(1, legacy_forward_evaluations), + 3, + ), + "reused_probe_output": True, + "selection_semantics": "legacy_v1_selection_policy", + "probe_control_scope": "full_runtime_controls_v2", + "mutual_stability_role": "shadow_diagnostic_only", + "rows": records, + } + selected_compute["auto_compute_plan"] = plan + selected_compute["inference_reused"] = True + return selected_output, selected_compute, plan + + +def compact_auto_compute_plan_metrics(plan: Any) -> Dict[str, Any]: + """Return terminal-safe metrics for the probe that supplied the output.""" + + if not isinstance(plan, dict): + return {} + rows = plan.get("rows") if isinstance(plan.get("rows"), list) else [] + try: + selected_index = int(plan.get("selected_index")) + except (TypeError, ValueError): + selected_index = -1 + if not 0 <= selected_index < len(rows): + selected_index = len(rows) - 1 + selected_row = rows[selected_index] if selected_index >= 0 else {} + shadow = ( + selected_row.get("mutual_stability_shadow", {}) + if isinstance(selected_row, dict) + else {} + ) + metrics: Dict[str, Any] = { + "auto_selected": plan.get("selected_reasoning_cycles"), + "auto_reason": plan.get("reason"), + "auto_forwards": ( + f"{plan.get('forward_evaluations')}/" + f"{plan.get('legacy_forward_evaluations')}" + ), + "auto_reused": bool(plan.get("reused_probe_output")), + } + if isinstance(shadow, dict) and shadow.get("js_divergence") is not None: + metrics["auto_shadow_js"] = shadow.get("js_divergence") + return metrics + + def _default_expansion_dim_for_model_size(model_size: str) -> int: if model_size in THIRD_EXPANSION_DIM_MODEL_SIZES: return 1024 @@ -1028,8 +1289,7 @@ def main(): compact_compute = dict(last_compute_metrics) plan = compact_compute.pop("auto_compute_plan", None) if isinstance(plan, dict): - compact_compute["auto_selected"] = plan.get("selected_reasoning_cycles") - compact_compute["auto_reason"] = plan.get("reason") + compact_compute.update(compact_auto_compute_plan_metrics(plan)) print( " last_compute=" + ", ".join(f"{k}={v}" for k, v in compact_compute.items()) @@ -1244,25 +1504,13 @@ def main(): effective_reasoning_cycles = args.reasoning_cycles auto_compute_plan = None if args.auto_compute and model_supports_runtime_compute(model): - auto_rows = evaluate_runtime_compute_budgets( + model_out, last_compute_metrics, auto_compute_plan = progressive_auto_compute_forward( model, x, available_labels, cycles=runtime_auto_compute_cycles(args.reasoning_cycles), - adaptive_compute=args.adaptive_compute, - exit_tol=args.adaptive_exit_tol, - ) - auto_compute_plan = select_auto_runtime_compute_budget( - auto_rows, confidence_target=args.auto_compute_confidence, entropy_target=args.auto_compute_entropy, - ) - effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") - with torch.no_grad(): - model_out, last_compute_metrics = forward_with_runtime_compute( - model, - x, - reasoning_cycles=effective_reasoning_cycles, adaptive_compute=args.adaptive_compute, exit_tol=args.adaptive_exit_tol, exit_entropy_threshold=args.adaptive_exit_entropy, @@ -1270,9 +1518,21 @@ def main(): prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, ) - logits = model_out[0, 0] # (10,) - if auto_compute_plan is not None: - last_compute_metrics["auto_compute_plan"] = auto_compute_plan + effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") + else: + with torch.no_grad(): + model_out, last_compute_metrics = forward_with_runtime_compute( + model, + x, + reasoning_cycles=effective_reasoning_cycles, + adaptive_compute=args.adaptive_compute, + exit_tol=args.adaptive_exit_tol, + exit_entropy_threshold=args.adaptive_exit_entropy, + prediction_stability_patience=args.prediction_stability_patience, + prediction_stability_tol=args.prediction_stability_tol, + auto_reasoning_context=context, + ) + logits = model_out[0, 0] # (10,) t_infer += max(0.0, time.perf_counter() - _t) idx = torch.tensor(available_labels, dtype=torch.long, device=logits.device) @@ -1424,9 +1684,9 @@ def main(): plan = last_compute_metrics.get("auto_compute_plan") if isinstance(last_compute_metrics, dict) else None plan_suffix = "" if isinstance(plan, dict): - plan_suffix = ( - f" auto_selected={plan.get('selected_reasoning_cycles')} " - f"reason={plan.get('reason')}" + plan_metrics = compact_auto_compute_plan_metrics(plan) + plan_suffix = " " + " ".join( + f"{key}={value}" for key, value in plan_metrics.items() ) print( f"{TerminalColors.SYSTEM}Compute:{TerminalColors.RESET} " diff --git a/source/chat_web_app.py b/source/chat_web_app.py index f212a12e5..d420541aa 100644 --- a/source/chat_web_app.py +++ b/source/chat_web_app.py @@ -101,7 +101,7 @@
-
+
Loading status...
Shadow registry - read only
Not loaded. Execution, activation, and promotion are unavailable.
@@ -125,7 +125,7 @@ function fmtNum(value,digits=3){const n=Number(value);return Number.isFinite(n)?n.toFixed(digits):null;} function timingText(t){if(!t)return'';let s=`${t.total??'?'} ms total - ${t.infer??'?'} ms infer - ${t.rank_pick??'?'} ms rank`;if(t.cycles_used!==undefined&&t.cycles_used!==null){s+=` - cycles ${t.cycles_used}`;}return s;} function reasoningCyclesValue(){const raw=els.reasoningCycles.value.trim();if(!raw)return null;const low=raw.toLowerCase();if(['auto','adaptive','smart'].includes(low))return 'auto';const n=Number(raw);return Number.isFinite(n)?n:raw;} -function computeText(compute){if(!compute||!compute.applied)return'';const parts=[];if(compute.reasoning_budget_mode==='auto'){parts.push('mode auto');}if(compute.requested_reasoning_cycles!==undefined&&compute.requested_reasoning_cycles!==null){parts.push(`requested ${compute.requested_reasoning_cycles}`);}if(compute.cycles_used!==undefined&&compute.cycles_used!==null){parts.push(`used ${compute.cycles_used}`);}if(compute.exit_reason){parts.push(`exit ${compute.exit_reason}`);}const streak=fmtNum(compute.prediction_streak);if(streak){parts.push(`stable ${streak}`);}const drift=fmtNum(compute.prediction_confidence_delta);if(drift){parts.push(`drift ${drift}`);}const ponder=fmtNum(compute.ponder_cost);if(ponder){parts.push(`ponder ${ponder}`);}const consistency=fmtNum(compute.consistency_loss);if(consistency){parts.push(`consistency ${consistency}`);}const entropy=fmtNum(compute.gating_entropy);if(entropy){parts.push(`gate entropy ${entropy}`);}const exitEntropy=fmtNum(compute.exit_entropy_threshold);if(exitEntropy){parts.push(`exit entropy ${exitEntropy}`);}if(compute.auto_reasoning_policy&&Array.isArray(compute.auto_reasoning_policy.reasons)){parts.push(`policy ${compute.auto_reasoning_policy.reasons.slice(0,3).join(',')}`);}if(compute.auto_compute_plan){parts.push(`budget ${compute.auto_compute_plan.selected_reasoning_cycles} (${compute.auto_compute_plan.reason})`);}return parts.length?`Compute: ${parts.join(' - ')}`:'';} +function computeText(compute){if(!compute||!compute.applied)return'';const parts=[];if(compute.reasoning_budget_mode==='auto'){parts.push('mode auto');}if(compute.requested_reasoning_cycles!==undefined&&compute.requested_reasoning_cycles!==null){parts.push(`requested ${compute.requested_reasoning_cycles}`);}if(compute.cycles_used!==undefined&&compute.cycles_used!==null){parts.push(`used ${compute.cycles_used}`);}if(compute.exit_reason){parts.push(`exit ${compute.exit_reason}`);}const streak=fmtNum(compute.prediction_streak);if(streak){parts.push(`stable ${streak}`);}const drift=fmtNum(compute.prediction_confidence_delta);if(drift){parts.push(`drift ${drift}`);}const ponder=fmtNum(compute.ponder_cost);if(ponder){parts.push(`ponder ${ponder}`);}const consistency=fmtNum(compute.consistency_loss);if(consistency){parts.push(`consistency ${consistency}`);}const entropy=fmtNum(compute.gating_entropy);if(entropy){parts.push(`gate entropy ${entropy}`);}const exitEntropy=fmtNum(compute.exit_entropy_threshold);if(exitEntropy){parts.push(`exit entropy ${exitEntropy}`);}if(compute.auto_reasoning_policy&&Array.isArray(compute.auto_reasoning_policy.reasons)){parts.push(`policy ${compute.auto_reasoning_policy.reasons.slice(0,3).join(',')}`);}if(compute.auto_compute_plan){const plan=compute.auto_compute_plan;parts.push(`budget ${plan.selected_reasoning_cycles} (${plan.reason})`);parts.push(`${plan.forward_evaluations}/${plan.legacy_forward_evaluations} forwards`);if(plan.reused_probe_output)parts.push('probe reused');const rows=Array.isArray(plan.rows)?plan.rows:[];const selectedIndex=Number(plan.selected_index);const selectedRow=Number.isInteger(selectedIndex)&&selectedIndex>=0&&selectedIndex{try{await navigator.clipboard.writeText(text);copy.textContent='Copied';setTimeout(()=>{copy.textContent='Copy';},1200);}catch(_){copy.textContent='Failed';}};who.appendChild(copy);}const body=document.createElement('div');body.textContent=text;card.appendChild(who);card.appendChild(body);const tt=timingText(timing);if(tt){const node=document.createElement('div');node.className='tim';node.textContent=tt;card.appendChild(node);}const ct=computeText(compute);if(ct){const node=document.createElement('div');node.className='tim';node.textContent=ct;card.appendChild(node);}if(Array.isArray(top)&&top.length){const details=document.createElement('details');const summary=document.createElement('summary');summary.textContent=`Top candidates (${top.length})`;details.appendChild(summary);top.forEach((candidate,index)=>{const row=document.createElement('div');const score=Number(candidate.score);const scoreText=Number.isFinite(score)?score.toFixed(3):'n/a';row.textContent=`${index+1}. (${scoreText}) ${String(candidate.text||'').slice(0,220)}`;details.appendChild(row);});card.appendChild(details);}els.msgs.appendChild(card);els.msgs.scrollTo({top:els.msgs.scrollHeight,behavior:'smooth'});if(persist){transcript.push({kind,text,timing,top,compute,ts:Date.now()});saveTranscript();}return card;} async function jget(path){const r=await fetch(path);const d=await r.json();if(!r.ok||d.ok===false)throw new Error(d.error||`HTTP ${r.status}`);return d;} async function jpost(path,payload){const r=await fetch(path,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(payload||{})});const d=await r.json();if(!r.ok||d.ok===false)throw new Error(d.error||`HTTP ${r.status}`);return d;} @@ -535,34 +535,36 @@ def chat( else chat_app._coerce_bool(auto_compute, default=False) ) if auto_enabled and chat_app.model_supports_runtime_compute(model): - sweep_rows = self._run_compute_sweep_rows( - model=model, - x=x, - labels=labels, + model_out, compute_metrics, compute_plan = chat_app.progressive_auto_compute_forward( + model, + x, + labels, cycles=self._auto_compute_cycles(effective_reasoning_cycles), - adaptive=chat_app._coerce_bool(resolved_adaptive_compute, default=False), + adaptive_compute=resolved_adaptive_compute, exit_tol=chat_app._coerce_nonnegative_float( resolved_exit_tol, default=chat_app.DEFAULT_ADAPTIVE_EXIT_TOL, ), - ) - compute_plan = self._select_auto_compute_budget(sweep_rows) - effective_reasoning_cycles = compute_plan.get("selected_reasoning_cycles") - with torch.no_grad(): - model_out, compute_metrics = chat_app.forward_with_runtime_compute( - model, - x, - reasoning_cycles=effective_reasoning_cycles, - adaptive_compute=resolved_adaptive_compute, - exit_tol=resolved_exit_tol, exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), auto_reasoning_context=context, ) - logits = model_out[0, 0] - if compute_plan is not None: - compute_metrics["auto_compute_plan"] = compute_plan + effective_reasoning_cycles = compute_plan.get("selected_reasoning_cycles") + else: + with torch.no_grad(): + model_out, compute_metrics = chat_app.forward_with_runtime_compute( + model, + x, + reasoning_cycles=effective_reasoning_cycles, + adaptive_compute=resolved_adaptive_compute, + exit_tol=resolved_exit_tol, + exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), + prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), + prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + auto_reasoning_context=context, + ) + logits = model_out[0, 0] t_infer += time.perf_counter() - tt idx = torch.tensor(labels, dtype=torch.long, device=logits.device) diff --git a/source/generate_studio_runtime_manifest.py b/source/generate_studio_runtime_manifest.py index f2340ac70..3d3715d41 100644 --- a/source/generate_studio_runtime_manifest.py +++ b/source/generate_studio_runtime_manifest.py @@ -42,6 +42,11 @@ ) CONTRACT_CONSTANTS = { + "source/chat_app.py": ( + "AUTO_COMPUTE_PLAN_SCHEMA_VERSION", + "AUTO_COMPUTE_STRATEGY", + "DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K", + ), "source/route_policy_ledger.py": ( "LEDGER_SCHEMA_VERSION", "SUPPORT_SCHEMA_VERSION", @@ -212,6 +217,8 @@ def build_manifest(repo_root: Path, *, include_git: bool = False) -> Dict[str, A "route_shadow_authenticity_proof_available": False, "route_shadow_trusted_timestamp_available": False, "automatic_policy_promotion_available": False, + "progressive_auto_compute_accepted_probe_reuse": True, + "mutual_stability_shadow_can_select_output": False, }, } if include_git: diff --git a/source/multimodel_runtime.py b/source/multimodel_runtime.py index ff62d5f2e..f02b947c1 100644 --- a/source/multimodel_runtime.py +++ b/source/multimodel_runtime.py @@ -743,6 +743,7 @@ def chat(self, session_id: str, prompt: str, settings: Dict[str, Any]) -> ChatRe adaptive_exit_entropy=settings.get("adaptive_exit_entropy"), prediction_stability_patience=settings.get("prediction_stability_patience"), prediction_stability_tol=settings.get("prediction_stability_tol"), + auto_compute=settings.get("auto_compute"), ) return ChatResult( kind="text", diff --git a/source/requirements_runtime_interface.txt b/source/requirements_runtime_interface.txt index e9a783ed0..0d8b71f6c 100644 --- a/source/requirements_runtime_interface.txt +++ b/source/requirements_runtime_interface.txt @@ -1,5 +1,5 @@ torch -flask==2.2.5 +flask==3.1.3 Pillow accelerate diffusers diff --git a/source/run_v51_prediction_stability_gate.py b/source/run_v51_prediction_stability_gate.py new file mode 100644 index 000000000..cf53ff412 --- /dev/null +++ b/source/run_v51_prediction_stability_gate.py @@ -0,0 +1,533 @@ +"""Run the release gate for v51 full-output prediction stability. + +The default evaluation is 8 native seeds x 512 requests. Each seed alternates +the fixed/adaptive measurement order, and the starting order alternates across +seeds, to reduce request-order latency bias without pooling the two timings. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +from typing import Any, Callable, Dict, Iterable, List, Mapping, Sequence + +import torch + +from benchmark_cognitive_leap_ultra_v51 import make_chained_task +from benchmark_v51_prediction_stability import ( + DEFAULT_ARTIFACT_DIR, + DEFAULT_WEIGHTS, + _load_model, + benchmark_serving_requests, +) +from device_utils import configure_torch_runtime, resolve_device + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_OUTPUT = DEFAULT_ARTIFACT_DIR / "prediction_stability_gate.json" +DEFAULT_SEEDS = (641, 643, 647, 653, 659, 661, 673, 677) +DEFAULT_SAMPLES_PER_SEED = 512 +GATE_SCHEMA_VERSION = 1 +MINIMUM_WEIGHTED_CYCLE_REDUCTION_PERCENT = 20.0 +MINIMUM_MEDIAN_LATENCY_REDUCTION_PERCENT = 0.0 + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _git_text(*args: str) -> str | None: + try: + completed = subprocess.run( + ["git", *args], + cwd=PROJECT_ROOT, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + return completed.stdout.strip() + + +def collect_provenance( + *, + device: Any, + device_info: Mapping[str, Any], +) -> Dict[str, Any]: + """Collect enough immutable context to reproduce or reject a gate run.""" + + source_paths = ( + Path(__file__).resolve(), + PROJECT_ROOT / "source" / "benchmark_v51_prediction_stability.py", + PROJECT_ROOT / "source" / "benchmark_cognitive_leap_ultra_v51.py", + PROJECT_ROOT / "source" / "model_variants.py", + ) + source_hashes = { + path.relative_to(PROJECT_ROOT).as_posix(): _sha256_file(path) + for path in source_paths + } + worktree_status = _git_text("status", "--porcelain", "--untracked-files=all") + resolved_device = str(device_info.get("resolved", device)) + device_metadata: Dict[str, Any] = { + "requested": str(device_info.get("requested", "unknown")), + "resolved": resolved_device, + "torch_device": str(device), + } + if resolved_device.startswith("cuda") and torch.cuda.is_available(): + device_metadata["name"] = torch.cuda.get_device_name(device) + device_metadata["capability"] = list(torch.cuda.get_device_capability(device)) + + return { + "git": { + "commit": _git_text("rev-parse", "HEAD"), + "branch": _git_text("rev-parse", "--abbrev-ref", "HEAD"), + "worktree_dirty": bool(worktree_status), + }, + "source_sha256": source_hashes, + "runtime": { + "python": { + "version": platform.python_version(), + "implementation": platform.python_implementation(), + "executable": sys.executable, + "platform": platform.platform(), + }, + "torch": { + "version": str(torch.__version__), + "cuda_version": str(torch.version.cuda) if torch.version.cuda else None, + "default_dtype": str(torch.get_default_dtype()), + "deterministic_algorithms": bool( + torch.are_deterministic_algorithms_enabled() + ), + }, + "device": device_metadata, + "threads": { + "torch_num_threads": int(torch.get_num_threads()), + "torch_num_interop_threads": int(torch.get_num_interop_threads()), + "logical_cpu_count": int(os.cpu_count() or 1), + }, + }, + } + + +def _finite_float(value: Any, *, label: str) -> float: + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{label} must be finite, got {value!r}") + return number + + +def aggregate_gate_results( + seed_results: Sequence[Mapping[str, Any]], +) -> Dict[str, Any]: + """Aggregate exact counts and evaluate every release-gate invariant.""" + + if not seed_results: + raise ValueError("At least one seed result is required") + minimum_cycle_reduction = _finite_float( + MINIMUM_WEIGHTED_CYCLE_REDUCTION_PERCENT, + label="minimum cycle reduction percent", + ) + minimum_median_latency_reduction = _finite_float( + MINIMUM_MEDIAN_LATENCY_REDUCTION_PERCENT, + label="minimum median latency reduction percent", + ) + + per_seed: List[Dict[str, Any]] = [] + total_requests = 0 + total_disagreements = 0 + total_fixed_cycles = 0.0 + total_adaptive_cycles = 0.0 + latency_reductions: List[float] = [] + total_fixed_latency_ms = 0.0 + total_adaptive_latency_ms = 0.0 + seen_seeds: set[int] = set() + + for row in seed_results: + seed = int(row["seed"]) + if seed in seen_seeds: + raise ValueError(f"Duplicate seed result: {seed}") + seen_seeds.add(seed) + metrics = row["metrics"] + comparison = metrics["comparison"] + fixed = metrics["fixed"] + adaptive = metrics["prediction_stability"] + request_count = int(comparison["request_count"]) + if request_count <= 0: + raise ValueError(f"Seed {seed} has no requests") + + fixed_correct = int(fixed["correct_predictions"]) + adaptive_correct = int(adaptive["correct_predictions"]) + disagreement_count = int(comparison["exact_disagreement_count"]) + fixed_cycles = _finite_float(fixed["cycles"], label="fixed cycles") + adaptive_cycles = _finite_float( + adaptive["total_cycles_used"], label="adaptive cycles" + ) + if not 0 <= fixed_correct <= request_count: + raise ValueError(f"Seed {seed} fixed correct count is invalid") + if not 0 <= adaptive_correct <= request_count: + raise ValueError(f"Seed {seed} adaptive correct count is invalid") + if not 0 <= disagreement_count <= request_count: + raise ValueError(f"Seed {seed} disagreement count is invalid") + if fixed_cycles <= 0.0 or adaptive_cycles < 0.0: + raise ValueError(f"Seed {seed} cycle totals are invalid") + order = comparison["measurement_order"] + fixed_first = int(order["fixed_then_adaptive"]) + adaptive_first = int(order["adaptive_then_fixed"]) + offset = int(order["offset"]) + expected_fixed_first = ( + (request_count + 1) // 2 if offset % 2 == 0 else request_count // 2 + ) + if ( + fixed_first != expected_fixed_first + or adaptive_first != request_count - expected_fixed_first + ): + raise ValueError(f"Seed {seed} measurement order is not counterbalanced") + fixed_mean_latency = _finite_float( + fixed["latency"]["mean_ms"], label="fixed latency" + ) + adaptive_mean_latency = _finite_float( + adaptive["latency"]["mean_ms"], label="adaptive latency" + ) + fixed_total_latency = _finite_float( + fixed["latency"].get("total_ms", fixed_mean_latency * request_count), + label="fixed total latency", + ) + adaptive_total_latency = _finite_float( + adaptive["latency"].get( + "total_ms", adaptive_mean_latency * request_count + ), + label="adaptive total latency", + ) + if fixed_total_latency <= 0.0: + raise ValueError(f"Seed {seed} fixed latency must be positive") + if adaptive_total_latency < 0.0: + raise ValueError(f"Seed {seed} adaptive latency cannot be negative") + latency_reduction = ( + 100.0 + * (fixed_total_latency - adaptive_total_latency) + / fixed_total_latency + ) + seed_fixed_cycle_budget = fixed_cycles * request_count + cycle_reduction = ( + 100.0 + * (seed_fixed_cycle_budget - adaptive_cycles) + / seed_fixed_cycle_budget + ) + accuracy_delta = (adaptive_correct - fixed_correct) / request_count + + total_requests += request_count + total_disagreements += disagreement_count + total_fixed_cycles += fixed_cycles * request_count + total_adaptive_cycles += adaptive_cycles + total_fixed_latency_ms += fixed_total_latency + total_adaptive_latency_ms += adaptive_total_latency + latency_reductions.append(latency_reduction) + per_seed.append( + { + "seed": seed, + "samples": request_count, + "fixed_correct": fixed_correct, + "adaptive_correct": adaptive_correct, + "accuracy_delta": round(accuracy_delta, 12), + "exact_disagreement_count": disagreement_count, + "cycle_reduction_percent": round(cycle_reduction, 6), + "fixed_mean_latency_ms": round( + fixed_total_latency / request_count, 6 + ), + "adaptive_mean_latency_ms": round( + adaptive_total_latency / request_count, 6 + ), + "mean_latency_reduction_percent": round( + latency_reduction, 6 + ), + "measurement_order": comparison["measurement_order"], + } + ) + + if total_fixed_cycles <= 0.0: + raise ValueError("Total fixed-cycle budget must be positive") + if total_fixed_latency_ms <= 0.0: + raise ValueError("Total fixed latency must be positive") + + weighted_cycle_reduction = ( + 100.0 * (total_fixed_cycles - total_adaptive_cycles) / total_fixed_cycles + ) + weighted_latency_reduction = ( + 100.0 + * (total_fixed_latency_ms - total_adaptive_latency_ms) + / total_fixed_latency_ms + ) + weighted_fixed_mean_latency = total_fixed_latency_ms / total_requests + weighted_adaptive_mean_latency = total_adaptive_latency_ms / total_requests + median_seed_latency_reduction = float(statistics.median(latency_reductions)) + negative_accuracy_seeds = [ + row["seed"] for row in per_seed if row["adaptive_correct"] < row["fixed_correct"] + ] + + checks = { + "zero_prediction_disagreements": { + "passed": total_disagreements == 0, + "observed": total_disagreements, + "required": 0, + }, + "no_negative_per_seed_accuracy_delta": { + "passed": not negative_accuracy_seeds, + "negative_seed_ids": negative_accuracy_seeds, + "required": "adaptive_correct >= fixed_correct for every seed", + }, + "minimum_weighted_cycle_reduction": { + "passed": weighted_cycle_reduction + >= minimum_cycle_reduction, + "observed_percent": round(weighted_cycle_reduction, 6), + "required_minimum_percent": minimum_cycle_reduction, + }, + "positive_median_per_seed_latency_reduction": { + "passed": median_seed_latency_reduction + > minimum_median_latency_reduction, + "observed_percent": round(median_seed_latency_reduction, 6), + "required_greater_than_percent": float( + minimum_median_latency_reduction + ), + }, + } + + return { + "summary": { + "seed_count": len(seed_results), + "total_samples": total_requests, + "exact_disagreement_count": total_disagreements, + "weighted_cycle_reduction_percent": round( + weighted_cycle_reduction, 6 + ), + "median_per_seed_latency_reduction_percent": round( + median_seed_latency_reduction, 6 + ), + "weighted_mean_latency_reduction_percent": round( + weighted_latency_reduction, 6 + ), + "weighted_fixed_mean_latency_ms": round( + weighted_fixed_mean_latency, 6 + ), + "weighted_adaptive_mean_latency_ms": round( + weighted_adaptive_mean_latency, 6 + ), + "total_fixed_cycle_budget": round(total_fixed_cycles, 6), + "total_adaptive_cycles_used": round(total_adaptive_cycles, 6), + "per_seed_accuracy_deltas": [ + {"seed": row["seed"], "accuracy_delta": row["accuracy_delta"]} + for row in per_seed + ], + }, + "per_seed_gate_metrics": per_seed, + "gates": { + "passed": all(bool(check["passed"]) for check in checks.values()), + "checks": checks, + }, + } + + +def run_gate( + *, + weights: Path, + seeds: Sequence[int], + samples_per_seed: int, + device: Any, + device_info: Mapping[str, Any], + fixed_cycles: int = 3, + max_cycles: int = 8, + stability_patience: int = 2, + stability_tol: float = 0.005, + distribution_top_k: int = 5, + model_loader: Callable[[Path, Any], Any] | None = None, + task_factory: Callable[..., Any] | None = None, + benchmark_fn: Callable[..., Dict[str, Any]] | None = None, + provenance: Mapping[str, Any] | None = None, +) -> Dict[str, Any]: + """Load one checkpoint, run every native seed, and evaluate the gate.""" + + weights = Path(weights).resolve() + if not weights.is_file(): + raise FileNotFoundError(f"Missing v51 checkpoint: {weights}") + normalized_seeds = [int(seed) for seed in seeds] + if not normalized_seeds: + raise ValueError("At least one seed is required") + if len(set(normalized_seeds)) != len(normalized_seeds): + raise ValueError("Seeds must be unique") + if int(samples_per_seed) <= 0: + raise ValueError("samples_per_seed must be positive") + normalized_fixed_cycles = int(fixed_cycles) + normalized_max_cycles = int(max_cycles) + normalized_patience = int(stability_patience) + normalized_tolerance = _finite_float( + stability_tol, label="stability tolerance" + ) + normalized_top_k = int(distribution_top_k) + if normalized_fixed_cycles <= 0 or normalized_max_cycles <= 0: + raise ValueError("fixed_cycles and max_cycles must be positive") + if normalized_patience < 0: + raise ValueError("stability_patience must be nonnegative") + if normalized_tolerance < 0.0: + raise ValueError("stability_tol must be nonnegative") + if normalized_top_k <= 0: + raise ValueError("distribution_top_k must be positive") + + loader = model_loader or _load_model + make_task = task_factory or make_chained_task + benchmark = benchmark_fn or benchmark_serving_requests + checkpoint_sha256 = _sha256_file(weights) + model = loader(weights, device) + seed_results: List[Dict[str, Any]] = [] + + for seed_index, seed in enumerate(normalized_seeds): + x, y = make_task(int(samples_per_seed), seed=seed) + x = x.to(device) + y = y.to(device) + metrics = benchmark( + model, + x, + y, + fixed_cycles=normalized_fixed_cycles, + max_cycles=normalized_max_cycles, + stability_patience=normalized_patience, + stability_tol=normalized_tolerance, + distribution_top_k=normalized_top_k, + order_offset=seed_index % 2, + ) + seed_results.append({"seed": seed, "metrics": metrics}) + + aggregate = aggregate_gate_results(seed_results) + return { + "schema_version": GATE_SCHEMA_VERSION, + "created_at": datetime.now(timezone.utc).isoformat(), + "checkpoint": { + "path": str(weights), + "sha256": checkpoint_sha256, + }, + "configuration": { + "seeds": normalized_seeds, + "samples_per_seed": int(samples_per_seed), + "total_samples": int(samples_per_seed) * len(normalized_seeds), + "fixed_cycles": normalized_fixed_cycles, + "max_cycles": normalized_max_cycles, + "stability_patience": normalized_patience, + "stability_tolerance": normalized_tolerance, + "distribution_top_k": normalized_top_k, + "counterbalance": "alternate_per_request_with_alternating_seed_offset", + }, + "provenance": dict(provenance) + if provenance is not None + else collect_provenance(device=device, device_info=device_info), + **aggregate, + "seed_results": seed_results, + } + + +def _strict_json(payload: Mapping[str, Any]) -> str: + return json.dumps( + payload, + indent=2, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run the v51 multi-seed prediction-stability release gate." + ) + parser.add_argument("--weights", default=str(DEFAULT_WEIGHTS)) + parser.add_argument("--output", default=str(DEFAULT_OUTPUT)) + parser.add_argument("--seeds", type=int, nargs="+", default=list(DEFAULT_SEEDS)) + parser.add_argument( + "--samples-per-seed", + "--samples_per_seed", + dest="samples_per_seed", + type=int, + default=DEFAULT_SAMPLES_PER_SEED, + ) + parser.add_argument("--fixed-cycles", "--fixed_cycles", type=int, default=3) + parser.add_argument("--max-cycles", "--max_cycles", type=int, default=8) + parser.add_argument( + "--stability-patience", "--stability_patience", type=int, default=2 + ) + parser.add_argument( + "--stability-tol", "--stability_tol", type=float, default=0.005 + ) + parser.add_argument( + "--distribution-top-k", "--distribution_top_k", type=int, default=5 + ) + parser.add_argument("--device", default="auto") + parser.add_argument( + "--device-preference", + "--device_preference", + default="cuda,npu,xpu,dml,mps,cpu", + ) + parser.add_argument( + "--torch-num-threads", "--torch_num_threads", type=int, default=0 + ) + parser.add_argument( + "--torch-interop-threads", "--torch_interop_threads", type=int, default=0 + ) + parser.add_argument( + "--strict-determinism", + action="store_true", + help="Ask torch to use deterministic algorithms where available.", + ) + parser.add_argument( + "--enforce-gates", + action="store_true", + help="Exit with status 2 when any release gate fails.", + ) + return parser + + +def main(argv: Iterable[str] | None = None) -> int: + args = build_parser().parse_args(argv) + configure_torch_runtime( + torch_num_threads=int(args.torch_num_threads), + torch_interop_threads=int(args.torch_interop_threads), + strict_determinism=bool(args.strict_determinism), + ) + device, device_info = resolve_device( + args.device, preference=args.device_preference + ) + payload = run_gate( + weights=Path(args.weights), + seeds=args.seeds, + samples_per_seed=int(args.samples_per_seed), + device=device, + device_info=device_info, + fixed_cycles=int(args.fixed_cycles), + max_cycles=int(args.max_cycles), + stability_patience=int(args.stability_patience), + stability_tol=float(args.stability_tol), + distribution_top_k=int(args.distribution_top_k), + ) + encoded = _strict_json(payload) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(encoded + "\n", encoding="utf-8") + sys.stdout.write(encoded + "\n") + if bool(args.enforce_gates) and not bool(payload["gates"]["passed"]): + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source/studio_runtime_manifest.json b/source/studio_runtime_manifest.json index ba58bdbfe..614418372 100644 --- a/source/studio_runtime_manifest.json +++ b/source/studio_runtime_manifest.json @@ -2,6 +2,11 @@ "active_runtime_tree": "source", "app_version": "2026.07.18", "contracts": { + "source/chat_app.py": { + "AUTO_COMPUTE_PLAN_SCHEMA_VERSION": "runtime-auto-compute-plan-v2", + "AUTO_COMPUTE_STRATEGY": "progressive_accepted_probe", + "DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K": 5 + }, "source/route_policy_explorer.py": { "REHEARSAL_RECEIPT_SCHEMA_VERSION": "route-exploration-rehearsal-receipt-v1", "REHEARSAL_SUPPORT_PROPOSAL_SCHEMA_VERSION": "route-exploration-rehearsal-support-proposal-v1", @@ -70,13 +75,13 @@ "modules": [ { "path": "source/chat_app.py", - "sha256": "c09e304356673cb49678ffbba4a53b00984659e911dcbfedef3c437aad07a3a7", - "size_bytes": 63948 + "sha256": "d90ee95c0a1b94a0c70b26e082f8106060dbab9a259b1ba1aab479c6b7237644", + "size_bytes": 74732 }, { "path": "source/chat_web_app.py", - "sha256": "0750072c52b10616cbd5c7c20e0ed8e335f54804fd863a47fd5824431f2a76ca", - "size_bytes": 62635 + "sha256": "6a55a2a103c4c7f0ec3a2d6549f073ef51dbac2bbe0ec6f9383876d97353e487", + "size_bytes": 63727 }, { "path": "source/model_variants.py", @@ -125,13 +130,13 @@ }, { "path": "source/multimodel_runtime.py", - "sha256": "0a7eb12fc60494f1b96b763cb98619d75d2b340d903c9abb861683cdd3046f7d", - "size_bytes": 263936 + "sha256": "f04a5710f936943bb777e10f1c988874bc377fd345c24011c0c766d068b6cfa1", + "size_bytes": 263991 }, { "path": "source/supermix_multimodel_web_app.py", - "sha256": "f73bd68afa75db328f688066471dd7b1c990047e32257c016cac31237e3ec0c3", - "size_bytes": 159501 + "sha256": "c9b58ad9cd7f5dce184a294d9e34fbbba6969986a98a2b760be97ada9f2cba8f", + "size_bytes": 161182 }, { "path": "source/supermix_multimodel_desktop_app.py", @@ -140,13 +145,13 @@ }, { "path": "runtime_python/chat_app.py", - "sha256": "c09e304356673cb49678ffbba4a53b00984659e911dcbfedef3c437aad07a3a7", - "size_bytes": 63948 + "sha256": "d90ee95c0a1b94a0c70b26e082f8106060dbab9a259b1ba1aab479c6b7237644", + "size_bytes": 74732 }, { "path": "runtime_python/chat_web_app.py", - "sha256": "6ee741f32ffed9e6818eebc33558f777b445415370fd783d1080404e24309122", - "size_bytes": 54592 + "sha256": "e16573872328ed4f07825a0156f461b951526e8b2f1b360d44a554fb6b8306b8", + "size_bytes": 55826 }, { "path": "runtime_python/chat_pipeline.py", @@ -181,6 +186,8 @@ ], "package_guards": { "automatic_policy_promotion_available": false, + "mutual_stability_shadow_can_select_output": false, + "progressive_auto_compute_accepted_probe_reuse": true, "route_ledger_requires_executed_assignment_namespace": true, "route_ledger_requires_issued_execution_assignment_record": true, "route_protocol_activation_available": false, diff --git a/source/supermix_multimodel_web_app.py b/source/supermix_multimodel_web_app.py index 004fd942b..752c1ab52 100644 --- a/source/supermix_multimodel_web_app.py +++ b/source/supermix_multimodel_web_app.py @@ -857,6 +857,14 @@ def _benchmark_rows_for_ui(data: Dict[str, Any]) -> List[Dict[str, Any]]:
+
+ + +
@@ -1416,6 +1424,23 @@ def _benchmark_rows_for_ui(data: Dict[str, Any]) -> List[Dict[str, Any]]: if (compute.prediction_confidence_delta !== undefined && compute.prediction_confidence_delta !== null) { bits.push(`Prediction drift ${escHtml(compute.prediction_confidence_delta)}`); } + const plan = compute.auto_compute_plan; + if (plan && typeof plan === 'object') { + bits.push(`Progressive ${escHtml(plan.selected_reasoning_cycles)} cycles`); + bits.push(`${escHtml(plan.forward_evaluations)}/${escHtml(plan.legacy_forward_evaluations)} forwards`); + if (plan.reused_probe_output) { + bits.push('Accepted probe reused'); + } + const rows = Array.isArray(plan.rows) ? plan.rows : []; + const selectedIndex = Number(plan.selected_index); + const selectedRow = Number.isInteger(selectedIndex) && selectedIndex >= 0 && selectedIndex < rows.length + ? rows[selectedIndex] + : (rows.length ? rows[rows.length - 1] : null); + const shadow = selectedRow ? selectedRow.mutual_stability_shadow : null; + if (shadow && shadow.js_divergence !== undefined && shadow.js_divergence !== null) { + bits.push(`Shadow JSD ${escHtml(shadow.js_divergence)}`); + } + } if (compute.applied === false && compute.supported === false) { bits.push('Compute controls unsupported'); } @@ -2240,8 +2265,10 @@ def _benchmark_rows_for_ui(data: Dict[str, Any]) -> List[Dict[str, Any]]: }; const requestedCycles = el('reasoningCycles').value; const requestedAdaptive = el('adaptiveCompute').value; + const requestedProgressive = el('progressiveAutoCompute').value; if (requestedCycles !== 'model') settings.reasoning_cycles = requestedCycles; if (requestedAdaptive !== 'model') settings.adaptive_compute = requestedAdaptive === 'on'; + if (requestedProgressive !== 'model') settings.auto_compute = requestedProgressive === 'on'; return { session_id: sessionId, message: text, diff --git a/test_multimodel_runtime.py b/test_multimodel_runtime.py index a37ac4816..b274b2ee6 100644 --- a/test_multimodel_runtime.py +++ b/test_multimodel_runtime.py @@ -78,6 +78,7 @@ def chat(self, **kwargs): "adaptive_exit_entropy": 0.25, "prediction_stability_patience": 3, "prediction_stability_tol": 0.01, + "auto_compute": True, }, ) @@ -87,6 +88,7 @@ def chat(self, **kwargs): assert captured["adaptive_exit_entropy"] == 0.25 assert captured["prediction_stability_patience"] == 3 assert captured["prediction_stability_tol"] == 0.01 + assert captured["auto_compute"] is True assert result.compute == { "applied": True, "cycles_used": 5.0, diff --git a/test_progressive_auto_compute_benchmark.py b/test_progressive_auto_compute_benchmark.py new file mode 100644 index 000000000..900de3bca --- /dev/null +++ b/test_progressive_auto_compute_benchmark.py @@ -0,0 +1,416 @@ +import hashlib +import json +import os +from pathlib import Path +import sys + +import pytest +import torch + + +sys.path.insert(0, os.path.join(os.getcwd(), "source")) + +import benchmark_progressive_auto_compute as benchmark + + +class _CycleModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.calls = [] + + def forward( + self, + x, + *, + reasoning_cycles=3, + adaptive_compute=False, + exit_tol=0.001, + exit_entropy_threshold=0.2, + prediction_stability_patience=2, + prediction_stability_tol=0.005, + ): + del ( + adaptive_compute, + exit_tol, + exit_entropy_threshold, + prediction_stability_patience, + prediction_stability_tol, + ) + cycle = int(reasoning_cycles) + self.calls.append(cycle) + logits = torch.zeros(len(x), 1, 10, dtype=x.dtype, device=x.device) + logits[:, :, 0] = {1: 0.0, 3: 2.0, 8: 3.0}[cycle] + return logits + + +class _BareModel: + training = False + + def eval(self): + return self + + +def _shadow_plan(*, cycles=3, forward_evaluations=2): + return { + "selected_reasoning_cycles": cycles, + "forward_evaluations": forward_evaluations, + "mutual_stability_role": "shadow_diagnostic_only", + "rows": [ + { + "mutual_stability_shadow": { + "role": "shadow_diagnostic_only", + "selection_enabled": False, + } + } + ], + } + + +def _legacy_result(output, *, cycles=3, forward_evaluations=4): + return ( + output, + {}, + { + "selected_reasoning_cycles": cycles, + "forward_evaluations": forward_evaluations, + }, + ) + + +def _progressive_result(output, *, cycles=3, forward_evaluations=2): + return ( + output, + {}, + _shadow_plan(cycles=cycles, forward_evaluations=forward_evaluations), + ) + + +def test_cli_defaults_use_four_held_out_seeds_and_256_requests(): + args = benchmark.build_parser().parse_args([]) + + assert args.seeds == [719, 727, 733, 739] + assert args.samples_per_seed == 64 + assert len(args.seeds) * args.samples_per_seed == 256 + assert not set(args.seeds).intersection( + benchmark.TRAINING_AND_ORIGINAL_TEST_SEEDS + ) + assert args.cycles == [1, 3, 8] + + +def test_literal_legacy_v1_and_progressive_accept_same_probe_and_output(): + model = _CycleModel().eval() + x = torch.zeros(1, 1, 4) + settings = { + "cycles": [1, 3, 8], + "confidence_target": 0.8, + "entropy_target": 0.0, + } + + legacy_output, _legacy_compute, legacy_plan = ( + benchmark.legacy_v1_auto_compute_forward(model, x, [0, 1], **settings) + ) + progressive_output, _progressive_compute, progressive_plan = ( + benchmark.progressive_auto_compute_forward( + model, x, [0, 1], **settings + ) + ) + + assert model.calls == [1, 3, 8, 3, 1, 3] + assert legacy_plan["selected_reasoning_cycles"] == 3 + assert progressive_plan["selected_reasoning_cycles"] == 3 + assert legacy_plan["forward_evaluations"] == 4 + assert progressive_plan["forward_evaluations"] == 2 + assert torch.equal(legacy_output, progressive_output) + assert benchmark._shadow_selection_is_disabled(progressive_plan) is True + + +@pytest.mark.parametrize( + ("offset", "expected_measured_calls"), + [ + (0, ["legacy", "progressive", "progressive", "legacy"]), + (1, ["progressive", "legacy", "legacy", "progressive"]), + ], +) +def test_request_benchmark_counterbalances_and_counts_forward_reduction( + monkeypatch, offset, expected_measured_calls +): + calls = [] + + def legacy_fn(_model, sample, _labels, **_kwargs): + calls.append("legacy") + return _legacy_result( + torch.tensor([[[2.0, 0.0]]], device=sample.device) + ) + + def progressive_fn(_model, sample, _labels, **_kwargs): + calls.append("progressive") + return _progressive_result( + torch.tensor([[[2.0, 0.0]]], device=sample.device) + ) + + ticks = iter(float(index) for index in range(20)) + monkeypatch.setattr(benchmark.time, "perf_counter", lambda: next(ticks)) + result = benchmark.benchmark_requests( + _BareModel(), + torch.zeros(2, 1, 4), + torch.zeros(2, dtype=torch.long), + available_labels=[0, 1], + order_offset=offset, + legacy_fn=legacy_fn, + progressive_fn=progressive_fn, + ) + + assert calls[2:] == expected_measured_calls + assert result["measurement_order"] == { + "offset": offset, + "legacy_then_progressive": 1, + "progressive_then_legacy": 1, + } + assert result["legacy_v1"]["forward_evaluations"] == 8 + assert result["progressive"]["forward_evaluations"] == 4 + assert result["comparison"]["forward_evaluation_reduction_percent"] == 50 + assert result["comparison"]["mean_latency_reduction_percent"] == 0 + assert result["comparison"]["exact_output_tensor_disagreement_count"] == 0 + assert result["gates"]["passed"] is True + + +def test_request_benchmark_fails_every_exact_equivalence_gate(monkeypatch): + def legacy_fn(_model, sample, _labels, **_kwargs): + return _legacy_result( + torch.tensor([[[2.0, 0.0]]], device=sample.device), cycles=3 + ) + + def progressive_fn(_model, sample, _labels, **_kwargs): + return _progressive_result( + torch.tensor([[[0.0, 2.0]]], device=sample.device), cycles=8 + ) + + ticks = iter(float(index) for index in range(20)) + monkeypatch.setattr(benchmark.time, "perf_counter", lambda: next(ticks)) + result = benchmark.benchmark_requests( + _BareModel(), + torch.zeros(1, 1, 4), + torch.zeros(1, dtype=torch.long), + available_labels=[0, 1], + legacy_fn=legacy_fn, + progressive_fn=progressive_fn, + ) + checks = result["gates"]["checks"] + + assert result["gates"]["passed"] is False + assert checks["exact_selected_cycle_agreement"]["passed"] is False + assert checks["exact_prediction_agreement"]["passed"] is False + assert checks["exact_output_tensor_agreement"]["passed"] is False + assert result["comparison"]["maximum_absolute_logit_difference"] == 2 + assert len(result["comparison"]["mismatch_details"]) == 1 + + +def test_shadow_diagnostic_cannot_silently_become_a_selector(monkeypatch): + def legacy_fn(_model, sample, _labels, **_kwargs): + return _legacy_result(torch.zeros(1, 1, 2, device=sample.device)) + + def progressive_fn(_model, sample, _labels, **_kwargs): + output, compute, plan = _progressive_result( + torch.zeros(1, 1, 2, device=sample.device) + ) + plan["rows"][0]["mutual_stability_shadow"]["selection_enabled"] = True + return output, compute, plan + + ticks = iter(float(index) for index in range(20)) + monkeypatch.setattr(benchmark.time, "perf_counter", lambda: next(ticks)) + result = benchmark.benchmark_requests( + _BareModel(), + torch.zeros(1, 1, 4), + torch.zeros(1, dtype=torch.long), + available_labels=[0, 1], + legacy_fn=legacy_fn, + progressive_fn=progressive_fn, + ) + + check = result["gates"]["checks"]["mutual_stability_is_shadow_only"] + assert check["passed"] is False + assert check["selection_policy_violation_count"] == 1 + + +def _fixture_metrics(*, samples, offset, legacy_ms=20.0, progressive_ms=12.0): + legacy_first = (samples + 1) // 2 if offset == 0 else samples // 2 + return { + "request_count": samples, + "measurement_order": { + "offset": offset, + "legacy_then_progressive": legacy_first, + "progressive_then_legacy": samples - legacy_first, + }, + "legacy_v1": { + "correct_predictions": samples, + "forward_evaluations": samples * 4, + "latency": {"total_ms": legacy_ms, "mean_ms": legacy_ms / samples}, + }, + "progressive": { + "correct_predictions": samples, + "forward_evaluations": samples * 2, + "latency": { + "total_ms": progressive_ms, + "mean_ms": progressive_ms / samples, + }, + }, + "comparison": { + "selected_cycle_pairs": {"3->3": samples}, + "exact_selected_cycle_disagreement_count": 0, + "exact_prediction_disagreement_count": 0, + "exact_output_tensor_disagreement_count": 0, + "maximum_absolute_logit_difference": 0.0, + "forward_evaluation_reduction_percent": 50.0, + }, + "gates": { + "passed": True, + "checks": { + "per_request_forward_evaluation_reduction": { + "non_reducing_request_count": 0 + }, + "mutual_stability_is_shadow_only": { + "selection_policy_violation_count": 0 + }, + }, + }, + } + + +def test_aggregate_uses_exact_counts_weighted_work_and_median_seed_latency(): + aggregate = benchmark.aggregate_results( + [ + {"seed": 719, "metrics": _fixture_metrics(samples=2, offset=0)}, + { + "seed": 727, + "metrics": _fixture_metrics( + samples=4, offset=1, legacy_ms=80, progressive_ms=60 + ), + }, + ] + ) + summary = aggregate["summary"] + + assert aggregate["gates"]["passed"] is True + assert summary["total_samples"] == 6 + assert summary["legacy_forward_evaluations"] == 24 + assert summary["progressive_forward_evaluations"] == 12 + assert summary["forward_evaluation_reduction_percent"] == 50 + assert summary["weighted_mean_latency_reduction_percent"] == 28 + assert summary["median_per_seed_latency_reduction_percent"] == 32.5 + assert summary["selected_cycle_pairs"] == {"3->3": 6} + + +def test_run_benchmark_loads_checkpoint_once_and_alternates_seed_offsets(tmp_path): + weights = tmp_path / "checkpoint.pth" + weights.write_bytes(b"real checkpoint identity") + loads = [] + tasks = [] + offsets = [] + + def loader(path, device): + loads.append((path, device)) + return _BareModel() + + def task_factory(samples, *, seed): + tasks.append((samples, seed)) + return torch.zeros(samples, 1, 4), torch.zeros(samples, dtype=torch.long) + + def benchmark_fn(_model, _x, y, **kwargs): + offsets.append(kwargs["order_offset"]) + return _fixture_metrics(samples=len(y), offset=kwargs["order_offset"]) + + result = benchmark.run_benchmark( + weights=weights, + seeds=[719, 727, 733], + samples_per_seed=3, + device=torch.device("cpu"), + device_info={"requested": "cpu", "resolved": "cpu"}, + model_loader=loader, + task_factory=task_factory, + benchmark_fn=benchmark_fn, + provenance={"fixture": True}, + ) + + assert len(loads) == 1 + assert tasks == [(3, 719), (3, 727), (3, 733)] + assert offsets == [0, 1, 0] + assert result["checkpoint"]["sha256"] == hashlib.sha256( + b"real checkpoint identity" + ).hexdigest() + assert result["configuration"]["total_samples"] == 9 + assert result["configuration"]["mutual_stability_role"] == ( + "shadow_diagnostic_only" + ) + assert result["gates"]["passed"] is True + + +@pytest.mark.parametrize("seed", [51, 52]) +def test_run_benchmark_rejects_training_or_original_test_seed(tmp_path, seed): + weights = tmp_path / "checkpoint.pth" + weights.write_bytes(b"checkpoint") + + with pytest.raises(ValueError, match="overlap"): + benchmark.run_benchmark( + weights=weights, + seeds=[seed], + samples_per_seed=1, + device=torch.device("cpu"), + device_info={"resolved": "cpu"}, + model_loader=lambda *_args: _BareModel(), + provenance={"fixture": True}, + ) + + +def test_provenance_captures_controller_model_data_runtime_and_git(monkeypatch): + def fake_git(*args): + if args == ("rev-parse", "HEAD"): + return "abc123" + if args == ("rev-parse", "--abbrev-ref", "HEAD"): + return "agent/benchmark" + return "" + + monkeypatch.setattr(benchmark, "_git_text", fake_git) + provenance = benchmark.collect_provenance( + device=torch.device("cpu"), + device_info={"requested": "auto", "resolved": "cpu"}, + ) + + assert provenance["git"] == { + "commit": "abc123", + "branch": "agent/benchmark", + "worktree_dirty": False, + } + assert set(provenance["source_sha256"]) == { + "source/benchmark_progressive_auto_compute.py", + "source/chat_app.py", + "source/benchmark_cognitive_leap_ultra_v51.py", + "source/model_variants.py", + } + assert all(len(value) == 64 for value in provenance["source_sha256"].values()) + assert provenance["runtime"]["torch"]["version"] == str(torch.__version__) + assert provenance["runtime"]["device"]["resolved"] == "cpu" + + +def test_main_enforces_gates_and_emits_strict_json(monkeypatch, tmp_path, capsys): + payload = {"gates": {"passed": False}, "finite": 1.0} + monkeypatch.setattr(benchmark, "configure_torch_runtime", lambda **_kwargs: None) + monkeypatch.setattr( + benchmark, + "resolve_device", + lambda *_args, **_kwargs: (torch.device("cpu"), {"resolved": "cpu"}), + ) + monkeypatch.setattr(benchmark, "run_benchmark", lambda **_kwargs: payload) + output = tmp_path / "benchmark.json" + + exit_code = benchmark.main( + ["--output", str(output), "--enforce-gates"] + ) + + assert exit_code == 2 + assert json.loads(capsys.readouterr().out) == payload + assert json.loads(output.read_text(encoding="utf-8")) == payload + + +def test_strict_json_rejects_non_finite_values(): + with pytest.raises(ValueError): + benchmark._strict_json({"bad": float("nan")}) diff --git a/test_runtime_auto_compute_controls.py b/test_runtime_auto_compute_controls.py index 9b8973733..79ee5fd86 100644 --- a/test_runtime_auto_compute_controls.py +++ b/test_runtime_auto_compute_controls.py @@ -1,4 +1,7 @@ +import json import os +from pathlib import Path +import subprocess import sys import torch @@ -18,6 +21,7 @@ def __init__(self): self.last_cycles_used = torch.tensor(0.0) self.last_ponder_cost = torch.tensor(0.0) self.last_consistency_loss = torch.tensor(0.0) + self.eval() def forward( self, @@ -43,6 +47,48 @@ def forward( return logits +class ProbabilityScheduleModel(nn.Module): + def __init__(self, probabilities): + super().__init__() + self.probabilities = dict(probabilities) + self.calls = [] + self.last_cycles_used = torch.tensor(0.0) + self.eval() + + def forward( + self, + x, + reasoning_cycles=None, + adaptive_compute=False, + exit_tol=chat_app.DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold=chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience=chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol=chat_app.DEFAULT_PREDICTION_STABILITY_TOL, + ): + cycles = int(reasoning_cycles or 1) + self.calls.append( + { + "reasoning_cycles": cycles, + "adaptive_compute": adaptive_compute, + "exit_tol": exit_tol, + "exit_entropy_threshold": exit_entropy_threshold, + "prediction_stability_patience": prediction_stability_patience, + "prediction_stability_tol": prediction_stability_tol, + } + ) + self.last_cycles_used = torch.tensor(float(cycles)) + probability = float(self.probabilities[cycles]) + tail = (1.0 - probability) / (chat_app.MODEL_CLASSES - 1) + probs = torch.full( + (chat_app.MODEL_CLASSES,), + tail, + dtype=torch.float64, + device=x.device, + ) + probs[0] = probability + return torch.log(probs).reshape(1, 1, -1).expand(x.shape[0], x.shape[1], -1) + + class LegacyModel(nn.Module): def __init__(self): super().__init__() @@ -115,6 +161,315 @@ def test_runtime_compute_budget_helpers_select_earliest_confident_budget(): assert plan["reason"] == "confidence_target" +def test_progressive_auto_compute_reuses_first_qualifying_probe(): + model = RuntimeAwareModel() + x = torch.zeros(1, 1, 128) + + output, compute, plan = chat_app.progressive_auto_compute_forward( + model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3, 8], + ) + + assert [call["reasoning_cycles"] for call in model.calls] == [1, 3] + assert int(output[0, 0].argmax().item()) == 0 + assert float(output[0, 0, 0].item()) == 3.0 + assert plan["schema_version"] == "runtime-auto-compute-plan-v2" + assert plan["strategy"] == "progressive_accepted_probe" + assert plan["selected_reasoning_cycles"] == 3 + assert plan["reason"] == "confidence_target" + assert plan["evaluated_cycles"] == [1, 3] + assert plan["skipped_cycles"] == [8] + assert plan["forward_evaluations"] == 2 + assert plan["legacy_forward_evaluations"] == 4 + assert plan["forward_reduction_percent"] == 50.0 + assert plan["reused_probe_output"] is True + assert plan["selection_semantics"] == "legacy_v1_selection_policy" + assert plan["probe_control_scope"] == "full_runtime_controls_v2" + assert compute["auto_compute_plan"] == plan + assert compute["inference_reused"] is True + + +def test_progressive_auto_compute_matches_v1_rounded_confidence_boundary(): + x = torch.zeros(1, 1, 128) + legacy_model = ProbabilityScheduleModel({1: 0.5499996, 3: 0.8}) + rows = chat_app.evaluate_runtime_compute_budgets( + legacy_model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3], + ) + legacy_plan = chat_app.select_auto_runtime_compute_budget( + rows, + confidence_target=0.55, + entropy_target=0.0, + ) + progressive_model = ProbabilityScheduleModel({1: 0.5499996, 3: 0.8}) + _, _, progressive_plan = chat_app.progressive_auto_compute_forward( + progressive_model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3], + confidence_target=0.55, + entropy_target=0.0, + ) + + assert rows[0]["confidence"] == 0.55 + assert legacy_plan["selected_reasoning_cycles"] == 1 + assert progressive_plan["selected_reasoning_cycles"] == 1 + assert [call["reasoning_cycles"] for call in progressive_model.calls] == [1] + + +def test_progressive_auto_compute_matches_v1_rounded_entropy_boundary(): + x = torch.zeros(1, 1, 128) + probability = 0.4728206703246546 + legacy_model = ProbabilityScheduleModel({1: probability, 3: 0.8}) + rows = chat_app.evaluate_runtime_compute_budgets( + legacy_model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3], + ) + legacy_plan = chat_app.select_auto_runtime_compute_budget( + rows, + confidence_target=1.0, + entropy_target=1.85, + ) + progressive_model = ProbabilityScheduleModel({1: probability, 3: 0.8}) + _, _, progressive_plan = chat_app.progressive_auto_compute_forward( + progressive_model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3], + confidence_target=1.0, + entropy_target=1.85, + ) + + assert rows[0]["entropy"] == 1.85 + assert legacy_plan["selected_reasoning_cycles"] == 1 + assert progressive_plan["selected_reasoning_cycles"] == 1 + assert progressive_plan["reason"] == "entropy_target" + + +def test_progressive_auto_compute_shadow_cost_cannot_change_fallback(monkeypatch): + model = ProbabilityScheduleModel({1: 0.1, 3: 0.1}) + x = torch.zeros(1, 1, 128) + ticks = iter([0.0, 0.030, 0.040, 0.041, 0.091]) + monkeypatch.setattr(chat_app.time, "perf_counter", lambda: next(ticks)) + js_calls = [] + + def expensive_shadow(*_args, **_kwargs): + js_calls.append(chat_app.time.perf_counter()) + return 0.0 + + monkeypatch.setattr(chat_app, "_topk_distribution_js_divergence", expensive_shadow) + _, _, plan = chat_app.progressive_auto_compute_forward( + model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3], + confidence_target=1.0, + entropy_target=0.0, + ) + + assert [row["latency_ms"] for row in plan["rows"]] == [30.0, 1.0] + assert js_calls == [0.091] + assert plan["selected_reasoning_cycles"] == 3 + + +def test_progressive_auto_compute_can_fallback_to_nonfinal_probe(): + model = ProbabilityScheduleModel({1: 0.6, 3: 0.2, 8: 0.4}) + x = torch.zeros(1, 1, 128) + _, _, plan = chat_app.progressive_auto_compute_forward( + model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3, 8], + confidence_target=1.0, + entropy_target=0.0, + ) + + assert plan["selected_index"] == 0 + assert plan["selected_reasoning_cycles"] == 1 + assert plan["rows"][-1]["requested_cycles"] == 8 + terminal_metrics = chat_app.compact_auto_compute_plan_metrics(plan) + assert terminal_metrics["auto_selected"] == 1 + assert terminal_metrics["auto_forwards"] == "3/4" + assert terminal_metrics["auto_reused"] is True + assert "auto_shadow_js" not in terminal_metrics + + +def test_topk_distribution_js_divergence_is_symmetric_and_bounded(): + left = torch.tensor([0.7, 0.2, 0.1]) + right = torch.tensor([0.1, 0.3, 0.6]) + + left_right = chat_app._topk_distribution_js_divergence(left, right, top_k=2) + right_left = chat_app._topk_distribution_js_divergence(right, left, top_k=2) + + assert chat_app._topk_distribution_js_divergence(left, left, top_k=2) == 0.0 + assert left_right == right_left + assert 0.0 <= left_right <= 0.6931471806 + + +def test_source_web_diagnostics_resolve_the_selected_probe_row(): + assert "rows[selectedIndex]" in chat_web_app.HTML + assert "rows[rows.length-1].mutual_stability_shadow" not in chat_web_app.HTML + + +def test_packaged_runtime_progressive_helper_and_api_smoke(): + root = Path(__file__).resolve().parent + script = r''' +import json +import sys +from pathlib import Path +import torch + +root = Path.cwd() +sys.path.insert(0, str(root / "runtime_python")) +import chat_app +import chat_web_app + +class Model(torch.nn.Module): + def __init__(self): + super().__init__() + self.calls = [] + self.last_cycles_used = torch.tensor(0.0) + + def forward(self, x, reasoning_cycles=None, **_kwargs): + cycles = int(reasoning_cycles or 1) + self.calls.append(cycles) + self.last_cycles_used = torch.tensor(float(cycles)) + logits = torch.zeros(x.shape[0], x.shape[1], chat_app.MODEL_CLASSES) + logits[..., 0] = float(cycles) + return logits + +model = Model().eval() +engine = chat_web_app.Engine(torch.device("cpu"), {"resolved": "cpu"}, {"pool_mode": "topk"}) +engine.model = model +engine.feature_mode = "legacy" +vec = [0.0] * 128 +engine.buckets = {0: [{"text": "packaged progressive answer", "count": 1, "vec": vec, "ctx_vec": vec}]} +engine.available_labels = [0] +client = chat_web_app.build_app(engine, "weights.pth", "meta.json").test_client() +response = client.post("/api/chat", json={"session_id": "packaged", "message": "test", "auto_compute": True}) +payload = response.get_json() +assert response.status_code == 200, payload +assert payload["auto_compute_plan"]["reused_probe_output"] is True +assert "rows[selectedIndex]" in chat_web_app.HTML +print(json.dumps({"cycles": model.calls, "strategy": payload["auto_compute_plan"]["strategy"]})) +''' + completed = subprocess.run( + [sys.executable, "-c", script], + cwd=root, + check=True, + capture_output=True, + text=True, + timeout=60, + ) + payload = json.loads(completed.stdout) + + assert payload["strategy"] == "progressive_accepted_probe" + assert payload["cycles"] == [1] + + +def test_progressive_auto_compute_applies_full_v2_probe_controls(): + model = ProbabilityScheduleModel({1: 0.8}) + x = torch.zeros(1, 1, 128) + _, _, plan = chat_app.progressive_auto_compute_forward( + model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1], + adaptive_compute=True, + exit_tol=0.02, + exit_entropy_threshold=0.7, + prediction_stability_patience=4, + prediction_stability_tol=0.123, + ) + + assert model.calls == [ + { + "reasoning_cycles": 1, + "adaptive_compute": True, + "exit_tol": 0.02, + "exit_entropy_threshold": 0.7, + "prediction_stability_patience": 4, + "prediction_stability_tol": 0.123, + } + ] + assert plan["probe_control_scope"] == "full_runtime_controls_v2" + + +def test_progressive_auto_compute_early_first_probe_and_no_target_fallback(): + x = torch.zeros(1, 1, 128) + early_model = RuntimeAwareModel() + early_output, _, early_plan = chat_app.progressive_auto_compute_forward( + early_model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3, 8], + confidence_target=0.2, + ) + + assert [call["reasoning_cycles"] for call in early_model.calls] == [1] + assert float(early_output[0, 0, 0].item()) == 1.0 + assert early_plan["forward_reduction_percent"] == 75.0 + + fallback_model = RuntimeAwareModel() + fallback_output, _, fallback_plan = chat_app.progressive_auto_compute_forward( + fallback_model, + x, + list(range(chat_app.MODEL_CLASSES)), + cycles=[1, 3, 8], + confidence_target=1.0, + entropy_target=0.0, + ) + + assert [call["reasoning_cycles"] for call in fallback_model.calls] == [1, 3, 8] + assert float(fallback_output[0, 0, 0].item()) == 8.0 + assert fallback_plan["reason"] == "best_confidence" + assert fallback_plan["selected_reasoning_cycles"] == 8 + assert fallback_plan["forward_evaluations"] == 3 + shadow = fallback_plan["rows"][-1]["mutual_stability_shadow"] + assert shadow["role"] == "shadow_diagnostic_only" + assert shadow["selection_enabled"] is False + assert shadow["top1_persistent"] is True + assert shadow["js_divergence"] is not None + + +def test_progressive_auto_compute_rejects_training_mode(): + model = RuntimeAwareModel().train() + x = torch.zeros(1, 1, 128) + + try: + chat_app.progressive_auto_compute_forward( + model, + x, + list(range(chat_app.MODEL_CLASSES)), + ) + except RuntimeError as exc: + assert "model.eval" in str(exc) + else: + raise AssertionError("training-mode progressive inference must fail closed") + + +def test_progressive_auto_compute_sanitizes_non_finite_thresholds(): + model = RuntimeAwareModel() + x = torch.zeros(1, 1, 128) + + _, _, plan = chat_app.progressive_auto_compute_forward( + model, + x, + list(range(chat_app.MODEL_CLASSES)), + confidence_target=float("nan"), + entropy_target=float("inf"), + ) + + assert plan["confidence_target"] == chat_app.DEFAULT_AUTO_COMPUTE_CONFIDENCE + assert plan["entropy_target"] == chat_app.DEFAULT_AUTO_COMPUTE_ENTROPY + + def test_web_engine_forwards_runtime_compute_controls_without_mutating_contract(): engine = chat_web_app.Engine( torch.device("cpu"), @@ -258,7 +613,8 @@ def test_chat_auto_compute_selects_confident_budget(): assert plan["selected_reasoning_cycles"] == 3 assert plan["reason"] == "confidence_target" assert result["compute"]["requested_reasoning_cycles"] == 3 - assert model.calls[-1]["reasoning_cycles"] == 3 + assert [call["reasoning_cycles"] for call in model.calls] == [1, 3] + assert plan["reused_probe_output"] is True assert engine.sessions["auto-session"], "Auto-compute chat should still append the final turn" @@ -285,11 +641,17 @@ def test_api_chat_accepts_auto_compute_payload(): payload = response.get_json() assert payload["auto_compute_plan"]["selected_reasoning_cycles"] == 3 assert payload["compute"]["auto_compute_plan"]["selected_reasoning_cycles"] == 3 + assert payload["auto_compute_plan"]["forward_evaluations"] == 2 + assert payload["auto_compute_plan"]["reused_probe_output"] is True if __name__ == "__main__": test_forward_with_runtime_compute_applies_supported_kwargs_only() test_runtime_compute_budget_helpers_select_earliest_confident_budget() + test_progressive_auto_compute_reuses_first_qualifying_probe() + test_progressive_auto_compute_early_first_probe_and_no_target_fallback() + test_progressive_auto_compute_rejects_training_mode() + test_progressive_auto_compute_sanitizes_non_finite_thresholds() test_web_engine_forwards_runtime_compute_controls_without_mutating_contract() test_api_chat_accepts_runtime_compute_payload() test_compute_sweep_reports_budget_rows_without_mutating_session() diff --git a/test_studio_runtime_manifest.py b/test_studio_runtime_manifest.py index 971f13774..f56ebc86f 100644 --- a/test_studio_runtime_manifest.py +++ b/test_studio_runtime_manifest.py @@ -59,12 +59,19 @@ def test_checked_runtime_manifest_is_current_and_complete(): "route_shadow_authenticity_proof_available": False, "route_shadow_trusted_timestamp_available": False, "automatic_policy_promotion_available": False, + "progressive_auto_compute_accepted_probe_reuse": True, + "mutual_stability_shadow_can_select_output": False, } def test_manifest_exposes_route_contract_versions_without_importing_runtime(): manifest = build_manifest(ROOT) contracts = manifest["contracts"] + assert contracts["source/chat_app.py"] == { + "AUTO_COMPUTE_PLAN_SCHEMA_VERSION": "runtime-auto-compute-plan-v2", + "AUTO_COMPUTE_STRATEGY": "progressive_accepted_probe", + "DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K": 5, + } assert contracts["source/route_policy_ledger.py"]["OUTCOME_CONTRACT_SCHEMA_VERSION"] == ( "route-outcome-contract-v1" ) diff --git a/test_supermix_multimodel_web_app.py b/test_supermix_multimodel_web_app.py index 6d74af3ae..431efc1ae 100644 --- a/test_supermix_multimodel_web_app.py +++ b/test_supermix_multimodel_web_app.py @@ -659,6 +659,7 @@ def test_index_contains_discovery_ui(): assert 'id="autoBudget"' in html assert 'id="reasoningCycles"' in html assert 'id="adaptiveCompute"' in html + assert 'id="progressiveAutoCompute"' in html assert 'id="sessionBudget"' in html assert 'id="sessionBudgetTargetRoutes"' in html assert 'data-mode="auto"' in html @@ -777,6 +778,10 @@ def test_index_contains_discovery_ui(): assert "function computePills" in html assert "requested_reasoning_cycles" in html assert "prediction_confidence_delta" in html + assert "auto_compute_plan" in html + assert "Accepted probe reused" in html + assert "rows[selectedIndex]" in html + assert "rows[rows.length - 1].mutual_stability_shadow" not in html assert "route_alternatives" in html assert "route_frontier" in html assert "recommended_agent_mode" in html diff --git a/test_v51_prediction_stability_gate.py b/test_v51_prediction_stability_gate.py new file mode 100644 index 000000000..b8c084d02 --- /dev/null +++ b/test_v51_prediction_stability_gate.py @@ -0,0 +1,317 @@ +import hashlib +import json +import os +from pathlib import Path +import sys + +import pytest +import torch + + +sys.path.insert(0, os.path.join(os.getcwd(), "source")) + +import benchmark_v51_prediction_stability as single_seed +import run_v51_prediction_stability_gate as gate + + +class _StubHead: + def __init__(self): + self.last_cycles_used = torch.tensor(2.0) + self.last_prediction_topk_js_divergence = torch.tensor(0.001) + self.last_prediction_topk_js_divergence_max = torch.tensor(0.002) + self.last_exit_reason = "prediction_stable" + + +class _StubModel: + def __init__(self): + self.layers = [None] * 11 + self.layers[10] = _StubHead() + self.calls = [] + + def __call__(self, _sample, *, reasoning_cycles, adaptive_compute=False, **_kwargs): + self.calls.append("adaptive" if adaptive_compute else "fixed") + logits = torch.tensor([[0.0, 1.0]]) + return logits + + +def test_gate_cli_defaults_to_native_4096_request_matrix(): + args = gate.build_parser().parse_args([]) + + assert args.seeds == [641, 643, 647, 653, 659, 661, 673, 677] + assert args.samples_per_seed == 512 + assert len(args.seeds) * args.samples_per_seed == 4096 + + +def _fixture_metrics( + *, + samples, + fixed_correct, + adaptive_correct, + disagreements, + fixed_cycles, + adaptive_total_cycles, + fixed_latency, + adaptive_latency, + latency_reduction, + cycle_reduction, + offset=0, +): + return { + "fixed": { + "cycles": fixed_cycles, + "correct_predictions": fixed_correct, + "latency": {"mean_ms": fixed_latency}, + }, + "prediction_stability": { + "correct_predictions": adaptive_correct, + "total_cycles_used": adaptive_total_cycles, + "latency": {"mean_ms": adaptive_latency}, + }, + "comparison": { + "request_count": samples, + "exact_disagreement_count": disagreements, + "cycle_reduction_percent": cycle_reduction, + "mean_latency_reduction_percent": latency_reduction, + "measurement_order": { + "offset": offset, + "fixed_then_adaptive": (samples + (1 - offset)) // 2, + "adaptive_then_fixed": (samples + offset) // 2, + }, + }, + } + + +@pytest.mark.parametrize( + ("offset", "expected_calls"), + [ + (0, ["fixed", "adaptive", "adaptive", "fixed"]), + (1, ["adaptive", "fixed", "fixed", "adaptive"]), + ], +) +def test_single_seed_benchmark_counterbalances_each_request( + monkeypatch, offset, expected_calls +): + model = _StubModel() + ticks = iter(float(index) for index in range(100)) + monkeypatch.setattr(single_seed.time, "perf_counter", lambda: next(ticks)) + x = torch.zeros(2, 1, 4) + y = torch.ones(2, dtype=torch.long) + + result = single_seed.benchmark_serving_requests( + model, + x, + y, + fixed_cycles=3, + max_cycles=8, + stability_patience=2, + stability_tol=0.005, + distribution_top_k=5, + order_offset=offset, + ) + + # Two unmeasured warmups precede the request calls. + assert model.calls[2:] == expected_calls + assert result["comparison"]["measurement_order"] == { + "offset": offset, + "fixed_then_adaptive": 1, + "adaptive_then_fixed": 1, + } + assert result["fixed"]["latency"]["mean_ms"] == 1000.0 + assert result["prediction_stability"]["latency"]["mean_ms"] == 1000.0 + assert result["comparison"]["exact_disagreement_count"] == 0 + assert result["prediction_stability"]["total_cycles_used"] == 4.0 + + +def test_aggregate_gate_uses_exact_counts_and_weighted_metrics(): + seed_results = [ + { + "seed": 641, + "metrics": _fixture_metrics( + samples=2, + fixed_correct=1, + adaptive_correct=1, + disagreements=0, + fixed_cycles=3, + adaptive_total_cycles=4, + fixed_latency=10, + adaptive_latency=9, + latency_reduction=10, + cycle_reduction=33.333, + ), + }, + { + "seed": 643, + "metrics": _fixture_metrics( + samples=4, + fixed_correct=3, + adaptive_correct=4, + disagreements=0, + fixed_cycles=3, + adaptive_total_cycles=6, + fixed_latency=20, + adaptive_latency=14, + latency_reduction=30, + cycle_reduction=50, + offset=1, + ), + }, + ] + + result = gate.aggregate_gate_results(seed_results) + + assert result["gates"]["passed"] is True + assert result["summary"]["exact_disagreement_count"] == 0 + assert result["summary"]["weighted_cycle_reduction_percent"] == pytest.approx( + 44.444444 + ) + assert result["summary"]["median_per_seed_latency_reduction_percent"] == 20 + assert result["summary"]["weighted_fixed_mean_latency_ms"] == pytest.approx( + 16.666667 + ) + assert result["summary"]["weighted_adaptive_mean_latency_ms"] == pytest.approx( + 12.333333 + ) + assert result["summary"]["per_seed_accuracy_deltas"] == [ + {"seed": 641, "accuracy_delta": 0.0}, + {"seed": 643, "accuracy_delta": 0.25}, + ] + + +def test_aggregate_gate_reports_every_failed_invariant(): + seed_results = [ + { + "seed": 641, + "metrics": _fixture_metrics( + samples=4, + fixed_correct=4, + adaptive_correct=3, + disagreements=1, + fixed_cycles=3, + adaptive_total_cycles=10, + fixed_latency=10, + adaptive_latency=11, + latency_reduction=-10, + cycle_reduction=16.667, + ), + } + ] + + result = gate.aggregate_gate_results(seed_results) + checks = result["gates"]["checks"] + + assert result["gates"]["passed"] is False + assert checks["zero_prediction_disagreements"]["passed"] is False + assert checks["no_negative_per_seed_accuracy_delta"]["negative_seed_ids"] == [641] + assert checks["minimum_weighted_cycle_reduction"]["passed"] is False + assert checks["positive_median_per_seed_latency_reduction"]["passed"] is False + + +def test_run_gate_loads_checkpoint_once_and_alternates_seed_offsets(tmp_path): + weights = tmp_path / "checkpoint.pth" + weights.write_bytes(b"one checkpoint") + loads = [] + tasks = [] + offsets = [] + + def model_loader(path, device): + loads.append((path, device)) + return object() + + def task_factory(samples, *, seed): + tasks.append((samples, seed)) + return torch.zeros(samples, 1), torch.zeros(samples, dtype=torch.long) + + def benchmark_fn(_model, _x, y, **kwargs): + offsets.append(kwargs["order_offset"]) + samples = len(y) + return _fixture_metrics( + samples=samples, + fixed_correct=samples, + adaptive_correct=samples, + disagreements=0, + fixed_cycles=3, + adaptive_total_cycles=2 * samples, + fixed_latency=10, + adaptive_latency=8, + latency_reduction=20, + cycle_reduction=33.333, + offset=kwargs["order_offset"], + ) + + result = gate.run_gate( + weights=weights, + seeds=[641, 643, 647], + samples_per_seed=3, + device=torch.device("cpu"), + device_info={"requested": "cpu", "resolved": "cpu"}, + model_loader=model_loader, + task_factory=task_factory, + benchmark_fn=benchmark_fn, + provenance={"fixture": True}, + ) + + assert len(loads) == 1 + assert tasks == [(3, 641), (3, 643), (3, 647)] + assert offsets == [0, 1, 0] + assert result["checkpoint"]["sha256"] == hashlib.sha256( + b"one checkpoint" + ).hexdigest() + assert result["configuration"]["total_samples"] == 9 + assert result["gates"]["passed"] is True + + +def test_provenance_captures_git_sources_runtime_device_and_threads(monkeypatch): + def fake_git(*args): + if args == ("rev-parse", "HEAD"): + return "abc123" + if args == ("rev-parse", "--abbrev-ref", "HEAD"): + return "main" + return "" + + monkeypatch.setattr(gate, "_git_text", fake_git) + + provenance = gate.collect_provenance( + device=torch.device("cpu"), + device_info={"requested": "auto", "resolved": "cpu"}, + ) + + assert provenance["git"] == { + "commit": "abc123", + "branch": "main", + "worktree_dirty": False, + } + assert set(provenance["source_sha256"]) == { + "source/run_v51_prediction_stability_gate.py", + "source/benchmark_v51_prediction_stability.py", + "source/benchmark_cognitive_leap_ultra_v51.py", + "source/model_variants.py", + } + assert all(len(value) == 64 for value in provenance["source_sha256"].values()) + assert provenance["runtime"]["torch"]["version"] == str(torch.__version__) + assert provenance["runtime"]["device"]["resolved"] == "cpu" + assert provenance["runtime"]["threads"]["torch_num_threads"] >= 1 + + +def test_enforce_gates_returns_two_and_stdout_remains_strict_json( + monkeypatch, tmp_path, capsys +): + payload = {"gates": {"passed": False}, "value": 1} + monkeypatch.setattr(gate, "configure_torch_runtime", lambda **_kwargs: None) + monkeypatch.setattr( + gate, + "resolve_device", + lambda *_args, **_kwargs: (torch.device("cpu"), {"resolved": "cpu"}), + ) + monkeypatch.setattr(gate, "run_gate", lambda **_kwargs: payload) + output = tmp_path / "gate.json" + + exit_code = gate.main(["--output", str(output), "--enforce-gates"]) + + assert exit_code == 2 + assert json.loads(capsys.readouterr().out) == payload + assert json.loads(output.read_text(encoding="utf-8")) == payload + + +def test_strict_json_rejects_non_finite_values(): + with pytest.raises(ValueError): + gate._strict_json({"bad": float("nan")}) From 205459417d4784275f11da8a4c3fc45b12db4808 Mon Sep 17 00:00:00 2001 From: Kai Piper Date: Wed, 22 Jul 2026 15:26:48 +0100 Subject: [PATCH 2/9] Guard adaptive exits with scoped decision fidelity --- launch_v51_ultra_chat_demo.ps1 | 4 +- runtime_python/chat_app.py | 213 ++- runtime_python/chat_web_app.py | 97 +- runtime_python/model_variants.py | 214 ++- source/benchmark_cognitive_leap_ultra_v51.py | 33 +- source/benchmark_progressive_auto_compute.py | 336 ++++- source/benchmark_v51_prediction_stability.py | 637 ++++++++- source/chat_app.py | 213 ++- source/chat_web_app.py | 88 +- source/generate_studio_runtime_manifest.py | 11 + source/materialize_v51_chat_demo.py | 6 + source/model_variants.py | 214 ++- source/multimodel_runtime.py | 2 + source/run_v51_chat_response_fidelity_gate.py | 1144 ++++++++++++++++ source/run_v51_prediction_stability_gate.py | 1176 ++++++++++++++++- source/studio_runtime_manifest.json | 45 +- source/supermix_multimodel_web_app.py | 42 + test_cognitive_leap_ultra_expert.py | 472 +++++++ test_multimodel_runtime.py | 4 + test_progressive_auto_compute_benchmark.py | 143 +- test_runtime_auto_compute_controls.py | 48 +- test_runtime_compute_controls.py | 337 +++++ test_studio_runtime_manifest.py | 11 + test_supermix_multimodel_web_app.py | 8 + test_v51_chat_response_fidelity_gate.py | 499 +++++++ test_v51_materialize_demo.py | 30 +- test_v51_prediction_stability_gate.py | 1096 ++++++++++++++- 27 files changed, 6900 insertions(+), 223 deletions(-) create mode 100644 source/run_v51_chat_response_fidelity_gate.py create mode 100644 test_v51_chat_response_fidelity_gate.py diff --git a/launch_v51_ultra_chat_demo.ps1 b/launch_v51_ultra_chat_demo.ps1 index 46b2fcefe..0c6d07f1d 100644 --- a/launch_v51_ultra_chat_demo.ps1 +++ b/launch_v51_ultra_chat_demo.ps1 @@ -59,7 +59,9 @@ try { "--adaptive_compute", "--adaptive_exit_entropy", "0.2", "--prediction_stability_patience", "2", - "--prediction_stability_tol", "0.005" + "--prediction_stability_tol", "0.005", + "--prediction_stability_margin", "0.0001", + "--prediction_stability_rank_depth", "3" ) | ForEach-Object { Quote-ProcessArg $_ } $process = Start-Process -FilePath "python" -ArgumentList $args -WorkingDirectory $Root -WindowStyle Hidden -PassThru -RedirectStandardOutput $Stdout -RedirectStandardError $Stderr Set-Content -LiteralPath $PidFile -Value $process.Id diff --git a/runtime_python/chat_app.py b/runtime_python/chat_app.py index f089d7585..238162214 100644 --- a/runtime_python/chat_app.py +++ b/runtime_python/chat_app.py @@ -62,12 +62,37 @@ def load_metadata(path: str) -> Dict: return {} +def _parse_metadata_buckets(raw: Any) -> Dict[int, List[Dict]]: + """Keep only non-empty buckets whose labels address a model output class.""" + + buckets: Dict[int, List[Dict]] = {} + if not isinstance(raw, dict): + return buckets + for raw_label, rows in raw.items(): + if isinstance(raw_label, bool): + continue + if isinstance(raw_label, int): + label = raw_label + elif isinstance(raw_label, str) and re.fullmatch(r"[+-]?\d+", raw_label.strip()): + try: + label = int(raw_label) + except (ValueError, OverflowError): + continue + else: + continue + if not 0 <= label < MODEL_CLASSES: + continue + if isinstance(rows, list) and rows: + buckets[label] = rows + return buckets + + def _int_or_default(value, default: int) -> int: if value is None: return int(default) try: return int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return int(default) @@ -130,6 +155,11 @@ def evaluate_runtime_compute_budgets( cycles=None, adaptive_compute: bool = False, exit_tol: Optional[float] = None, + exit_entropy_threshold: Any = None, + prediction_stability_patience: Any = None, + prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> List[Dict[str, object]]: labels = list(available_labels) or list(range(MODEL_CLASSES)) idx_device = x.device if isinstance(x, torch.Tensor) else None @@ -148,7 +178,13 @@ def evaluate_runtime_compute_budgets( reasoning_cycles=requested_cycles, adaptive_compute=adaptive_compute, exit_tol=resolved_exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=prediction_stability_margin, return_diagnostics=True, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) logits = logits_tensor[0, 0] avail_logits = logits.index_select(0, idx.to(logits.device)) @@ -223,6 +259,8 @@ def select_auto_runtime_compute_budget( DEFAULT_ADAPTIVE_EXIT_ENTROPY = 0.2 DEFAULT_PREDICTION_STABILITY_PATIENCE = 2 DEFAULT_PREDICTION_STABILITY_TOL = 5e-3 +DEFAULT_PREDICTION_STABILITY_MARGIN = 1e-4 +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = 3 AUTO_REASONING_CYCLE_BUCKETS = (1, 3, 8, 16) @@ -313,7 +351,7 @@ def _coerce_optional_positive_int( value = stripped try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return default parsed = max(1, parsed) if max_value is not None: @@ -385,7 +423,7 @@ def estimate_auto_reasoning_cycles(context: str, max_value: Optional[int] = None def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[float] = None) -> float: try: parsed = float(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): parsed = float(default) if not math.isfinite(parsed): parsed = float(default) @@ -395,10 +433,40 @@ def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[fl return parsed +def _coerce_prediction_stability_margin( + value: Any, + default: float = DEFAULT_PREDICTION_STABILITY_MARGIN, +) -> float: + """Preserve explicit zero but never turn malformed input into guard-off.""" + + try: + parsed = float(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return float(default) + if not math.isfinite(parsed) or parsed < 0.0: + return float(default) + return parsed + + +def _coerce_prediction_stability_rank_depth( + value: Any, + default: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, +) -> int: + """Preserve explicit zero while failing malformed/negative values closed.""" + + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return int(default) + if parsed < 0: + return int(default) + return parsed + + def _coerce_nonnegative_int(value: Any, default: int, max_value: Optional[int] = None) -> int: try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): parsed = int(default) parsed = max(0, parsed) if max_value is not None: @@ -450,7 +518,7 @@ def _to_optional_scalar(value: Any) -> Optional[float]: if hasattr(value, "detach"): return float(value.detach().cpu().item()) return float(value) - except (TypeError, ValueError, RuntimeError): + except (TypeError, ValueError, OverflowError, RuntimeError): return None @@ -469,6 +537,11 @@ def collect_runtime_compute_metrics( "gating_entropy": None, "prediction_streak": None, "prediction_confidence_delta": None, + "prediction_margin": None, + "prediction_decision_margin": None, + "prediction_rank_depth": None, + "prediction_class_count": None, + "prediction_class_selection_valid": None, "exit_reason": None, } attr_map = { @@ -478,6 +551,11 @@ def collect_runtime_compute_metrics( "last_gating_entropy": "gating_entropy", "last_prediction_streak": "prediction_streak", "last_prediction_confidence_delta": "prediction_confidence_delta", + "last_prediction_margin": "prediction_margin", + "last_prediction_decision_margin": "prediction_decision_margin", + "last_prediction_rank_depth": "prediction_rank_depth", + "last_prediction_class_count": "prediction_class_count", + "last_prediction_class_selection_valid": "prediction_class_selection_valid", } modules = list(model.modules()) if hasattr(model, "modules") else [model] for module in modules: @@ -491,6 +569,10 @@ def collect_runtime_compute_metrics( reason = getattr(module, "last_exit_reason", None) if isinstance(reason, str) and reason.strip(): metrics["exit_reason"] = reason.strip() + if metrics["prediction_class_selection_valid"] is not None: + metrics["prediction_class_selection_valid"] = bool( + metrics["prediction_class_selection_valid"] + ) metrics.update({ "supported": model_supports_runtime_compute(model), "requested_reasoning_cycles": requested_reasoning_cycles, @@ -518,6 +600,9 @@ def forward_with_runtime_compute( prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, auto_reasoning_context: str = "", return_diagnostics: Optional[bool] = None, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_class_indices: Any = None, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ): """Forward a model while applying optional v50 runtime-compute controls. @@ -548,6 +633,14 @@ def forward_with_runtime_compute( prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + stability_margin = _coerce_prediction_stability_margin( + prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + stability_rank_depth = _coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) kwargs: Dict[str, Any] = {} if cycles is not None and _model_forward_accepts_kwarg(model, "reasoning_cycles"): @@ -562,8 +655,29 @@ def forward_with_runtime_compute( kwargs["prediction_stability_patience"] = stability_patience if _model_forward_accepts_kwarg(model, "prediction_stability_tol"): kwargs["prediction_stability_tol"] = stability_tol + if _model_forward_accepts_kwarg(model, "prediction_stability_margin"): + kwargs["prediction_stability_margin"] = stability_margin + if _model_forward_accepts_kwarg(model, "prediction_stability_rank_depth"): + kwargs["prediction_stability_rank_depth"] = stability_rank_depth + if ( + prediction_class_indices is not None + and _model_forward_accepts_kwarg(model, "prediction_class_indices") + ): + if isinstance(prediction_class_indices, torch.Tensor): + prediction_class_indices = prediction_class_indices.detach().cpu().tolist() + elif isinstance(prediction_class_indices, tuple): + prediction_class_indices = list(prediction_class_indices) + kwargs["prediction_class_indices"] = prediction_class_indices output = model(x, **kwargs) if kwargs else model(x) + prediction_verifier_active = bool( + adaptive + and stability_patience > 0 + and stability_rank_depth > 0 + and "adaptive_compute" in kwargs + and "prediction_stability_patience" in kwargs + and "prediction_stability_rank_depth" in kwargs + ) metrics = collect_runtime_compute_metrics( model, requested_reasoning_cycles=cycles, @@ -571,6 +685,17 @@ def forward_with_runtime_compute( exit_tol=tol, applied_kwargs=kwargs, ) + if not prediction_verifier_active: + for key in ( + "prediction_streak", + "prediction_confidence_delta", + "prediction_margin", + "prediction_decision_margin", + "prediction_rank_depth", + "prediction_class_count", + "prediction_class_selection_valid", + ): + metrics[key] = None diagnostics = { "requested_reasoning_cycles": cycles, "selected_reasoning_cycles": cycles, @@ -582,6 +707,14 @@ def forward_with_runtime_compute( "exit_entropy_threshold": entropy_threshold, "prediction_stability_patience": stability_patience, "prediction_stability_tol": stability_tol, + "prediction_stability_margin": stability_margin, + "prediction_stability_rank_depth": stability_rank_depth, + "prediction_verifier_active": prediction_verifier_active, + "prediction_class_indices": ( + kwargs.get("prediction_class_indices") + if "prediction_class_indices" in kwargs + else None + ), "applied": bool(kwargs), "applied_kwargs": dict(kwargs), **metrics, @@ -647,6 +780,8 @@ def progressive_auto_compute_forward( prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, auto_reasoning_context: str = "", distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: """Evaluate auto-compute budgets progressively and reuse the accepted probe. @@ -696,6 +831,9 @@ def progressive_auto_compute_forward( prediction_stability_patience=prediction_stability_patience, prediction_stability_tol=prediction_stability_tol, auto_reasoning_context=auto_reasoning_context, + prediction_stability_margin=prediction_stability_margin, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) logits = model_out[0, 0] available_logits = logits.index_select(0, idx.to(logits.device)) @@ -822,7 +960,7 @@ def compact_auto_compute_plan_metrics(plan: Any) -> Dict[str, Any]: rows = plan.get("rows") if isinstance(plan.get("rows"), list) else [] try: selected_index = int(plan.get("selected_index")) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): selected_index = -1 if not 0 <= selected_index < len(rows): selected_index = len(rows) - 1 @@ -887,6 +1025,8 @@ def _print_chat_help() -> None: print(" /exit_entropy Set v50 adaptive-compute entropy exit threshold") print(" /stability Stable prediction cycles required (0 disables)") print(" /stability_tol Maximum confidence drift across the stable streak") + print(" /stability_margin Minimum top-two probability margin for stable exit") + print(" /stability_rank_depth Ordered prediction ranks verified (0 disables)") print(" /top Show top reranked candidates each turn (0 disables)") print(" /timing on|off Toggle per-turn timing output") print(" /auto_compute on|off Toggle confidence/entropy based cycle selection") @@ -948,6 +1088,18 @@ def main(): default=DEFAULT_PREDICTION_STABILITY_TOL, help="Maximum confidence drift allowed across a stable prediction streak.", ) + ap.add_argument( + "--prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + help="Minimum full-output top-two probability margin required for stable exit.", + ) + ap.add_argument( + "--prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + help="Ordered prediction ranks required to remain stable (0 disables the rank verifier).", + ) ap.add_argument("--device", default="auto") ap.add_argument( "--device_preference", @@ -1140,15 +1292,7 @@ def main(): if missing or unexpected: raise RuntimeError(f"State dict mismatch. Missing={missing}, Unexpected={unexpected}") - buckets_raw = meta.get("buckets", {}) - buckets: Dict[int, List[Dict]] = {} - for k, v in buckets_raw.items(): - try: - label = int(k) - except ValueError: - continue - if isinstance(v, list) and v: - buckets[label] = v + buckets = _parse_metadata_buckets(meta.get("buckets", {})) available_labels = sorted(buckets.keys()) if not available_labels: @@ -1203,6 +1347,14 @@ def main(): args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + args.prediction_stability_margin = _coerce_prediction_stability_margin( + args.prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + args.prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) print(f"\n{TerminalColors.SYSTEM}--- Session Info ---") print(f"Loaded: {Path(args.weights).name} [{resolved_model_size}] | Available labels: {len(available_labels)}") @@ -1217,6 +1369,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) @@ -1272,6 +1426,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) print( @@ -1312,6 +1468,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'} " f"auto_conf={float(args.auto_compute_confidence):.3f} " f"auto_entropy={float(args.auto_compute_entropy):.3f}" @@ -1398,6 +1556,26 @@ def main(): ) except Exception: print(f"{TerminalColors.SYSTEM}Usage: /stability_tol {TerminalColors.RESET}") + elif cmd == "stability_margin": + try: + args.prediction_stability_margin = _coerce_prediction_stability_margin( + arg, + args.prediction_stability_margin, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability margin set to " + f"{float(args.prediction_stability_margin):.4g}.{TerminalColors.RESET}" + ) + except Exception: + print(f"{TerminalColors.SYSTEM}Usage: /stability_margin {TerminalColors.RESET}") + elif cmd == "stability_rank_depth": + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + arg, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability rank depth set to " + f"{int(args.prediction_stability_rank_depth)}.{TerminalColors.RESET}" + ) elif cmd == "top": try: args.show_top_responses = max(0, int(arg)) @@ -1517,6 +1695,8 @@ def main(): prediction_stability_patience=args.prediction_stability_patience, prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, ) effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") else: @@ -1531,6 +1711,9 @@ def main(): prediction_stability_patience=args.prediction_stability_patience, prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_class_indices=available_labels, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, ) logits = model_out[0, 0] # (10,) t_infer += max(0.0, time.perf_counter() - _t) diff --git a/runtime_python/chat_web_app.py b/runtime_python/chat_web_app.py index 2e94dc6a6..9806a8002 100644 --- a/runtime_python/chat_web_app.py +++ b/runtime_python/chat_web_app.py @@ -229,7 +229,9 @@
+
+
Verify ordered top ranks; 0 intentionally disables rank verification.
@@ -270,6 +272,7 @@ function fmtNum(value,digits=3){const n=Number(value);return Number.isFinite(n)?n.toFixed(digits):null;} function reasoningCyclesValue(){const raw=el('reasoningCycles').value.trim();if(!raw)return null;const low=raw.toLowerCase();if(['auto','adaptive','smart'].includes(low))return 'auto';const n=Number(raw);return Number.isFinite(n)?n:raw;} +function addOptionalFiniteNumber(payload,key,input){const raw=input.value.trim();if(raw==='')return;const value=Number(raw);if(Number.isFinite(value))payload[key]=value;} function add(kind,text,timing,top,compute){ const d=document.createElement('div'); d.className='msg '+kind; @@ -290,12 +293,21 @@ } if(compute&&compute.applied){ const parts=[]; + if(compute.prediction_verifier_active===true) parts.push('verifier active'); + else if(compute.adaptive_compute) parts.push('verifier inactive'); if(compute.reasoning_budget_mode==='auto') parts.push('mode auto'); if(compute.requested_reasoning_cycles!==undefined&&compute.requested_reasoning_cycles!==null) parts.push(`requested ${compute.requested_reasoning_cycles}`); if(compute.cycles_used!==undefined&&compute.cycles_used!==null) parts.push(`used ${compute.cycles_used}`); if(compute.exit_reason) parts.push(`exit ${compute.exit_reason}`); const streak=fmtNum(compute.prediction_streak); if(streak) parts.push(`stable ${streak}`); const drift=fmtNum(compute.prediction_confidence_delta); if(drift) parts.push(`drift ${drift}`); + const observedMargin=fmtNum(compute.prediction_margin,6); if(observedMargin) parts.push(`top-1 margin ${observedMargin}`); + const decisionMargin=fmtNum(compute.prediction_decision_margin,6); if(decisionMargin) parts.push(`decision margin ${decisionMargin}`); + const marginFloor=fmtNum(compute.prediction_stability_margin,6); if(marginFloor) parts.push(`margin floor ${marginFloor}`); + if(compute.prediction_rank_depth!==undefined&&compute.prediction_rank_depth!==null) parts.push(`verified depth ${compute.prediction_rank_depth}`); + if(compute.prediction_stability_rank_depth!==undefined&&compute.prediction_stability_rank_depth!==null) parts.push(`requested depth ${compute.prediction_stability_rank_depth}`); + if(compute.prediction_class_count!==undefined&&compute.prediction_class_count!==null) parts.push(`verifier ${compute.prediction_class_count} classes`); + if(compute.prediction_class_selection_valid===false) parts.push('verifier scope invalid'); const ponder=fmtNum(compute.ponder_cost); if(ponder) parts.push(`ponder ${ponder}`); const consistency=fmtNum(compute.consistency_loss); if(consistency) parts.push(`consistency ${consistency}`); const entropy=fmtNum(compute.gating_entropy); if(entropy) parts.push(`gate entropy ${entropy}`); @@ -368,6 +380,8 @@ if(d.status.adaptive_exit_entropy !== undefined) el('exitEntropy').value = d.status.adaptive_exit_entropy; if(d.status.prediction_stability_patience !== undefined) el('stabilityPatience').value = d.status.prediction_stability_patience; if(d.status.prediction_stability_tol !== undefined) el('stabilityTol').value = d.status.prediction_stability_tol; + if(d.status.prediction_stability_margin !== undefined) el('stabilityMargin').value = d.status.prediction_stability_margin; + if(d.status.prediction_stability_rank_depth !== undefined) el('stabilityRankDepth').value = d.status.prediction_stability_rank_depth; }catch(e){ el('statusBox').textContent='TELEMETRY ERROR: '+e.message; } } function renderRouteShadowRegistry(snapshot){ @@ -408,14 +422,20 @@ promptEl.style.height = 'auto'; try{ const cycles = reasoningCyclesValue(); - const d=await jpost('/api/chat',{session_id:sid,message:text,style_mode:el('style').value,response_temperature:Number(el('rt').value),show_top_responses:Number(el('showTop').value),reasoning_cycles:cycles,adaptive_compute:el('adaptiveCompute').value==='on',auto_compute:el('autoCompute').value==='on',adaptive_exit_tol:Number(el('exitTol').value),adaptive_exit_entropy:Number(el('exitEntropy').value),prediction_stability_patience:Number(el('stabilityPatience').value),prediction_stability_tol:Number(el('stabilityTol').value)}); + const payload={session_id:sid,message:text,style_mode:el('style').value,response_temperature:Number(el('rt').value),show_top_responses:Number(el('showTop').value),reasoning_cycles:cycles,adaptive_compute:el('adaptiveCompute').value==='on',auto_compute:el('autoCompute').value==='on',adaptive_exit_tol:Number(el('exitTol').value),adaptive_exit_entropy:Number(el('exitEntropy').value),prediction_stability_patience:Number(el('stabilityPatience').value),prediction_stability_tol:Number(el('stabilityTol').value)}; + addOptionalFiniteNumber(payload,'prediction_stability_margin',el('stabilityMargin')); + addOptionalFiniteNumber(payload,'prediction_stability_rank_depth',el('stabilityRankDepth')); + const d=await jpost('/api/chat',payload); add('bot',d.response,d.timing_ms,d.top_candidates,d.compute); }catch(e){ add('bot','CORE ERROR: '+e.message); } } async function sweepCompute(){ const text=promptEl.value.trim(); if(!text) return; try{ - const d=await jpost('/api/compute_sweep',{session_id:sid,message:text,cycles:[1,3,8],adaptive_compute:el('adaptiveCompute').value==='on',adaptive_exit_tol:Number(el('exitTol').value),adaptive_exit_entropy:Number(el('exitEntropy').value),prediction_stability_patience:Number(el('stabilityPatience').value),prediction_stability_tol:Number(el('stabilityTol').value)}); + const payload={session_id:sid,message:text,cycles:[1,3,8],adaptive_compute:el('adaptiveCompute').value==='on',adaptive_exit_tol:Number(el('exitTol').value),adaptive_exit_entropy:Number(el('exitEntropy').value),prediction_stability_patience:Number(el('stabilityPatience').value),prediction_stability_tol:Number(el('stabilityTol').value)}; + addOptionalFiniteNumber(payload,'prediction_stability_margin',el('stabilityMargin')); + addOptionalFiniteNumber(payload,'prediction_stability_rank_depth',el('stabilityRankDepth')); + const d=await jpost('/api/compute_sweep',payload); const lines=['Compute sweep for draft prompt:']; d.rows.forEach((row)=>{ const conf=fmtNum(row.confidence); @@ -446,6 +466,8 @@ "adaptive_exit_entropy", "prediction_stability_patience", "prediction_stability_tol", + "prediction_stability_margin", + "prediction_stability_rank_depth", "auto_compute", ) @@ -458,6 +480,8 @@ def _library_runtime_compute_defaults() -> Dict[str, Any]: "adaptive_exit_entropy": chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY, "prediction_stability_patience": chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE, "prediction_stability_tol": chat_app.DEFAULT_PREDICTION_STABILITY_TOL, + "prediction_stability_margin": chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN, + "prediction_stability_rank_depth": chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, "auto_compute": False, } @@ -492,6 +516,12 @@ def _normalize_runtime_compute_defaults(values: Dict[str, Any]) -> Dict[str, Any values.get("prediction_stability_tol"), chat_app.DEFAULT_PREDICTION_STABILITY_TOL, ), + "prediction_stability_margin": chat_app._coerce_prediction_stability_margin( + values.get("prediction_stability_margin"), + ), + "prediction_stability_rank_depth": chat_app._coerce_prediction_stability_rank_depth( + values.get("prediction_stability_rank_depth"), + ), "auto_compute": chat_app._coerce_bool(values.get("auto_compute")), } @@ -505,6 +535,8 @@ def _runtime_compute_cli_overrides(args: argparse.Namespace) -> Dict[str, Any]: "adaptive_exit_entropy": getattr(args, "adaptive_exit_entropy", None), "prediction_stability_patience": getattr(args, "prediction_stability_patience", None), "prediction_stability_tol": getattr(args, "prediction_stability_tol", None), + "prediction_stability_margin": getattr(args, "prediction_stability_margin", None), + "prediction_stability_rank_depth": getattr(args, "prediction_stability_rank_depth", None), "auto_compute": getattr(args, "auto_compute", None), } return {key: value for key, value in values.items() if value is not None} @@ -571,6 +603,8 @@ def status(self) -> Dict[str, Any]: "adaptive_exit_entropy": chat_app._coerce_nonnegative_float(self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), "prediction_stability_patience": chat_app._coerce_nonnegative_int(self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE, chat_app.MAX_RUNTIME_REASONING_CYCLES), "prediction_stability_tol": chat_app._coerce_nonnegative_float(self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + "prediction_stability_margin": chat_app._coerce_prediction_stability_margin(self.defaults.get("prediction_stability_margin", chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN)), + "prediction_stability_rank_depth": chat_app._coerce_prediction_stability_rank_depth(self.defaults.get("prediction_stability_rank_depth", chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH)), "auto_compute": chat_app._coerce_bool(self.defaults.get("auto_compute", False)), } @@ -616,16 +650,7 @@ def route_shadow_registry_snapshot(self) -> Dict[str, Any]: } def _parse_buckets(self, meta: Dict[str, Any]) -> None: - buckets: Dict[int, List[Dict[str, Any]]] = {} - raw = meta.get("buckets", {}) - if isinstance(raw, dict): - for k, v in raw.items(): - try: - label = int(k) - except Exception: - continue - if isinstance(v, list) and v: - buckets[label] = v + buckets = chat_app._parse_metadata_buckets(meta.get("buckets", {})) self.buckets = buckets self.available_labels = sorted(buckets.keys()) or list(range(chat_app.MODEL_CLASSES)) @@ -705,6 +730,11 @@ def _run_compute_sweep_rows( cycles: Any, adaptive: bool, exit_tol: Optional[float], + exit_entropy_threshold: Any = None, + prediction_stability_patience: Any = None, + prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> List[Dict[str, Any]]: return chat_app.evaluate_runtime_compute_budgets( model, @@ -713,6 +743,11 @@ def _run_compute_sweep_rows( cycles=cycles, adaptive_compute=adaptive, exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=prediction_stability_margin, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) def _select_auto_compute_budget(self, rows: List[Dict[str, Any]]) -> Dict[str, Any]: @@ -728,6 +763,8 @@ def compute_sweep( adaptive_exit_entropy: Any = None, prediction_stability_patience: Any = None, prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> Dict[str, Any]: if not user_text.strip(): raise ValueError("Empty message") @@ -752,6 +789,16 @@ def compute_sweep( x = chat_app.text_to_model_input(context, feature_mode=feature_mode).to(self.device) idx = torch.tensor(labels, dtype=torch.long, device=self.device) rows: List[Dict[str, Any]] = [] + resolved_prediction_stability_margin = chat_app._coerce_prediction_stability_margin( + prediction_stability_margin + if prediction_stability_margin is not None + else self.defaults.get("prediction_stability_margin", chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN) + ) + resolved_prediction_stability_rank_depth = chat_app._coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth + if prediction_stability_rank_depth is not None + else self.defaults.get("prediction_stability_rank_depth", chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH) + ) with torch.no_grad(): for cycle_count in requested_cycles: @@ -765,6 +812,9 @@ def compute_sweep( exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + prediction_stability_margin=resolved_prediction_stability_margin, + prediction_class_indices=labels, + prediction_stability_rank_depth=resolved_prediction_stability_rank_depth, ) latency_ms = round((time.perf_counter() - t0) * 1000, 1) logits = model_out[0, 0] @@ -803,6 +853,8 @@ def chat( prediction_stability_patience: Any = None, prediction_stability_tol: Any = None, auto_compute: Optional[bool] = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> Dict[str, Any]: if not user_text.strip(): raise ValueError("Empty message") @@ -832,6 +884,16 @@ def chat( if adaptive_exit_tol is None else adaptive_exit_tol ) + resolved_prediction_stability_margin = chat_app._coerce_prediction_stability_margin( + prediction_stability_margin + if prediction_stability_margin is not None + else self.defaults.get("prediction_stability_margin", chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN) + ) + resolved_prediction_stability_rank_depth = chat_app._coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth + if prediction_stability_rank_depth is not None + else self.defaults.get("prediction_stability_rank_depth", chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH) + ) effective_reasoning_cycles = ( self.defaults.get("reasoning_cycles") if reasoning_cycles is None @@ -857,6 +919,8 @@ def chat( exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + prediction_stability_margin=resolved_prediction_stability_margin, + prediction_stability_rank_depth=resolved_prediction_stability_rank_depth, auto_reasoning_context=context, ) effective_reasoning_cycles = compute_plan.get("selected_reasoning_cycles") @@ -871,7 +935,10 @@ def chat( exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + prediction_stability_margin=resolved_prediction_stability_margin, auto_reasoning_context=context, + prediction_class_indices=labels, + prediction_stability_rank_depth=resolved_prediction_stability_rank_depth, ) logits = model_out[0, 0] t_infer += time.perf_counter() - tt @@ -1038,6 +1105,8 @@ def api_chat(): prediction_stability_patience=p.get('prediction_stability_patience'), prediction_stability_tol=p.get('prediction_stability_tol'), auto_compute=p.get('auto_compute'), + prediction_stability_margin=p.get('prediction_stability_margin'), + prediction_stability_rank_depth=p.get('prediction_stability_rank_depth'), )) except Exception as e: return jsonify({"ok": False, "error": str(e)}), 400 @@ -1057,6 +1126,8 @@ def api_compute_sweep(): adaptive_exit_entropy=p.get('adaptive_exit_entropy'), prediction_stability_patience=p.get('prediction_stability_patience'), prediction_stability_tol=p.get('prediction_stability_tol'), + prediction_stability_margin=p.get('prediction_stability_margin'), + prediction_stability_rank_depth=p.get('prediction_stability_rank_depth'), )) except Exception as e: return jsonify({"ok": False, "error": str(e)}), 400 @@ -1113,6 +1184,8 @@ def main() -> None: ap.add_argument('--adaptive_exit_entropy', type=float, default=None) ap.add_argument('--prediction_stability_patience', type=int, default=None) ap.add_argument('--prediction_stability_tol', type=float, default=None) + ap.add_argument('--prediction_stability_margin', type=float, default=None) + ap.add_argument('--prediction_stability_rank_depth', type=int, default=None) auto_compute_group = ap.add_mutually_exclusive_group() auto_compute_group.add_argument( '--auto_compute', diff --git a/runtime_python/model_variants.py b/runtime_python/model_variants.py index 8fde3df37..e6ff4efb7 100644 --- a/runtime_python/model_variants.py +++ b/runtime_python/model_variants.py @@ -6029,6 +6029,19 @@ def __init__( self.register_buffer("last_gating_entropy", torch.tensor(0.0)) self.register_buffer("last_prediction_streak", torch.tensor(0.0), persistent=False) self.register_buffer("last_prediction_confidence_delta", torch.tensor(0.0), persistent=False) + self.register_buffer("last_prediction_margin", torch.tensor(0.0), persistent=False) + self.register_buffer( + "last_prediction_decision_margin", torch.tensor(0.0), persistent=False + ) + self.register_buffer( + "last_prediction_rank_depth", torch.tensor(1.0), persistent=False + ) + self.register_buffer( + "last_prediction_class_count", torch.tensor(float(out_dim)), persistent=False + ) + self.register_buffer( + "last_prediction_class_selection_valid", torch.tensor(True), persistent=False + ) self.register_buffer( "last_prediction_topk_js_divergence", torch.tensor(0.0), persistent=False ) @@ -6135,6 +6148,72 @@ def _kl(left: torch.Tensor, right: torch.Tensor) -> torch.Tensor: + _kl(current_compact, compact_midpoint) ) + @staticmethod + def _normalize_prediction_class_indices( + prediction_class_indices, + *, + out_dim: int, + device: torch.device, + ): + """Validate an index list or full-length bool mask without raising. + + ``None`` and any complete permutation of all output classes use the + untouched all-class verifier path. Invalid scopes return no index + tensor and ``False`` so inference can continue while the + prediction-stability exit fails closed. + """ + + if prediction_class_indices is None: + return None, int(out_dim), int(out_dim) >= 2 + + try: + raw_selection = ( + prediction_class_indices.detach() + if torch.is_tensor(prediction_class_indices) + else torch.as_tensor(prediction_class_indices) + ) + except (TypeError, ValueError, RuntimeError): + return None, 0, False + + if raw_selection.ndim != 1: + return None, 0, False + + if raw_selection.dtype == torch.bool: + if int(raw_selection.numel()) != int(out_dim): + return None, 0, False + class_indices = raw_selection.nonzero(as_tuple=False).flatten() + elif raw_selection.dtype.is_floating_point or raw_selection.dtype.is_complex: + return None, 0, False + else: + class_indices = raw_selection + + try: + class_indices = class_indices.to(device=device, dtype=torch.long) + except (TypeError, ValueError, RuntimeError): + return None, 0, False + + class_count = int(class_indices.numel()) + if class_count == 0: + return None, 0, False + try: + selection_is_invalid = ( + int(class_indices.min().item()) < 0 + or int(class_indices.max().item()) >= int(out_dim) + or int(torch.unique(class_indices).numel()) != class_count + ) + except (TypeError, ValueError, RuntimeError): + return None, 0, False + if selection_is_invalid: + return None, 0, False + + # A valid complete selection is equivalent to ``None``. Avoiding an + # index_select here preserves the historical all-class numerics bit for + # bit, including for a complete selection supplied in another order. + if class_count == int(out_dim): + return None, class_count, class_count >= 2 + + return class_indices, class_count, class_count >= 2 + def forward( self, x, @@ -6145,6 +6224,10 @@ def forward( prediction_stability_patience: int = 2, prediction_stability_tol: float = 5e-3, prediction_stability_top_k: int = 5, + prediction_stability_margin: float = 0.0, + prediction_output_transform: Optional[nn.Module] = None, + prediction_class_indices=None, + prediction_stability_rank_depth: int = 1, ): N = x.shape[0] * x.shape[1] if x.dim() == 3 else x.shape[0] shape_prefix = x.shape[:-1] @@ -6174,13 +6257,48 @@ def forward( cycles_used = n_cycles stability_patience = max(0, int(prediction_stability_patience)) stability_tol = max(0.0, float(prediction_stability_tol)) - use_prediction_stability = adaptive_compute and not self.training and stability_patience > 0 + try: + stability_margin = float(prediction_stability_margin) + except (TypeError, ValueError): + stability_margin = float("inf") + if not math.isfinite(stability_margin) or stability_margin < 0.0: + stability_margin = float("inf") + ( + verifier_class_indices, + verifier_class_count, + verifier_class_selection_valid, + ) = self._normalize_prediction_class_indices( + prediction_class_indices, + out_dim=self.out_dim, + device=x_flat.device, + ) + try: + requested_rank_depth = int(prediction_stability_rank_depth) + except (TypeError, ValueError, OverflowError, RuntimeError): + requested_rank_depth = 0 + effective_rank_depth = min( + max(0, requested_rank_depth), + max(0, int(verifier_class_count) - 1), + ) + verifier_decision_margin_valid = ( + verifier_class_selection_valid and effective_rank_depth >= 1 + ) + use_prediction_stability = ( + adaptive_compute + and not self.training + and stability_patience > 0 + and verifier_decision_margin_valid + ) weighted_recursive = torch.zeros(N, self.out_dim, device=x_flat.device, dtype=x_flat.dtype) previous_prediction = None prediction_streak = torch.zeros(N, device=x_flat.device, dtype=torch.long) stable_confidence_min = None stable_confidence_max = None prediction_confidence_delta = torch.zeros(N, device=x_flat.device, dtype=x_flat.dtype) + prediction_margin = torch.zeros(N, device=x_flat.device, dtype=x_flat.dtype) + prediction_decision_margin = torch.zeros( + N, device=x_flat.device, dtype=x_flat.dtype + ) prediction_topk_js_divergence = torch.zeros( N, device=x_flat.device, dtype=x_flat.dtype ) @@ -6246,7 +6364,19 @@ def forward( + self.shared_scale * shared_out + (self.alpha + 1e-4) * prefix_recursive ) - prefix_probs = F.softmax(prefix_output, dim=-1) + # Verify the exact distribution returned by the complete model, + # including its learned post-head calibration layer. + verifier_output = ( + prediction_output_transform(prefix_output) + if prediction_output_transform is not None + else prefix_output + ) + scoped_verifier_output = ( + verifier_output.index_select(-1, verifier_class_indices) + if verifier_class_indices is not None + else verifier_output + ) + prefix_probs = F.softmax(scoped_verifier_output, dim=-1) if previous_prediction_distribution is not None: prediction_topk_js_divergence = self._topk_js_divergence( previous_prediction_distribution, @@ -6259,6 +6389,28 @@ def forward( ) previous_prediction_distribution = prefix_probs prediction_confidence, prediction = prefix_probs.max(dim=-1) + margin_verifiable = ( + verifier_class_selection_valid + and int(prefix_probs.shape[-1]) >= 2 + ) + if margin_verifiable: + top_two_probabilities = prefix_probs.topk(k=2, dim=-1).values + prediction_margin = ( + top_two_probabilities[:, 0] - top_two_probabilities[:, 1] + ) + decision_probabilities = prefix_probs.topk( + k=effective_rank_depth + 1, + dim=-1, + ).values + prediction_decision_margin = ( + decision_probabilities[:, :-1] + - decision_probabilities[:, 1:] + ).min(dim=-1).values + else: + prediction_margin = torch.zeros_like(prediction_confidence) + prediction_decision_margin = torch.zeros_like( + prediction_confidence + ) if previous_prediction is None: prediction_streak = torch.ones_like(prediction_streak) stable_confidence_min = prediction_confidence @@ -6286,8 +6438,16 @@ def forward( prediction_streak >= stability_patience ) & ( prediction_confidence_delta <= stability_tol + ) & ( + prediction_decision_margin >= stability_margin + ) & ( + margin_verifiable + ) + prediction_stable = ( + verifier_class_selection_valid + and stable_mask.numel() > 0 + and bool(stable_mask.all().item()) ) - prediction_stable = bool(stable_mask.all().item()) # Inference-only verifier and convergence exits. Prediction # stability is checked first so diagnostics identify the new path @@ -6325,18 +6485,51 @@ def forward( self.last_cycles_used = torch.tensor( float(cycles_used), device=x_flat.device ) - self.last_prediction_streak = prediction_streak.float().mean() - self.last_prediction_confidence_delta = prediction_confidence_delta.mean() + self.last_prediction_streak = ( + prediction_streak.float().mean() + if prediction_streak.numel() + else prediction_margin.new_zeros(()) + ) + self.last_prediction_confidence_delta = ( + prediction_confidence_delta.mean() + if prediction_confidence_delta.numel() + else prediction_margin.new_zeros(()) + ) + # Batch-level telemetry reports the least decisive member because the + # all-samples early-exit rule is only as safe as its smallest margin. + self.last_prediction_margin = ( + prediction_margin.min().detach() + if prediction_margin.numel() + else prediction_margin.new_zeros(()) + ) + self.last_prediction_decision_margin = ( + prediction_decision_margin.min().detach() + if prediction_decision_margin.numel() + else prediction_decision_margin.new_zeros(()) + ) + self.last_prediction_rank_depth = torch.tensor( + float(effective_rank_depth), device=x_flat.device + ) + self.last_prediction_class_count = torch.tensor( + float(verifier_class_count), device=x_flat.device + ) + self.last_prediction_class_selection_valid = torch.tensor( + bool(verifier_class_selection_valid), device=x_flat.device + ) self.last_prediction_topk_js_divergence = ( prediction_topk_js_divergence.mean().detach() + if prediction_topk_js_divergence.numel() + else prediction_margin.new_zeros(()) ) self.last_prediction_topk_js_divergence_max = ( prediction_topk_js_divergence_max.mean().detach() + if prediction_topk_js_divergence_max.numel() + else prediction_margin.new_zeros(()) ) self.last_exit_reason = exit_reason # Calculate gating entropy for MoE cores - if gate_probs_list: + if gate_probs_list and N > 0: all_gate_probs = torch.stack(gate_probs_list, dim=0) # (Steps, N, n_cores) mean_gate_probs = all_gate_probs.mean(dim=(0, 1)) # (n_cores,) gating_entropy = -torch.sum(mean_gate_probs * torch.log(mean_gate_probs + 1e-9)) @@ -6363,7 +6556,7 @@ def forward( + (self.alpha + 1e-4) * recursive_out ) - return output.view(*shape_prefix, -1) + return output.view(*shape_prefix, self.out_dim) class ChampionNetCognitiveLeapUltraExpert(nn.Module): @@ -6406,6 +6599,9 @@ def forward( prediction_stability_patience: int = 2, prediction_stability_tol: float = 5e-3, prediction_stability_top_k: int = 5, + prediction_stability_margin: float = 0.0, + prediction_class_indices=None, + prediction_stability_rank_depth: int = 1, ): for layer in self.layers: if isinstance(layer, CognitiveLeapUltraExpertHead): @@ -6418,6 +6614,10 @@ def forward( prediction_stability_patience=prediction_stability_patience, prediction_stability_tol=prediction_stability_tol, prediction_stability_top_k=prediction_stability_top_k, + prediction_stability_margin=prediction_stability_margin, + prediction_output_transform=self.layers[11], + prediction_class_indices=prediction_class_indices, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) else: x = layer(x) diff --git a/source/benchmark_cognitive_leap_ultra_v51.py b/source/benchmark_cognitive_leap_ultra_v51.py index 060d39eee..4ed16ebb3 100644 --- a/source/benchmark_cognitive_leap_ultra_v51.py +++ b/source/benchmark_cognitive_leap_ultra_v51.py @@ -33,6 +33,8 @@ N_CLASSES = 10 N_OPS = 4 IN_DIM = 128 +DEFAULT_PREDICTION_STABILITY_MARGIN = 1e-4 +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = 3 def make_chained_task(n: int, seed: int) -> tuple[torch.Tensor, torch.Tensor]: @@ -134,7 +136,7 @@ def evaluate( y_test: torch.Tensor, device: torch.device, **forward_kwargs: Any, -) -> Dict[str, float]: +) -> Dict[str, Any]: model.eval() x = x_test.to(device) y = y_test.to(device) @@ -144,12 +146,34 @@ def evaluate( loss = F.cross_entropy(logits, y) acc = (logits.argmax(dim=-1) == y).float().mean() head = model.layers[10] + observed_rank_depth = int(head.last_prediction_rank_depth.item()) + prediction_verifier_active = bool(forward_kwargs.get("adaptive_compute", False)) and ( + int(forward_kwargs.get("prediction_stability_patience", 2)) > 0 + and bool(head.last_prediction_class_selection_valid.item()) + and observed_rank_depth > 0 + ) return { "loss": round(float(loss.item()), 4), "accuracy": round(float(acc.item()), 4), "latency_ms": round(latency_ms, 2), "cycles_used": round(float(head.last_cycles_used.item()), 2), "gating_entropy": round(float(head.last_gating_entropy.item()), 4), + "prediction_verifier_active": prediction_verifier_active, + "prediction_margin": ( + round(float(head.last_prediction_margin.item()), 8) + if prediction_verifier_active + else None + ), + "prediction_decision_margin": ( + round(float(head.last_prediction_decision_margin.item()), 8) + if prediction_verifier_active + else None + ), + "prediction_rank_depth": ( + observed_rank_depth + if prediction_verifier_active + else None + ), } @@ -208,6 +232,8 @@ def main() -> None: adaptive_compute=True, exit_tol=1e-4, exit_entropy_threshold=0.0, + prediction_stability_margin=DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ) out_dir = Path(args.output_dir) @@ -233,6 +259,11 @@ def main() -> None: "eval_default": eval_default, "test_time_scaling": eval_cycles, "adaptive_compute": eval_adaptive, + "adaptive_compute_configuration": { + "prediction_stability_margin": DEFAULT_PREDICTION_STABILITY_MARGIN, + "prediction_stability_rank_depth": DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + "margin_scope": "checkpoint_calibrated_not_universal", + }, } metrics_path.write_text(json.dumps(results, indent=2), encoding="utf-8") diff --git a/source/benchmark_progressive_auto_compute.py b/source/benchmark_progressive_auto_compute.py index 0407ade7d..642079000 100644 --- a/source/benchmark_progressive_auto_compute.py +++ b/source/benchmark_progressive_auto_compute.py @@ -42,7 +42,9 @@ DEFAULT_AUTO_COMPUTE_CONFIDENCE, DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, DEFAULT_AUTO_COMPUTE_ENTROPY, + DEFAULT_PREDICTION_STABILITY_MARGIN, DEFAULT_PREDICTION_STABILITY_PATIENCE, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, DEFAULT_PREDICTION_STABILITY_TOL, evaluate_runtime_compute_budgets, forward_with_runtime_compute, @@ -153,6 +155,85 @@ def _finite_float(value: Any, *, label: str) -> float: return number +def _nonnegative_finite_float(value: Any, *, label: str) -> float: + number = _finite_float(value, label=label) + if number < 0.0: + raise ValueError(f"{label} must be nonnegative, got {value!r}") + return number + + +def _observed_prediction_margin_summary( + values: Sequence[float], +) -> Dict[str, Any]: + """Summarize final-call margins; the minimum is the safety-facing value.""" + + if not values: + return { + "metric": "top_1_minus_top_2_probability", + "observation_count": 0, + "minimum": None, + "mean": None, + "maximum": None, + } + tensor = torch.tensor(list(values), dtype=torch.float64) + return { + "metric": "top_1_minus_top_2_probability", + "observation_count": len(values), + "minimum": round(float(tensor.min().item()), 8), + "mean": round(float(tensor.mean().item()), 8), + "maximum": round(float(tensor.max().item()), 8), + } + + +def _observed_prediction_decision_margin_summary( + values: Sequence[float], +) -> Dict[str, Any]: + """Summarize the rank-boundary margin used by the active verifier.""" + + if not values: + return { + "metric": "minimum_adjacent_probability_gap_through_rank_depth", + "observation_count": 0, + "minimum": None, + "mean": None, + "maximum": None, + } + tensor = torch.tensor(list(values), dtype=torch.float64) + return { + "metric": "minimum_adjacent_probability_gap_through_rank_depth", + "observation_count": len(values), + "minimum": round(float(tensor.min().item()), 8), + "mean": round(float(tensor.mean().item()), 8), + "maximum": round(float(tensor.max().item()), 8), + } + + +def _prediction_margin_from_compute(compute: Mapping[str, Any]) -> float | None: + if compute.get("prediction_verifier_active") is not True: + return None + value = compute.get("prediction_margin") + if value is None: + return None + return _nonnegative_finite_float( + value, + label="observed prediction stability margin", + ) + + +def _prediction_decision_margin_from_compute( + compute: Mapping[str, Any], +) -> float | None: + if compute.get("prediction_verifier_active") is not True: + return None + value = compute.get("prediction_decision_margin") + if value is None: + return None + return _nonnegative_finite_float( + value, + label="observed prediction decision margin", + ) + + def _synchronize_for_timing(device: torch.device) -> None: device_type = str(device).split(":", 1)[0].lower() if device_type == "cuda" and torch.cuda.is_available(): @@ -189,12 +270,18 @@ def legacy_v1_auto_compute_forward( exit_entropy_threshold: Any = DEFAULT_ADAPTIVE_EXIT_ENTROPY, prediction_stability_patience: Any = DEFAULT_PREDICTION_STABILITY_PATIENCE, prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, auto_reasoning_context: str = "", distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, ) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: """Execute the retired v1 probe-all/select/rerun controller literally.""" del distribution_top_k # The v1 selector had no distribution-stability input. + stability_margin = _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ) rows = evaluate_runtime_compute_budgets( model, x, @@ -202,6 +289,11 @@ def legacy_v1_auto_compute_forward( cycles=cycles, adaptive_compute=bool(adaptive_compute), exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=stability_margin, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) selection = select_auto_runtime_compute_budget( rows, @@ -218,6 +310,9 @@ def legacy_v1_auto_compute_forward( exit_entropy_threshold=exit_entropy_threshold, prediction_stability_patience=prediction_stability_patience, prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=stability_margin, + prediction_stability_rank_depth=prediction_stability_rank_depth, + prediction_class_indices=available_labels, auto_reasoning_context=auto_reasoning_context, ) plan = { @@ -290,6 +385,8 @@ def benchmark_requests( exit_entropy_threshold: float = DEFAULT_ADAPTIVE_EXIT_ENTROPY, prediction_stability_patience: int = DEFAULT_PREDICTION_STABILITY_PATIENCE, prediction_stability_tol: float = DEFAULT_PREDICTION_STABILITY_TOL, + prediction_stability_margin: float = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, order_offset: int = 0, legacy_fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]] @@ -323,10 +420,17 @@ def benchmark_requests( "prediction_stability_tol": _finite_float( prediction_stability_tol, label="prediction stability tolerance" ), + "prediction_stability_margin": _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ), + "prediction_stability_rank_depth": int(prediction_stability_rank_depth), "distribution_top_k": int(distribution_top_k), } if settings["prediction_stability_patience"] < 0: raise ValueError("prediction_stability_patience must be nonnegative") + if settings["prediction_stability_rank_depth"] < 0: + raise ValueError("prediction_stability_rank_depth must be nonnegative") if settings["distribution_top_k"] <= 0: raise ValueError("distribution_top_k must be positive") @@ -344,6 +448,10 @@ def benchmark_requests( legacy_latencies: List[float] = [] progressive_latencies: List[float] = [] + legacy_prediction_margins: List[float] = [] + progressive_prediction_margins: List[float] = [] + legacy_prediction_decision_margins: List[float] = [] + progressive_prediction_decision_margins: List[float] = [] measurement_orders: Counter[str] = Counter() selected_cycle_pairs: Counter[str] = Counter() selected_cycle_disagreements = 0 @@ -362,12 +470,22 @@ def timed_call( fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]], sample: torch.Tensor, values: List[float], + observed_margins: List[float], + observed_decision_margins: List[float], ) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: _synchronize_for_timing(sample.device) started = time.perf_counter() result = fn(model, sample, labels, **settings) _synchronize_for_timing(sample.device) values.append((time.perf_counter() - started) * 1000.0) + observed_margin = _prediction_margin_from_compute(result[1]) + if observed_margin is not None: + observed_margins.append(observed_margin) + observed_decision_margin = _prediction_decision_margin_from_compute( + result[1] + ) + if observed_decision_margin is not None: + observed_decision_margins.append(observed_decision_margin) return result for index in range(request_count): @@ -375,16 +493,36 @@ def timed_call( legacy_first = (index + int(order_offset)) % 2 == 0 if legacy_first: measurement_orders["legacy_then_progressive"] += 1 - legacy_result = timed_call(legacy, sample, legacy_latencies) + legacy_result = timed_call( + legacy, + sample, + legacy_latencies, + legacy_prediction_margins, + legacy_prediction_decision_margins, + ) progressive_result = timed_call( - progressive, sample, progressive_latencies + progressive, + sample, + progressive_latencies, + progressive_prediction_margins, + progressive_prediction_decision_margins, ) else: measurement_orders["progressive_then_legacy"] += 1 progressive_result = timed_call( - progressive, sample, progressive_latencies + progressive, + sample, + progressive_latencies, + progressive_prediction_margins, + progressive_prediction_decision_margins, + ) + legacy_result = timed_call( + legacy, + sample, + legacy_latencies, + legacy_prediction_margins, + legacy_prediction_decision_margins, ) - legacy_result = timed_call(legacy, sample, legacy_latencies) legacy_output, _legacy_compute, legacy_plan = legacy_result progressive_output, _progressive_compute, progressive_plan = ( @@ -505,12 +643,28 @@ def timed_call( "strategy": "probe_all_select_and_rerun", "correct_predictions": legacy_correct, "forward_evaluations": legacy_forward_evaluations, + "observed_prediction_margin": _observed_prediction_margin_summary( + legacy_prediction_margins + ), + "observed_prediction_decision_margin": ( + _observed_prediction_decision_margin_summary( + legacy_prediction_decision_margins + ) + ), "latency": legacy_latency, }, "progressive": { "strategy": "progressive_accepted_probe", "correct_predictions": progressive_correct, "forward_evaluations": progressive_forward_evaluations, + "observed_prediction_margin": _observed_prediction_margin_summary( + progressive_prediction_margins + ), + "observed_prediction_decision_margin": ( + _observed_prediction_decision_margin_summary( + progressive_prediction_decision_margins + ) + ), "latency": progressive_latency, }, "comparison": { @@ -556,6 +710,14 @@ def aggregate_results(seed_results: Sequence[Mapping[str, Any]]) -> Dict[str, An per_seed_latency_reductions: List[float] = [] per_seed: List[Dict[str, Any]] = [] seen_seeds: set[int] = set() + legacy_margin_observations = 0 + progressive_margin_observations = 0 + legacy_decision_margin_observations = 0 + progressive_decision_margin_observations = 0 + minimum_legacy_prediction_margin = math.inf + minimum_progressive_prediction_margin = math.inf + minimum_legacy_prediction_decision_margin = math.inf + minimum_progressive_prediction_decision_margin = math.inf for seed_result in seed_results: seed = int(seed_result["seed"]) @@ -582,6 +744,81 @@ def aggregate_results(seed_results: Sequence[Mapping[str, Any]]) -> Dict[str, An legacy = metrics["legacy_v1"] progressive = metrics["progressive"] comparison = metrics["comparison"] + legacy_margin = legacy.get("observed_prediction_margin", {}) + progressive_margin = progressive.get("observed_prediction_margin", {}) + legacy_decision_margin = legacy.get( + "observed_prediction_decision_margin", {} + ) + progressive_decision_margin = progressive.get( + "observed_prediction_decision_margin", {} + ) + legacy_margin_count = int(legacy_margin.get("observation_count", 0)) + progressive_margin_count = int( + progressive_margin.get("observation_count", 0) + ) + legacy_margin_minimum = legacy_margin.get("minimum") + progressive_margin_minimum = progressive_margin.get("minimum") + legacy_decision_margin_count = int( + legacy_decision_margin.get("observation_count", 0) + ) + progressive_decision_margin_count = int( + progressive_decision_margin.get("observation_count", 0) + ) + legacy_decision_margin_minimum = legacy_decision_margin.get("minimum") + progressive_decision_margin_minimum = progressive_decision_margin.get( + "minimum" + ) + if legacy_margin_count < 0 or progressive_margin_count < 0: + raise ValueError(f"Seed {seed} margin observation count is invalid") + if legacy_margin_count: + legacy_margin_minimum = _nonnegative_finite_float( + legacy_margin_minimum, + label="minimum legacy prediction stability margin", + ) + minimum_legacy_prediction_margin = min( + minimum_legacy_prediction_margin, + legacy_margin_minimum, + ) + legacy_margin_observations += legacy_margin_count + if progressive_margin_count: + progressive_margin_minimum = _nonnegative_finite_float( + progressive_margin_minimum, + label="minimum progressive prediction stability margin", + ) + minimum_progressive_prediction_margin = min( + minimum_progressive_prediction_margin, + progressive_margin_minimum, + ) + progressive_margin_observations += progressive_margin_count + if ( + legacy_decision_margin_count < 0 + or progressive_decision_margin_count < 0 + ): + raise ValueError( + f"Seed {seed} decision-margin observation count is invalid" + ) + if legacy_decision_margin_count: + legacy_decision_margin_minimum = _nonnegative_finite_float( + legacy_decision_margin_minimum, + label="minimum legacy prediction decision margin", + ) + minimum_legacy_prediction_decision_margin = min( + minimum_legacy_prediction_decision_margin, + legacy_decision_margin_minimum, + ) + legacy_decision_margin_observations += legacy_decision_margin_count + if progressive_decision_margin_count: + progressive_decision_margin_minimum = _nonnegative_finite_float( + progressive_decision_margin_minimum, + label="minimum progressive prediction decision margin", + ) + minimum_progressive_prediction_decision_margin = min( + minimum_progressive_prediction_decision_margin, + progressive_decision_margin_minimum, + ) + progressive_decision_margin_observations += ( + progressive_decision_margin_count + ) legacy_latency = _finite_float( legacy["latency"]["total_ms"], label="legacy latency" ) @@ -652,6 +889,14 @@ def aggregate_results(seed_results: Sequence[Mapping[str, Any]]) -> Dict[str, An seed_latency_reduction, 6 ), "measurement_order": dict(order), + "minimum_observed_prediction_margin": { + "legacy_v1": legacy_margin_minimum, + "progressive": progressive_margin_minimum, + }, + "minimum_observed_prediction_decision_margin": { + "legacy_v1": legacy_decision_margin_minimum, + "progressive": progressive_decision_margin_minimum, + }, } ) @@ -730,6 +975,48 @@ def aggregate_results(seed_results: Sequence[Mapping[str, Any]]) -> Dict[str, An total_progressive_latency_ms / total_requests, 6 ), "selected_cycle_pairs": dict(sorted(selected_cycle_pairs.items())), + "observed_prediction_margin": { + "metric": "top_1_minus_top_2_probability", + "legacy_v1": { + "observation_count": legacy_margin_observations, + "minimum": ( + round(minimum_legacy_prediction_margin, 8) + if legacy_margin_observations + else None + ), + }, + "progressive": { + "observation_count": progressive_margin_observations, + "minimum": ( + round(minimum_progressive_prediction_margin, 8) + if progressive_margin_observations + else None + ), + }, + }, + "observed_prediction_decision_margin": { + "metric": ( + "minimum_adjacent_probability_gap_through_rank_depth" + ), + "legacy_v1": { + "observation_count": legacy_decision_margin_observations, + "minimum": ( + round(minimum_legacy_prediction_decision_margin, 8) + if legacy_decision_margin_observations + else None + ), + }, + "progressive": { + "observation_count": ( + progressive_decision_margin_observations + ), + "minimum": ( + round(minimum_progressive_prediction_decision_margin, 8) + if progressive_decision_margin_observations + else None + ), + }, + }, }, "per_seed_summary": per_seed, "gates": { @@ -754,6 +1041,8 @@ def run_benchmark( exit_entropy_threshold: float = DEFAULT_ADAPTIVE_EXIT_ENTROPY, prediction_stability_patience: int = DEFAULT_PREDICTION_STABILITY_PATIENCE, prediction_stability_tol: float = DEFAULT_PREDICTION_STABILITY_TOL, + prediction_stability_margin: float = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, model_loader: Callable[[Path, Any], Any] | None = None, task_factory: Callable[..., Tuple[torch.Tensor, torch.Tensor]] | None = None, @@ -789,6 +1078,15 @@ def run_benchmark( raise ValueError("confidence_target must be between 0 and 1") if normalized_entropy_target < 0.0: raise ValueError("entropy_target must be nonnegative") + normalized_prediction_stability_margin = _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ) + normalized_prediction_stability_rank_depth = int( + prediction_stability_rank_depth + ) + if normalized_prediction_stability_rank_depth < 0: + raise ValueError("prediction_stability_rank_depth must be nonnegative") loader = model_loader or _load_model task = task_factory or make_chained_task @@ -812,6 +1110,12 @@ def run_benchmark( exit_entropy_threshold=float(exit_entropy_threshold), prediction_stability_patience=int(prediction_stability_patience), prediction_stability_tol=float(prediction_stability_tol), + prediction_stability_margin=( + normalized_prediction_stability_margin + ), + prediction_stability_rank_depth=( + normalized_prediction_stability_rank_depth + ), distribution_top_k=int(distribution_top_k), order_offset=seed_index % 2, ) @@ -847,6 +1151,12 @@ def run_benchmark( prediction_stability_patience ), "prediction_stability_tol": float(prediction_stability_tol), + "prediction_stability_margin": ( + normalized_prediction_stability_margin + ), + "prediction_stability_rank_depth": ( + normalized_prediction_stability_rank_depth + ), "distribution_top_k": int(distribution_top_k), "counterbalance": ( "alternate_per_request_with_alternating_seed_offset" @@ -921,6 +1231,20 @@ def build_parser() -> argparse.ArgumentParser: type=float, default=DEFAULT_PREDICTION_STABILITY_TOL, ) + parser.add_argument( + "--prediction-stability-margin", + "--prediction_stability_margin", + dest="prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + parser.add_argument( + "--prediction-stability-rank-depth", + "--prediction_stability_rank_depth", + dest="prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) parser.add_argument( "--distribution-top-k", type=int, @@ -965,6 +1289,10 @@ def main(argv: Iterable[str] | None = None) -> int: exit_entropy_threshold=float(args.exit_entropy_threshold), prediction_stability_patience=int(args.prediction_stability_patience), prediction_stability_tol=float(args.prediction_stability_tol), + prediction_stability_margin=float(args.prediction_stability_margin), + prediction_stability_rank_depth=int( + args.prediction_stability_rank_depth + ), distribution_top_k=int(args.distribution_top_k), ) encoded = _strict_json(payload) diff --git a/source/benchmark_v51_prediction_stability.py b/source/benchmark_v51_prediction_stability.py index fea1fab11..3fede0a7c 100644 --- a/source/benchmark_v51_prediction_stability.py +++ b/source/benchmark_v51_prediction_stability.py @@ -1,7 +1,8 @@ """Benchmark v51 prediction-stability exits under request-sized inference. -This intentionally disables the older latent/entropy exits so the comparison -isolates the new full-output stability verifier against a fixed-cycle baseline. +The standalone defaults disable the older latent/entropy exits to isolate the +full-output verifier. The multi-seed release gate also calls this benchmark +with the authoritative deployed runtime controls and metadata class scope. """ from __future__ import annotations @@ -10,13 +11,16 @@ from collections import Counter from datetime import datetime, timezone import json +import math +import numbers from pathlib import Path import time -from typing import Any, Dict, List +from typing import Any, Dict, List, Sequence import torch from benchmark_cognitive_leap_ultra_v51 import make_chained_task +import chat_app from device_utils import configure_torch_runtime, resolve_device from model_variants import ChampionNetCognitiveLeapUltraExpert @@ -25,6 +29,26 @@ DEFAULT_ARTIFACT_DIR = PROJECT_ROOT / "output" / "benchmark_v51_cognitive_leap_ultra_latest" DEFAULT_WEIGHTS = DEFAULT_ARTIFACT_DIR / "cognitive_leap_ultra_v51_trained.pth" DEFAULT_OUTPUT = DEFAULT_ARTIFACT_DIR / "prediction_stability_results.json" +DEFAULT_ADAPTIVE_EXIT_TOL = chat_app.DEFAULT_ADAPTIVE_EXIT_TOL +DEFAULT_ADAPTIVE_EXIT_ENTROPY = chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY +DEFAULT_PREDICTION_STABILITY_PATIENCE = ( + chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE +) +DEFAULT_PREDICTION_STABILITY_TOL = chat_app.DEFAULT_PREDICTION_STABILITY_TOL +DEFAULT_PREDICTION_STABILITY_MARGIN = chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = ( + chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH +) +DEFAULT_DECISION_TOP_K = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH +KNOWN_ADAPTIVE_EXIT_REASONS = frozenset( + { + "prediction_stable", + "latent_converged", + "low_entropy", + "halt_mass", + "max_cycles", + } +) def _load_model(weights: Path, device: torch.device) -> ChampionNetCognitiveLeapUltraExpert: @@ -36,8 +60,19 @@ def _load_model(weights: Path, device: torch.device) -> ChampionNetCognitiveLeap return model -def _latency_summary(values: List[float]) -> Dict[str, float]: +def _finite_values(values: Sequence[float], *, label: str) -> torch.Tensor: + if not values: + raise ValueError(f"{label} requires at least one value") tensor = torch.tensor(values, dtype=torch.float64) + if not bool(torch.isfinite(tensor).all().item()): + raise ValueError(f"{label} must contain only finite values") + return tensor + + +def _latency_summary(values: List[float]) -> Dict[str, float]: + tensor = _finite_values(values, label="latency summary") + if bool((tensor <= 0.0).any().item()): + raise ValueError("latency values must be positive") return { "total_ms": round(float(tensor.sum().item()), 6), "mean_ms": round(float(tensor.mean().item()), 3), @@ -46,16 +81,171 @@ def _latency_summary(values: List[float]) -> Dict[str, float]: } -def _scalar_summary(values: List[float]) -> Dict[str, float]: - tensor = torch.tensor(values, dtype=torch.float64) +def _scalar_summary(values: List[float], *, digits: int = 8) -> Dict[str, float]: + tensor = _finite_values(values, label="scalar summary") return { - "mean": round(float(tensor.mean().item()), 8), - "p50": round(float(torch.quantile(tensor, 0.50).item()), 8), - "p95": round(float(torch.quantile(tensor, 0.95).item()), 8), - "max": round(float(tensor.max().item()), 8), + "min": round(float(tensor.min().item()), digits), + "mean": round(float(tensor.mean().item()), digits), + "p50": round(float(torch.quantile(tensor, 0.50).item()), digits), + "p95": round(float(torch.quantile(tensor, 0.95).item()), digits), + "max": round(float(tensor.max().item()), digits), } +def _nonnegative_finite_float(value: Any, *, label: str) -> float: + try: + number = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError( + f"{label} must be finite and nonnegative, got {value!r}" + ) from exc + if not math.isfinite(number) or number < 0.0: + raise ValueError(f"{label} must be finite and nonnegative, got {value!r}") + return number + + +def _strict_bool_scalar(value: Any, *, label: str) -> bool: + if torch.is_tensor(value): + if value.numel() != 1: + raise ValueError(f"{label} must be a scalar boolean") + value = value.item() + if not isinstance(value, bool): + raise ValueError(f"{label} must be a boolean, got {value!r}") + return value + + +def _positive_integral_scalar(value: Any, *, label: str) -> int: + if torch.is_tensor(value): + if value.numel() != 1: + raise ValueError(f"{label} must be a scalar integer") + value = value.item() + if isinstance(value, bool): + raise ValueError(f"{label} must be a positive integer, got {value!r}") + number = _nonnegative_finite_float(value, label=label) + integer = int(number) + if integer <= 0 or number != float(integer): + raise ValueError(f"{label} must be a positive integer, got {value!r}") + return integer + + +def _normalize_prediction_class_indices_input( + prediction_class_indices: Any, +) -> List[int] | List[bool] | None: + """Normalize one optional verifier scope without guessing output width.""" + + if prediction_class_indices is None: + return None + if torch.is_tensor(prediction_class_indices): + raw = prediction_class_indices.detach().cpu() + if raw.ndim != 1: + raise ValueError("prediction_class_indices must be one-dimensional") + if raw.dtype == torch.bool: + values: List[int] | List[bool] = [bool(value) for value in raw.tolist()] + elif raw.dtype.is_floating_point or raw.dtype.is_complex: + raise ValueError("prediction_class_indices must contain integers or be a bool mask") + else: + values = [int(value) for value in raw.tolist()] + else: + if isinstance(prediction_class_indices, (str, bytes)): + raise ValueError("prediction_class_indices must be an integer sequence") + try: + raw_values = list(prediction_class_indices) + except TypeError as exc: + raise ValueError( + "prediction_class_indices must be an integer sequence" + ) from exc + if raw_values and all(isinstance(value, bool) for value in raw_values): + values = [bool(value) for value in raw_values] + else: + if any(isinstance(value, bool) for value in raw_values) or any( + not isinstance(value, numbers.Integral) for value in raw_values + ): + raise ValueError( + "prediction_class_indices must contain only integers or be a bool mask" + ) + values = [int(value) for value in raw_values] + + if not values: + raise ValueError("prediction_class_indices cannot be empty") + if all(isinstance(value, bool) for value in values): + if not any(values): + raise ValueError("prediction_class_indices bool mask must select a class") + return values + integer_values = [int(value) for value in values] + if any(value < 0 for value in integer_values): + raise ValueError("prediction_class_indices cannot contain negative indices") + if len(set(integer_values)) != len(integer_values): + raise ValueError("prediction_class_indices cannot contain duplicates") + return integer_values + + +def _request_logits(output: Any, *, label: str) -> torch.Tensor: + if not torch.is_tensor(output): + raise ValueError(f"{label} output must be a tensor") + detached = output.detach() + if detached.ndim < 1 or int(detached.shape[-1]) < 1: + raise ValueError(f"{label} output must have a non-empty class dimension") + if int(detached.numel()) != int(detached.shape[-1]): + raise ValueError(f"{label} output must contain exactly one serving request") + logits = detached.reshape(-1).to(device="cpu", dtype=torch.float64) + if not bool(torch.isfinite(logits).all().item()): + raise ValueError(f"{label} logits must be finite") + return logits + + +def _resolve_prediction_scope( + normalized_scope: List[int] | List[bool] | None, + *, + output_class_count: int, + decision_top_k: int, +) -> List[int]: + if normalized_scope is None: + resolved = list(range(output_class_count)) + elif all(isinstance(value, bool) for value in normalized_scope): + if len(normalized_scope) != output_class_count: + raise ValueError( + "prediction_class_indices bool mask length must match output classes" + ) + resolved = [index for index, selected in enumerate(normalized_scope) if selected] + else: + resolved = [int(value) for value in normalized_scope] + if max(resolved) >= output_class_count: + raise ValueError("prediction_class_indices contains an out-of-range class") + if len(resolved) < decision_top_k: + raise ValueError( + f"decision_top_k={decision_top_k} requires at least {decision_top_k} scoped classes" + ) + return resolved + + +def _distribution_distances( + fixed_logits: torch.Tensor, + adaptive_logits: torch.Tensor, +) -> tuple[float, float]: + fixed_distribution = torch.softmax(fixed_logits, dim=-1) + adaptive_distribution = torch.softmax(adaptive_logits, dim=-1) + midpoint = 0.5 * (fixed_distribution + adaptive_distribution) + fixed_kl = torch.sum( + torch.special.xlogy(fixed_distribution, fixed_distribution) + - torch.special.xlogy(fixed_distribution, midpoint) + ) + adaptive_kl = torch.sum( + torch.special.xlogy(adaptive_distribution, adaptive_distribution) + - torch.special.xlogy(adaptive_distribution, midpoint) + ) + js_divergence = float((0.5 * (fixed_kl + adaptive_kl)).item()) + total_variation = float( + (0.5 * torch.sum(torch.abs(fixed_distribution - adaptive_distribution))).item() + ) + # Floating-point cancellation can produce a tiny negative JS value. + if js_divergence < 0.0 and js_divergence > -1e-15: + js_divergence = 0.0 + return ( + _nonnegative_finite_float(js_divergence, label="distribution JSD"), + _nonnegative_finite_float(total_variation, label="total variation distance"), + ) + + def _synchronize_for_timing(device: torch.device) -> None: """Finish queued accelerator work so request latency is measured honestly.""" @@ -79,8 +269,54 @@ def benchmark_serving_requests( stability_patience: int, stability_tol: float, distribution_top_k: int, + prediction_stability_margin: float = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + decision_top_k: int = DEFAULT_DECISION_TOP_K, + prediction_class_indices: Any = None, + exit_tol: float = 0.0, + exit_entropy_threshold: float = 0.0, order_offset: int = 0, ) -> Dict[str, Any]: + request_count = int(len(y)) + if request_count <= 0: + raise ValueError("At least one serving request is required") + if int(len(x)) != request_count: + raise ValueError("x and y must contain the same number of requests") + normalized_fixed_cycles = int(fixed_cycles) + normalized_max_cycles = int(max_cycles) + normalized_patience = int(stability_patience) + normalized_distribution_top_k = int(distribution_top_k) + normalized_decision_top_k = int(decision_top_k) + normalized_rank_depth = int(prediction_stability_rank_depth) + normalized_stability_tol = _nonnegative_finite_float( + stability_tol, + label="stability tolerance", + ) + normalized_exit_tol = _nonnegative_finite_float( + exit_tol, + label="adaptive exit tolerance", + ) + normalized_exit_entropy_threshold = _nonnegative_finite_float( + exit_entropy_threshold, + label="adaptive exit entropy threshold", + ) + stability_margin = _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ) + if normalized_fixed_cycles <= 0 or normalized_max_cycles <= 0: + raise ValueError("fixed_cycles and max_cycles must be positive") + if normalized_patience < 0: + raise ValueError("stability_patience must be nonnegative") + if normalized_distribution_top_k <= 0: + raise ValueError("distribution_top_k must be positive") + if normalized_decision_top_k <= 0: + raise ValueError("decision_top_k must be positive") + if normalized_rank_depth <= 0: + raise ValueError("prediction_stability_rank_depth must be positive") + normalized_scope = _normalize_prediction_class_indices_input( + prediction_class_indices + ) head = model.layers[10] fixed_predictions: List[int] = [] adaptive_predictions: List[int] = [] @@ -89,62 +325,162 @@ def benchmark_serving_requests( adaptive_cycles: List[float] = [] adaptive_latest_js_drift: List[float] = [] adaptive_max_js_drift: List[float] = [] + adaptive_prediction_margins: List[float] = [] + adaptive_decision_margins: List[float] = [] + prediction_stable_decision_margins: List[float] = [] + adaptive_rank_depths: List[int] = [] + adaptive_prediction_class_counts: List[int] = [] + adaptive_requested_class_indices: List[List[int] | None] = [] + absolute_logit_deltas: List[float] = [] + distribution_js_divergences: List[float] = [] + distribution_total_variations: List[float] = [] + top_k_order_disagreement_count = 0 + top_k_set_disagreement_count = 0 + resolved_scope: List[int] | None = None + output_class_count: int | None = None exit_reasons: Counter[str] = Counter() measurement_orders: Counter[str] = Counter() + normalized_requested_class_indices = ( + None + if normalized_scope is None + else [ + index + for index, selected in enumerate(normalized_scope) + if selected + ] + if all(isinstance(value, bool) for value in normalized_scope) + else [int(value) for value in normalized_scope] + ) + + def adaptive_kwargs() -> Dict[str, Any]: + kwargs: Dict[str, Any] = { + "reasoning_cycles": normalized_max_cycles, + "adaptive_compute": True, + "exit_tol": normalized_exit_tol, + "exit_entropy_threshold": normalized_exit_entropy_threshold, + "prediction_stability_patience": normalized_patience, + "prediction_stability_tol": normalized_stability_tol, + "prediction_stability_top_k": normalized_distribution_top_k, + "prediction_stability_margin": stability_margin, + "prediction_stability_rank_depth": normalized_rank_depth, + } + if normalized_scope is not None: + kwargs["prediction_class_indices"] = normalized_scope + return kwargs + + def record_requested_scope(kwargs: Dict[str, Any]) -> None: + raw_scope = kwargs.get("prediction_class_indices") + if raw_scope is None: + adaptive_requested_class_indices.append(None) + return + if all(isinstance(value, bool) for value in raw_scope): + adaptive_requested_class_indices.append( + [index for index, selected in enumerate(raw_scope) if selected] + ) + return + adaptive_requested_class_indices.append([int(value) for value in raw_scope]) + + def read_adaptive_scope_telemetry(*, label: str) -> int: + selection_valid = _strict_bool_scalar( + getattr(head, "last_prediction_class_selection_valid", None), + label=f"{label} prediction class selection validity", + ) + if not selection_valid: + raise ValueError(f"{label} prediction class selection is invalid") + return _positive_integral_scalar( + getattr(head, "last_prediction_class_count", None), + label=f"{label} prediction class count", + ) + # Keep initialization and one-time kernel setup outside measured requests. # End the warmup on the opposite mode from the first measured request so # both seed offsets receive the same cross-mode transition. def warm_adaptive() -> None: - model( - x[:1], - reasoning_cycles=max_cycles, - adaptive_compute=True, - exit_tol=0.0, - exit_entropy_threshold=0.0, - prediction_stability_patience=stability_patience, - prediction_stability_tol=stability_tol, - prediction_stability_top_k=distribution_top_k, + kwargs = adaptive_kwargs() + record_requested_scope(kwargs) + model(x[:1], **kwargs) + adaptive_prediction_class_counts.append( + read_adaptive_scope_telemetry(label="adaptive warmup") ) if int(order_offset) % 2 == 0: - model(x[:1], reasoning_cycles=fixed_cycles) + model(x[:1], reasoning_cycles=normalized_fixed_cycles) warm_adaptive() else: warm_adaptive() - model(x[:1], reasoning_cycles=fixed_cycles) + model(x[:1], reasoning_cycles=normalized_fixed_cycles) def run_fixed(sample: torch.Tensor) -> torch.Tensor: _synchronize_for_timing(sample.device) started = time.perf_counter() - output = model(sample, reasoning_cycles=fixed_cycles) + output = model(sample, reasoning_cycles=normalized_fixed_cycles) _synchronize_for_timing(sample.device) - fixed_latencies.append((time.perf_counter() - started) * 1000.0) + fixed_latencies.append( + _nonnegative_finite_float( + (time.perf_counter() - started) * 1000.0, + label="fixed request latency", + ) + ) return output def run_adaptive(sample: torch.Tensor) -> torch.Tensor: _synchronize_for_timing(sample.device) started = time.perf_counter() - output = model( - sample, - reasoning_cycles=max_cycles, - adaptive_compute=True, - exit_tol=0.0, - exit_entropy_threshold=0.0, - prediction_stability_patience=stability_patience, - prediction_stability_tol=stability_tol, - prediction_stability_top_k=distribution_top_k, - ) + kwargs = adaptive_kwargs() + record_requested_scope(kwargs) + output = model(sample, **kwargs) _synchronize_for_timing(sample.device) - adaptive_latencies.append((time.perf_counter() - started) * 1000.0) - adaptive_cycles.append(float(head.last_cycles_used.item())) + adaptive_latencies.append( + _nonnegative_finite_float( + (time.perf_counter() - started) * 1000.0, + label="adaptive request latency", + ) + ) + adaptive_cycles.append( + _nonnegative_finite_float( + head.last_cycles_used.item(), label="adaptive cycles used" + ) + ) adaptive_latest_js_drift.append( - float(head.last_prediction_topk_js_divergence.item()) + _nonnegative_finite_float( + head.last_prediction_topk_js_divergence.item(), + label="latest prediction distribution drift", + ) ) adaptive_max_js_drift.append( - float(head.last_prediction_topk_js_divergence_max.item()) + _nonnegative_finite_float( + head.last_prediction_topk_js_divergence_max.item(), + label="maximum prediction distribution drift", + ) + ) + adaptive_prediction_margins.append( + _nonnegative_finite_float( + head.last_prediction_margin.item(), + label="observed prediction stability margin", + ) + ) + observed_decision_margin = _nonnegative_finite_float( + head.last_prediction_decision_margin.item(), + label="observed prediction decision margin", + ) + adaptive_decision_margins.append(observed_decision_margin) + observed_rank_depth = int(head.last_prediction_rank_depth.item()) + if observed_rank_depth <= 0: + raise ValueError("observed prediction rank depth must be positive") + adaptive_rank_depths.append(observed_rank_depth) + adaptive_prediction_class_counts.append( + read_adaptive_scope_telemetry(label="adaptive request") ) - exit_reasons[str(head.last_exit_reason)] += 1 + raw_exit_reason = getattr(head, "last_exit_reason", None) + if not isinstance(raw_exit_reason, str) or not raw_exit_reason.strip(): + raise ValueError("adaptive exit reason must be a non-empty string") + exit_reason = raw_exit_reason.strip() + if exit_reason not in KNOWN_ADAPTIVE_EXIT_REASONS: + raise ValueError(f"unknown adaptive exit reason: {exit_reason}") + exit_reasons[exit_reason] += 1 + if exit_reason == "prediction_stable": + prediction_stable_decision_margins.append(observed_decision_margin) return output for index in range(len(y)): @@ -159,34 +495,161 @@ def run_adaptive(sample: torch.Tensor) -> torch.Tensor: adaptive_output = run_adaptive(sample) fixed_output = run_fixed(sample) - fixed_predictions.append(int(fixed_output.argmax(dim=-1).item())) - adaptive_predictions.append(int(adaptive_output.argmax(dim=-1).item())) - - truth = y.detach().cpu() + fixed_logits = _request_logits(fixed_output, label="fixed") + adaptive_logits = _request_logits(adaptive_output, label="adaptive") + if fixed_logits.shape != adaptive_logits.shape: + raise ValueError("fixed and adaptive logits must have the same shape") + current_output_class_count = int(fixed_logits.numel()) + if output_class_count is None: + output_class_count = current_output_class_count + resolved_scope = _resolve_prediction_scope( + normalized_scope, + output_class_count=output_class_count, + decision_top_k=normalized_decision_top_k, + ) + elif current_output_class_count != output_class_count: + raise ValueError("model output class count changed between requests") + assert resolved_scope is not None + scope_tensor = torch.tensor(resolved_scope, dtype=torch.long) + fixed_scoped = fixed_logits.index_select(0, scope_tensor) + adaptive_scoped = adaptive_logits.index_select(0, scope_tensor) + fixed_order_local = torch.argsort( + fixed_scoped, descending=True, stable=True + )[:normalized_decision_top_k] + adaptive_order_local = torch.argsort( + adaptive_scoped, descending=True, stable=True + )[:normalized_decision_top_k] + fixed_order = [ + resolved_scope[int(index.item())] for index in fixed_order_local + ] + adaptive_order = [ + resolved_scope[int(index.item())] for index in adaptive_order_local + ] + fixed_predictions.append(fixed_order[0]) + adaptive_predictions.append(adaptive_order[0]) + top_k_order_disagreement_count += int(fixed_order != adaptive_order) + top_k_set_disagreement_count += int(set(fixed_order) != set(adaptive_order)) + absolute_logit_deltas.extend( + float(value) + for value in torch.abs(fixed_scoped - adaptive_scoped).tolist() + ) + js_divergence, total_variation = _distribution_distances( + fixed_scoped, adaptive_scoped + ) + distribution_js_divergences.append(js_divergence) + distribution_total_variations.append(total_variation) + + assert output_class_count is not None and resolved_scope is not None + truth = y.detach().cpu().reshape(-1) + if truth.numel() != request_count: + raise ValueError("y must contain one target per serving request") + if truth.dtype.is_floating_point or truth.dtype.is_complex: + raise ValueError("y must contain integer class indices") + if bool(((truth < 0) | (truth >= output_class_count)).any().item()): + raise ValueError("y contains an out-of-range class index") fixed_tensor = torch.tensor(fixed_predictions) adaptive_tensor = torch.tensor(adaptive_predictions) fixed_correct = int((fixed_tensor == truth).sum().item()) adaptive_correct = int((adaptive_tensor == truth).sum().item()) - disagreement_count = int((fixed_tensor != adaptive_tensor).sum().item()) - request_count = int(len(y)) + top1_disagreement_count = int((fixed_tensor != adaptive_tensor).sum().item()) fixed_accuracy = fixed_correct / max(1, request_count) adaptive_accuracy = adaptive_correct / max(1, request_count) - agreement = 1.0 - (disagreement_count / max(1, request_count)) + agreement = 1.0 - (top1_disagreement_count / max(1, request_count)) fixed_latency = _latency_summary(fixed_latencies) adaptive_latency = _latency_summary(adaptive_latencies) - mean_cycles = sum(adaptive_cycles) / max(1, len(adaptive_cycles)) + mean_cycles = float( + _finite_values(adaptive_cycles, label="adaptive cycles").mean().item() + ) + expected_effective_rank_depth = min( + normalized_rank_depth, max(1, len(resolved_scope) - 1) + ) + if set(adaptive_rank_depths) != {expected_effective_rank_depth}: + raise ValueError( + "adaptive verifier reported a rank depth inconsistent with the verified scope" + ) + if set(adaptive_prediction_class_counts) != {len(resolved_scope)}: + raise ValueError( + "adaptive verifier reported a class count inconsistent with the verified scope" + ) + if set( + None if value is None else tuple(value) + for value in adaptive_requested_class_indices + ) != { + None + if normalized_requested_class_indices is None + else tuple(normalized_requested_class_indices) + }: + raise ValueError( + "adaptive verifier calls did not preserve the exact requested class indices" + ) + scoped_truth_count = int( + torch.isin(truth, torch.tensor(resolved_scope, dtype=truth.dtype)).sum().item() + ) + verified_scope = { + "verified": True, + "mode": ( + "all_output_classes" + if normalized_scope is None + else "prediction_class_indices" + ), + "output_class_count": output_class_count, + "class_count": len(resolved_scope), + "class_indices": resolved_scope, + "requested_normalized_class_indices": normalized_requested_class_indices, + "adaptive_requested_class_indices_verified": True, + "truth_labels_in_scope": scoped_truth_count, + "truth_labels_outside_scope": request_count - scoped_truth_count, + "adaptive_verifier_rank_depth_verified": True, + "adaptive_verifier_class_scope_verified": True, + "observed_prediction_class_counts": sorted( + set(adaptive_prediction_class_counts) + ), + "effective_prediction_rank_depth": expected_effective_rank_depth, + } + logit_delta_tensor = _finite_values( + absolute_logit_deltas, label="absolute logit deltas" + ) return { "fixed": { - "cycles": int(fixed_cycles), + "adaptive_compute": False, + "cycles": normalized_fixed_cycles, "accuracy": round(fixed_accuracy, 6), "correct_predictions": fixed_correct, "latency": fixed_latency, }, "prediction_stability": { - "max_cycles": int(max_cycles), - "patience": int(stability_patience), - "confidence_tolerance": float(stability_tol), + "adaptive_compute": True, + "max_cycles": normalized_max_cycles, + "exit_tolerance": normalized_exit_tol, + "exit_entropy_threshold": normalized_exit_entropy_threshold, + "patience": normalized_patience, + "confidence_tolerance": normalized_stability_tol, + "prediction_margin": { + "metric": "top_1_minus_top_2_probability", + "configured_minimum": stability_margin, + "observed": _scalar_summary(adaptive_prediction_margins), + }, + "decision_margin": { + "metric": "minimum_adjacent_probability_gap_through_rank_depth", + "configured_minimum": stability_margin, + "configured_rank_depth": normalized_rank_depth, + "observed_rank_depths": sorted(set(adaptive_rank_depths)), + "observed": _scalar_summary(adaptive_decision_margins), + "prediction_stable_observed": { + "observation_count": len(prediction_stable_decision_margins), + "minimum": ( + min(prediction_stable_decision_margins) + if prediction_stable_decision_margins + else None + ), + "summary": ( + _scalar_summary(prediction_stable_decision_margins) + if prediction_stable_decision_margins + else None + ), + }, + }, "accuracy": round(adaptive_accuracy, 6), "correct_predictions": adaptive_correct, "mean_cycles_used": round(mean_cycles, 4), @@ -198,7 +661,7 @@ def run_adaptive(sample: torch.Tensor) -> torch.Tensor: "exit_reasons": dict(sorted(exit_reasons.items())), "distribution_drift": { "metric": "top_k_jensen_shannon_divergence_nats", - "top_k": int(distribution_top_k), + "top_k": normalized_distribution_top_k, "role": "shadow_diagnostic_only", "latest_cycle_pair": _scalar_summary(adaptive_latest_js_drift), "max_cycle_pair": _scalar_summary(adaptive_max_js_drift), @@ -208,8 +671,31 @@ def run_adaptive(sample: torch.Tensor) -> torch.Tensor: "comparison": { "accuracy_delta": round(adaptive_accuracy - fixed_accuracy, 6), "prediction_agreement": round(agreement, 6), - "exact_disagreement_count": disagreement_count, + "top1_disagreement_count": top1_disagreement_count, + # Compatibility alias retained for existing artifact consumers. + "exact_disagreement_count": top1_disagreement_count, "request_count": request_count, + "decision_fidelity": { + "verified_scope": verified_scope, + "top_k": normalized_decision_top_k, + "top_k_order_disagreement_count": top_k_order_disagreement_count, + "top_k_set_disagreement_count": top_k_set_disagreement_count, + "absolute_logit_delta": { + "mean": round(float(logit_delta_tensor.mean().item()), 12), + "max": round(float(logit_delta_tensor.max().item()), 12), + "role": "diagnostic_only_not_gated", + }, + "distribution_distance": { + "jensen_shannon_divergence_nats": _scalar_summary( + distribution_js_divergences, digits=12 + ), + "total_variation_distance": _scalar_summary( + distribution_total_variations, digits=12 + ), + "role": "diagnostic_only_not_gated", + }, + "tensor_equality_required": False, + }, "measurement_order": { "offset": int(order_offset), "fixed_then_adaptive": measurement_orders["fixed_then_adaptive"], @@ -237,8 +723,44 @@ def main() -> None: parser.add_argument("--seed", type=int, default=52) parser.add_argument("--fixed_cycles", type=int, default=3) parser.add_argument("--max_cycles", type=int, default=8) - parser.add_argument("--stability_patience", type=int, default=2) - parser.add_argument("--stability_tol", type=float, default=0.005) + parser.add_argument( + "--stability_patience", + type=int, + default=DEFAULT_PREDICTION_STABILITY_PATIENCE, + ) + parser.add_argument( + "--stability_tol", type=float, default=DEFAULT_PREDICTION_STABILITY_TOL + ) + parser.add_argument( + "--prediction-stability-margin", + "--prediction_stability_margin", + dest="prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + parser.add_argument( + "--prediction-stability-rank-depth", + "--prediction_stability_rank_depth", + dest="prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) + parser.add_argument( + "--decision-top-k", + "--decision_top_k", + dest="decision_top_k", + type=int, + default=DEFAULT_DECISION_TOP_K, + ) + parser.add_argument( + "--prediction-class-indices", + "--prediction_class_indices", + dest="prediction_class_indices", + type=int, + nargs="+", + default=None, + help="Evaluate and verify one explicit class scope (single scope only).", + ) parser.add_argument("--distribution_top_k", type=int, default=5) parser.add_argument("--order_offset", type=int, default=0) parser.add_argument("--device", default="auto") @@ -266,6 +788,13 @@ def main() -> None: stability_patience=max(0, int(args.stability_patience)), stability_tol=max(0.0, float(args.stability_tol)), distribution_top_k=max(1, int(args.distribution_top_k)), + prediction_stability_margin=_nonnegative_finite_float( + args.prediction_stability_margin, + label="prediction stability margin", + ), + prediction_stability_rank_depth=int(args.prediction_stability_rank_depth), + decision_top_k=int(args.decision_top_k), + prediction_class_indices=args.prediction_class_indices, order_offset=int(args.order_offset), ) payload = { diff --git a/source/chat_app.py b/source/chat_app.py index f089d7585..238162214 100644 --- a/source/chat_app.py +++ b/source/chat_app.py @@ -62,12 +62,37 @@ def load_metadata(path: str) -> Dict: return {} +def _parse_metadata_buckets(raw: Any) -> Dict[int, List[Dict]]: + """Keep only non-empty buckets whose labels address a model output class.""" + + buckets: Dict[int, List[Dict]] = {} + if not isinstance(raw, dict): + return buckets + for raw_label, rows in raw.items(): + if isinstance(raw_label, bool): + continue + if isinstance(raw_label, int): + label = raw_label + elif isinstance(raw_label, str) and re.fullmatch(r"[+-]?\d+", raw_label.strip()): + try: + label = int(raw_label) + except (ValueError, OverflowError): + continue + else: + continue + if not 0 <= label < MODEL_CLASSES: + continue + if isinstance(rows, list) and rows: + buckets[label] = rows + return buckets + + def _int_or_default(value, default: int) -> int: if value is None: return int(default) try: return int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return int(default) @@ -130,6 +155,11 @@ def evaluate_runtime_compute_budgets( cycles=None, adaptive_compute: bool = False, exit_tol: Optional[float] = None, + exit_entropy_threshold: Any = None, + prediction_stability_patience: Any = None, + prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> List[Dict[str, object]]: labels = list(available_labels) or list(range(MODEL_CLASSES)) idx_device = x.device if isinstance(x, torch.Tensor) else None @@ -148,7 +178,13 @@ def evaluate_runtime_compute_budgets( reasoning_cycles=requested_cycles, adaptive_compute=adaptive_compute, exit_tol=resolved_exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=prediction_stability_margin, return_diagnostics=True, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) logits = logits_tensor[0, 0] avail_logits = logits.index_select(0, idx.to(logits.device)) @@ -223,6 +259,8 @@ def select_auto_runtime_compute_budget( DEFAULT_ADAPTIVE_EXIT_ENTROPY = 0.2 DEFAULT_PREDICTION_STABILITY_PATIENCE = 2 DEFAULT_PREDICTION_STABILITY_TOL = 5e-3 +DEFAULT_PREDICTION_STABILITY_MARGIN = 1e-4 +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = 3 AUTO_REASONING_CYCLE_BUCKETS = (1, 3, 8, 16) @@ -313,7 +351,7 @@ def _coerce_optional_positive_int( value = stripped try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return default parsed = max(1, parsed) if max_value is not None: @@ -385,7 +423,7 @@ def estimate_auto_reasoning_cycles(context: str, max_value: Optional[int] = None def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[float] = None) -> float: try: parsed = float(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): parsed = float(default) if not math.isfinite(parsed): parsed = float(default) @@ -395,10 +433,40 @@ def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[fl return parsed +def _coerce_prediction_stability_margin( + value: Any, + default: float = DEFAULT_PREDICTION_STABILITY_MARGIN, +) -> float: + """Preserve explicit zero but never turn malformed input into guard-off.""" + + try: + parsed = float(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return float(default) + if not math.isfinite(parsed) or parsed < 0.0: + return float(default) + return parsed + + +def _coerce_prediction_stability_rank_depth( + value: Any, + default: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, +) -> int: + """Preserve explicit zero while failing malformed/negative values closed.""" + + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return int(default) + if parsed < 0: + return int(default) + return parsed + + def _coerce_nonnegative_int(value: Any, default: int, max_value: Optional[int] = None) -> int: try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): parsed = int(default) parsed = max(0, parsed) if max_value is not None: @@ -450,7 +518,7 @@ def _to_optional_scalar(value: Any) -> Optional[float]: if hasattr(value, "detach"): return float(value.detach().cpu().item()) return float(value) - except (TypeError, ValueError, RuntimeError): + except (TypeError, ValueError, OverflowError, RuntimeError): return None @@ -469,6 +537,11 @@ def collect_runtime_compute_metrics( "gating_entropy": None, "prediction_streak": None, "prediction_confidence_delta": None, + "prediction_margin": None, + "prediction_decision_margin": None, + "prediction_rank_depth": None, + "prediction_class_count": None, + "prediction_class_selection_valid": None, "exit_reason": None, } attr_map = { @@ -478,6 +551,11 @@ def collect_runtime_compute_metrics( "last_gating_entropy": "gating_entropy", "last_prediction_streak": "prediction_streak", "last_prediction_confidence_delta": "prediction_confidence_delta", + "last_prediction_margin": "prediction_margin", + "last_prediction_decision_margin": "prediction_decision_margin", + "last_prediction_rank_depth": "prediction_rank_depth", + "last_prediction_class_count": "prediction_class_count", + "last_prediction_class_selection_valid": "prediction_class_selection_valid", } modules = list(model.modules()) if hasattr(model, "modules") else [model] for module in modules: @@ -491,6 +569,10 @@ def collect_runtime_compute_metrics( reason = getattr(module, "last_exit_reason", None) if isinstance(reason, str) and reason.strip(): metrics["exit_reason"] = reason.strip() + if metrics["prediction_class_selection_valid"] is not None: + metrics["prediction_class_selection_valid"] = bool( + metrics["prediction_class_selection_valid"] + ) metrics.update({ "supported": model_supports_runtime_compute(model), "requested_reasoning_cycles": requested_reasoning_cycles, @@ -518,6 +600,9 @@ def forward_with_runtime_compute( prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, auto_reasoning_context: str = "", return_diagnostics: Optional[bool] = None, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_class_indices: Any = None, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ): """Forward a model while applying optional v50 runtime-compute controls. @@ -548,6 +633,14 @@ def forward_with_runtime_compute( prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + stability_margin = _coerce_prediction_stability_margin( + prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + stability_rank_depth = _coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) kwargs: Dict[str, Any] = {} if cycles is not None and _model_forward_accepts_kwarg(model, "reasoning_cycles"): @@ -562,8 +655,29 @@ def forward_with_runtime_compute( kwargs["prediction_stability_patience"] = stability_patience if _model_forward_accepts_kwarg(model, "prediction_stability_tol"): kwargs["prediction_stability_tol"] = stability_tol + if _model_forward_accepts_kwarg(model, "prediction_stability_margin"): + kwargs["prediction_stability_margin"] = stability_margin + if _model_forward_accepts_kwarg(model, "prediction_stability_rank_depth"): + kwargs["prediction_stability_rank_depth"] = stability_rank_depth + if ( + prediction_class_indices is not None + and _model_forward_accepts_kwarg(model, "prediction_class_indices") + ): + if isinstance(prediction_class_indices, torch.Tensor): + prediction_class_indices = prediction_class_indices.detach().cpu().tolist() + elif isinstance(prediction_class_indices, tuple): + prediction_class_indices = list(prediction_class_indices) + kwargs["prediction_class_indices"] = prediction_class_indices output = model(x, **kwargs) if kwargs else model(x) + prediction_verifier_active = bool( + adaptive + and stability_patience > 0 + and stability_rank_depth > 0 + and "adaptive_compute" in kwargs + and "prediction_stability_patience" in kwargs + and "prediction_stability_rank_depth" in kwargs + ) metrics = collect_runtime_compute_metrics( model, requested_reasoning_cycles=cycles, @@ -571,6 +685,17 @@ def forward_with_runtime_compute( exit_tol=tol, applied_kwargs=kwargs, ) + if not prediction_verifier_active: + for key in ( + "prediction_streak", + "prediction_confidence_delta", + "prediction_margin", + "prediction_decision_margin", + "prediction_rank_depth", + "prediction_class_count", + "prediction_class_selection_valid", + ): + metrics[key] = None diagnostics = { "requested_reasoning_cycles": cycles, "selected_reasoning_cycles": cycles, @@ -582,6 +707,14 @@ def forward_with_runtime_compute( "exit_entropy_threshold": entropy_threshold, "prediction_stability_patience": stability_patience, "prediction_stability_tol": stability_tol, + "prediction_stability_margin": stability_margin, + "prediction_stability_rank_depth": stability_rank_depth, + "prediction_verifier_active": prediction_verifier_active, + "prediction_class_indices": ( + kwargs.get("prediction_class_indices") + if "prediction_class_indices" in kwargs + else None + ), "applied": bool(kwargs), "applied_kwargs": dict(kwargs), **metrics, @@ -647,6 +780,8 @@ def progressive_auto_compute_forward( prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, auto_reasoning_context: str = "", distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: """Evaluate auto-compute budgets progressively and reuse the accepted probe. @@ -696,6 +831,9 @@ def progressive_auto_compute_forward( prediction_stability_patience=prediction_stability_patience, prediction_stability_tol=prediction_stability_tol, auto_reasoning_context=auto_reasoning_context, + prediction_stability_margin=prediction_stability_margin, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) logits = model_out[0, 0] available_logits = logits.index_select(0, idx.to(logits.device)) @@ -822,7 +960,7 @@ def compact_auto_compute_plan_metrics(plan: Any) -> Dict[str, Any]: rows = plan.get("rows") if isinstance(plan.get("rows"), list) else [] try: selected_index = int(plan.get("selected_index")) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): selected_index = -1 if not 0 <= selected_index < len(rows): selected_index = len(rows) - 1 @@ -887,6 +1025,8 @@ def _print_chat_help() -> None: print(" /exit_entropy Set v50 adaptive-compute entropy exit threshold") print(" /stability Stable prediction cycles required (0 disables)") print(" /stability_tol Maximum confidence drift across the stable streak") + print(" /stability_margin Minimum top-two probability margin for stable exit") + print(" /stability_rank_depth Ordered prediction ranks verified (0 disables)") print(" /top Show top reranked candidates each turn (0 disables)") print(" /timing on|off Toggle per-turn timing output") print(" /auto_compute on|off Toggle confidence/entropy based cycle selection") @@ -948,6 +1088,18 @@ def main(): default=DEFAULT_PREDICTION_STABILITY_TOL, help="Maximum confidence drift allowed across a stable prediction streak.", ) + ap.add_argument( + "--prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + help="Minimum full-output top-two probability margin required for stable exit.", + ) + ap.add_argument( + "--prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + help="Ordered prediction ranks required to remain stable (0 disables the rank verifier).", + ) ap.add_argument("--device", default="auto") ap.add_argument( "--device_preference", @@ -1140,15 +1292,7 @@ def main(): if missing or unexpected: raise RuntimeError(f"State dict mismatch. Missing={missing}, Unexpected={unexpected}") - buckets_raw = meta.get("buckets", {}) - buckets: Dict[int, List[Dict]] = {} - for k, v in buckets_raw.items(): - try: - label = int(k) - except ValueError: - continue - if isinstance(v, list) and v: - buckets[label] = v + buckets = _parse_metadata_buckets(meta.get("buckets", {})) available_labels = sorted(buckets.keys()) if not available_labels: @@ -1203,6 +1347,14 @@ def main(): args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + args.prediction_stability_margin = _coerce_prediction_stability_margin( + args.prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + args.prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) print(f"\n{TerminalColors.SYSTEM}--- Session Info ---") print(f"Loaded: {Path(args.weights).name} [{resolved_model_size}] | Available labels: {len(available_labels)}") @@ -1217,6 +1369,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) @@ -1272,6 +1426,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) print( @@ -1312,6 +1468,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'} " f"auto_conf={float(args.auto_compute_confidence):.3f} " f"auto_entropy={float(args.auto_compute_entropy):.3f}" @@ -1398,6 +1556,26 @@ def main(): ) except Exception: print(f"{TerminalColors.SYSTEM}Usage: /stability_tol {TerminalColors.RESET}") + elif cmd == "stability_margin": + try: + args.prediction_stability_margin = _coerce_prediction_stability_margin( + arg, + args.prediction_stability_margin, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability margin set to " + f"{float(args.prediction_stability_margin):.4g}.{TerminalColors.RESET}" + ) + except Exception: + print(f"{TerminalColors.SYSTEM}Usage: /stability_margin {TerminalColors.RESET}") + elif cmd == "stability_rank_depth": + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + arg, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability rank depth set to " + f"{int(args.prediction_stability_rank_depth)}.{TerminalColors.RESET}" + ) elif cmd == "top": try: args.show_top_responses = max(0, int(arg)) @@ -1517,6 +1695,8 @@ def main(): prediction_stability_patience=args.prediction_stability_patience, prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, ) effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") else: @@ -1531,6 +1711,9 @@ def main(): prediction_stability_patience=args.prediction_stability_patience, prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_class_indices=available_labels, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, ) logits = model_out[0, 0] # (10,) t_infer += max(0.0, time.perf_counter() - _t) diff --git a/source/chat_web_app.py b/source/chat_web_app.py index d420541aa..8a5bf0814 100644 --- a/source/chat_web_app.py +++ b/source/chat_web_app.py @@ -100,7 +100,9 @@
-
+
+
Verify the ordered top ranks; 0 intentionally disables rank verification.
+
Loading status...
@@ -114,7 +116,7 @@