diff --git a/.github/workflows/runtime-quality-gates.yml b/.github/workflows/runtime-quality-gates.yml index 5bdfb223..7a3b7f25 100644 --- a/.github/workflows/runtime-quality-gates.yml +++ b/.github/workflows/runtime-quality-gates.yml @@ -37,8 +37,9 @@ jobs: - name: Install focused test dependencies run: | python -m pip install --upgrade pip - python -m pip install pytest flask pillow + python -m pip install pytest flask pillow matplotlib python -m pip install torch --index-url https://download.pytorch.org/whl/cpu + python -m pip install peft transformers - name: Compile runtime surfaces run: | @@ -113,8 +114,12 @@ jobs: if ($seal.execution_enabled -ne $false) { throw 'Frozen shadow seal unexpectedly enables execution' } $campaignId = $seal.public_package.campaign_seal.seal.campaign_id $clusterPath = Join-Path $PWD 'build\ci_route_shadow\cluster.json' - @{ cluster_identifier = 'ci-shadow-cluster-1' } | ConvertTo-Json -Compress | Set-Content -LiteralPath $clusterPath -Encoding utf8 - $commit = (& .\dist\ci_route_shadow\SupermixRouteShadow.exe commit --registry $registryPath --campaign $campaignId --seed-input $seedPath --cluster-input $clusterPath --compact) | ConvertFrom-Json + $clusterIdentifier = python -c "from source.route_policy_ledger import hash_session_identity; print(hash_session_identity('ci-shadow-cluster-1'))" + if ($LASTEXITCODE -ne 0) { throw 'CI shadow cluster hash generation failed' } + @{ cluster_identifier = $clusterIdentifier } | ConvertTo-Json -Compress | Set-Content -LiteralPath $clusterPath -Encoding utf8 + $commitRaw = & .\dist\ci_route_shadow\SupermixRouteShadow.exe commit --registry $registryPath --campaign $campaignId --seed-input $seedPath --cluster-input $clusterPath --compact + if ($LASTEXITCODE -ne 0) { throw 'Frozen shadow commitment failed' } + $commit = $commitRaw | ConvertFrom-Json if ($commit.chosen_arm_revealed -ne $false) { throw 'Frozen shadow commitment revealed an arm early' } $closure = (& .\dist\ci_route_shadow\SupermixRouteShadow.exe close --registry $registryPath --campaign $campaignId --compact) | ConvertFrom-Json if ($closure.closure.frozen_commitment_count -ne 1) { throw 'Frozen shadow closure count mismatch' } diff --git a/README.md b/README.md index 731a59b2..4016e682 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,15 @@ It is intentionally a mixed workspace, not a minimal source-only model repo. ## Current status -As of July 18, 2026: +As of July 22, 2026: - `source/` is the active Supermix Studio runtime and packaging tree - the curated desktop build selects `11` core model artifacts and leaves expansion to the model store - the route control plane includes durable lifecycle evidence, Policy Lab diagnostics, bounded-exposure rehearsal, and a fail-closed stateful experiment protocol preflight +- v51 local inference supports progressive accepted-probe reuse plus a post-head, + allowed-label-scoped decision verifier that checks the ordered top-3 boundary before + an adaptive early exit - `runtime_python/` remains a legacy compatibility snapshot for the smaller chat runtime; it is not the source of truth for the multimodel Studio route control plane - the Windows installer contract version is `2026.07.18` @@ -52,6 +55,8 @@ As of July 18, 2026: - native-image experimental checkpoints - training pipelines for frontier, omni, lite, and specialist model lines - benchmark sweeps across common text benchmarks +- release-gated v51 adaptive compute with source/package parity and frozen-prompt + response-fidelity checks - export and publishing workflows for GitHub releases and Hugging Face model/dataset repos ## Quick start @@ -333,18 +338,44 @@ executed envelope. ## Research and experiment notes -The v51 stability benchmark supports a research-derived, shadow-only -distribution-drift diagnostic: +The accepted v51 decision-fidelity configuration is checkpoint- and +workload-calibrated: ordered top-3 persistence, a `0.0005` minimum adjacent +probability gap through the top-3/outside boundary, and exact fallback to the +trained three-cycle reference output whenever an earlier decision is not +certified. Legacy latent, entropy, and ACT signals cannot bypass this post-head +verifier. + +The clean CPU release gate at commit `81c4dbe7` evaluated 4,096 held-out +requests in each of two modes (isolated verifier and exact release-runtime +defaults): + +- zero top-1, ordered top-3, top-3-set, or exact-output disagreements in both + modes, with zero per-seed accuracy deltas; +- 3,941 certified cycle-2 exits and 155 exact cycle-3 reference fallbacks, + for 2.0378 mean cycles and a 32.07% cycle reduction from fixed cycle 3; +- positive counterbalanced latency results in both modes: 4.76% weighted / 3.71% + median-per-seed reduction for release runtime and 7.23% / 5.57% for the + isolated verifier; +- the frozen 16-prompt source/package response gate had zero response, + top-five-order, runtime-contract, or packaged-behavior mismatches; and +- progressive accepted-probe auto compute matched the legacy controller exactly + on 256 requests while reducing forward evaluations by 31.25% and weighted + latency by 30.09%. ```powershell -python source/benchmark_v51_prediction_stability.py --distribution_top_k 5 +python source/run_v51_prediction_stability_gate.py --device cpu --torch-num-threads 8 --torch-interop-threads 1 --strict-determinism --samples-per-seed 512 --enforce-gates +python source/run_v51_chat_response_fidelity_gate.py --device cpu --torch-num-threads 8 --torch-interop-threads 1 --strict-determinism --enforce-gates +python source/benchmark_progressive_auto_compute.py --device cpu --torch-num-threads 8 --torch-interop-threads 1 --strict-determinism --enforce-gates ``` -It reports consecutive-prefix top-k Jensen-Shannon divergence with a retained -`other` mass bucket. The metric is telemetry only and does not authorize an -early exit. A 480-request CPU pilot retained 96/96 agreement for the current -patience-2/tolerance-0.005 setting with 28.8% fewer cycles; the release gate is a -larger fresh-seed confirmation, not that pilot result. +The stricter gate rejected the earlier `0.0001` candidate: release mode showed +5 top-1, 18 ordered-top-3, and 10 top-3-set disagreements; isolated mode showed +2, 6, and 4. That failure led to the common exit guard, ordered-rank state, the +calibrated margin, bounded disagreement evidence, and reference-budget fallback. +Top-k Jensen-Shannon divergence remains shadow telemetry only and cannot +authorize an exit. These are deterministic results for this checkpoint, +synthetic task, seeds, and frozen prompt matrix—not a universal reasoning or +chat-quality guarantee. This repo is a living experiment workspace. It contains finished artifacts, release-ready packaging, and in-progress work at the same time. diff --git a/chat_web_app.py b/chat_web_app.py index fda53e06..59a0267f 100644 --- a/chat_web_app.py +++ b/chat_web_app.py @@ -36,7 +36,10 @@ # Keep the root compatibility surface behaviorally identical for callers that # validate CLI default inheritance after another test imported this wrapper # first. The canonical helper remains private in the source implementation. -for _name in ("_runtime_compute_cli_overrides",): +for _name in ( + "_normalize_runtime_compute_defaults", + "_runtime_compute_cli_overrides", +): globals()[_name] = getattr(_IMPL, _name) diff --git a/launch_v51_ultra_chat_demo.ps1 b/launch_v51_ultra_chat_demo.ps1 index 46b2fcef..0baa852b 100644 --- a/launch_v51_ultra_chat_demo.ps1 +++ b/launch_v51_ultra_chat_demo.ps1 @@ -59,7 +59,9 @@ try { "--adaptive_compute", "--adaptive_exit_entropy", "0.2", "--prediction_stability_patience", "2", - "--prediction_stability_tol", "0.005" + "--prediction_stability_tol", "0.005", + "--prediction_stability_margin", "0.0005", + "--prediction_stability_rank_depth", "3" ) | ForEach-Object { Quote-ProcessArg $_ } $process = Start-Process -FilePath "python" -ArgumentList $args -WorkingDirectory $Root -WindowStyle Hidden -PassThru -RedirectStandardOutput $Stdout -RedirectStandardError $Stderr Set-Content -LiteralPath $PidFile -Value $process.Id diff --git a/runtime_python/chat_app.py b/runtime_python/chat_app.py index d5e43143..cec08dad 100644 --- a/runtime_python/chat_app.py +++ b/runtime_python/chat_app.py @@ -1,6 +1,7 @@ import argparse import inspect import json +import math import re import time from pathlib import Path @@ -61,12 +62,37 @@ def load_metadata(path: str) -> Dict: return {} +def _parse_metadata_buckets(raw: Any) -> Dict[int, List[Dict]]: + """Keep only non-empty buckets whose labels address a model output class.""" + + buckets: Dict[int, List[Dict]] = {} + if not isinstance(raw, dict): + return buckets + for raw_label, rows in raw.items(): + if isinstance(raw_label, bool): + continue + if isinstance(raw_label, int): + label = raw_label + elif isinstance(raw_label, str) and re.fullmatch(r"[+-]?\d+", raw_label.strip()): + try: + label = int(raw_label) + except (ValueError, OverflowError): + continue + else: + continue + if not 0 <= label < MODEL_CLASSES: + continue + if isinstance(rows, list) and rows: + buckets[label] = rows + return buckets + + def _int_or_default(value, default: int) -> int: if value is None: return int(default) try: return int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return int(default) @@ -74,6 +100,9 @@ def _int_or_default(value, default: int) -> int: DEFAULT_ADAPTIVE_EXIT_TOL = 1e-3 DEFAULT_AUTO_COMPUTE_CONFIDENCE = 0.55 DEFAULT_AUTO_COMPUTE_ENTROPY = 1.85 +AUTO_COMPUTE_PLAN_SCHEMA_VERSION = "runtime-auto-compute-plan-v2" +AUTO_COMPUTE_STRATEGY = "progressive_accepted_probe" +DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K = 5 def resolve_runtime_compute_cycles(cycles, default: Optional[List[int]] = None, limit: int = 8) -> List[int]: @@ -126,6 +155,11 @@ def evaluate_runtime_compute_budgets( cycles=None, adaptive_compute: bool = False, exit_tol: Optional[float] = None, + exit_entropy_threshold: Any = None, + prediction_stability_patience: Any = None, + prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> List[Dict[str, object]]: labels = list(available_labels) or list(range(MODEL_CLASSES)) idx_device = x.device if isinstance(x, torch.Tensor) else None @@ -144,7 +178,13 @@ def evaluate_runtime_compute_budgets( reasoning_cycles=requested_cycles, adaptive_compute=adaptive_compute, exit_tol=resolved_exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=prediction_stability_margin, return_diagnostics=True, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) logits = logits_tensor[0, 0] avail_logits = logits.index_select(0, idx.to(logits.device)) @@ -219,6 +259,9 @@ def select_auto_runtime_compute_budget( DEFAULT_ADAPTIVE_EXIT_ENTROPY = 0.2 DEFAULT_PREDICTION_STABILITY_PATIENCE = 2 DEFAULT_PREDICTION_STABILITY_TOL = 5e-3 +# Calibrated for the released v51 checkpoint and workload; not a universal margin. +DEFAULT_PREDICTION_STABILITY_MARGIN = 5e-4 +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = 3 AUTO_REASONING_CYCLE_BUCKETS = (1, 3, 8, 16) @@ -309,7 +352,7 @@ def _coerce_optional_positive_int( value = stripped try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return default parsed = max(1, parsed) if max_value is not None: @@ -381,7 +424,9 @@ def estimate_auto_reasoning_cycles(context: str, max_value: Optional[int] = None def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[float] = None) -> float: try: parsed = float(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): + parsed = float(default) + if not math.isfinite(parsed): parsed = float(default) parsed = max(0.0, parsed) if max_value is not None: @@ -389,10 +434,40 @@ def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[fl return parsed +def _coerce_prediction_stability_margin( + value: Any, + default: float = DEFAULT_PREDICTION_STABILITY_MARGIN, +) -> float: + """Preserve explicit zero but never turn malformed input into guard-off.""" + + try: + parsed = float(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return float(default) + if not math.isfinite(parsed) or parsed < 0.0: + return float(default) + return parsed + + +def _coerce_prediction_stability_rank_depth( + value: Any, + default: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, +) -> int: + """Preserve explicit zero while failing malformed/negative values closed.""" + + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return int(default) + if parsed < 0: + return int(default) + return parsed + + def _coerce_nonnegative_int(value: Any, default: int, max_value: Optional[int] = None) -> int: try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): parsed = int(default) parsed = max(0, parsed) if max_value is not None: @@ -444,7 +519,7 @@ def _to_optional_scalar(value: Any) -> Optional[float]: if hasattr(value, "detach"): return float(value.detach().cpu().item()) return float(value) - except (TypeError, ValueError, RuntimeError): + except (TypeError, ValueError, OverflowError, RuntimeError): return None @@ -463,6 +538,12 @@ def collect_runtime_compute_metrics( "gating_entropy": None, "prediction_streak": None, "prediction_confidence_delta": None, + "prediction_margin": None, + "prediction_decision_margin": None, + "decision_reference_cycles": None, + "prediction_rank_depth": None, + "prediction_class_count": None, + "prediction_class_selection_valid": None, "exit_reason": None, } attr_map = { @@ -472,6 +553,12 @@ def collect_runtime_compute_metrics( "last_gating_entropy": "gating_entropy", "last_prediction_streak": "prediction_streak", "last_prediction_confidence_delta": "prediction_confidence_delta", + "last_prediction_margin": "prediction_margin", + "last_prediction_decision_margin": "prediction_decision_margin", + "last_decision_reference_cycles": "decision_reference_cycles", + "last_prediction_rank_depth": "prediction_rank_depth", + "last_prediction_class_count": "prediction_class_count", + "last_prediction_class_selection_valid": "prediction_class_selection_valid", } modules = list(model.modules()) if hasattr(model, "modules") else [model] for module in modules: @@ -485,6 +572,10 @@ def collect_runtime_compute_metrics( reason = getattr(module, "last_exit_reason", None) if isinstance(reason, str) and reason.strip(): metrics["exit_reason"] = reason.strip() + if metrics["prediction_class_selection_valid"] is not None: + metrics["prediction_class_selection_valid"] = bool( + metrics["prediction_class_selection_valid"] + ) metrics.update({ "supported": model_supports_runtime_compute(model), "requested_reasoning_cycles": requested_reasoning_cycles, @@ -512,6 +603,9 @@ def forward_with_runtime_compute( prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, auto_reasoning_context: str = "", return_diagnostics: Optional[bool] = None, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_class_indices: Any = None, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ): """Forward a model while applying optional v50 runtime-compute controls. @@ -542,6 +636,14 @@ def forward_with_runtime_compute( prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + stability_margin = _coerce_prediction_stability_margin( + prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + stability_rank_depth = _coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) kwargs: Dict[str, Any] = {} if cycles is not None and _model_forward_accepts_kwarg(model, "reasoning_cycles"): @@ -556,8 +658,29 @@ def forward_with_runtime_compute( kwargs["prediction_stability_patience"] = stability_patience if _model_forward_accepts_kwarg(model, "prediction_stability_tol"): kwargs["prediction_stability_tol"] = stability_tol + if _model_forward_accepts_kwarg(model, "prediction_stability_margin"): + kwargs["prediction_stability_margin"] = stability_margin + if _model_forward_accepts_kwarg(model, "prediction_stability_rank_depth"): + kwargs["prediction_stability_rank_depth"] = stability_rank_depth + if ( + prediction_class_indices is not None + and _model_forward_accepts_kwarg(model, "prediction_class_indices") + ): + if isinstance(prediction_class_indices, torch.Tensor): + prediction_class_indices = prediction_class_indices.detach().cpu().tolist() + elif isinstance(prediction_class_indices, tuple): + prediction_class_indices = list(prediction_class_indices) + kwargs["prediction_class_indices"] = prediction_class_indices output = model(x, **kwargs) if kwargs else model(x) + prediction_verifier_active = bool( + adaptive + and stability_patience > 0 + and stability_rank_depth > 0 + and "adaptive_compute" in kwargs + and "prediction_stability_patience" in kwargs + and "prediction_stability_rank_depth" in kwargs + ) metrics = collect_runtime_compute_metrics( model, requested_reasoning_cycles=cycles, @@ -565,6 +688,18 @@ def forward_with_runtime_compute( exit_tol=tol, applied_kwargs=kwargs, ) + if not prediction_verifier_active: + for key in ( + "prediction_streak", + "prediction_confidence_delta", + "prediction_margin", + "prediction_decision_margin", + "decision_reference_cycles", + "prediction_rank_depth", + "prediction_class_count", + "prediction_class_selection_valid", + ): + metrics[key] = None diagnostics = { "requested_reasoning_cycles": cycles, "selected_reasoning_cycles": cycles, @@ -576,6 +711,14 @@ def forward_with_runtime_compute( "exit_entropy_threshold": entropy_threshold, "prediction_stability_patience": stability_patience, "prediction_stability_tol": stability_tol, + "prediction_stability_margin": stability_margin, + "prediction_stability_rank_depth": stability_rank_depth, + "prediction_verifier_active": prediction_verifier_active, + "prediction_class_indices": ( + kwargs.get("prediction_class_indices") + if "prediction_class_indices" in kwargs + else None + ), "applied": bool(kwargs), "applied_kwargs": dict(kwargs), **metrics, @@ -585,6 +728,266 @@ def forward_with_runtime_compute( return output, diagnostics +def _topk_distribution_js_divergence( + previous: torch.Tensor, + current: torch.Tensor, + top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> float: + """Return midpoint-support JSD while preserving tail probability mass.""" + + left = previous.detach().to(dtype=torch.float64).reshape(-1) + right = current.detach().to(dtype=torch.float64).reshape(-1) + if left.numel() != right.numel() or left.numel() == 0: + return 0.0 + k = min(max(1, int(top_k)), int(left.numel())) + midpoint = 0.5 * (left + right) + indices = torch.topk(midpoint, k=k).indices + left_top = left.index_select(0, indices) + right_top = right.index_select(0, indices) + left_compact = torch.cat( + [left_top, (1.0 - left_top.sum()).clamp_min(0.0).reshape(1)] + ) + right_compact = torch.cat( + [right_top, (1.0 - right_top.sum()).clamp_min(0.0).reshape(1)] + ) + compact_midpoint = 0.5 * (left_compact + right_compact) + epsilon = torch.finfo(left_compact.dtype).tiny + + def _kl(source: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return ( + source + * ( + torch.log(source.clamp_min(epsilon)) + - torch.log(target.clamp_min(epsilon)) + ) + ).sum() + + value = 0.5 * ( + _kl(left_compact, compact_midpoint) + + _kl(right_compact, compact_midpoint) + ) + return round(max(0.0, float(value.item())), 10) + + +def progressive_auto_compute_forward( + model: Any, + x: torch.Tensor, + available_labels: List[int], + *, + cycles: Any = None, + confidence_target: Any = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: Any = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: Any = False, + exit_tol: Any = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: Any = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: Any = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, + auto_reasoning_context: str = "", + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, +) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + """Evaluate auto-compute budgets progressively and reuse the accepted probe. + + This preserves the v1 selection policy and row order exactly: the first + confidence/entropy target wins, otherwise the best-confidence row wins + after the full ladder. Probes use the complete current runtime-control set, + and the accepted probe output is returned directly, removing the old N+1 + rerun. Cross-budget persistence and JSD are telemetry only and never select + or reject an output. + """ + + if bool(getattr(model, "training", False)): + raise RuntimeError("progressive auto compute requires model.eval()") + labels = list(available_labels) or list(range(MODEL_CLASSES)) + resolved_cycles = resolve_runtime_compute_cycles(cycles) + confidence_target = min( + 1.0, + _coerce_nonnegative_float( + confidence_target, + DEFAULT_AUTO_COMPUTE_CONFIDENCE, + ), + ) + entropy_target = _coerce_nonnegative_float( + entropy_target, + DEFAULT_AUTO_COMPUTE_ENTROPY, + ) + adaptive = _coerce_bool(adaptive_compute) + idx = torch.tensor(labels, dtype=torch.long, device=x.device) + records: List[Dict[str, Any]] = [] + cached_outputs: List[torch.Tensor] = [] + previous_probs: Optional[torch.Tensor] = None + previous_label: Optional[int] = None + previous_confidence: Optional[float] = None + accepted_index: Optional[int] = None + reason = "best_confidence" + + for requested_cycles in resolved_cycles: + started = time.perf_counter() + with torch.no_grad(): + model_out, compute = forward_with_runtime_compute( + model, + x, + reasoning_cycles=requested_cycles, + adaptive_compute=adaptive, + exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + auto_reasoning_context=auto_reasoning_context, + prediction_stability_margin=prediction_stability_margin, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, + ) + logits = model_out[0, 0] + available_logits = logits.index_select(0, idx.to(logits.device)) + probs = torch.softmax(available_logits, dim=0) + confidence_tensor, predicted_position = torch.max(probs, dim=0) + entropy_tensor = -(probs * torch.log(probs.clamp_min(1e-8))).sum() + # v1 serialized these values to six decimals before the selector read + # them. Keep that boundary exact so values such as 0.5499996 still + # satisfy a 0.55 target just as they did before. + confidence = round(float(confidence_tensor.item()), 6) + entropy = round(float(entropy_tensor.item()), 6) + predicted_label = int(labels[int(predicted_position.item())]) + # Freeze selection latency before shadow-only diagnostics. Otherwise a + # costly JSD calculation could alter the legacy fallback tie-break. + probe_latency_ms = round((time.perf_counter() - started) * 1000.0, 1) + has_previous = previous_probs is not None + shadow = { + "role": "shadow_diagnostic_only", + "metric": "midpoint_support_top_k_jensen_shannon_divergence_nats", + "top_k": min(max(1, int(distribution_top_k)), len(labels)), + "has_previous_probe": has_previous, + "top1_persistent": ( + bool(previous_label == predicted_label) if has_previous else None + ), + "confidence_delta": ( + round(abs(confidence - float(previous_confidence)), 8) + if previous_confidence is not None + else None + ), + "js_divergence": ( + _topk_distribution_js_divergence( + previous_probs, + probs, + distribution_top_k, + ) + if previous_probs is not None + else None + ), + "selection_enabled": False, + } + row = { + "requested_cycles": int(requested_cycles), + "latency_ms": probe_latency_ms, + "cycles_used": compute.get("cycles_used"), + "predicted_label": predicted_label, + "confidence": confidence, + "entropy": entropy, + "confidence_target_met": bool(confidence >= confidence_target), + "entropy_target_met": bool(entropy <= entropy_target), + "mutual_stability_shadow": shadow, + "compute": compute, + } + records.append(row) + cached_outputs.append(model_out) + + if confidence >= confidence_target: + accepted_index = len(records) - 1 + reason = "confidence_target" + break + if entropy <= entropy_target: + accepted_index = len(records) - 1 + reason = "entropy_target" + break + previous_probs = probs.detach() + previous_label = predicted_label + previous_confidence = confidence + + if accepted_index is None: + accepted_index = max( + range(len(records)), + key=lambda index: ( + float(records[index].get("confidence", 0.0)), + -float(records[index].get("entropy", 999.0)), + -float(records[index].get("latency_ms", 0.0)), + ), + ) + + selected_row = records[accepted_index] + selected_output = cached_outputs[accepted_index] + selected_compute = dict(selected_row["compute"]) + evaluated_cycles = [int(row["requested_cycles"]) for row in records] + skipped_cycles = [ + int(value) for value in resolved_cycles[len(evaluated_cycles) :] + ] + legacy_forward_evaluations = len(resolved_cycles) + 1 + forward_evaluations = len(records) + plan = { + "schema_version": AUTO_COMPUTE_PLAN_SCHEMA_VERSION, + "enabled": True, + "strategy": AUTO_COMPUTE_STRATEGY, + "selected_reasoning_cycles": int(selected_row["requested_cycles"]), + "selected_index": int(accepted_index), + "accepted_probe_index": int(accepted_index), + "reason": reason, + "confidence_target": confidence_target, + "entropy_target": entropy_target, + "candidate_cycles": [int(value) for value in resolved_cycles], + "evaluated_cycles": evaluated_cycles, + "skipped_cycles": skipped_cycles, + "forward_evaluations": forward_evaluations, + "legacy_forward_evaluations": legacy_forward_evaluations, + "forward_reduction_percent": round( + 100.0 + * (legacy_forward_evaluations - forward_evaluations) + / max(1, legacy_forward_evaluations), + 3, + ), + "reused_probe_output": True, + "selection_semantics": "legacy_v1_selection_policy", + "probe_control_scope": "full_runtime_controls_v2", + "mutual_stability_role": "shadow_diagnostic_only", + "rows": records, + } + selected_compute["auto_compute_plan"] = plan + selected_compute["inference_reused"] = True + return selected_output, selected_compute, plan + + +def compact_auto_compute_plan_metrics(plan: Any) -> Dict[str, Any]: + """Return terminal-safe metrics for the probe that supplied the output.""" + + if not isinstance(plan, dict): + return {} + rows = plan.get("rows") if isinstance(plan.get("rows"), list) else [] + try: + selected_index = int(plan.get("selected_index")) + except (TypeError, ValueError, OverflowError, RuntimeError): + selected_index = -1 + if not 0 <= selected_index < len(rows): + selected_index = len(rows) - 1 + selected_row = rows[selected_index] if selected_index >= 0 else {} + shadow = ( + selected_row.get("mutual_stability_shadow", {}) + if isinstance(selected_row, dict) + else {} + ) + metrics: Dict[str, Any] = { + "auto_selected": plan.get("selected_reasoning_cycles"), + "auto_reason": plan.get("reason"), + "auto_forwards": ( + f"{plan.get('forward_evaluations')}/" + f"{plan.get('legacy_forward_evaluations')}" + ), + "auto_reused": bool(plan.get("reused_probe_output")), + } + if isinstance(shadow, dict) and shadow.get("js_divergence") is not None: + metrics["auto_shadow_js"] = shadow.get("js_divergence") + return metrics + + def _default_expansion_dim_for_model_size(model_size: str) -> int: if model_size in THIRD_EXPANSION_DIM_MODEL_SIZES: return 1024 @@ -626,6 +1029,8 @@ def _print_chat_help() -> None: print(" /exit_entropy Set v50 adaptive-compute entropy exit threshold") print(" /stability Stable prediction cycles required (0 disables)") print(" /stability_tol Maximum confidence drift across the stable streak") + print(" /stability_margin Minimum top-two probability margin for stable exit") + print(" /stability_rank_depth Ordered prediction ranks verified (0 disables)") print(" /top Show top reranked candidates each turn (0 disables)") print(" /timing on|off Toggle per-turn timing output") print(" /auto_compute on|off Toggle confidence/entropy based cycle selection") @@ -687,6 +1092,18 @@ def main(): default=DEFAULT_PREDICTION_STABILITY_TOL, help="Maximum confidence drift allowed across a stable prediction streak.", ) + ap.add_argument( + "--prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + help="Minimum full-output top-two probability margin required for stable exit.", + ) + ap.add_argument( + "--prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + help="Ordered prediction ranks required to remain stable (0 disables the rank verifier).", + ) ap.add_argument("--device", default="auto") ap.add_argument( "--device_preference", @@ -879,15 +1296,7 @@ def main(): if missing or unexpected: raise RuntimeError(f"State dict mismatch. Missing={missing}, Unexpected={unexpected}") - buckets_raw = meta.get("buckets", {}) - buckets: Dict[int, List[Dict]] = {} - for k, v in buckets_raw.items(): - try: - label = int(k) - except ValueError: - continue - if isinstance(v, list) and v: - buckets[label] = v + buckets = _parse_metadata_buckets(meta.get("buckets", {})) available_labels = sorted(buckets.keys()) if not available_labels: @@ -942,6 +1351,14 @@ def main(): args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + args.prediction_stability_margin = _coerce_prediction_stability_margin( + args.prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + args.prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) print(f"\n{TerminalColors.SYSTEM}--- Session Info ---") print(f"Loaded: {Path(args.weights).name} [{resolved_model_size}] | Available labels: {len(available_labels)}") @@ -956,6 +1373,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) @@ -1011,6 +1430,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) print( @@ -1028,8 +1449,7 @@ def main(): compact_compute = dict(last_compute_metrics) plan = compact_compute.pop("auto_compute_plan", None) if isinstance(plan, dict): - compact_compute["auto_selected"] = plan.get("selected_reasoning_cycles") - compact_compute["auto_reason"] = plan.get("reason") + compact_compute.update(compact_auto_compute_plan_metrics(plan)) print( " last_compute=" + ", ".join(f"{k}={v}" for k, v in compact_compute.items()) @@ -1052,6 +1472,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'} " f"auto_conf={float(args.auto_compute_confidence):.3f} " f"auto_entropy={float(args.auto_compute_entropy):.3f}" @@ -1138,6 +1560,26 @@ def main(): ) except Exception: print(f"{TerminalColors.SYSTEM}Usage: /stability_tol {TerminalColors.RESET}") + elif cmd == "stability_margin": + try: + args.prediction_stability_margin = _coerce_prediction_stability_margin( + arg, + args.prediction_stability_margin, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability margin set to " + f"{float(args.prediction_stability_margin):.4g}.{TerminalColors.RESET}" + ) + except Exception: + print(f"{TerminalColors.SYSTEM}Usage: /stability_margin {TerminalColors.RESET}") + elif cmd == "stability_rank_depth": + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + arg, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability rank depth set to " + f"{int(args.prediction_stability_rank_depth)}.{TerminalColors.RESET}" + ) elif cmd == "top": try: args.show_top_responses = max(0, int(arg)) @@ -1244,35 +1686,40 @@ def main(): effective_reasoning_cycles = args.reasoning_cycles auto_compute_plan = None if args.auto_compute and model_supports_runtime_compute(model): - auto_rows = evaluate_runtime_compute_budgets( + model_out, last_compute_metrics, auto_compute_plan = progressive_auto_compute_forward( model, x, available_labels, cycles=runtime_auto_compute_cycles(args.reasoning_cycles), - adaptive_compute=args.adaptive_compute, - exit_tol=args.adaptive_exit_tol, - ) - auto_compute_plan = select_auto_runtime_compute_budget( - auto_rows, confidence_target=args.auto_compute_confidence, entropy_target=args.auto_compute_entropy, - ) - effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") - with torch.no_grad(): - model_out, last_compute_metrics = forward_with_runtime_compute( - model, - x, - reasoning_cycles=effective_reasoning_cycles, adaptive_compute=args.adaptive_compute, exit_tol=args.adaptive_exit_tol, exit_entropy_threshold=args.adaptive_exit_entropy, prediction_stability_patience=args.prediction_stability_patience, prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, ) - logits = model_out[0, 0] # (10,) - if auto_compute_plan is not None: - last_compute_metrics["auto_compute_plan"] = auto_compute_plan + effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") + else: + with torch.no_grad(): + model_out, last_compute_metrics = forward_with_runtime_compute( + model, + x, + reasoning_cycles=effective_reasoning_cycles, + adaptive_compute=args.adaptive_compute, + exit_tol=args.adaptive_exit_tol, + exit_entropy_threshold=args.adaptive_exit_entropy, + prediction_stability_patience=args.prediction_stability_patience, + prediction_stability_tol=args.prediction_stability_tol, + auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_class_indices=available_labels, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, + ) + logits = model_out[0, 0] # (10,) t_infer += max(0.0, time.perf_counter() - _t) idx = torch.tensor(available_labels, dtype=torch.long, device=logits.device) @@ -1424,9 +1871,9 @@ def main(): plan = last_compute_metrics.get("auto_compute_plan") if isinstance(last_compute_metrics, dict) else None plan_suffix = "" if isinstance(plan, dict): - plan_suffix = ( - f" auto_selected={plan.get('selected_reasoning_cycles')} " - f"reason={plan.get('reason')}" + plan_metrics = compact_auto_compute_plan_metrics(plan) + plan_suffix = " " + " ".join( + f"{key}={value}" for key, value in plan_metrics.items() ) print( f"{TerminalColors.SYSTEM}Compute:{TerminalColors.RESET} " diff --git a/runtime_python/chat_web_app.py b/runtime_python/chat_web_app.py index 3104db5d..b075fbd7 100644 --- a/runtime_python/chat_web_app.py +++ b/runtime_python/chat_web_app.py @@ -220,7 +220,7 @@
-
+
@@ -229,7 +229,9 @@
+
+
Verify ordered top ranks; 0 intentionally disables rank verification.
@@ -270,6 +272,7 @@ function fmtNum(value,digits=3){const n=Number(value);return Number.isFinite(n)?n.toFixed(digits):null;} function reasoningCyclesValue(){const raw=el('reasoningCycles').value.trim();if(!raw)return null;const low=raw.toLowerCase();if(['auto','adaptive','smart'].includes(low))return 'auto';const n=Number(raw);return Number.isFinite(n)?n:raw;} +function addOptionalFiniteNumber(payload,key,input){const raw=input.value.trim();if(raw==='')return;const value=Number(raw);if(Number.isFinite(value))payload[key]=value;} function add(kind,text,timing,top,compute){ const d=document.createElement('div'); d.className='msg '+kind; @@ -290,18 +293,39 @@ } if(compute&&compute.applied){ const parts=[]; + if(compute.prediction_verifier_active===true) parts.push('verifier active'); + else if(compute.adaptive_compute) parts.push('verifier inactive'); if(compute.reasoning_budget_mode==='auto') parts.push('mode auto'); if(compute.requested_reasoning_cycles!==undefined&&compute.requested_reasoning_cycles!==null) parts.push(`requested ${compute.requested_reasoning_cycles}`); if(compute.cycles_used!==undefined&&compute.cycles_used!==null) parts.push(`used ${compute.cycles_used}`); + if(compute.decision_reference_cycles!==undefined&&compute.decision_reference_cycles!==null) parts.push(`reference ${compute.decision_reference_cycles}`); if(compute.exit_reason) parts.push(`exit ${compute.exit_reason}`); const streak=fmtNum(compute.prediction_streak); if(streak) parts.push(`stable ${streak}`); const drift=fmtNum(compute.prediction_confidence_delta); if(drift) parts.push(`drift ${drift}`); + const observedMargin=fmtNum(compute.prediction_margin,6); if(observedMargin) parts.push(`top-1 margin ${observedMargin}`); + const decisionMargin=fmtNum(compute.prediction_decision_margin,6); if(decisionMargin) parts.push(`decision margin ${decisionMargin}`); + const marginFloor=fmtNum(compute.prediction_stability_margin,6); if(marginFloor) parts.push(`margin floor ${marginFloor}`); + if(compute.prediction_rank_depth!==undefined&&compute.prediction_rank_depth!==null) parts.push(`verified depth ${compute.prediction_rank_depth}`); + if(compute.prediction_stability_rank_depth!==undefined&&compute.prediction_stability_rank_depth!==null) parts.push(`requested depth ${compute.prediction_stability_rank_depth}`); + if(compute.prediction_class_count!==undefined&&compute.prediction_class_count!==null) parts.push(`verifier ${compute.prediction_class_count} classes`); + if(compute.prediction_class_selection_valid===false) parts.push('verifier scope invalid'); const ponder=fmtNum(compute.ponder_cost); if(ponder) parts.push(`ponder ${ponder}`); const consistency=fmtNum(compute.consistency_loss); if(consistency) parts.push(`consistency ${consistency}`); const entropy=fmtNum(compute.gating_entropy); if(entropy) parts.push(`gate entropy ${entropy}`); const exitEntropy=fmtNum(compute.exit_entropy_threshold); if(exitEntropy) parts.push(`exit entropy ${exitEntropy}`); if(compute.auto_reasoning_policy&&Array.isArray(compute.auto_reasoning_policy.reasons)) parts.push(`policy ${compute.auto_reasoning_policy.reasons.slice(0,3).join(',')}`); - if(compute.auto_compute_plan) parts.push(`budget ${compute.auto_compute_plan.selected_reasoning_cycles} (${compute.auto_compute_plan.reason})`); + if(compute.auto_compute_plan){ + const plan=compute.auto_compute_plan; + parts.push(`budget ${plan.selected_reasoning_cycles} (${plan.reason})`); + parts.push(`${plan.forward_evaluations}/${plan.legacy_forward_evaluations} forwards`); + if(plan.reused_probe_output) parts.push('probe reused'); + const rows=Array.isArray(plan.rows)?plan.rows:[]; + const selectedIndex=Number(plan.selected_index); + const selectedRow=Number.isInteger(selectedIndex)&&selectedIndex>=0&&selectedIndex{ const conf=fmtNum(row.confidence); @@ -435,6 +467,8 @@ "adaptive_exit_entropy", "prediction_stability_patience", "prediction_stability_tol", + "prediction_stability_margin", + "prediction_stability_rank_depth", "auto_compute", ) @@ -447,6 +481,8 @@ def _library_runtime_compute_defaults() -> Dict[str, Any]: "adaptive_exit_entropy": chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY, "prediction_stability_patience": chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE, "prediction_stability_tol": chat_app.DEFAULT_PREDICTION_STABILITY_TOL, + "prediction_stability_margin": chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN, + "prediction_stability_rank_depth": chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, "auto_compute": False, } @@ -481,6 +517,12 @@ def _normalize_runtime_compute_defaults(values: Dict[str, Any]) -> Dict[str, Any values.get("prediction_stability_tol"), chat_app.DEFAULT_PREDICTION_STABILITY_TOL, ), + "prediction_stability_margin": chat_app._coerce_prediction_stability_margin( + values.get("prediction_stability_margin"), + ), + "prediction_stability_rank_depth": chat_app._coerce_prediction_stability_rank_depth( + values.get("prediction_stability_rank_depth"), + ), "auto_compute": chat_app._coerce_bool(values.get("auto_compute")), } @@ -494,6 +536,8 @@ def _runtime_compute_cli_overrides(args: argparse.Namespace) -> Dict[str, Any]: "adaptive_exit_entropy": getattr(args, "adaptive_exit_entropy", None), "prediction_stability_patience": getattr(args, "prediction_stability_patience", None), "prediction_stability_tol": getattr(args, "prediction_stability_tol", None), + "prediction_stability_margin": getattr(args, "prediction_stability_margin", None), + "prediction_stability_rank_depth": getattr(args, "prediction_stability_rank_depth", None), "auto_compute": getattr(args, "auto_compute", None), } return {key: value for key, value in values.items() if value is not None} @@ -560,6 +604,8 @@ def status(self) -> Dict[str, Any]: "adaptive_exit_entropy": chat_app._coerce_nonnegative_float(self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), "prediction_stability_patience": chat_app._coerce_nonnegative_int(self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE, chat_app.MAX_RUNTIME_REASONING_CYCLES), "prediction_stability_tol": chat_app._coerce_nonnegative_float(self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + "prediction_stability_margin": chat_app._coerce_prediction_stability_margin(self.defaults.get("prediction_stability_margin", chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN)), + "prediction_stability_rank_depth": chat_app._coerce_prediction_stability_rank_depth(self.defaults.get("prediction_stability_rank_depth", chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH)), "auto_compute": chat_app._coerce_bool(self.defaults.get("auto_compute", False)), } @@ -605,16 +651,7 @@ def route_shadow_registry_snapshot(self) -> Dict[str, Any]: } def _parse_buckets(self, meta: Dict[str, Any]) -> None: - buckets: Dict[int, List[Dict[str, Any]]] = {} - raw = meta.get("buckets", {}) - if isinstance(raw, dict): - for k, v in raw.items(): - try: - label = int(k) - except Exception: - continue - if isinstance(v, list) and v: - buckets[label] = v + buckets = chat_app._parse_metadata_buckets(meta.get("buckets", {})) self.buckets = buckets self.available_labels = sorted(buckets.keys()) or list(range(chat_app.MODEL_CLASSES)) @@ -694,6 +731,11 @@ def _run_compute_sweep_rows( cycles: Any, adaptive: bool, exit_tol: Optional[float], + exit_entropy_threshold: Any = None, + prediction_stability_patience: Any = None, + prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> List[Dict[str, Any]]: return chat_app.evaluate_runtime_compute_budgets( model, @@ -702,6 +744,11 @@ def _run_compute_sweep_rows( cycles=cycles, adaptive_compute=adaptive, exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=prediction_stability_margin, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) def _select_auto_compute_budget(self, rows: List[Dict[str, Any]]) -> Dict[str, Any]: @@ -717,6 +764,8 @@ def compute_sweep( adaptive_exit_entropy: Any = None, prediction_stability_patience: Any = None, prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> Dict[str, Any]: if not user_text.strip(): raise ValueError("Empty message") @@ -741,6 +790,16 @@ def compute_sweep( x = chat_app.text_to_model_input(context, feature_mode=feature_mode).to(self.device) idx = torch.tensor(labels, dtype=torch.long, device=self.device) rows: List[Dict[str, Any]] = [] + resolved_prediction_stability_margin = chat_app._coerce_prediction_stability_margin( + prediction_stability_margin + if prediction_stability_margin is not None + else self.defaults.get("prediction_stability_margin", chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN) + ) + resolved_prediction_stability_rank_depth = chat_app._coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth + if prediction_stability_rank_depth is not None + else self.defaults.get("prediction_stability_rank_depth", chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH) + ) with torch.no_grad(): for cycle_count in requested_cycles: @@ -754,6 +813,9 @@ def compute_sweep( exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + prediction_stability_margin=resolved_prediction_stability_margin, + prediction_class_indices=labels, + prediction_stability_rank_depth=resolved_prediction_stability_rank_depth, ) latency_ms = round((time.perf_counter() - t0) * 1000, 1) logits = model_out[0, 0] @@ -792,6 +854,8 @@ def chat( prediction_stability_patience: Any = None, prediction_stability_tol: Any = None, auto_compute: Optional[bool] = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> Dict[str, Any]: if not user_text.strip(): raise ValueError("Empty message") @@ -821,6 +885,16 @@ def chat( if adaptive_exit_tol is None else adaptive_exit_tol ) + resolved_prediction_stability_margin = chat_app._coerce_prediction_stability_margin( + prediction_stability_margin + if prediction_stability_margin is not None + else self.defaults.get("prediction_stability_margin", chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN) + ) + resolved_prediction_stability_rank_depth = chat_app._coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth + if prediction_stability_rank_depth is not None + else self.defaults.get("prediction_stability_rank_depth", chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH) + ) effective_reasoning_cycles = ( self.defaults.get("reasoning_cycles") if reasoning_cycles is None @@ -833,34 +907,41 @@ def chat( else chat_app._coerce_bool(auto_compute, default=False) ) if auto_enabled and chat_app.model_supports_runtime_compute(model): - sweep_rows = self._run_compute_sweep_rows( - model=model, - x=x, - labels=labels, + model_out, compute_metrics, compute_plan = chat_app.progressive_auto_compute_forward( + model, + x, + labels, cycles=self._auto_compute_cycles(effective_reasoning_cycles), - adaptive=chat_app._coerce_bool(resolved_adaptive_compute, default=False), + adaptive_compute=resolved_adaptive_compute, exit_tol=chat_app._coerce_nonnegative_float( resolved_exit_tol, default=chat_app.DEFAULT_ADAPTIVE_EXIT_TOL, ), - ) - compute_plan = self._select_auto_compute_budget(sweep_rows) - effective_reasoning_cycles = compute_plan.get("selected_reasoning_cycles") - with torch.no_grad(): - model_out, compute_metrics = chat_app.forward_with_runtime_compute( - model, - x, - reasoning_cycles=effective_reasoning_cycles, - adaptive_compute=resolved_adaptive_compute, - exit_tol=resolved_exit_tol, exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + prediction_stability_margin=resolved_prediction_stability_margin, + prediction_stability_rank_depth=resolved_prediction_stability_rank_depth, auto_reasoning_context=context, ) - logits = model_out[0, 0] - if compute_plan is not None: - compute_metrics["auto_compute_plan"] = compute_plan + effective_reasoning_cycles = compute_plan.get("selected_reasoning_cycles") + else: + with torch.no_grad(): + model_out, compute_metrics = chat_app.forward_with_runtime_compute( + model, + x, + reasoning_cycles=effective_reasoning_cycles, + adaptive_compute=resolved_adaptive_compute, + exit_tol=resolved_exit_tol, + exit_entropy_threshold=adaptive_exit_entropy if adaptive_exit_entropy is not None else self.defaults.get("adaptive_exit_entropy", chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY), + prediction_stability_patience=prediction_stability_patience if prediction_stability_patience is not None else self.defaults.get("prediction_stability_patience", chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE), + prediction_stability_tol=prediction_stability_tol if prediction_stability_tol is not None else self.defaults.get("prediction_stability_tol", chat_app.DEFAULT_PREDICTION_STABILITY_TOL), + prediction_stability_margin=resolved_prediction_stability_margin, + auto_reasoning_context=context, + prediction_class_indices=labels, + prediction_stability_rank_depth=resolved_prediction_stability_rank_depth, + ) + logits = model_out[0, 0] t_infer += time.perf_counter() - tt idx = torch.tensor(labels, dtype=torch.long, device=logits.device) @@ -1025,6 +1106,8 @@ def api_chat(): prediction_stability_patience=p.get('prediction_stability_patience'), prediction_stability_tol=p.get('prediction_stability_tol'), auto_compute=p.get('auto_compute'), + prediction_stability_margin=p.get('prediction_stability_margin'), + prediction_stability_rank_depth=p.get('prediction_stability_rank_depth'), )) except Exception as e: return jsonify({"ok": False, "error": str(e)}), 400 @@ -1044,6 +1127,8 @@ def api_compute_sweep(): adaptive_exit_entropy=p.get('adaptive_exit_entropy'), prediction_stability_patience=p.get('prediction_stability_patience'), prediction_stability_tol=p.get('prediction_stability_tol'), + prediction_stability_margin=p.get('prediction_stability_margin'), + prediction_stability_rank_depth=p.get('prediction_stability_rank_depth'), )) except Exception as e: return jsonify({"ok": False, "error": str(e)}), 400 @@ -1100,6 +1185,8 @@ def main() -> None: ap.add_argument('--adaptive_exit_entropy', type=float, default=None) ap.add_argument('--prediction_stability_patience', type=int, default=None) ap.add_argument('--prediction_stability_tol', type=float, default=None) + ap.add_argument('--prediction_stability_margin', type=float, default=None) + ap.add_argument('--prediction_stability_rank_depth', type=int, default=None) auto_compute_group = ap.add_mutually_exclusive_group() auto_compute_group.add_argument( '--auto_compute', diff --git a/runtime_python/model_variants.py b/runtime_python/model_variants.py index 8fde3df3..9ff767d4 100644 --- a/runtime_python/model_variants.py +++ b/runtime_python/model_variants.py @@ -6029,6 +6029,24 @@ def __init__( self.register_buffer("last_gating_entropy", torch.tensor(0.0)) self.register_buffer("last_prediction_streak", torch.tensor(0.0), persistent=False) self.register_buffer("last_prediction_confidence_delta", torch.tensor(0.0), persistent=False) + self.register_buffer("last_prediction_margin", torch.tensor(0.0), persistent=False) + self.register_buffer( + "last_prediction_decision_margin", torch.tensor(0.0), persistent=False + ) + self.register_buffer( + "last_prediction_rank_depth", torch.tensor(1.0), persistent=False + ) + self.register_buffer( + "last_prediction_class_count", torch.tensor(float(out_dim)), persistent=False + ) + self.register_buffer( + "last_prediction_class_selection_valid", torch.tensor(True), persistent=False + ) + self.register_buffer( + "last_decision_reference_cycles", + torch.tensor(float(n_cycles)), + persistent=False, + ) self.register_buffer( "last_prediction_topk_js_divergence", torch.tensor(0.0), persistent=False ) @@ -6130,10 +6148,82 @@ def _kl(left: torch.Tensor, right: torch.Tensor) -> torch.Tensor: ) return terms.sum(dim=-1) - return 0.5 * ( - _kl(previous_compact, compact_midpoint) - + _kl(current_compact, compact_midpoint) - ) + # JSD is nonnegative by definition. Float32 cancellation can leave a + # few negative nanonats for nearly identical distributions, which must + # not turn shadow telemetry into an inference failure. + return ( + 0.5 + * ( + _kl(previous_compact, compact_midpoint) + + _kl(current_compact, compact_midpoint) + ) + ).clamp_min(0.0) + + @staticmethod + def _normalize_prediction_class_indices( + prediction_class_indices, + *, + out_dim: int, + device: torch.device, + ): + """Validate an index list or full-length bool mask without raising. + + ``None`` and any complete permutation of all output classes use the + untouched all-class verifier path. Invalid scopes return no index + tensor and ``False`` so inference can continue while the + prediction-stability exit fails closed. + """ + + if prediction_class_indices is None: + return None, int(out_dim), int(out_dim) >= 2 + + try: + raw_selection = ( + prediction_class_indices.detach() + if torch.is_tensor(prediction_class_indices) + else torch.as_tensor(prediction_class_indices) + ) + except (TypeError, ValueError, RuntimeError): + return None, 0, False + + if raw_selection.ndim != 1: + return None, 0, False + + if raw_selection.dtype == torch.bool: + if int(raw_selection.numel()) != int(out_dim): + return None, 0, False + class_indices = raw_selection.nonzero(as_tuple=False).flatten() + elif raw_selection.dtype.is_floating_point or raw_selection.dtype.is_complex: + return None, 0, False + else: + class_indices = raw_selection + + try: + class_indices = class_indices.to(device=device, dtype=torch.long) + except (TypeError, ValueError, RuntimeError): + return None, 0, False + + class_count = int(class_indices.numel()) + if class_count == 0: + return None, 0, False + try: + selection_is_invalid = ( + int(class_indices.min().item()) < 0 + or int(class_indices.max().item()) >= int(out_dim) + or int(torch.unique(class_indices).numel()) != class_count + ) + except (TypeError, ValueError, RuntimeError): + return None, 0, False + if selection_is_invalid: + return None, 0, False + + # A valid complete selection is equivalent to ``None``. Avoiding an + # index_select here preserves the historical all-class numerics bit for + # bit, including for a complete selection supplied in another order. + if class_count == int(out_dim): + return None, class_count, class_count >= 2 + + return class_indices, class_count, class_count >= 2 def forward( self, @@ -6145,6 +6235,10 @@ def forward( prediction_stability_patience: int = 2, prediction_stability_tol: float = 5e-3, prediction_stability_top_k: int = 5, + prediction_stability_margin: float = 0.0, + prediction_output_transform: Optional[nn.Module] = None, + prediction_class_indices=None, + prediction_stability_rank_depth: int = 1, ): N = x.shape[0] * x.shape[1] if x.dim() == 3 else x.shape[0] shape_prefix = x.shape[:-1] @@ -6174,13 +6268,49 @@ def forward( cycles_used = n_cycles stability_patience = max(0, int(prediction_stability_patience)) stability_tol = max(0.0, float(prediction_stability_tol)) - use_prediction_stability = adaptive_compute and not self.training and stability_patience > 0 + try: + stability_margin = float(prediction_stability_margin) + except (TypeError, ValueError): + stability_margin = float("inf") + if not math.isfinite(stability_margin) or stability_margin < 0.0: + stability_margin = float("inf") + ( + verifier_class_indices, + verifier_class_count, + verifier_class_selection_valid, + ) = self._normalize_prediction_class_indices( + prediction_class_indices, + out_dim=self.out_dim, + device=x_flat.device, + ) + try: + requested_rank_depth = int(prediction_stability_rank_depth) + except (TypeError, ValueError, OverflowError, RuntimeError): + requested_rank_depth = 0 + effective_rank_depth = min( + max(0, requested_rank_depth), + max(0, int(verifier_class_count) - 1), + ) + verifier_decision_margin_valid = ( + verifier_class_selection_valid and effective_rank_depth >= 1 + ) + use_prediction_stability = ( + adaptive_compute + and not self.training + and stability_patience > 0 + and verifier_decision_margin_valid + ) + decision_reference_cycles = max(1, int(self.n_cycles)) weighted_recursive = torch.zeros(N, self.out_dim, device=x_flat.device, dtype=x_flat.dtype) - previous_prediction = None + previous_prediction_rank_tuple = None prediction_streak = torch.zeros(N, device=x_flat.device, dtype=torch.long) stable_confidence_min = None stable_confidence_max = None prediction_confidence_delta = torch.zeros(N, device=x_flat.device, dtype=x_flat.dtype) + prediction_margin = torch.zeros(N, device=x_flat.device, dtype=x_flat.dtype) + prediction_decision_margin = torch.zeros( + N, device=x_flat.device, dtype=x_flat.dtype + ) prediction_topk_js_divergence = torch.zeros( N, device=x_flat.device, dtype=x_flat.dtype ) @@ -6246,7 +6376,19 @@ def forward( + self.shared_scale * shared_out + (self.alpha + 1e-4) * prefix_recursive ) - prefix_probs = F.softmax(prefix_output, dim=-1) + # Verify the exact distribution returned by the complete model, + # including its learned post-head calibration layer. + verifier_output = ( + prediction_output_transform(prefix_output) + if prediction_output_transform is not None + else prefix_output + ) + scoped_verifier_output = ( + verifier_output.index_select(-1, verifier_class_indices) + if verifier_class_indices is not None + else verifier_output + ) + prefix_probs = F.softmax(scoped_verifier_output, dim=-1) if previous_prediction_distribution is not None: prediction_topk_js_divergence = self._topk_js_divergence( previous_prediction_distribution, @@ -6258,13 +6400,45 @@ def forward( prediction_topk_js_divergence, ) previous_prediction_distribution = prefix_probs - prediction_confidence, prediction = prefix_probs.max(dim=-1) - if previous_prediction is None: + prediction_confidence, _ = prefix_probs.max(dim=-1) + prediction_rank_tuple = prefix_probs.topk( + k=effective_rank_depth, + dim=-1, + ).indices + margin_verifiable = ( + verifier_class_selection_valid + and int(prefix_probs.shape[-1]) >= 2 + ) + if margin_verifiable: + top_two_probabilities = prefix_probs.topk(k=2, dim=-1).values + prediction_margin = ( + top_two_probabilities[:, 0] - top_two_probabilities[:, 1] + ) + decision_probabilities = prefix_probs.topk( + k=effective_rank_depth + 1, + dim=-1, + ).values + prediction_decision_margin = ( + decision_probabilities[:, :-1] + - decision_probabilities[:, 1:] + ).min(dim=-1).values + else: + prediction_margin = torch.zeros_like(prediction_confidence) + prediction_decision_margin = torch.zeros_like( + prediction_confidence + ) + if previous_prediction_rank_tuple is None: prediction_streak = torch.ones_like(prediction_streak) stable_confidence_min = prediction_confidence stable_confidence_max = prediction_confidence else: - same_prediction = prediction.eq(previous_prediction) + # A certified decision preserves the complete ordered + # candidate tuple, not just its top-1 member. Any swap or + # replacement through the requested rank depth restarts + # the persistence window. + same_prediction = prediction_rank_tuple.eq( + previous_prediction_rank_tuple + ).all(dim=-1) prediction_streak = torch.where( same_prediction, prediction_streak + 1, @@ -6281,31 +6455,50 @@ def forward( prediction_confidence, ) prediction_confidence_delta = stable_confidence_max - stable_confidence_min - previous_prediction = prediction + previous_prediction_rank_tuple = prediction_rank_tuple stable_mask = ( prediction_streak >= stability_patience ) & ( prediction_confidence_delta <= stability_tol + ) & ( + prediction_decision_margin >= stability_margin + ) & ( + margin_verifiable + ) + prediction_stable = ( + verifier_class_selection_valid + and stable_mask.numel() > 0 + and bool(stable_mask.all().item()) ) - prediction_stable = bool(stable_mask.all().item()) # Inference-only verifier and convergence exits. Prediction - # stability is checked first so diagnostics identify the new path - # when multiple exit criteria become true on the same cycle. + # stability exclusively controls early stopping while its exact + # post-head verifier is active. Legacy latent, entropy, and halt + # signals remain available only when that verifier is disabled or + # invalid, so they cannot bypass a failed decision certificate. converged = False if adaptive_compute and not self.training: - if prediction_stable: - converged = True - exit_reason = "prediction_stable" - elif delta is not None and delta.item() < exit_tol: - converged = True - exit_reason = "latent_converged" - elif entropy.mean().item() < exit_entropy_threshold: - converged = True - exit_reason = "low_entropy" - elif remaining.mean().item() < 0.05: - converged = True - exit_reason = "halt_mass" + if use_prediction_stability: + if prediction_stable and (t + 1) < decision_reference_cycles: + converged = True + exit_reason = "prediction_stable" + elif (t + 1) >= decision_reference_cycles: + # The head was trained against ``self.n_cycles``. If + # the verifier cannot certify a strictly earlier + # answer, use that exact reference-budget mixture + # instead of drifting into untrained extra cycles. + converged = True + exit_reason = "decision_reference_budget" + else: + if delta is not None and delta.item() < exit_tol: + converged = True + exit_reason = "latent_converged" + elif entropy.mean().item() < exit_entropy_threshold: + converged = True + exit_reason = "low_entropy" + elif remaining.mean().item() < 0.05: + converged = True + exit_reason = "halt_mass" # Differentiable ACT halting h_t = torch.sigmoid(self.halt_head(z_h)) # (N, 1) @@ -6325,18 +6518,54 @@ def forward( self.last_cycles_used = torch.tensor( float(cycles_used), device=x_flat.device ) - self.last_prediction_streak = prediction_streak.float().mean() - self.last_prediction_confidence_delta = prediction_confidence_delta.mean() + self.last_prediction_streak = ( + prediction_streak.float().mean() + if prediction_streak.numel() + else prediction_margin.new_zeros(()) + ) + self.last_prediction_confidence_delta = ( + prediction_confidence_delta.mean() + if prediction_confidence_delta.numel() + else prediction_margin.new_zeros(()) + ) + # Batch-level telemetry reports the least decisive member because the + # all-samples early-exit rule is only as safe as its smallest margin. + self.last_prediction_margin = ( + prediction_margin.min().detach() + if prediction_margin.numel() + else prediction_margin.new_zeros(()) + ) + self.last_prediction_decision_margin = ( + prediction_decision_margin.min().detach() + if prediction_decision_margin.numel() + else prediction_decision_margin.new_zeros(()) + ) + self.last_prediction_rank_depth = torch.tensor( + float(effective_rank_depth), device=x_flat.device + ) + self.last_prediction_class_count = torch.tensor( + float(verifier_class_count), device=x_flat.device + ) + self.last_prediction_class_selection_valid = torch.tensor( + bool(verifier_class_selection_valid), device=x_flat.device + ) + self.last_decision_reference_cycles = torch.tensor( + float(decision_reference_cycles), device=x_flat.device + ) self.last_prediction_topk_js_divergence = ( prediction_topk_js_divergence.mean().detach() + if prediction_topk_js_divergence.numel() + else prediction_margin.new_zeros(()) ) self.last_prediction_topk_js_divergence_max = ( prediction_topk_js_divergence_max.mean().detach() + if prediction_topk_js_divergence_max.numel() + else prediction_margin.new_zeros(()) ) self.last_exit_reason = exit_reason # Calculate gating entropy for MoE cores - if gate_probs_list: + if gate_probs_list and N > 0: all_gate_probs = torch.stack(gate_probs_list, dim=0) # (Steps, N, n_cores) mean_gate_probs = all_gate_probs.mean(dim=(0, 1)) # (n_cores,) gating_entropy = -torch.sum(mean_gate_probs * torch.log(mean_gate_probs + 1e-9)) @@ -6363,7 +6592,7 @@ def forward( + (self.alpha + 1e-4) * recursive_out ) - return output.view(*shape_prefix, -1) + return output.view(*shape_prefix, self.out_dim) class ChampionNetCognitiveLeapUltraExpert(nn.Module): @@ -6406,6 +6635,9 @@ def forward( prediction_stability_patience: int = 2, prediction_stability_tol: float = 5e-3, prediction_stability_top_k: int = 5, + prediction_stability_margin: float = 0.0, + prediction_class_indices=None, + prediction_stability_rank_depth: int = 1, ): for layer in self.layers: if isinstance(layer, CognitiveLeapUltraExpertHead): @@ -6418,6 +6650,10 @@ def forward( prediction_stability_patience=prediction_stability_patience, prediction_stability_tol=prediction_stability_tol, prediction_stability_top_k=prediction_stability_top_k, + prediction_stability_margin=prediction_stability_margin, + prediction_output_transform=self.layers[11], + prediction_class_indices=prediction_class_indices, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) else: x = layer(x) diff --git a/runtime_python/requirements_runtime_interface.txt b/runtime_python/requirements_runtime_interface.txt index 5499bafd..5e6a5671 100644 --- a/runtime_python/requirements_runtime_interface.txt +++ b/runtime_python/requirements_runtime_interface.txt @@ -1,5 +1,5 @@ torch -flask +flask==3.1.3 pywebview # Optional Windows GPU backend diff --git a/source/RESEARCH_UPGRADES.md b/source/RESEARCH_UPGRADES.md index 6d5d0b84..a389794e 100644 --- a/source/RESEARCH_UPGRADES.md +++ b/source/RESEARCH_UPGRADES.md @@ -646,11 +646,14 @@ Additional primary sources reviewed: Repo changes: -- Added an inference-only full-output verifier to the v51 recurrent head. At - each cycle it constructs the output that would be returned if reasoning - stopped at that point, then tracks the top prediction and confidence range. -- Adaptive inference can now stop when the prediction remains unchanged for a - configurable patience window and confidence drift stays below a configurable +- Added an inference-only post-head verifier to the v51 recurrent head. At each + cycle it constructs the exact output that would be returned if reasoning + stopped at that point, applies softmax only over the caller's verified + allowed-label scope, and tracks the ordered decision boundary through the + configured rank depth. +- Adaptive inference can now stop when the ordered top-k decision remains + unchanged for a configurable patience window, its minimum adjacent margin + clears the calibrated floor, and confidence drift stays below a configurable tolerance. This criterion is separate from latent convergence, low entropy, and ACT remaining-mass exits. - Added `prediction_stability_patience` and `prediction_stability_tol` across @@ -1297,40 +1300,75 @@ Research and deployment boundary: Those controls remain external prerequisites, not properties implied by a verified registry. -## July 2026: v51 prediction-stability pilot and distribution-drift shadow metric +## July 2026: v51 decision-fidelity release gate and reference fallback Recent primary sources reviewed: -1. LESS Is More: Adaptive Early Exit for Diffusion Language Models - https://arxiv.org/abs/2606.16908 -2. Stop When Reasoning Converges: Adaptive Test-Time Scaling with PUMA +1. Understanding and Mitigating Premature Confidence for Better LLM Reasoning + https://arxiv.org/abs/2605.24396 +2. MarginGate: Sparse Margin-Triggered Verification for Batch-Invariant LLM Inference + https://arxiv.org/abs/2605.30218 +3. Stop When Reasoning Converges: Semantic-Preserving Early Exit for Reasoning Models https://arxiv.org/abs/2605.17672 -3. LLMRouterBench: A Massive Benchmark and Unified Framework for LLM Routing - https://arxiv.org/abs/2601.07206 -4. TwinRouterBench: A Systematic Benchmark for Dynamic LLM Routing - https://arxiv.org/abs/2605.18859 +4. LESS Is More: Mutual-Stability Sampling for Diffusion Language Models + https://arxiv.org/abs/2606.16908 +5. UCCI: Calibrated Uncertainty for Cost-Optimal LLM Cascade Routing + https://arxiv.org/abs/2605.18796 +6. Conformal Thinking: Risk Control for Reasoning on a Compute Budget + https://arxiv.org/abs/2602.03814 +7. MARS: Margin-Adversarial Risk-controlled Stopping for Parallel LLM Test-time Scaling + https://arxiv.org/abs/2606.12935 -Implementation and pilot evidence: +Implementation and release evidence: - `benchmark_v51_prediction_stability.py` now records top-k Jensen-Shannon divergence between consecutive full-prefix output distributions. The shared top-k support is chosen from the midpoint distribution and all remaining probability is retained in one `other` bucket. This is diagnostic telemetry only: it cannot trigger an exit or change a model answer. -- A CPU pilot screened five stopping configurations over three unseen seeds and - 32 examples per seed (480 requests total). Patience 2 / tolerance 0.005 kept - 96/96 prediction agreement and zero observed accuracy delta while using 2.135 - mean cycles, a 28.8% reduction from the three-cycle baseline. Patience 1 - changed one prediction and is rejected. Stricter patience/tolerance settings - increased work without improving observed agreement. -- Latency measurements were not counterbalanced, so they are screening evidence - rather than a release claim. The next gate is 512 held-out examples over eight - fresh seeds with zero disagreements, no negative per-seed accuracy delta, at - least 20% mean cycle reduction, and positive median latency reduction before - changing any runtime default. -- The research transfer remains provisional. LESS studies diffusion language - models, PUMA combines semantic convergence with answer verification, and the - router benchmarks emphasize held-out dynamic evaluation plus strong simple - baselines. Supermix therefore exposes distribution drift as a shadow metric - beside its existing output-persistence verifier rather than treating a new - paper or a small pilot as activation evidence. +- The first strict 4,096-request dual-mode gate rejected the `0.0001` margin: + release mode observed 5 top-1, 18 ordered-top-3, and 10 top-3-set + disagreements; isolated mode observed 2, 6, and 4. Exact replay attributed + 13 release top-3 failures to legacy `latent_converged` bypasses and five to + cycle-2 verifier exits. The largest failing certified decision margin was + `0.000412636`, so `0.0005` is the smallest round candidate screened here. +- The accepted implementation persists the complete ordered rank tuple, makes + the post-head verifier the sole authority for early exit, and falls back to + the exact trained three-cycle ACT mixture with + `decision_reference_budget` when no strictly earlier decision is certified. + Bounded disagreement records retain sample index, target, decisions, exit + reason, cycles, margins, and verifier telemetry without retaining raw inputs. +- The clean v5 CPU gate at commit `81c4dbe7` used eight seeds and 512 requests + per seed in both isolated-verifier and exact release-runtime modes. Both had + zero top-1, ordered-top-3, top-3-set, and exact-output disagreements, and all + per-seed accuracy deltas were zero. Of 4,096 requests, 3,941 exited at cycle 2 + and 155 used the exact cycle-3 reference fallback: mean cycles were 2.0378 and + cycle reduction was 32.0719%. Release-runtime weighted/median-per-seed latency + reductions were 4.7567%/3.7062%; isolated reductions were + 7.2346%/5.5732%. +- The frozen 16-prompt response-fidelity gate ran source and a `python -I` + isolated packaged engine with verified module provenance. It observed zero + response, ordered-top-five, compute-contract, or packaged-behavior mismatches; + 15 prompts certified at cycle 2 and one used the exact reference fallback. + The progressive accepted-probe controller separately matched the legacy + controller exactly on 256 requests while reducing forward evaluations by + 31.25% and weighted latency by 30.095%. + +Research and statistical boundary: + +- Premature Confidence motivates distrust of early commitment, but its tested + remedy is training-time confidence shaping. PUMA studies semantic convergence + in generated reasoning, LESS studies diffusion-language-model token + commitment, MarginGate studies margin-aware numerical routing, UCCI studies + calibrated cascades, and MARS analyzes a different switching process. Their + results motivate this design; none directly proves the Supermix stopping rule. +- `0.0005` is a checkpoint/workload operating point, not a universal margin. + The zero-error observation over 4,096 requests still has a one-sided 95% + binomial upper bound of roughly 0.073%. Conformal Thinking is the rationale + for reporting finite-sample risk rather than turning a held-out zero into a + universal guarantee. +- JSD and total variation remain shadow diagnostics. In the rejected replay + they did not uniquely identify every decision-boundary failure, so they are + not substitutes for ordered decisions, protected adjacent gaps, or exact + fallback. The release evidence is scoped to this checkpoint, synthetic task, + seed matrix, CPU configuration, and frozen prompt matrix. diff --git a/source/benchmark_cognitive_leap_ultra_v51.py b/source/benchmark_cognitive_leap_ultra_v51.py index 060d39ee..16a961a6 100644 --- a/source/benchmark_cognitive_leap_ultra_v51.py +++ b/source/benchmark_cognitive_leap_ultra_v51.py @@ -33,6 +33,9 @@ N_CLASSES = 10 N_OPS = 4 IN_DIM = 128 +# Calibrated for this v51 checkpoint/workload, not a universal safety threshold. +DEFAULT_PREDICTION_STABILITY_MARGIN = 5e-4 +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = 3 def make_chained_task(n: int, seed: int) -> tuple[torch.Tensor, torch.Tensor]: @@ -134,7 +137,7 @@ def evaluate( y_test: torch.Tensor, device: torch.device, **forward_kwargs: Any, -) -> Dict[str, float]: +) -> Dict[str, Any]: model.eval() x = x_test.to(device) y = y_test.to(device) @@ -144,12 +147,34 @@ def evaluate( loss = F.cross_entropy(logits, y) acc = (logits.argmax(dim=-1) == y).float().mean() head = model.layers[10] + observed_rank_depth = int(head.last_prediction_rank_depth.item()) + prediction_verifier_active = bool(forward_kwargs.get("adaptive_compute", False)) and ( + int(forward_kwargs.get("prediction_stability_patience", 2)) > 0 + and bool(head.last_prediction_class_selection_valid.item()) + and observed_rank_depth > 0 + ) return { "loss": round(float(loss.item()), 4), "accuracy": round(float(acc.item()), 4), "latency_ms": round(latency_ms, 2), "cycles_used": round(float(head.last_cycles_used.item()), 2), "gating_entropy": round(float(head.last_gating_entropy.item()), 4), + "prediction_verifier_active": prediction_verifier_active, + "prediction_margin": ( + round(float(head.last_prediction_margin.item()), 8) + if prediction_verifier_active + else None + ), + "prediction_decision_margin": ( + round(float(head.last_prediction_decision_margin.item()), 8) + if prediction_verifier_active + else None + ), + "prediction_rank_depth": ( + observed_rank_depth + if prediction_verifier_active + else None + ), } @@ -208,6 +233,8 @@ def main() -> None: adaptive_compute=True, exit_tol=1e-4, exit_entropy_threshold=0.0, + prediction_stability_margin=DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ) out_dir = Path(args.output_dir) @@ -233,6 +260,11 @@ def main() -> None: "eval_default": eval_default, "test_time_scaling": eval_cycles, "adaptive_compute": eval_adaptive, + "adaptive_compute_configuration": { + "prediction_stability_margin": DEFAULT_PREDICTION_STABILITY_MARGIN, + "prediction_stability_rank_depth": DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + "margin_scope": "checkpoint_calibrated_not_universal", + }, } metrics_path.write_text(json.dumps(results, indent=2), encoding="utf-8") diff --git a/source/benchmark_progressive_auto_compute.py b/source/benchmark_progressive_auto_compute.py new file mode 100644 index 00000000..64207900 --- /dev/null +++ b/source/benchmark_progressive_auto_compute.py @@ -0,0 +1,1309 @@ +"""Benchmark progressive accepted-probe auto compute against legacy v1. + +The legacy controller evaluates every candidate budget, selects with the v1 +confidence/entropy policy, and reruns the selected budget. The progressive +controller applies the same policy but returns the accepted probe directly. +This benchmark requires exact selected-budget, prediction, and output-tensor +agreement while measuring the avoided forward evaluations and counterbalanced +wall-clock latency on deterministic held-out v51 requests. + +Cross-budget top-k JSD remains shadow telemetry. It is audited here but never +participates in selection or any output-agreement gate. +""" + +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timezone +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import time +from typing import Any, Callable, Dict, Iterable, List, Mapping, Sequence, Tuple + +import torch + +from benchmark_cognitive_leap_ultra_v51 import make_chained_task +from benchmark_v51_prediction_stability import ( + DEFAULT_ARTIFACT_DIR, + DEFAULT_WEIGHTS, + _load_model, +) +from chat_app import ( + DEFAULT_ADAPTIVE_EXIT_ENTROPY, + DEFAULT_ADAPTIVE_EXIT_TOL, + DEFAULT_AUTO_COMPUTE_CONFIDENCE, + DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + DEFAULT_AUTO_COMPUTE_ENTROPY, + DEFAULT_PREDICTION_STABILITY_MARGIN, + DEFAULT_PREDICTION_STABILITY_PATIENCE, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + DEFAULT_PREDICTION_STABILITY_TOL, + evaluate_runtime_compute_budgets, + forward_with_runtime_compute, + progressive_auto_compute_forward, + resolve_runtime_compute_cycles, + select_auto_runtime_compute_budget, +) +from device_utils import configure_torch_runtime, resolve_device + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_OUTPUT = DEFAULT_ARTIFACT_DIR / "progressive_auto_compute_benchmark.json" +DEFAULT_SEEDS = (719, 727, 733, 739) +DEFAULT_SAMPLES_PER_SEED = 64 +DEFAULT_CYCLES = (1, 3, 8) +TRAINING_AND_ORIGINAL_TEST_SEEDS = frozenset((51, 52)) +BENCHMARK_SCHEMA_VERSION = "progressive-auto-compute-benchmark-v1" +MAX_MISMATCH_DETAILS = 32 + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _git_text(*args: str) -> str | None: + try: + completed = subprocess.run( + ["git", *args], + cwd=PROJECT_ROOT, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + return completed.stdout.strip() + + +def collect_provenance( + *, + device: Any, + device_info: Mapping[str, Any], +) -> Dict[str, Any]: + """Capture the controller, model, data, runtime, and repository identity.""" + + source_paths = ( + Path(__file__).resolve(), + PROJECT_ROOT / "source" / "chat_app.py", + PROJECT_ROOT / "source" / "benchmark_cognitive_leap_ultra_v51.py", + PROJECT_ROOT / "source" / "model_variants.py", + ) + source_hashes = { + path.relative_to(PROJECT_ROOT).as_posix(): _sha256_file(path) + for path in source_paths + } + status = _git_text("status", "--porcelain", "--untracked-files=all") + resolved_device = str(device_info.get("resolved", device)) + device_metadata: Dict[str, Any] = { + "requested": str(device_info.get("requested", "unknown")), + "resolved": resolved_device, + "torch_device": str(device), + } + if resolved_device.startswith("cuda") and torch.cuda.is_available(): + device_metadata["name"] = torch.cuda.get_device_name(device) + device_metadata["capability"] = list(torch.cuda.get_device_capability(device)) + + return { + "git": { + "commit": _git_text("rev-parse", "HEAD"), + "branch": _git_text("rev-parse", "--abbrev-ref", "HEAD"), + "worktree_dirty": bool(status), + }, + "source_sha256": source_hashes, + "runtime": { + "python": { + "version": platform.python_version(), + "implementation": platform.python_implementation(), + "executable": sys.executable, + "platform": platform.platform(), + }, + "torch": { + "version": str(torch.__version__), + "cuda_version": str(torch.version.cuda) if torch.version.cuda else None, + "default_dtype": str(torch.get_default_dtype()), + "deterministic_algorithms": bool( + torch.are_deterministic_algorithms_enabled() + ), + }, + "device": device_metadata, + "threads": { + "torch_num_threads": int(torch.get_num_threads()), + "torch_num_interop_threads": int(torch.get_num_interop_threads()), + "logical_cpu_count": int(os.cpu_count() or 1), + }, + }, + } + + +def _finite_float(value: Any, *, label: str) -> float: + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{label} must be finite, got {value!r}") + return number + + +def _nonnegative_finite_float(value: Any, *, label: str) -> float: + number = _finite_float(value, label=label) + if number < 0.0: + raise ValueError(f"{label} must be nonnegative, got {value!r}") + return number + + +def _observed_prediction_margin_summary( + values: Sequence[float], +) -> Dict[str, Any]: + """Summarize final-call margins; the minimum is the safety-facing value.""" + + if not values: + return { + "metric": "top_1_minus_top_2_probability", + "observation_count": 0, + "minimum": None, + "mean": None, + "maximum": None, + } + tensor = torch.tensor(list(values), dtype=torch.float64) + return { + "metric": "top_1_minus_top_2_probability", + "observation_count": len(values), + "minimum": round(float(tensor.min().item()), 8), + "mean": round(float(tensor.mean().item()), 8), + "maximum": round(float(tensor.max().item()), 8), + } + + +def _observed_prediction_decision_margin_summary( + values: Sequence[float], +) -> Dict[str, Any]: + """Summarize the rank-boundary margin used by the active verifier.""" + + if not values: + return { + "metric": "minimum_adjacent_probability_gap_through_rank_depth", + "observation_count": 0, + "minimum": None, + "mean": None, + "maximum": None, + } + tensor = torch.tensor(list(values), dtype=torch.float64) + return { + "metric": "minimum_adjacent_probability_gap_through_rank_depth", + "observation_count": len(values), + "minimum": round(float(tensor.min().item()), 8), + "mean": round(float(tensor.mean().item()), 8), + "maximum": round(float(tensor.max().item()), 8), + } + + +def _prediction_margin_from_compute(compute: Mapping[str, Any]) -> float | None: + if compute.get("prediction_verifier_active") is not True: + return None + value = compute.get("prediction_margin") + if value is None: + return None + return _nonnegative_finite_float( + value, + label="observed prediction stability margin", + ) + + +def _prediction_decision_margin_from_compute( + compute: Mapping[str, Any], +) -> float | None: + if compute.get("prediction_verifier_active") is not True: + return None + value = compute.get("prediction_decision_margin") + if value is None: + return None + return _nonnegative_finite_float( + value, + label="observed prediction decision margin", + ) + + +def _synchronize_for_timing(device: torch.device) -> None: + device_type = str(device).split(":", 1)[0].lower() + if device_type == "cuda" and torch.cuda.is_available(): + torch.cuda.synchronize(device) + elif device_type == "xpu" and hasattr(torch, "xpu"): + torch.xpu.synchronize() # type: ignore[attr-defined] + elif device_type == "mps" and hasattr(torch, "mps"): + torch.mps.synchronize() + + +def _latency_summary(values: Sequence[float]) -> Dict[str, float]: + if not values: + raise ValueError("At least one latency observation is required") + tensor = torch.tensor(list(values), dtype=torch.float64) + return { + "total_ms": round(float(tensor.sum().item()), 6), + "mean_ms": round(float(tensor.mean().item()), 6), + "p50_ms": round(float(torch.quantile(tensor, 0.50).item()), 6), + "p95_ms": round(float(torch.quantile(tensor, 0.95).item()), 6), + } + + +@torch.no_grad() +def legacy_v1_auto_compute_forward( + model: Any, + x: torch.Tensor, + available_labels: List[int], + *, + cycles: Any = None, + confidence_target: Any = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: Any = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: Any = False, + exit_tol: Any = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: Any = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: Any = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + auto_reasoning_context: str = "", + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + """Execute the retired v1 probe-all/select/rerun controller literally.""" + + del distribution_top_k # The v1 selector had no distribution-stability input. + stability_margin = _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ) + rows = evaluate_runtime_compute_budgets( + model, + x, + available_labels, + cycles=cycles, + adaptive_compute=bool(adaptive_compute), + exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=stability_margin, + prediction_stability_rank_depth=prediction_stability_rank_depth, + ) + selection = select_auto_runtime_compute_budget( + rows, + confidence_target=float(confidence_target), + entropy_target=float(entropy_target), + ) + selected_cycles = int(selection["selected_reasoning_cycles"]) + output, compute = forward_with_runtime_compute( + model, + x, + reasoning_cycles=selected_cycles, + adaptive_compute=adaptive_compute, + exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=stability_margin, + prediction_stability_rank_depth=prediction_stability_rank_depth, + prediction_class_indices=available_labels, + auto_reasoning_context=auto_reasoning_context, + ) + plan = { + **selection, + "schema_version": "runtime-auto-compute-plan-v1-benchmark-reference", + "strategy": "probe_all_select_and_rerun", + "candidate_cycles": [ + int(value) for value in resolve_runtime_compute_cycles(cycles) + ], + "evaluated_cycles": [int(row["requested_cycles"]) for row in rows], + "forward_evaluations": len(rows) + 1, + "reused_probe_output": False, + "mutual_stability_role": "not_present_in_v1", + } + compute = dict(compute) + compute["auto_compute_plan"] = plan + compute["inference_reused"] = False + return output, compute, plan + + +def _prediction(output: torch.Tensor, labels: Sequence[int]) -> int: + logits = output[0, 0] + indices = torch.tensor(list(labels), dtype=torch.long, device=logits.device) + position = int(logits.index_select(0, indices).argmax(dim=0).item()) + return int(labels[position]) + + +def _shadow_selection_is_disabled(plan: Mapping[str, Any]) -> bool: + if plan.get("mutual_stability_role") != "shadow_diagnostic_only": + return False + for row in plan.get("rows", []): + if not isinstance(row, Mapping): + return False + shadow = row.get("mutual_stability_shadow") + if not isinstance(shadow, Mapping): + return False + if shadow.get("role") != "shadow_diagnostic_only": + return False + if shadow.get("selection_enabled") is not False: + return False + return True + + +def _max_abs_difference( + left: torch.Tensor, right: torch.Tensor +) -> float | None: + if tuple(left.shape) != tuple(right.shape): + return None + return float( + (left.detach().to(dtype=torch.float64, device="cpu") + - right.detach().to(dtype=torch.float64, device="cpu")) + .abs() + .max() + .item() + ) + + +@torch.no_grad() +def benchmark_requests( + model: Any, + x: torch.Tensor, + y: torch.Tensor, + *, + available_labels: Sequence[int] = tuple(range(10)), + cycles: Any = DEFAULT_CYCLES, + confidence_target: float = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: float = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: bool = False, + exit_tol: float = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: float = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: int = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: float = DEFAULT_PREDICTION_STABILITY_TOL, + prediction_stability_margin: float = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + order_offset: int = 0, + legacy_fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]] + | None = None, + progressive_fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]] + | None = None, +) -> Dict[str, Any]: + """Run a paired, request-level, counterbalanced controller comparison.""" + + if bool(getattr(model, "training", False)): + raise RuntimeError("controller benchmark requires model.eval()") + request_count = int(len(y)) + if request_count <= 0 or int(len(x)) != request_count: + raise ValueError("x and y must contain the same positive request count") + labels = [int(label) for label in available_labels] + if not labels: + raise ValueError("available_labels must not be empty") + resolved_cycles = resolve_runtime_compute_cycles(cycles) + settings = { + "cycles": resolved_cycles, + "confidence_target": _finite_float( + confidence_target, label="confidence target" + ), + "entropy_target": _finite_float(entropy_target, label="entropy target"), + "adaptive_compute": bool(adaptive_compute), + "exit_tol": _finite_float(exit_tol, label="exit tolerance"), + "exit_entropy_threshold": _finite_float( + exit_entropy_threshold, label="exit entropy threshold" + ), + "prediction_stability_patience": int(prediction_stability_patience), + "prediction_stability_tol": _finite_float( + prediction_stability_tol, label="prediction stability tolerance" + ), + "prediction_stability_margin": _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ), + "prediction_stability_rank_depth": int(prediction_stability_rank_depth), + "distribution_top_k": int(distribution_top_k), + } + if settings["prediction_stability_patience"] < 0: + raise ValueError("prediction_stability_patience must be nonnegative") + if settings["prediction_stability_rank_depth"] < 0: + raise ValueError("prediction_stability_rank_depth must be nonnegative") + if settings["distribution_top_k"] <= 0: + raise ValueError("distribution_top_k must be positive") + + legacy = legacy_fn or legacy_v1_auto_compute_forward + progressive = progressive_fn or progressive_auto_compute_forward + first_sample = x[:1] + + # Match the transition immediately before each seed's first measured mode. + if int(order_offset) % 2 == 0: + legacy(model, first_sample, labels, **settings) + progressive(model, first_sample, labels, **settings) + else: + progressive(model, first_sample, labels, **settings) + legacy(model, first_sample, labels, **settings) + + legacy_latencies: List[float] = [] + progressive_latencies: List[float] = [] + legacy_prediction_margins: List[float] = [] + progressive_prediction_margins: List[float] = [] + legacy_prediction_decision_margins: List[float] = [] + progressive_prediction_decision_margins: List[float] = [] + measurement_orders: Counter[str] = Counter() + selected_cycle_pairs: Counter[str] = Counter() + selected_cycle_disagreements = 0 + prediction_disagreements = 0 + output_disagreements = 0 + legacy_correct = 0 + progressive_correct = 0 + legacy_forward_evaluations = 0 + progressive_forward_evaluations = 0 + progressive_not_better_count = 0 + shadow_selection_violations = 0 + maximum_abs_logit_difference = 0.0 + mismatch_details: List[Dict[str, Any]] = [] + + def timed_call( + fn: Callable[..., Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]], + sample: torch.Tensor, + values: List[float], + observed_margins: List[float], + observed_decision_margins: List[float], + ) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + _synchronize_for_timing(sample.device) + started = time.perf_counter() + result = fn(model, sample, labels, **settings) + _synchronize_for_timing(sample.device) + values.append((time.perf_counter() - started) * 1000.0) + observed_margin = _prediction_margin_from_compute(result[1]) + if observed_margin is not None: + observed_margins.append(observed_margin) + observed_decision_margin = _prediction_decision_margin_from_compute( + result[1] + ) + if observed_decision_margin is not None: + observed_decision_margins.append(observed_decision_margin) + return result + + for index in range(request_count): + sample = x[index : index + 1] + legacy_first = (index + int(order_offset)) % 2 == 0 + if legacy_first: + measurement_orders["legacy_then_progressive"] += 1 + legacy_result = timed_call( + legacy, + sample, + legacy_latencies, + legacy_prediction_margins, + legacy_prediction_decision_margins, + ) + progressive_result = timed_call( + progressive, + sample, + progressive_latencies, + progressive_prediction_margins, + progressive_prediction_decision_margins, + ) + else: + measurement_orders["progressive_then_legacy"] += 1 + progressive_result = timed_call( + progressive, + sample, + progressive_latencies, + progressive_prediction_margins, + progressive_prediction_decision_margins, + ) + legacy_result = timed_call( + legacy, + sample, + legacy_latencies, + legacy_prediction_margins, + legacy_prediction_decision_margins, + ) + + legacy_output, _legacy_compute, legacy_plan = legacy_result + progressive_output, _progressive_compute, progressive_plan = ( + progressive_result + ) + legacy_cycles = int(legacy_plan["selected_reasoning_cycles"]) + progressive_cycles = int(progressive_plan["selected_reasoning_cycles"]) + legacy_prediction = _prediction(legacy_output, labels) + progressive_prediction = _prediction(progressive_output, labels) + truth = int(y[index].item()) + shapes_match = tuple(legacy_output.shape) == tuple(progressive_output.shape) + outputs_equal = bool( + shapes_match and torch.equal(legacy_output, progressive_output) + ) + max_abs_difference = _max_abs_difference( + legacy_output, progressive_output + ) + if max_abs_difference is not None: + maximum_abs_logit_difference = max( + maximum_abs_logit_difference, max_abs_difference + ) + + legacy_evaluations = int(legacy_plan["forward_evaluations"]) + progressive_evaluations = int(progressive_plan["forward_evaluations"]) + legacy_forward_evaluations += legacy_evaluations + progressive_forward_evaluations += progressive_evaluations + progressive_not_better_count += int( + progressive_evaluations >= legacy_evaluations + ) + shadow_ok = _shadow_selection_is_disabled(progressive_plan) + shadow_selection_violations += int(not shadow_ok) + selected_cycle_pairs[f"{legacy_cycles}->{progressive_cycles}"] += 1 + selected_cycle_disagreements += int(legacy_cycles != progressive_cycles) + prediction_disagreements += int( + legacy_prediction != progressive_prediction + ) + output_disagreements += int(not outputs_equal) + legacy_correct += int(legacy_prediction == truth) + progressive_correct += int(progressive_prediction == truth) + + if ( + legacy_cycles != progressive_cycles + or legacy_prediction != progressive_prediction + or not outputs_equal + or not shadow_ok + ) and len(mismatch_details) < MAX_MISMATCH_DETAILS: + mismatch_details.append( + { + "request_index": index, + "truth": truth, + "legacy_selected_cycles": legacy_cycles, + "progressive_selected_cycles": progressive_cycles, + "legacy_prediction": legacy_prediction, + "progressive_prediction": progressive_prediction, + "output_shapes_match": shapes_match, + "outputs_exactly_equal": outputs_equal, + "max_abs_logit_difference": max_abs_difference, + "shadow_selection_disabled": shadow_ok, + } + ) + + legacy_latency = _latency_summary(legacy_latencies) + progressive_latency = _latency_summary(progressive_latencies) + forward_reduction = ( + 100.0 + * (legacy_forward_evaluations - progressive_forward_evaluations) + / max(1, legacy_forward_evaluations) + ) + latency_reduction = ( + 100.0 + * (legacy_latency["total_ms"] - progressive_latency["total_ms"]) + / max(1e-12, legacy_latency["total_ms"]) + ) + checks = { + "exact_selected_cycle_agreement": { + "passed": selected_cycle_disagreements == 0, + "observed_disagreements": selected_cycle_disagreements, + "required": 0, + }, + "exact_prediction_agreement": { + "passed": prediction_disagreements == 0, + "observed_disagreements": prediction_disagreements, + "required": 0, + }, + "exact_output_tensor_agreement": { + "passed": output_disagreements == 0, + "observed_disagreements": output_disagreements, + "required": 0, + }, + "positive_forward_evaluation_reduction": { + "passed": progressive_forward_evaluations < legacy_forward_evaluations, + "legacy_forward_evaluations": legacy_forward_evaluations, + "progressive_forward_evaluations": progressive_forward_evaluations, + }, + "per_request_forward_evaluation_reduction": { + "passed": progressive_not_better_count == 0, + "non_reducing_request_count": progressive_not_better_count, + "required": 0, + }, + "mutual_stability_is_shadow_only": { + "passed": shadow_selection_violations == 0, + "selection_policy_violation_count": shadow_selection_violations, + "required": 0, + }, + } + return { + "request_count": request_count, + "measurement_order": { + "offset": int(order_offset), + "legacy_then_progressive": measurement_orders[ + "legacy_then_progressive" + ], + "progressive_then_legacy": measurement_orders[ + "progressive_then_legacy" + ], + }, + "legacy_v1": { + "strategy": "probe_all_select_and_rerun", + "correct_predictions": legacy_correct, + "forward_evaluations": legacy_forward_evaluations, + "observed_prediction_margin": _observed_prediction_margin_summary( + legacy_prediction_margins + ), + "observed_prediction_decision_margin": ( + _observed_prediction_decision_margin_summary( + legacy_prediction_decision_margins + ) + ), + "latency": legacy_latency, + }, + "progressive": { + "strategy": "progressive_accepted_probe", + "correct_predictions": progressive_correct, + "forward_evaluations": progressive_forward_evaluations, + "observed_prediction_margin": _observed_prediction_margin_summary( + progressive_prediction_margins + ), + "observed_prediction_decision_margin": ( + _observed_prediction_decision_margin_summary( + progressive_prediction_decision_margins + ) + ), + "latency": progressive_latency, + }, + "comparison": { + "selected_cycle_pairs": dict(sorted(selected_cycle_pairs.items())), + "exact_selected_cycle_disagreement_count": ( + selected_cycle_disagreements + ), + "exact_prediction_disagreement_count": prediction_disagreements, + "exact_output_tensor_disagreement_count": output_disagreements, + "maximum_absolute_logit_difference": round( + maximum_abs_logit_difference, 12 + ), + "forward_evaluation_reduction_percent": round( + forward_reduction, 6 + ), + "mean_latency_reduction_percent": round(latency_reduction, 6), + "mismatch_details": mismatch_details, + }, + "gates": { + "passed": all(bool(check["passed"]) for check in checks.values()), + "checks": checks, + }, + } + + +def aggregate_results(seed_results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: + if not seed_results: + raise ValueError("At least one seed result is required") + total_requests = 0 + total_selected_cycle_disagreements = 0 + total_prediction_disagreements = 0 + total_output_disagreements = 0 + total_legacy_correct = 0 + total_progressive_correct = 0 + total_legacy_forward_evaluations = 0 + total_progressive_forward_evaluations = 0 + total_legacy_latency_ms = 0.0 + total_progressive_latency_ms = 0.0 + maximum_abs_logit_difference = 0.0 + shadow_selection_violations = 0 + non_reducing_requests = 0 + selected_cycle_pairs: Counter[str] = Counter() + per_seed_latency_reductions: List[float] = [] + per_seed: List[Dict[str, Any]] = [] + seen_seeds: set[int] = set() + legacy_margin_observations = 0 + progressive_margin_observations = 0 + legacy_decision_margin_observations = 0 + progressive_decision_margin_observations = 0 + minimum_legacy_prediction_margin = math.inf + minimum_progressive_prediction_margin = math.inf + minimum_legacy_prediction_decision_margin = math.inf + minimum_progressive_prediction_decision_margin = math.inf + + for seed_result in seed_results: + seed = int(seed_result["seed"]) + if seed in seen_seeds: + raise ValueError(f"Duplicate seed result: {seed}") + seen_seeds.add(seed) + metrics = seed_result["metrics"] + request_count = int(metrics["request_count"]) + if request_count <= 0: + raise ValueError(f"Seed {seed} has no requests") + order = metrics["measurement_order"] + legacy_first = int(order["legacy_then_progressive"]) + progressive_first = int(order["progressive_then_legacy"]) + offset = int(order["offset"]) + expected_legacy_first = ( + (request_count + 1) // 2 if offset % 2 == 0 else request_count // 2 + ) + if ( + legacy_first != expected_legacy_first + or progressive_first != request_count - expected_legacy_first + ): + raise ValueError(f"Seed {seed} measurement order is not counterbalanced") + + legacy = metrics["legacy_v1"] + progressive = metrics["progressive"] + comparison = metrics["comparison"] + legacy_margin = legacy.get("observed_prediction_margin", {}) + progressive_margin = progressive.get("observed_prediction_margin", {}) + legacy_decision_margin = legacy.get( + "observed_prediction_decision_margin", {} + ) + progressive_decision_margin = progressive.get( + "observed_prediction_decision_margin", {} + ) + legacy_margin_count = int(legacy_margin.get("observation_count", 0)) + progressive_margin_count = int( + progressive_margin.get("observation_count", 0) + ) + legacy_margin_minimum = legacy_margin.get("minimum") + progressive_margin_minimum = progressive_margin.get("minimum") + legacy_decision_margin_count = int( + legacy_decision_margin.get("observation_count", 0) + ) + progressive_decision_margin_count = int( + progressive_decision_margin.get("observation_count", 0) + ) + legacy_decision_margin_minimum = legacy_decision_margin.get("minimum") + progressive_decision_margin_minimum = progressive_decision_margin.get( + "minimum" + ) + if legacy_margin_count < 0 or progressive_margin_count < 0: + raise ValueError(f"Seed {seed} margin observation count is invalid") + if legacy_margin_count: + legacy_margin_minimum = _nonnegative_finite_float( + legacy_margin_minimum, + label="minimum legacy prediction stability margin", + ) + minimum_legacy_prediction_margin = min( + minimum_legacy_prediction_margin, + legacy_margin_minimum, + ) + legacy_margin_observations += legacy_margin_count + if progressive_margin_count: + progressive_margin_minimum = _nonnegative_finite_float( + progressive_margin_minimum, + label="minimum progressive prediction stability margin", + ) + minimum_progressive_prediction_margin = min( + minimum_progressive_prediction_margin, + progressive_margin_minimum, + ) + progressive_margin_observations += progressive_margin_count + if ( + legacy_decision_margin_count < 0 + or progressive_decision_margin_count < 0 + ): + raise ValueError( + f"Seed {seed} decision-margin observation count is invalid" + ) + if legacy_decision_margin_count: + legacy_decision_margin_minimum = _nonnegative_finite_float( + legacy_decision_margin_minimum, + label="minimum legacy prediction decision margin", + ) + minimum_legacy_prediction_decision_margin = min( + minimum_legacy_prediction_decision_margin, + legacy_decision_margin_minimum, + ) + legacy_decision_margin_observations += legacy_decision_margin_count + if progressive_decision_margin_count: + progressive_decision_margin_minimum = _nonnegative_finite_float( + progressive_decision_margin_minimum, + label="minimum progressive prediction decision margin", + ) + minimum_progressive_prediction_decision_margin = min( + minimum_progressive_prediction_decision_margin, + progressive_decision_margin_minimum, + ) + progressive_decision_margin_observations += ( + progressive_decision_margin_count + ) + legacy_latency = _finite_float( + legacy["latency"]["total_ms"], label="legacy latency" + ) + progressive_latency = _finite_float( + progressive["latency"]["total_ms"], label="progressive latency" + ) + if legacy_latency <= 0.0 or progressive_latency < 0.0: + raise ValueError(f"Seed {seed} latency totals are invalid") + seed_latency_reduction = ( + 100.0 * (legacy_latency - progressive_latency) / legacy_latency + ) + per_seed_latency_reductions.append(seed_latency_reduction) + total_requests += request_count + total_selected_cycle_disagreements += int( + comparison["exact_selected_cycle_disagreement_count"] + ) + total_prediction_disagreements += int( + comparison["exact_prediction_disagreement_count"] + ) + total_output_disagreements += int( + comparison["exact_output_tensor_disagreement_count"] + ) + total_legacy_correct += int(legacy["correct_predictions"]) + total_progressive_correct += int(progressive["correct_predictions"]) + total_legacy_forward_evaluations += int(legacy["forward_evaluations"]) + total_progressive_forward_evaluations += int( + progressive["forward_evaluations"] + ) + total_legacy_latency_ms += legacy_latency + total_progressive_latency_ms += progressive_latency + maximum_abs_logit_difference = max( + maximum_abs_logit_difference, + _finite_float( + comparison["maximum_absolute_logit_difference"], + label="maximum absolute logit difference", + ), + ) + for pair, count in comparison["selected_cycle_pairs"].items(): + selected_cycle_pairs[str(pair)] += int(count) + checks = metrics["gates"]["checks"] + shadow_selection_violations += int( + checks["mutual_stability_is_shadow_only"][ + "selection_policy_violation_count" + ] + ) + non_reducing_requests += int( + checks["per_request_forward_evaluation_reduction"][ + "non_reducing_request_count" + ] + ) + per_seed.append( + { + "seed": seed, + "samples": request_count, + "selected_cycle_disagreements": int( + comparison["exact_selected_cycle_disagreement_count"] + ), + "prediction_disagreements": int( + comparison["exact_prediction_disagreement_count"] + ), + "output_tensor_disagreements": int( + comparison["exact_output_tensor_disagreement_count"] + ), + "forward_evaluation_reduction_percent": float( + comparison["forward_evaluation_reduction_percent"] + ), + "mean_latency_reduction_percent": round( + seed_latency_reduction, 6 + ), + "measurement_order": dict(order), + "minimum_observed_prediction_margin": { + "legacy_v1": legacy_margin_minimum, + "progressive": progressive_margin_minimum, + }, + "minimum_observed_prediction_decision_margin": { + "legacy_v1": legacy_decision_margin_minimum, + "progressive": progressive_decision_margin_minimum, + }, + } + ) + + forward_reduction = ( + 100.0 + * (total_legacy_forward_evaluations - total_progressive_forward_evaluations) + / max(1, total_legacy_forward_evaluations) + ) + weighted_latency_reduction = ( + 100.0 + * (total_legacy_latency_ms - total_progressive_latency_ms) + / max(1e-12, total_legacy_latency_ms) + ) + checks = { + "exact_selected_cycle_agreement": { + "passed": total_selected_cycle_disagreements == 0, + "observed_disagreements": total_selected_cycle_disagreements, + "required": 0, + }, + "exact_prediction_agreement": { + "passed": total_prediction_disagreements == 0, + "observed_disagreements": total_prediction_disagreements, + "required": 0, + }, + "exact_output_tensor_agreement": { + "passed": total_output_disagreements == 0, + "observed_disagreements": total_output_disagreements, + "required": 0, + }, + "positive_forward_evaluation_reduction": { + "passed": ( + total_progressive_forward_evaluations + < total_legacy_forward_evaluations + ), + "legacy_forward_evaluations": total_legacy_forward_evaluations, + "progressive_forward_evaluations": total_progressive_forward_evaluations, + }, + "per_request_forward_evaluation_reduction": { + "passed": non_reducing_requests == 0, + "non_reducing_request_count": non_reducing_requests, + "required": 0, + }, + "mutual_stability_is_shadow_only": { + "passed": shadow_selection_violations == 0, + "selection_policy_violation_count": shadow_selection_violations, + "required": 0, + }, + } + return { + "summary": { + "seed_count": len(seed_results), + "total_samples": total_requests, + "exact_selected_cycle_disagreement_count": ( + total_selected_cycle_disagreements + ), + "exact_prediction_disagreement_count": total_prediction_disagreements, + "exact_output_tensor_disagreement_count": total_output_disagreements, + "maximum_absolute_logit_difference": round( + maximum_abs_logit_difference, 12 + ), + "legacy_correct_predictions": total_legacy_correct, + "progressive_correct_predictions": total_progressive_correct, + "legacy_forward_evaluations": total_legacy_forward_evaluations, + "progressive_forward_evaluations": total_progressive_forward_evaluations, + "forward_evaluation_reduction_percent": round(forward_reduction, 6), + "weighted_mean_latency_reduction_percent": round( + weighted_latency_reduction, 6 + ), + "median_per_seed_latency_reduction_percent": round( + float(statistics.median(per_seed_latency_reductions)), 6 + ), + "weighted_legacy_mean_latency_ms": round( + total_legacy_latency_ms / total_requests, 6 + ), + "weighted_progressive_mean_latency_ms": round( + total_progressive_latency_ms / total_requests, 6 + ), + "selected_cycle_pairs": dict(sorted(selected_cycle_pairs.items())), + "observed_prediction_margin": { + "metric": "top_1_minus_top_2_probability", + "legacy_v1": { + "observation_count": legacy_margin_observations, + "minimum": ( + round(minimum_legacy_prediction_margin, 8) + if legacy_margin_observations + else None + ), + }, + "progressive": { + "observation_count": progressive_margin_observations, + "minimum": ( + round(minimum_progressive_prediction_margin, 8) + if progressive_margin_observations + else None + ), + }, + }, + "observed_prediction_decision_margin": { + "metric": ( + "minimum_adjacent_probability_gap_through_rank_depth" + ), + "legacy_v1": { + "observation_count": legacy_decision_margin_observations, + "minimum": ( + round(minimum_legacy_prediction_decision_margin, 8) + if legacy_decision_margin_observations + else None + ), + }, + "progressive": { + "observation_count": ( + progressive_decision_margin_observations + ), + "minimum": ( + round(minimum_progressive_prediction_decision_margin, 8) + if progressive_decision_margin_observations + else None + ), + }, + }, + }, + "per_seed_summary": per_seed, + "gates": { + "passed": all(bool(check["passed"]) for check in checks.values()), + "checks": checks, + }, + } + + +def run_benchmark( + *, + weights: Path, + seeds: Sequence[int], + samples_per_seed: int, + device: Any, + device_info: Mapping[str, Any], + cycles: Sequence[int] = DEFAULT_CYCLES, + confidence_target: float = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: float = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: bool = False, + exit_tol: float = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: float = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: int = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: float = DEFAULT_PREDICTION_STABILITY_TOL, + prediction_stability_margin: float = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + model_loader: Callable[[Path, Any], Any] | None = None, + task_factory: Callable[..., Tuple[torch.Tensor, torch.Tensor]] | None = None, + benchmark_fn: Callable[..., Dict[str, Any]] | None = None, + provenance: Mapping[str, Any] | None = None, +) -> Dict[str, Any]: + weights = Path(weights).resolve() + if not weights.is_file(): + raise FileNotFoundError(f"Missing v51 checkpoint: {weights}") + normalized_seeds = [int(seed) for seed in seeds] + if not normalized_seeds: + raise ValueError("At least one held-out seed is required") + if len(set(normalized_seeds)) != len(normalized_seeds): + raise ValueError("Seeds must be unique") + overlapping_seeds = sorted(TRAINING_AND_ORIGINAL_TEST_SEEDS.intersection(normalized_seeds)) + if overlapping_seeds: + raise ValueError( + "Held-out seeds overlap v51 training/original-test seeds: " + + ", ".join(str(seed) for seed in overlapping_seeds) + ) + if int(samples_per_seed) <= 0: + raise ValueError("samples_per_seed must be positive") + resolved_cycles = resolve_runtime_compute_cycles(cycles) + if len(resolved_cycles) < 2: + raise ValueError("At least two distinct compute-cycle candidates are required") + normalized_confidence_target = _finite_float( + confidence_target, label="confidence target" + ) + normalized_entropy_target = _finite_float( + entropy_target, label="entropy target" + ) + if not 0.0 <= normalized_confidence_target <= 1.0: + raise ValueError("confidence_target must be between 0 and 1") + if normalized_entropy_target < 0.0: + raise ValueError("entropy_target must be nonnegative") + normalized_prediction_stability_margin = _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ) + normalized_prediction_stability_rank_depth = int( + prediction_stability_rank_depth + ) + if normalized_prediction_stability_rank_depth < 0: + raise ValueError("prediction_stability_rank_depth must be nonnegative") + + loader = model_loader or _load_model + task = task_factory or make_chained_task + benchmark = benchmark_fn or benchmark_requests + checkpoint_sha256 = _sha256_file(weights) + model = loader(weights, device) + if hasattr(model, "eval"): + model.eval() + seed_results: List[Dict[str, Any]] = [] + for seed_index, seed in enumerate(normalized_seeds): + x, y = task(int(samples_per_seed), seed=seed) + metrics = benchmark( + model, + x.to(device), + y.to(device), + cycles=resolved_cycles, + confidence_target=normalized_confidence_target, + entropy_target=normalized_entropy_target, + adaptive_compute=bool(adaptive_compute), + exit_tol=float(exit_tol), + exit_entropy_threshold=float(exit_entropy_threshold), + prediction_stability_patience=int(prediction_stability_patience), + prediction_stability_tol=float(prediction_stability_tol), + prediction_stability_margin=( + normalized_prediction_stability_margin + ), + prediction_stability_rank_depth=( + normalized_prediction_stability_rank_depth + ), + distribution_top_k=int(distribution_top_k), + order_offset=seed_index % 2, + ) + seed_results.append({"seed": seed, "metrics": metrics}) + + aggregate = aggregate_results(seed_results) + return { + "schema_version": BENCHMARK_SCHEMA_VERSION, + "created_at": datetime.now(timezone.utc).isoformat(), + "checkpoint": { + "path": str(weights), + "sha256": checkpoint_sha256, + }, + "configuration": { + "seeds": normalized_seeds, + "samples_per_seed": int(samples_per_seed), + "total_samples": int(samples_per_seed) * len(normalized_seeds), + "data": { + "generator": "make_chained_task", + "kind": "deterministic_synthetic_chained_arithmetic", + "split": "held_out_rng_seeds", + "excluded_training_and_original_test_seeds": sorted( + TRAINING_AND_ORIGINAL_TEST_SEEDS + ), + }, + "cycles": resolved_cycles, + "confidence_target": normalized_confidence_target, + "entropy_target": normalized_entropy_target, + "adaptive_compute": bool(adaptive_compute), + "exit_tol": float(exit_tol), + "exit_entropy_threshold": float(exit_entropy_threshold), + "prediction_stability_patience": int( + prediction_stability_patience + ), + "prediction_stability_tol": float(prediction_stability_tol), + "prediction_stability_margin": ( + normalized_prediction_stability_margin + ), + "prediction_stability_rank_depth": ( + normalized_prediction_stability_rank_depth + ), + "distribution_top_k": int(distribution_top_k), + "counterbalance": ( + "alternate_per_request_with_alternating_seed_offset" + ), + "latency_is_release_gate": False, + "mutual_stability_role": "shadow_diagnostic_only", + }, + "provenance": dict(provenance) + if provenance is not None + else collect_provenance(device=device, device_info=device_info), + **aggregate, + "seed_results": seed_results, + } + + +def _strict_json(payload: Mapping[str, Any]) -> str: + return json.dumps( + payload, + indent=2, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Compare progressive accepted-probe auto compute with legacy v1 " + "on deterministic held-out v51 requests." + ) + ) + parser.add_argument("--weights", default=str(DEFAULT_WEIGHTS)) + parser.add_argument("--output", default=str(DEFAULT_OUTPUT)) + parser.add_argument("--seeds", nargs="+", type=int, default=list(DEFAULT_SEEDS)) + parser.add_argument( + "--samples-per-seed", + "--samples_per_seed", + dest="samples_per_seed", + type=int, + default=DEFAULT_SAMPLES_PER_SEED, + ) + parser.add_argument("--cycles", nargs="+", type=int, default=list(DEFAULT_CYCLES)) + parser.add_argument( + "--confidence-target", + "--confidence_target", + type=float, + default=DEFAULT_AUTO_COMPUTE_CONFIDENCE, + ) + parser.add_argument( + "--entropy-target", + "--entropy_target", + type=float, + default=DEFAULT_AUTO_COMPUTE_ENTROPY, + ) + parser.add_argument("--adaptive-compute", action="store_true") + parser.add_argument( + "--exit-tol", type=float, default=DEFAULT_ADAPTIVE_EXIT_TOL + ) + parser.add_argument( + "--exit-entropy-threshold", + type=float, + default=DEFAULT_ADAPTIVE_EXIT_ENTROPY, + ) + parser.add_argument( + "--prediction-stability-patience", + type=int, + default=DEFAULT_PREDICTION_STABILITY_PATIENCE, + ) + parser.add_argument( + "--prediction-stability-tol", + type=float, + default=DEFAULT_PREDICTION_STABILITY_TOL, + ) + parser.add_argument( + "--prediction-stability-margin", + "--prediction_stability_margin", + dest="prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + parser.add_argument( + "--prediction-stability-rank-depth", + "--prediction_stability_rank_depth", + dest="prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) + parser.add_argument( + "--distribution-top-k", + type=int, + default=DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + ) + parser.add_argument("--device", default="auto") + parser.add_argument( + "--device-preference", default="cuda,npu,xpu,dml,mps,cpu" + ) + parser.add_argument("--torch-num-threads", type=int, default=0) + parser.add_argument("--torch-interop-threads", type=int, default=0) + parser.add_argument("--strict-determinism", action="store_true") + parser.add_argument( + "--enforce-gates", + action="store_true", + help="Exit with status 2 unless every exact-agreement gate passes.", + ) + return parser + + +def main(argv: Iterable[str] | None = None) -> int: + args = build_parser().parse_args(argv) + configure_torch_runtime( + torch_num_threads=int(args.torch_num_threads), + torch_interop_threads=int(args.torch_interop_threads), + strict_determinism=bool(args.strict_determinism), + ) + device, device_info = resolve_device( + args.device, preference=args.device_preference + ) + payload = run_benchmark( + weights=Path(args.weights), + seeds=args.seeds, + samples_per_seed=int(args.samples_per_seed), + device=device, + device_info=device_info, + cycles=args.cycles, + confidence_target=float(args.confidence_target), + entropy_target=float(args.entropy_target), + adaptive_compute=bool(args.adaptive_compute), + exit_tol=float(args.exit_tol), + exit_entropy_threshold=float(args.exit_entropy_threshold), + prediction_stability_patience=int(args.prediction_stability_patience), + prediction_stability_tol=float(args.prediction_stability_tol), + prediction_stability_margin=float(args.prediction_stability_margin), + prediction_stability_rank_depth=int( + args.prediction_stability_rank_depth + ), + distribution_top_k=int(args.distribution_top_k), + ) + encoded = _strict_json(payload) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(encoded + "\n", encoding="utf-8") + sys.stdout.write(encoded + "\n") + if bool(args.enforce_gates) and not bool(payload["gates"]["passed"]): + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/source/benchmark_v51_prediction_stability.py b/source/benchmark_v51_prediction_stability.py index d3887fcd..1b21f88c 100644 --- a/source/benchmark_v51_prediction_stability.py +++ b/source/benchmark_v51_prediction_stability.py @@ -1,7 +1,8 @@ """Benchmark v51 prediction-stability exits under request-sized inference. -This intentionally disables the older latent/entropy exits so the comparison -isolates the new full-output stability verifier against a fixed-cycle baseline. +The standalone defaults disable the older latent/entropy exits to isolate the +full-output verifier. The multi-seed release gate also calls this benchmark +with the authoritative deployed runtime controls and metadata class scope. """ from __future__ import annotations @@ -10,13 +11,16 @@ from collections import Counter from datetime import datetime, timezone import json +import math +import numbers from pathlib import Path import time -from typing import Any, Dict, List +from typing import Any, Dict, List, Sequence import torch from benchmark_cognitive_leap_ultra_v51 import make_chained_task +import chat_app from device_utils import configure_torch_runtime, resolve_device from model_variants import ChampionNetCognitiveLeapUltraExpert @@ -25,6 +29,28 @@ DEFAULT_ARTIFACT_DIR = PROJECT_ROOT / "output" / "benchmark_v51_cognitive_leap_ultra_latest" DEFAULT_WEIGHTS = DEFAULT_ARTIFACT_DIR / "cognitive_leap_ultra_v51_trained.pth" DEFAULT_OUTPUT = DEFAULT_ARTIFACT_DIR / "prediction_stability_results.json" +DEFAULT_ADAPTIVE_EXIT_TOL = chat_app.DEFAULT_ADAPTIVE_EXIT_TOL +DEFAULT_ADAPTIVE_EXIT_ENTROPY = chat_app.DEFAULT_ADAPTIVE_EXIT_ENTROPY +DEFAULT_PREDICTION_STABILITY_PATIENCE = ( + chat_app.DEFAULT_PREDICTION_STABILITY_PATIENCE +) +DEFAULT_PREDICTION_STABILITY_TOL = chat_app.DEFAULT_PREDICTION_STABILITY_TOL +DEFAULT_PREDICTION_STABILITY_MARGIN = chat_app.DEFAULT_PREDICTION_STABILITY_MARGIN +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = ( + chat_app.DEFAULT_PREDICTION_STABILITY_RANK_DEPTH +) +DEFAULT_DECISION_TOP_K = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH +MAX_TOP_K_DISAGREEMENT_EXAMPLES = 32 +KNOWN_ADAPTIVE_EXIT_REASONS = frozenset( + { + "prediction_stable", + "decision_reference_budget", + "latent_converged", + "low_entropy", + "halt_mass", + "max_cycles", + } +) def _load_model(weights: Path, device: torch.device) -> ChampionNetCognitiveLeapUltraExpert: @@ -36,25 +62,206 @@ def _load_model(weights: Path, device: torch.device) -> ChampionNetCognitiveLeap return model -def _latency_summary(values: List[float]) -> Dict[str, float]: +def _finite_values(values: Sequence[float], *, label: str) -> torch.Tensor: + if not values: + raise ValueError(f"{label} requires at least one value") tensor = torch.tensor(values, dtype=torch.float64) + if not bool(torch.isfinite(tensor).all().item()): + raise ValueError(f"{label} must contain only finite values") + return tensor + + +def _latency_summary(values: List[float]) -> Dict[str, float]: + tensor = _finite_values(values, label="latency summary") + if bool((tensor <= 0.0).any().item()): + raise ValueError("latency values must be positive") return { + "total_ms": round(float(tensor.sum().item()), 6), "mean_ms": round(float(tensor.mean().item()), 3), "p50_ms": round(float(torch.quantile(tensor, 0.50).item()), 3), "p95_ms": round(float(torch.quantile(tensor, 0.95).item()), 3), } -def _scalar_summary(values: List[float]) -> Dict[str, float]: - tensor = torch.tensor(values, dtype=torch.float64) +def _scalar_summary(values: List[float], *, digits: int = 8) -> Dict[str, float]: + tensor = _finite_values(values, label="scalar summary") return { - "mean": round(float(tensor.mean().item()), 8), - "p50": round(float(torch.quantile(tensor, 0.50).item()), 8), - "p95": round(float(torch.quantile(tensor, 0.95).item()), 8), - "max": round(float(tensor.max().item()), 8), + "min": round(float(tensor.min().item()), digits), + "mean": round(float(tensor.mean().item()), digits), + "p50": round(float(torch.quantile(tensor, 0.50).item()), digits), + "p95": round(float(torch.quantile(tensor, 0.95).item()), digits), + "max": round(float(tensor.max().item()), digits), } +def _nonnegative_finite_float(value: Any, *, label: str) -> float: + try: + number = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError( + f"{label} must be finite and nonnegative, got {value!r}" + ) from exc + if not math.isfinite(number) or number < 0.0: + raise ValueError(f"{label} must be finite and nonnegative, got {value!r}") + return number + + +def _strict_bool_scalar(value: Any, *, label: str) -> bool: + if torch.is_tensor(value): + if value.numel() != 1: + raise ValueError(f"{label} must be a scalar boolean") + value = value.item() + if not isinstance(value, bool): + raise ValueError(f"{label} must be a boolean, got {value!r}") + return value + + +def _positive_integral_scalar(value: Any, *, label: str) -> int: + if torch.is_tensor(value): + if value.numel() != 1: + raise ValueError(f"{label} must be a scalar integer") + value = value.item() + if isinstance(value, bool): + raise ValueError(f"{label} must be a positive integer, got {value!r}") + number = _nonnegative_finite_float(value, label=label) + integer = int(number) + if integer <= 0 or number != float(integer): + raise ValueError(f"{label} must be a positive integer, got {value!r}") + return integer + + +def _normalize_prediction_class_indices_input( + prediction_class_indices: Any, +) -> List[int] | List[bool] | None: + """Normalize one optional verifier scope without guessing output width.""" + + if prediction_class_indices is None: + return None + if torch.is_tensor(prediction_class_indices): + raw = prediction_class_indices.detach().cpu() + if raw.ndim != 1: + raise ValueError("prediction_class_indices must be one-dimensional") + if raw.dtype == torch.bool: + values: List[int] | List[bool] = [bool(value) for value in raw.tolist()] + elif raw.dtype.is_floating_point or raw.dtype.is_complex: + raise ValueError("prediction_class_indices must contain integers or be a bool mask") + else: + values = [int(value) for value in raw.tolist()] + else: + if isinstance(prediction_class_indices, (str, bytes)): + raise ValueError("prediction_class_indices must be an integer sequence") + try: + raw_values = list(prediction_class_indices) + except TypeError as exc: + raise ValueError( + "prediction_class_indices must be an integer sequence" + ) from exc + if raw_values and all(isinstance(value, bool) for value in raw_values): + values = [bool(value) for value in raw_values] + else: + if any(isinstance(value, bool) for value in raw_values) or any( + not isinstance(value, numbers.Integral) for value in raw_values + ): + raise ValueError( + "prediction_class_indices must contain only integers or be a bool mask" + ) + values = [int(value) for value in raw_values] + + if not values: + raise ValueError("prediction_class_indices cannot be empty") + if all(isinstance(value, bool) for value in values): + if not any(values): + raise ValueError("prediction_class_indices bool mask must select a class") + return values + integer_values = [int(value) for value in values] + if any(value < 0 for value in integer_values): + raise ValueError("prediction_class_indices cannot contain negative indices") + if len(set(integer_values)) != len(integer_values): + raise ValueError("prediction_class_indices cannot contain duplicates") + return integer_values + + +def _request_logits(output: Any, *, label: str) -> torch.Tensor: + if not torch.is_tensor(output): + raise ValueError(f"{label} output must be a tensor") + detached = output.detach() + if detached.ndim < 1 or int(detached.shape[-1]) < 1: + raise ValueError(f"{label} output must have a non-empty class dimension") + if int(detached.numel()) != int(detached.shape[-1]): + raise ValueError(f"{label} output must contain exactly one serving request") + logits = detached.reshape(-1).to(device="cpu", dtype=torch.float64) + if not bool(torch.isfinite(logits).all().item()): + raise ValueError(f"{label} logits must be finite") + return logits + + +def _resolve_prediction_scope( + normalized_scope: List[int] | List[bool] | None, + *, + output_class_count: int, + decision_top_k: int, +) -> List[int]: + if normalized_scope is None: + resolved = list(range(output_class_count)) + elif all(isinstance(value, bool) for value in normalized_scope): + if len(normalized_scope) != output_class_count: + raise ValueError( + "prediction_class_indices bool mask length must match output classes" + ) + resolved = [index for index, selected in enumerate(normalized_scope) if selected] + else: + resolved = [int(value) for value in normalized_scope] + if max(resolved) >= output_class_count: + raise ValueError("prediction_class_indices contains an out-of-range class") + if len(resolved) < decision_top_k: + raise ValueError( + f"decision_top_k={decision_top_k} requires at least {decision_top_k} scoped classes" + ) + return resolved + + +def _distribution_distances( + fixed_logits: torch.Tensor, + adaptive_logits: torch.Tensor, +) -> tuple[float, float]: + fixed_distribution = torch.softmax(fixed_logits, dim=-1) + adaptive_distribution = torch.softmax(adaptive_logits, dim=-1) + midpoint = 0.5 * (fixed_distribution + adaptive_distribution) + fixed_kl = torch.sum( + torch.special.xlogy(fixed_distribution, fixed_distribution) + - torch.special.xlogy(fixed_distribution, midpoint) + ) + adaptive_kl = torch.sum( + torch.special.xlogy(adaptive_distribution, adaptive_distribution) + - torch.special.xlogy(adaptive_distribution, midpoint) + ) + js_divergence = float((0.5 * (fixed_kl + adaptive_kl)).item()) + total_variation = float( + (0.5 * torch.sum(torch.abs(fixed_distribution - adaptive_distribution))).item() + ) + # JSD is nonnegative by definition. Float32 cancellation can produce a + # small negative value for nearly identical distributions; retain a strict + # bound so materially invalid telemetry still fails closed. + if -1e-6 < js_divergence < 0.0: + js_divergence = 0.0 + return ( + _nonnegative_finite_float(js_divergence, label="distribution JSD"), + _nonnegative_finite_float(total_variation, label="total variation distance"), + ) + + +def _synchronize_for_timing(device: torch.device) -> None: + """Finish queued accelerator work so request latency is measured honestly.""" + + device_type = str(device).split(":", 1)[0].lower() + if device_type == "cuda" and torch.cuda.is_available(): + torch.cuda.synchronize(device) + elif device_type == "xpu" and hasattr(torch, "xpu"): + torch.xpu.synchronize() # type: ignore[attr-defined] + elif device_type == "mps" and hasattr(torch, "mps"): + torch.mps.synchronize() + + @torch.no_grad() def benchmark_serving_requests( model: ChampionNetCognitiveLeapUltraExpert, @@ -66,7 +273,54 @@ def benchmark_serving_requests( stability_patience: int, stability_tol: float, distribution_top_k: int, + prediction_stability_margin: float = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + decision_top_k: int = DEFAULT_DECISION_TOP_K, + prediction_class_indices: Any = None, + exit_tol: float = 0.0, + exit_entropy_threshold: float = 0.0, + order_offset: int = 0, ) -> Dict[str, Any]: + request_count = int(len(y)) + if request_count <= 0: + raise ValueError("At least one serving request is required") + if int(len(x)) != request_count: + raise ValueError("x and y must contain the same number of requests") + normalized_fixed_cycles = int(fixed_cycles) + normalized_max_cycles = int(max_cycles) + normalized_patience = int(stability_patience) + normalized_distribution_top_k = int(distribution_top_k) + normalized_decision_top_k = int(decision_top_k) + normalized_rank_depth = int(prediction_stability_rank_depth) + normalized_stability_tol = _nonnegative_finite_float( + stability_tol, + label="stability tolerance", + ) + normalized_exit_tol = _nonnegative_finite_float( + exit_tol, + label="adaptive exit tolerance", + ) + normalized_exit_entropy_threshold = _nonnegative_finite_float( + exit_entropy_threshold, + label="adaptive exit entropy threshold", + ) + stability_margin = _nonnegative_finite_float( + prediction_stability_margin, + label="prediction stability margin", + ) + if normalized_fixed_cycles <= 0 or normalized_max_cycles <= 0: + raise ValueError("fixed_cycles and max_cycles must be positive") + if normalized_patience < 0: + raise ValueError("stability_patience must be nonnegative") + if normalized_distribution_top_k <= 0: + raise ValueError("distribution_top_k must be positive") + if normalized_decision_top_k <= 0: + raise ValueError("decision_top_k must be positive") + if normalized_rank_depth <= 0: + raise ValueError("prediction_stability_rank_depth must be positive") + normalized_scope = _normalize_prediction_class_indices_input( + prediction_class_indices + ) head = model.layers[10] fixed_predictions: List[int] = [] adaptive_predictions: List[int] = [] @@ -75,73 +329,408 @@ def benchmark_serving_requests( adaptive_cycles: List[float] = [] adaptive_latest_js_drift: List[float] = [] adaptive_max_js_drift: List[float] = [] + adaptive_prediction_margins: List[float] = [] + adaptive_decision_margins: List[float] = [] + adaptive_decision_reference_cycles: List[int] = [] + prediction_stable_decision_margins: List[float] = [] + adaptive_rank_depths: List[int] = [] + adaptive_prediction_class_counts: List[int] = [] + adaptive_requested_class_indices: List[List[int] | None] = [] + absolute_logit_deltas: List[float] = [] + distribution_js_divergences: List[float] = [] + distribution_total_variations: List[float] = [] + pending_top_k_disagreement_examples: List[Dict[str, Any]] = [] + top_k_order_disagreement_count = 0 + top_k_set_disagreement_count = 0 + resolved_scope: List[int] | None = None + output_class_count: int | None = None exit_reasons: Counter[str] = Counter() + measurement_orders: Counter[str] = Counter() - # Keep initialization and one-time kernel setup outside measured requests. - model(x[:1], reasoning_cycles=fixed_cycles) - model( - x[:1], - reasoning_cycles=max_cycles, - adaptive_compute=True, - exit_tol=0.0, - exit_entropy_threshold=0.0, - prediction_stability_patience=stability_patience, - prediction_stability_tol=stability_tol, - prediction_stability_top_k=distribution_top_k, + normalized_requested_class_indices = ( + None + if normalized_scope is None + else [ + index + for index, selected in enumerate(normalized_scope) + if selected + ] + if all(isinstance(value, bool) for value in normalized_scope) + else [int(value) for value in normalized_scope] ) - for index in range(len(y)): - sample = x[index : index + 1] + def adaptive_kwargs() -> Dict[str, Any]: + kwargs: Dict[str, Any] = { + "reasoning_cycles": normalized_max_cycles, + "adaptive_compute": True, + "exit_tol": normalized_exit_tol, + "exit_entropy_threshold": normalized_exit_entropy_threshold, + "prediction_stability_patience": normalized_patience, + "prediction_stability_tol": normalized_stability_tol, + "prediction_stability_top_k": normalized_distribution_top_k, + "prediction_stability_margin": stability_margin, + "prediction_stability_rank_depth": normalized_rank_depth, + } + if normalized_scope is not None: + kwargs["prediction_class_indices"] = normalized_scope + return kwargs + + def record_requested_scope(kwargs: Dict[str, Any]) -> None: + raw_scope = kwargs.get("prediction_class_indices") + if raw_scope is None: + adaptive_requested_class_indices.append(None) + return + if all(isinstance(value, bool) for value in raw_scope): + adaptive_requested_class_indices.append( + [index for index, selected in enumerate(raw_scope) if selected] + ) + return + adaptive_requested_class_indices.append([int(value) for value in raw_scope]) + def read_adaptive_scope_telemetry(*, label: str) -> int: + selection_valid = _strict_bool_scalar( + getattr(head, "last_prediction_class_selection_valid", None), + label=f"{label} prediction class selection validity", + ) + if not selection_valid: + raise ValueError(f"{label} prediction class selection is invalid") + return _positive_integral_scalar( + getattr(head, "last_prediction_class_count", None), + label=f"{label} prediction class count", + ) + + # Keep initialization and one-time kernel setup outside measured requests. + # End the warmup on the opposite mode from the first measured request so + # both seed offsets receive the same cross-mode transition. + def warm_adaptive() -> None: + kwargs = adaptive_kwargs() + record_requested_scope(kwargs) + model(x[:1], **kwargs) + adaptive_prediction_class_counts.append( + read_adaptive_scope_telemetry(label="adaptive warmup") + ) + + if int(order_offset) % 2 == 0: + model(x[:1], reasoning_cycles=normalized_fixed_cycles) + warm_adaptive() + else: + warm_adaptive() + model(x[:1], reasoning_cycles=normalized_fixed_cycles) + + def run_fixed(sample: torch.Tensor) -> torch.Tensor: + _synchronize_for_timing(sample.device) started = time.perf_counter() - fixed_output = model(sample, reasoning_cycles=fixed_cycles) - fixed_latencies.append((time.perf_counter() - started) * 1000.0) - fixed_predictions.append(int(fixed_output.argmax(dim=-1).item())) + output = model(sample, reasoning_cycles=normalized_fixed_cycles) + _synchronize_for_timing(sample.device) + fixed_latencies.append( + _nonnegative_finite_float( + (time.perf_counter() - started) * 1000.0, + label="fixed request latency", + ) + ) + return output + def run_adaptive(sample: torch.Tensor) -> tuple[torch.Tensor, Dict[str, Any]]: + _synchronize_for_timing(sample.device) started = time.perf_counter() - adaptive_output = model( - sample, - reasoning_cycles=max_cycles, - adaptive_compute=True, - exit_tol=0.0, - exit_entropy_threshold=0.0, - prediction_stability_patience=stability_patience, - prediction_stability_tol=stability_tol, - prediction_stability_top_k=distribution_top_k, - ) - adaptive_latencies.append((time.perf_counter() - started) * 1000.0) - adaptive_predictions.append(int(adaptive_output.argmax(dim=-1).item())) - adaptive_cycles.append(float(head.last_cycles_used.item())) - adaptive_latest_js_drift.append( - float(head.last_prediction_topk_js_divergence.item()) - ) - adaptive_max_js_drift.append( - float(head.last_prediction_topk_js_divergence_max.item()) - ) - exit_reasons[str(head.last_exit_reason)] += 1 - - truth = y.detach().cpu() + kwargs = adaptive_kwargs() + record_requested_scope(kwargs) + output = model(sample, **kwargs) + _synchronize_for_timing(sample.device) + adaptive_latencies.append( + _nonnegative_finite_float( + (time.perf_counter() - started) * 1000.0, + label="adaptive request latency", + ) + ) + cycles_used = _nonnegative_finite_float( + head.last_cycles_used.item(), label="adaptive cycles used" + ) + adaptive_cycles.append(cycles_used) + observed_decision_reference_cycles = _positive_integral_scalar( + getattr(head, "last_decision_reference_cycles", None), + label="observed decision reference cycles", + ) + adaptive_decision_reference_cycles.append( + observed_decision_reference_cycles + ) + latest_js_drift = _nonnegative_finite_float( + head.last_prediction_topk_js_divergence.item(), + label="latest prediction distribution drift", + ) + adaptive_latest_js_drift.append(latest_js_drift) + max_js_drift = _nonnegative_finite_float( + head.last_prediction_topk_js_divergence_max.item(), + label="maximum prediction distribution drift", + ) + adaptive_max_js_drift.append(max_js_drift) + observed_prediction_margin = _nonnegative_finite_float( + head.last_prediction_margin.item(), + label="observed prediction stability margin", + ) + adaptive_prediction_margins.append(observed_prediction_margin) + observed_decision_margin = _nonnegative_finite_float( + head.last_prediction_decision_margin.item(), + label="observed prediction decision margin", + ) + adaptive_decision_margins.append(observed_decision_margin) + observed_rank_depth = int(head.last_prediction_rank_depth.item()) + if observed_rank_depth <= 0: + raise ValueError("observed prediction rank depth must be positive") + adaptive_rank_depths.append(observed_rank_depth) + observed_class_count = read_adaptive_scope_telemetry( + label="adaptive request" + ) + adaptive_prediction_class_counts.append(observed_class_count) + observed_prediction_streak = _nonnegative_finite_float( + head.last_prediction_streak.item(), + label="observed prediction stability streak", + ) + observed_confidence_delta = _nonnegative_finite_float( + head.last_prediction_confidence_delta.item(), + label="observed prediction confidence delta", + ) + raw_exit_reason = getattr(head, "last_exit_reason", None) + if not isinstance(raw_exit_reason, str) or not raw_exit_reason.strip(): + raise ValueError("adaptive exit reason must be a non-empty string") + exit_reason = raw_exit_reason.strip() + if exit_reason not in KNOWN_ADAPTIVE_EXIT_REASONS: + raise ValueError(f"unknown adaptive exit reason: {exit_reason}") + exit_reasons[exit_reason] += 1 + if exit_reason == "prediction_stable": + prediction_stable_decision_margins.append(observed_decision_margin) + request_telemetry = { + "adaptive_exit_reason": exit_reason, + "adaptive_cycles_used": int(cycles_used), + "prediction_margin": round(observed_prediction_margin, 8), + "decision_margin": round(observed_decision_margin, 8), + "verifier": { + "prediction_streak": round(observed_prediction_streak, 8), + "confidence_delta": round(observed_confidence_delta, 8), + "decision_reference_cycles": observed_decision_reference_cycles, + "observed_rank_depth": observed_rank_depth, + "observed_class_count": observed_class_count, + "class_selection_valid": True, + "latest_top_k_js_divergence": round(latest_js_drift, 8), + "maximum_top_k_js_divergence": round(max_js_drift, 8), + }, + } + return output, request_telemetry + + for index in range(len(y)): + sample = x[index : index + 1] + fixed_first = (index + int(order_offset)) % 2 == 0 + if fixed_first: + measurement_orders["fixed_then_adaptive"] += 1 + fixed_output = run_fixed(sample) + adaptive_output, adaptive_request_telemetry = run_adaptive(sample) + else: + measurement_orders["adaptive_then_fixed"] += 1 + adaptive_output, adaptive_request_telemetry = run_adaptive(sample) + fixed_output = run_fixed(sample) + + fixed_logits = _request_logits(fixed_output, label="fixed") + adaptive_logits = _request_logits(adaptive_output, label="adaptive") + if fixed_logits.shape != adaptive_logits.shape: + raise ValueError("fixed and adaptive logits must have the same shape") + current_output_class_count = int(fixed_logits.numel()) + if output_class_count is None: + output_class_count = current_output_class_count + resolved_scope = _resolve_prediction_scope( + normalized_scope, + output_class_count=output_class_count, + decision_top_k=normalized_decision_top_k, + ) + elif current_output_class_count != output_class_count: + raise ValueError("model output class count changed between requests") + assert resolved_scope is not None + scope_tensor = torch.tensor(resolved_scope, dtype=torch.long) + fixed_scoped = fixed_logits.index_select(0, scope_tensor) + adaptive_scoped = adaptive_logits.index_select(0, scope_tensor) + fixed_order_local = torch.argsort( + fixed_scoped, descending=True, stable=True + )[:normalized_decision_top_k] + adaptive_order_local = torch.argsort( + adaptive_scoped, descending=True, stable=True + )[:normalized_decision_top_k] + fixed_order = [ + resolved_scope[int(index.item())] for index in fixed_order_local + ] + adaptive_order = [ + resolved_scope[int(index.item())] for index in adaptive_order_local + ] + fixed_predictions.append(fixed_order[0]) + adaptive_predictions.append(adaptive_order[0]) + top_k_order_disagreed = fixed_order != adaptive_order + top_k_order_disagreement_count += int(top_k_order_disagreed) + top_k_set_disagreement_count += int(set(fixed_order) != set(adaptive_order)) + if ( + top_k_order_disagreed + and len(pending_top_k_disagreement_examples) + < MAX_TOP_K_DISAGREEMENT_EXAMPLES + ): + pending_top_k_disagreement_examples.append( + { + "sample_index": int(index), + "fixed_ordered_top_k": fixed_order, + "adaptive_ordered_top_k": adaptive_order, + **adaptive_request_telemetry, + } + ) + absolute_logit_deltas.extend( + float(value) + for value in torch.abs(fixed_scoped - adaptive_scoped).tolist() + ) + js_divergence, total_variation = _distribution_distances( + fixed_scoped, adaptive_scoped + ) + distribution_js_divergences.append(js_divergence) + distribution_total_variations.append(total_variation) + + assert output_class_count is not None and resolved_scope is not None + truth = y.detach().cpu().reshape(-1) + if truth.numel() != request_count: + raise ValueError("y must contain one target per serving request") + if truth.dtype.is_floating_point or truth.dtype.is_complex: + raise ValueError("y must contain integer class indices") + if bool(((truth < 0) | (truth >= output_class_count)).any().item()): + raise ValueError("y contains an out-of-range class index") + top_k_disagreement_examples = [ + { + "sample_index": example["sample_index"], + "target": int(truth[example["sample_index"]].item()), + "fixed_ordered_top_k": example["fixed_ordered_top_k"], + "adaptive_ordered_top_k": example["adaptive_ordered_top_k"], + "adaptive_exit_reason": example["adaptive_exit_reason"], + "adaptive_cycles_used": example["adaptive_cycles_used"], + "prediction_margin": example["prediction_margin"], + "decision_margin": example["decision_margin"], + "verifier": example["verifier"], + } + for example in pending_top_k_disagreement_examples + ] fixed_tensor = torch.tensor(fixed_predictions) adaptive_tensor = torch.tensor(adaptive_predictions) - fixed_accuracy = float((fixed_tensor == truth).float().mean().item()) - adaptive_accuracy = float((adaptive_tensor == truth).float().mean().item()) - agreement = float((fixed_tensor == adaptive_tensor).float().mean().item()) + fixed_correct = int((fixed_tensor == truth).sum().item()) + adaptive_correct = int((adaptive_tensor == truth).sum().item()) + top1_disagreement_count = int((fixed_tensor != adaptive_tensor).sum().item()) + fixed_accuracy = fixed_correct / max(1, request_count) + adaptive_accuracy = adaptive_correct / max(1, request_count) + agreement = 1.0 - (top1_disagreement_count / max(1, request_count)) fixed_latency = _latency_summary(fixed_latencies) adaptive_latency = _latency_summary(adaptive_latencies) - mean_cycles = sum(adaptive_cycles) / max(1, len(adaptive_cycles)) + mean_cycles = float( + _finite_values(adaptive_cycles, label="adaptive cycles").mean().item() + ) + observed_decision_reference_cycles = sorted( + set(adaptive_decision_reference_cycles) + ) + if ( + len(adaptive_decision_reference_cycles) != request_count + or len(observed_decision_reference_cycles) != 1 + ): + raise ValueError( + "adaptive verifier must report one consistent decision reference budget" + ) + expected_effective_rank_depth = min( + normalized_rank_depth, max(1, len(resolved_scope) - 1) + ) + if set(adaptive_rank_depths) != {expected_effective_rank_depth}: + raise ValueError( + "adaptive verifier reported a rank depth inconsistent with the verified scope" + ) + if set(adaptive_prediction_class_counts) != {len(resolved_scope)}: + raise ValueError( + "adaptive verifier reported a class count inconsistent with the verified scope" + ) + if set( + None if value is None else tuple(value) + for value in adaptive_requested_class_indices + ) != { + None + if normalized_requested_class_indices is None + else tuple(normalized_requested_class_indices) + }: + raise ValueError( + "adaptive verifier calls did not preserve the exact requested class indices" + ) + scoped_truth_count = int( + torch.isin(truth, torch.tensor(resolved_scope, dtype=truth.dtype)).sum().item() + ) + verified_scope = { + "verified": True, + "mode": ( + "all_output_classes" + if normalized_scope is None + else "prediction_class_indices" + ), + "output_class_count": output_class_count, + "class_count": len(resolved_scope), + "class_indices": resolved_scope, + "requested_normalized_class_indices": normalized_requested_class_indices, + "adaptive_requested_class_indices_verified": True, + "truth_labels_in_scope": scoped_truth_count, + "truth_labels_outside_scope": request_count - scoped_truth_count, + "adaptive_verifier_rank_depth_verified": True, + "adaptive_verifier_class_scope_verified": True, + "observed_prediction_class_counts": sorted( + set(adaptive_prediction_class_counts) + ), + "effective_prediction_rank_depth": expected_effective_rank_depth, + } + logit_delta_tensor = _finite_values( + absolute_logit_deltas, label="absolute logit deltas" + ) return { "fixed": { - "cycles": int(fixed_cycles), + "adaptive_compute": False, + "cycles": normalized_fixed_cycles, "accuracy": round(fixed_accuracy, 6), + "correct_predictions": fixed_correct, "latency": fixed_latency, }, "prediction_stability": { - "max_cycles": int(max_cycles), - "patience": int(stability_patience), - "confidence_tolerance": float(stability_tol), + "adaptive_compute": True, + "max_cycles": normalized_max_cycles, + "decision_reference_cycles": { + "metric": "trained_fixed_compute_reference_budget", + "observed_values": observed_decision_reference_cycles, + "all_requests_reported": True, + }, + "exit_tolerance": normalized_exit_tol, + "exit_entropy_threshold": normalized_exit_entropy_threshold, + "patience": normalized_patience, + "confidence_tolerance": normalized_stability_tol, + "prediction_margin": { + "metric": "top_1_minus_top_2_probability", + "configured_minimum": stability_margin, + "observed": _scalar_summary(adaptive_prediction_margins), + }, + "decision_margin": { + "metric": "minimum_adjacent_probability_gap_through_rank_depth", + "configured_minimum": stability_margin, + "configured_rank_depth": normalized_rank_depth, + "observed_rank_depths": sorted(set(adaptive_rank_depths)), + "observed": _scalar_summary(adaptive_decision_margins), + "prediction_stable_observed": { + "observation_count": len(prediction_stable_decision_margins), + "minimum": ( + min(prediction_stable_decision_margins) + if prediction_stable_decision_margins + else None + ), + "summary": ( + _scalar_summary(prediction_stable_decision_margins) + if prediction_stable_decision_margins + else None + ), + }, + }, "accuracy": round(adaptive_accuracy, 6), + "correct_predictions": adaptive_correct, "mean_cycles_used": round(mean_cycles, 4), + "total_cycles_used": round(sum(adaptive_cycles), 6), "cycle_counts": { str(cycle): count for cycle, count in sorted(Counter(adaptive_cycles).items()) @@ -149,7 +738,7 @@ def benchmark_serving_requests( "exit_reasons": dict(sorted(exit_reasons.items())), "distribution_drift": { "metric": "top_k_jensen_shannon_divergence_nats", - "top_k": int(distribution_top_k), + "top_k": normalized_distribution_top_k, "role": "shadow_diagnostic_only", "latest_cycle_pair": _scalar_summary(adaptive_latest_js_drift), "max_cycle_pair": _scalar_summary(adaptive_max_js_drift), @@ -159,6 +748,46 @@ def benchmark_serving_requests( "comparison": { "accuracy_delta": round(adaptive_accuracy - fixed_accuracy, 6), "prediction_agreement": round(agreement, 6), + "top1_disagreement_count": top1_disagreement_count, + # Compatibility alias retained for existing artifact consumers. + "exact_disagreement_count": top1_disagreement_count, + "request_count": request_count, + "decision_fidelity": { + "verified_scope": verified_scope, + "top_k": normalized_decision_top_k, + "top_k_order_disagreement_count": top_k_order_disagreement_count, + "top_k_set_disagreement_count": top_k_set_disagreement_count, + "top_k_disagreement_evidence": { + "limit": MAX_TOP_K_DISAGREEMENT_EXAMPLES, + "total_count": top_k_order_disagreement_count, + "recorded_count": len(top_k_disagreement_examples), + "truncated": ( + top_k_order_disagreement_count + > len(top_k_disagreement_examples) + ), + "examples": top_k_disagreement_examples, + }, + "absolute_logit_delta": { + "mean": round(float(logit_delta_tensor.mean().item()), 12), + "max": round(float(logit_delta_tensor.max().item()), 12), + "role": "diagnostic_only_not_gated", + }, + "distribution_distance": { + "jensen_shannon_divergence_nats": _scalar_summary( + distribution_js_divergences, digits=12 + ), + "total_variation_distance": _scalar_summary( + distribution_total_variations, digits=12 + ), + "role": "diagnostic_only_not_gated", + }, + "tensor_equality_required": False, + }, + "measurement_order": { + "offset": int(order_offset), + "fixed_then_adaptive": measurement_orders["fixed_then_adaptive"], + "adaptive_then_fixed": measurement_orders["adaptive_then_fixed"], + }, "cycle_reduction_percent": round( 100.0 * (float(fixed_cycles) - mean_cycles) / max(1.0, float(fixed_cycles)), 3, @@ -181,9 +810,46 @@ def main() -> None: parser.add_argument("--seed", type=int, default=52) parser.add_argument("--fixed_cycles", type=int, default=3) parser.add_argument("--max_cycles", type=int, default=8) - parser.add_argument("--stability_patience", type=int, default=2) - parser.add_argument("--stability_tol", type=float, default=0.005) + parser.add_argument( + "--stability_patience", + type=int, + default=DEFAULT_PREDICTION_STABILITY_PATIENCE, + ) + parser.add_argument( + "--stability_tol", type=float, default=DEFAULT_PREDICTION_STABILITY_TOL + ) + parser.add_argument( + "--prediction-stability-margin", + "--prediction_stability_margin", + dest="prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + parser.add_argument( + "--prediction-stability-rank-depth", + "--prediction_stability_rank_depth", + dest="prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) + parser.add_argument( + "--decision-top-k", + "--decision_top_k", + dest="decision_top_k", + type=int, + default=DEFAULT_DECISION_TOP_K, + ) + parser.add_argument( + "--prediction-class-indices", + "--prediction_class_indices", + dest="prediction_class_indices", + type=int, + nargs="+", + default=None, + help="Evaluate and verify one explicit class scope (single scope only).", + ) parser.add_argument("--distribution_top_k", type=int, default=5) + parser.add_argument("--order_offset", type=int, default=0) parser.add_argument("--device", default="auto") parser.add_argument("--device_preference", default="cuda,npu,xpu,dml,mps,cpu") parser.add_argument("--torch_num_threads", type=int, default=0) @@ -209,6 +875,14 @@ def main() -> None: stability_patience=max(0, int(args.stability_patience)), stability_tol=max(0.0, float(args.stability_tol)), distribution_top_k=max(1, int(args.distribution_top_k)), + prediction_stability_margin=_nonnegative_finite_float( + args.prediction_stability_margin, + label="prediction stability margin", + ), + prediction_stability_rank_depth=int(args.prediction_stability_rank_depth), + decision_top_k=int(args.decision_top_k), + prediction_class_indices=args.prediction_class_indices, + order_offset=int(args.order_offset), ) payload = { "created_at": datetime.now(timezone.utc).isoformat(), @@ -220,8 +894,10 @@ def main() -> None: } output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(payload, indent=2), encoding="utf-8") - print(json.dumps(payload, indent=2)) + output.write_text( + json.dumps(payload, indent=2, allow_nan=False), encoding="utf-8" + ) + print(json.dumps(payload, indent=2, allow_nan=False)) print(f"Results written to {output}") diff --git a/source/chat_app.py b/source/chat_app.py index d5e43143..cec08dad 100644 --- a/source/chat_app.py +++ b/source/chat_app.py @@ -1,6 +1,7 @@ import argparse import inspect import json +import math import re import time from pathlib import Path @@ -61,12 +62,37 @@ def load_metadata(path: str) -> Dict: return {} +def _parse_metadata_buckets(raw: Any) -> Dict[int, List[Dict]]: + """Keep only non-empty buckets whose labels address a model output class.""" + + buckets: Dict[int, List[Dict]] = {} + if not isinstance(raw, dict): + return buckets + for raw_label, rows in raw.items(): + if isinstance(raw_label, bool): + continue + if isinstance(raw_label, int): + label = raw_label + elif isinstance(raw_label, str) and re.fullmatch(r"[+-]?\d+", raw_label.strip()): + try: + label = int(raw_label) + except (ValueError, OverflowError): + continue + else: + continue + if not 0 <= label < MODEL_CLASSES: + continue + if isinstance(rows, list) and rows: + buckets[label] = rows + return buckets + + def _int_or_default(value, default: int) -> int: if value is None: return int(default) try: return int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return int(default) @@ -74,6 +100,9 @@ def _int_or_default(value, default: int) -> int: DEFAULT_ADAPTIVE_EXIT_TOL = 1e-3 DEFAULT_AUTO_COMPUTE_CONFIDENCE = 0.55 DEFAULT_AUTO_COMPUTE_ENTROPY = 1.85 +AUTO_COMPUTE_PLAN_SCHEMA_VERSION = "runtime-auto-compute-plan-v2" +AUTO_COMPUTE_STRATEGY = "progressive_accepted_probe" +DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K = 5 def resolve_runtime_compute_cycles(cycles, default: Optional[List[int]] = None, limit: int = 8) -> List[int]: @@ -126,6 +155,11 @@ def evaluate_runtime_compute_budgets( cycles=None, adaptive_compute: bool = False, exit_tol: Optional[float] = None, + exit_entropy_threshold: Any = None, + prediction_stability_patience: Any = None, + prediction_stability_tol: Any = None, + prediction_stability_margin: Any = None, + prediction_stability_rank_depth: Any = None, ) -> List[Dict[str, object]]: labels = list(available_labels) or list(range(MODEL_CLASSES)) idx_device = x.device if isinstance(x, torch.Tensor) else None @@ -144,7 +178,13 @@ def evaluate_runtime_compute_budgets( reasoning_cycles=requested_cycles, adaptive_compute=adaptive_compute, exit_tol=resolved_exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + prediction_stability_margin=prediction_stability_margin, return_diagnostics=True, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, ) logits = logits_tensor[0, 0] avail_logits = logits.index_select(0, idx.to(logits.device)) @@ -219,6 +259,9 @@ def select_auto_runtime_compute_budget( DEFAULT_ADAPTIVE_EXIT_ENTROPY = 0.2 DEFAULT_PREDICTION_STABILITY_PATIENCE = 2 DEFAULT_PREDICTION_STABILITY_TOL = 5e-3 +# Calibrated for the released v51 checkpoint and workload; not a universal margin. +DEFAULT_PREDICTION_STABILITY_MARGIN = 5e-4 +DEFAULT_PREDICTION_STABILITY_RANK_DEPTH = 3 AUTO_REASONING_CYCLE_BUCKETS = (1, 3, 8, 16) @@ -309,7 +352,7 @@ def _coerce_optional_positive_int( value = stripped try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): return default parsed = max(1, parsed) if max_value is not None: @@ -381,7 +424,9 @@ def estimate_auto_reasoning_cycles(context: str, max_value: Optional[int] = None def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[float] = None) -> float: try: parsed = float(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): + parsed = float(default) + if not math.isfinite(parsed): parsed = float(default) parsed = max(0.0, parsed) if max_value is not None: @@ -389,10 +434,40 @@ def _coerce_nonnegative_float(value: Any, default: float, max_value: Optional[fl return parsed +def _coerce_prediction_stability_margin( + value: Any, + default: float = DEFAULT_PREDICTION_STABILITY_MARGIN, +) -> float: + """Preserve explicit zero but never turn malformed input into guard-off.""" + + try: + parsed = float(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return float(default) + if not math.isfinite(parsed) or parsed < 0.0: + return float(default) + return parsed + + +def _coerce_prediction_stability_rank_depth( + value: Any, + default: int = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, +) -> int: + """Preserve explicit zero while failing malformed/negative values closed.""" + + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError, RuntimeError): + return int(default) + if parsed < 0: + return int(default) + return parsed + + def _coerce_nonnegative_int(value: Any, default: int, max_value: Optional[int] = None) -> int: try: parsed = int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError, RuntimeError): parsed = int(default) parsed = max(0, parsed) if max_value is not None: @@ -444,7 +519,7 @@ def _to_optional_scalar(value: Any) -> Optional[float]: if hasattr(value, "detach"): return float(value.detach().cpu().item()) return float(value) - except (TypeError, ValueError, RuntimeError): + except (TypeError, ValueError, OverflowError, RuntimeError): return None @@ -463,6 +538,12 @@ def collect_runtime_compute_metrics( "gating_entropy": None, "prediction_streak": None, "prediction_confidence_delta": None, + "prediction_margin": None, + "prediction_decision_margin": None, + "decision_reference_cycles": None, + "prediction_rank_depth": None, + "prediction_class_count": None, + "prediction_class_selection_valid": None, "exit_reason": None, } attr_map = { @@ -472,6 +553,12 @@ def collect_runtime_compute_metrics( "last_gating_entropy": "gating_entropy", "last_prediction_streak": "prediction_streak", "last_prediction_confidence_delta": "prediction_confidence_delta", + "last_prediction_margin": "prediction_margin", + "last_prediction_decision_margin": "prediction_decision_margin", + "last_decision_reference_cycles": "decision_reference_cycles", + "last_prediction_rank_depth": "prediction_rank_depth", + "last_prediction_class_count": "prediction_class_count", + "last_prediction_class_selection_valid": "prediction_class_selection_valid", } modules = list(model.modules()) if hasattr(model, "modules") else [model] for module in modules: @@ -485,6 +572,10 @@ def collect_runtime_compute_metrics( reason = getattr(module, "last_exit_reason", None) if isinstance(reason, str) and reason.strip(): metrics["exit_reason"] = reason.strip() + if metrics["prediction_class_selection_valid"] is not None: + metrics["prediction_class_selection_valid"] = bool( + metrics["prediction_class_selection_valid"] + ) metrics.update({ "supported": model_supports_runtime_compute(model), "requested_reasoning_cycles": requested_reasoning_cycles, @@ -512,6 +603,9 @@ def forward_with_runtime_compute( prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, auto_reasoning_context: str = "", return_diagnostics: Optional[bool] = None, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_class_indices: Any = None, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, ): """Forward a model while applying optional v50 runtime-compute controls. @@ -542,6 +636,14 @@ def forward_with_runtime_compute( prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + stability_margin = _coerce_prediction_stability_margin( + prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + stability_rank_depth = _coerce_prediction_stability_rank_depth( + prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) kwargs: Dict[str, Any] = {} if cycles is not None and _model_forward_accepts_kwarg(model, "reasoning_cycles"): @@ -556,8 +658,29 @@ def forward_with_runtime_compute( kwargs["prediction_stability_patience"] = stability_patience if _model_forward_accepts_kwarg(model, "prediction_stability_tol"): kwargs["prediction_stability_tol"] = stability_tol + if _model_forward_accepts_kwarg(model, "prediction_stability_margin"): + kwargs["prediction_stability_margin"] = stability_margin + if _model_forward_accepts_kwarg(model, "prediction_stability_rank_depth"): + kwargs["prediction_stability_rank_depth"] = stability_rank_depth + if ( + prediction_class_indices is not None + and _model_forward_accepts_kwarg(model, "prediction_class_indices") + ): + if isinstance(prediction_class_indices, torch.Tensor): + prediction_class_indices = prediction_class_indices.detach().cpu().tolist() + elif isinstance(prediction_class_indices, tuple): + prediction_class_indices = list(prediction_class_indices) + kwargs["prediction_class_indices"] = prediction_class_indices output = model(x, **kwargs) if kwargs else model(x) + prediction_verifier_active = bool( + adaptive + and stability_patience > 0 + and stability_rank_depth > 0 + and "adaptive_compute" in kwargs + and "prediction_stability_patience" in kwargs + and "prediction_stability_rank_depth" in kwargs + ) metrics = collect_runtime_compute_metrics( model, requested_reasoning_cycles=cycles, @@ -565,6 +688,18 @@ def forward_with_runtime_compute( exit_tol=tol, applied_kwargs=kwargs, ) + if not prediction_verifier_active: + for key in ( + "prediction_streak", + "prediction_confidence_delta", + "prediction_margin", + "prediction_decision_margin", + "decision_reference_cycles", + "prediction_rank_depth", + "prediction_class_count", + "prediction_class_selection_valid", + ): + metrics[key] = None diagnostics = { "requested_reasoning_cycles": cycles, "selected_reasoning_cycles": cycles, @@ -576,6 +711,14 @@ def forward_with_runtime_compute( "exit_entropy_threshold": entropy_threshold, "prediction_stability_patience": stability_patience, "prediction_stability_tol": stability_tol, + "prediction_stability_margin": stability_margin, + "prediction_stability_rank_depth": stability_rank_depth, + "prediction_verifier_active": prediction_verifier_active, + "prediction_class_indices": ( + kwargs.get("prediction_class_indices") + if "prediction_class_indices" in kwargs + else None + ), "applied": bool(kwargs), "applied_kwargs": dict(kwargs), **metrics, @@ -585,6 +728,266 @@ def forward_with_runtime_compute( return output, diagnostics +def _topk_distribution_js_divergence( + previous: torch.Tensor, + current: torch.Tensor, + top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, +) -> float: + """Return midpoint-support JSD while preserving tail probability mass.""" + + left = previous.detach().to(dtype=torch.float64).reshape(-1) + right = current.detach().to(dtype=torch.float64).reshape(-1) + if left.numel() != right.numel() or left.numel() == 0: + return 0.0 + k = min(max(1, int(top_k)), int(left.numel())) + midpoint = 0.5 * (left + right) + indices = torch.topk(midpoint, k=k).indices + left_top = left.index_select(0, indices) + right_top = right.index_select(0, indices) + left_compact = torch.cat( + [left_top, (1.0 - left_top.sum()).clamp_min(0.0).reshape(1)] + ) + right_compact = torch.cat( + [right_top, (1.0 - right_top.sum()).clamp_min(0.0).reshape(1)] + ) + compact_midpoint = 0.5 * (left_compact + right_compact) + epsilon = torch.finfo(left_compact.dtype).tiny + + def _kl(source: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return ( + source + * ( + torch.log(source.clamp_min(epsilon)) + - torch.log(target.clamp_min(epsilon)) + ) + ).sum() + + value = 0.5 * ( + _kl(left_compact, compact_midpoint) + + _kl(right_compact, compact_midpoint) + ) + return round(max(0.0, float(value.item())), 10) + + +def progressive_auto_compute_forward( + model: Any, + x: torch.Tensor, + available_labels: List[int], + *, + cycles: Any = None, + confidence_target: Any = DEFAULT_AUTO_COMPUTE_CONFIDENCE, + entropy_target: Any = DEFAULT_AUTO_COMPUTE_ENTROPY, + adaptive_compute: Any = False, + exit_tol: Any = DEFAULT_ADAPTIVE_EXIT_TOL, + exit_entropy_threshold: Any = DEFAULT_ADAPTIVE_EXIT_ENTROPY, + prediction_stability_patience: Any = DEFAULT_PREDICTION_STABILITY_PATIENCE, + prediction_stability_tol: Any = DEFAULT_PREDICTION_STABILITY_TOL, + auto_reasoning_context: str = "", + distribution_top_k: int = DEFAULT_AUTO_COMPUTE_DISTRIBUTION_TOP_K, + prediction_stability_margin: Any = DEFAULT_PREDICTION_STABILITY_MARGIN, + prediction_stability_rank_depth: Any = DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, +) -> Tuple[torch.Tensor, Dict[str, Any], Dict[str, Any]]: + """Evaluate auto-compute budgets progressively and reuse the accepted probe. + + This preserves the v1 selection policy and row order exactly: the first + confidence/entropy target wins, otherwise the best-confidence row wins + after the full ladder. Probes use the complete current runtime-control set, + and the accepted probe output is returned directly, removing the old N+1 + rerun. Cross-budget persistence and JSD are telemetry only and never select + or reject an output. + """ + + if bool(getattr(model, "training", False)): + raise RuntimeError("progressive auto compute requires model.eval()") + labels = list(available_labels) or list(range(MODEL_CLASSES)) + resolved_cycles = resolve_runtime_compute_cycles(cycles) + confidence_target = min( + 1.0, + _coerce_nonnegative_float( + confidence_target, + DEFAULT_AUTO_COMPUTE_CONFIDENCE, + ), + ) + entropy_target = _coerce_nonnegative_float( + entropy_target, + DEFAULT_AUTO_COMPUTE_ENTROPY, + ) + adaptive = _coerce_bool(adaptive_compute) + idx = torch.tensor(labels, dtype=torch.long, device=x.device) + records: List[Dict[str, Any]] = [] + cached_outputs: List[torch.Tensor] = [] + previous_probs: Optional[torch.Tensor] = None + previous_label: Optional[int] = None + previous_confidence: Optional[float] = None + accepted_index: Optional[int] = None + reason = "best_confidence" + + for requested_cycles in resolved_cycles: + started = time.perf_counter() + with torch.no_grad(): + model_out, compute = forward_with_runtime_compute( + model, + x, + reasoning_cycles=requested_cycles, + adaptive_compute=adaptive, + exit_tol=exit_tol, + exit_entropy_threshold=exit_entropy_threshold, + prediction_stability_patience=prediction_stability_patience, + prediction_stability_tol=prediction_stability_tol, + auto_reasoning_context=auto_reasoning_context, + prediction_stability_margin=prediction_stability_margin, + prediction_class_indices=labels, + prediction_stability_rank_depth=prediction_stability_rank_depth, + ) + logits = model_out[0, 0] + available_logits = logits.index_select(0, idx.to(logits.device)) + probs = torch.softmax(available_logits, dim=0) + confidence_tensor, predicted_position = torch.max(probs, dim=0) + entropy_tensor = -(probs * torch.log(probs.clamp_min(1e-8))).sum() + # v1 serialized these values to six decimals before the selector read + # them. Keep that boundary exact so values such as 0.5499996 still + # satisfy a 0.55 target just as they did before. + confidence = round(float(confidence_tensor.item()), 6) + entropy = round(float(entropy_tensor.item()), 6) + predicted_label = int(labels[int(predicted_position.item())]) + # Freeze selection latency before shadow-only diagnostics. Otherwise a + # costly JSD calculation could alter the legacy fallback tie-break. + probe_latency_ms = round((time.perf_counter() - started) * 1000.0, 1) + has_previous = previous_probs is not None + shadow = { + "role": "shadow_diagnostic_only", + "metric": "midpoint_support_top_k_jensen_shannon_divergence_nats", + "top_k": min(max(1, int(distribution_top_k)), len(labels)), + "has_previous_probe": has_previous, + "top1_persistent": ( + bool(previous_label == predicted_label) if has_previous else None + ), + "confidence_delta": ( + round(abs(confidence - float(previous_confidence)), 8) + if previous_confidence is not None + else None + ), + "js_divergence": ( + _topk_distribution_js_divergence( + previous_probs, + probs, + distribution_top_k, + ) + if previous_probs is not None + else None + ), + "selection_enabled": False, + } + row = { + "requested_cycles": int(requested_cycles), + "latency_ms": probe_latency_ms, + "cycles_used": compute.get("cycles_used"), + "predicted_label": predicted_label, + "confidence": confidence, + "entropy": entropy, + "confidence_target_met": bool(confidence >= confidence_target), + "entropy_target_met": bool(entropy <= entropy_target), + "mutual_stability_shadow": shadow, + "compute": compute, + } + records.append(row) + cached_outputs.append(model_out) + + if confidence >= confidence_target: + accepted_index = len(records) - 1 + reason = "confidence_target" + break + if entropy <= entropy_target: + accepted_index = len(records) - 1 + reason = "entropy_target" + break + previous_probs = probs.detach() + previous_label = predicted_label + previous_confidence = confidence + + if accepted_index is None: + accepted_index = max( + range(len(records)), + key=lambda index: ( + float(records[index].get("confidence", 0.0)), + -float(records[index].get("entropy", 999.0)), + -float(records[index].get("latency_ms", 0.0)), + ), + ) + + selected_row = records[accepted_index] + selected_output = cached_outputs[accepted_index] + selected_compute = dict(selected_row["compute"]) + evaluated_cycles = [int(row["requested_cycles"]) for row in records] + skipped_cycles = [ + int(value) for value in resolved_cycles[len(evaluated_cycles) :] + ] + legacy_forward_evaluations = len(resolved_cycles) + 1 + forward_evaluations = len(records) + plan = { + "schema_version": AUTO_COMPUTE_PLAN_SCHEMA_VERSION, + "enabled": True, + "strategy": AUTO_COMPUTE_STRATEGY, + "selected_reasoning_cycles": int(selected_row["requested_cycles"]), + "selected_index": int(accepted_index), + "accepted_probe_index": int(accepted_index), + "reason": reason, + "confidence_target": confidence_target, + "entropy_target": entropy_target, + "candidate_cycles": [int(value) for value in resolved_cycles], + "evaluated_cycles": evaluated_cycles, + "skipped_cycles": skipped_cycles, + "forward_evaluations": forward_evaluations, + "legacy_forward_evaluations": legacy_forward_evaluations, + "forward_reduction_percent": round( + 100.0 + * (legacy_forward_evaluations - forward_evaluations) + / max(1, legacy_forward_evaluations), + 3, + ), + "reused_probe_output": True, + "selection_semantics": "legacy_v1_selection_policy", + "probe_control_scope": "full_runtime_controls_v2", + "mutual_stability_role": "shadow_diagnostic_only", + "rows": records, + } + selected_compute["auto_compute_plan"] = plan + selected_compute["inference_reused"] = True + return selected_output, selected_compute, plan + + +def compact_auto_compute_plan_metrics(plan: Any) -> Dict[str, Any]: + """Return terminal-safe metrics for the probe that supplied the output.""" + + if not isinstance(plan, dict): + return {} + rows = plan.get("rows") if isinstance(plan.get("rows"), list) else [] + try: + selected_index = int(plan.get("selected_index")) + except (TypeError, ValueError, OverflowError, RuntimeError): + selected_index = -1 + if not 0 <= selected_index < len(rows): + selected_index = len(rows) - 1 + selected_row = rows[selected_index] if selected_index >= 0 else {} + shadow = ( + selected_row.get("mutual_stability_shadow", {}) + if isinstance(selected_row, dict) + else {} + ) + metrics: Dict[str, Any] = { + "auto_selected": plan.get("selected_reasoning_cycles"), + "auto_reason": plan.get("reason"), + "auto_forwards": ( + f"{plan.get('forward_evaluations')}/" + f"{plan.get('legacy_forward_evaluations')}" + ), + "auto_reused": bool(plan.get("reused_probe_output")), + } + if isinstance(shadow, dict) and shadow.get("js_divergence") is not None: + metrics["auto_shadow_js"] = shadow.get("js_divergence") + return metrics + + def _default_expansion_dim_for_model_size(model_size: str) -> int: if model_size in THIRD_EXPANSION_DIM_MODEL_SIZES: return 1024 @@ -626,6 +1029,8 @@ def _print_chat_help() -> None: print(" /exit_entropy Set v50 adaptive-compute entropy exit threshold") print(" /stability Stable prediction cycles required (0 disables)") print(" /stability_tol Maximum confidence drift across the stable streak") + print(" /stability_margin Minimum top-two probability margin for stable exit") + print(" /stability_rank_depth Ordered prediction ranks verified (0 disables)") print(" /top Show top reranked candidates each turn (0 disables)") print(" /timing on|off Toggle per-turn timing output") print(" /auto_compute on|off Toggle confidence/entropy based cycle selection") @@ -687,6 +1092,18 @@ def main(): default=DEFAULT_PREDICTION_STABILITY_TOL, help="Maximum confidence drift allowed across a stable prediction streak.", ) + ap.add_argument( + "--prediction_stability_margin", + type=float, + default=DEFAULT_PREDICTION_STABILITY_MARGIN, + help="Minimum full-output top-two probability margin required for stable exit.", + ) + ap.add_argument( + "--prediction_stability_rank_depth", + type=int, + default=DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + help="Ordered prediction ranks required to remain stable (0 disables the rank verifier).", + ) ap.add_argument("--device", default="auto") ap.add_argument( "--device_preference", @@ -879,15 +1296,7 @@ def main(): if missing or unexpected: raise RuntimeError(f"State dict mismatch. Missing={missing}, Unexpected={unexpected}") - buckets_raw = meta.get("buckets", {}) - buckets: Dict[int, List[Dict]] = {} - for k, v in buckets_raw.items(): - try: - label = int(k) - except ValueError: - continue - if isinstance(v, list) and v: - buckets[label] = v + buckets = _parse_metadata_buckets(meta.get("buckets", {})) available_labels = sorted(buckets.keys()) if not available_labels: @@ -942,6 +1351,14 @@ def main(): args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL, ) + args.prediction_stability_margin = _coerce_prediction_stability_margin( + args.prediction_stability_margin, + DEFAULT_PREDICTION_STABILITY_MARGIN, + ) + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + args.prediction_stability_rank_depth, + DEFAULT_PREDICTION_STABILITY_RANK_DEPTH, + ) print(f"\n{TerminalColors.SYSTEM}--- Session Info ---") print(f"Loaded: {Path(args.weights).name} [{resolved_model_size}] | Available labels: {len(available_labels)}") @@ -956,6 +1373,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) @@ -1011,6 +1430,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'}" ) print( @@ -1028,8 +1449,7 @@ def main(): compact_compute = dict(last_compute_metrics) plan = compact_compute.pop("auto_compute_plan", None) if isinstance(plan, dict): - compact_compute["auto_selected"] = plan.get("selected_reasoning_cycles") - compact_compute["auto_reason"] = plan.get("reason") + compact_compute.update(compact_auto_compute_plan_metrics(plan)) print( " last_compute=" + ", ".join(f"{k}={v}" for k, v in compact_compute.items()) @@ -1052,6 +1472,8 @@ def main(): f"exit_entropy={_coerce_nonnegative_float(args.adaptive_exit_entropy, DEFAULT_ADAPTIVE_EXIT_ENTROPY):.4g} " f"stability={_coerce_nonnegative_int(args.prediction_stability_patience, DEFAULT_PREDICTION_STABILITY_PATIENCE)} " f"stability_tol={_coerce_nonnegative_float(args.prediction_stability_tol, DEFAULT_PREDICTION_STABILITY_TOL):.4g} " + f"stability_margin={_coerce_prediction_stability_margin(args.prediction_stability_margin, DEFAULT_PREDICTION_STABILITY_MARGIN):.4g} " + f"stability_rank_depth={_coerce_prediction_stability_rank_depth(args.prediction_stability_rank_depth)} " f"auto={'on' if args.auto_compute else 'off'} " f"auto_conf={float(args.auto_compute_confidence):.3f} " f"auto_entropy={float(args.auto_compute_entropy):.3f}" @@ -1138,6 +1560,26 @@ def main(): ) except Exception: print(f"{TerminalColors.SYSTEM}Usage: /stability_tol {TerminalColors.RESET}") + elif cmd == "stability_margin": + try: + args.prediction_stability_margin = _coerce_prediction_stability_margin( + arg, + args.prediction_stability_margin, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability margin set to " + f"{float(args.prediction_stability_margin):.4g}.{TerminalColors.RESET}" + ) + except Exception: + print(f"{TerminalColors.SYSTEM}Usage: /stability_margin {TerminalColors.RESET}") + elif cmd == "stability_rank_depth": + args.prediction_stability_rank_depth = _coerce_prediction_stability_rank_depth( + arg, + ) + print( + f"{TerminalColors.SYSTEM}Prediction stability rank depth set to " + f"{int(args.prediction_stability_rank_depth)}.{TerminalColors.RESET}" + ) elif cmd == "top": try: args.show_top_responses = max(0, int(arg)) @@ -1244,35 +1686,40 @@ def main(): effective_reasoning_cycles = args.reasoning_cycles auto_compute_plan = None if args.auto_compute and model_supports_runtime_compute(model): - auto_rows = evaluate_runtime_compute_budgets( + model_out, last_compute_metrics, auto_compute_plan = progressive_auto_compute_forward( model, x, available_labels, cycles=runtime_auto_compute_cycles(args.reasoning_cycles), - adaptive_compute=args.adaptive_compute, - exit_tol=args.adaptive_exit_tol, - ) - auto_compute_plan = select_auto_runtime_compute_budget( - auto_rows, confidence_target=args.auto_compute_confidence, entropy_target=args.auto_compute_entropy, - ) - effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") - with torch.no_grad(): - model_out, last_compute_metrics = forward_with_runtime_compute( - model, - x, - reasoning_cycles=effective_reasoning_cycles, adaptive_compute=args.adaptive_compute, exit_tol=args.adaptive_exit_tol, exit_entropy_threshold=args.adaptive_exit_entropy, prediction_stability_patience=args.prediction_stability_patience, prediction_stability_tol=args.prediction_stability_tol, auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, ) - logits = model_out[0, 0] # (10,) - if auto_compute_plan is not None: - last_compute_metrics["auto_compute_plan"] = auto_compute_plan + effective_reasoning_cycles = auto_compute_plan.get("selected_reasoning_cycles") + else: + with torch.no_grad(): + model_out, last_compute_metrics = forward_with_runtime_compute( + model, + x, + reasoning_cycles=effective_reasoning_cycles, + adaptive_compute=args.adaptive_compute, + exit_tol=args.adaptive_exit_tol, + exit_entropy_threshold=args.adaptive_exit_entropy, + prediction_stability_patience=args.prediction_stability_patience, + prediction_stability_tol=args.prediction_stability_tol, + auto_reasoning_context=context, + prediction_stability_margin=args.prediction_stability_margin, + prediction_class_indices=available_labels, + prediction_stability_rank_depth=args.prediction_stability_rank_depth, + ) + logits = model_out[0, 0] # (10,) t_infer += max(0.0, time.perf_counter() - _t) idx = torch.tensor(available_labels, dtype=torch.long, device=logits.device) @@ -1424,9 +1871,9 @@ def main(): plan = last_compute_metrics.get("auto_compute_plan") if isinstance(last_compute_metrics, dict) else None plan_suffix = "" if isinstance(plan, dict): - plan_suffix = ( - f" auto_selected={plan.get('selected_reasoning_cycles')} " - f"reason={plan.get('reason')}" + plan_metrics = compact_auto_compute_plan_metrics(plan) + plan_suffix = " " + " ".join( + f"{key}={value}" for key, value in plan_metrics.items() ) print( f"{TerminalColors.SYSTEM}Compute:{TerminalColors.RESET} " diff --git a/source/chat_web_app.py b/source/chat_web_app.py index f212a12e..6ca660b1 100644 --- a/source/chat_web_app.py +++ b/source/chat_web_app.py @@ -100,8 +100,10 @@
-
-
+
+
Verify the ordered top ranks; 0 intentionally disables rank verification.
+
+
Loading status...
Shadow registry - read only
Not loaded. Execution, activation, and promotion are unavailable.
@@ -114,7 +116,7 @@