diff --git a/edge/__pycache__/edge_server.cpython-312.pyc b/edge/__pycache__/edge_server.cpython-312.pyc deleted file mode 100644 index e0bea39..0000000 Binary files a/edge/__pycache__/edge_server.cpython-312.pyc and /dev/null differ diff --git a/pipeline/config.py b/pipeline/config.py index d0be3a2..7bb158c 100644 --- a/pipeline/config.py +++ b/pipeline/config.py @@ -156,6 +156,7 @@ def require_mimic() -> None: RPT_TOOL_GATE_PIVOT = REPORTS / "tool_gate_pivot.json" RPT_TOOL_TARGETS = REPORTS / "tool_target_candidates.json" RPT_TOOL_CAUSAL_PARITY = REPORTS / "tool_causal_parity.json" +RPT_TOOL_VRAM_PROBE = REPORTS / "tool_vram_probe.json" # -------------------------------------------------------------------------- # Keep every temporary file on D:. The C: drive is tight (~13 GB), and a @@ -604,6 +605,53 @@ def _flag(name: str, default: bool) -> bool: LLM_REVISION = os.getenv("PM_LLM_REVISION", "main") LLM_QUANT = os.getenv("PM_LLM_QUANT", "nf4") # nf4 | none LLM_PREFLIGHT_MODEL = "Qwen/Qwen2.5-0.5B-Instruct" # ~350 MB toolchain probe +# WHERE THE EMBEDDING TABLE LIVES ONCE LOADED. +# +# This model unties its embeddings (`tie_word_embeddings: false`) over a 152,064 +# token vocabulary, so `embed_tokens` and `lm_head` are 545 M parameters EACH and +# bitsandbytes quantises neither -- `nn.Embedding` is not `nn.Linear`, and an +# untied `lm_head` is what `get_keys_to_not_convert` returns. 1040 MiB apiece, in +# 16-bit, on the scarcest resource in the machine. Only the embedding is a +# candidate to move: `lm_head` runs a 152,064-way matmul every decode step. +# +# ⚠️ `cpu` PAYS, BUT NOT WHERE IT LOOKS LIKE IT DOES. Moving the table frees +# 1039 MiB in torch's books and returns only 130 MiB to the driver -- the rest is +# a hole inside a partially-used segment, and `empty_cache()` cannot hand that +# back without `expandable_segments`, a no-op on Windows. Measured 2026-09-07: +# +# after load driver_free=1022 alloc=5309 reserved=6050 +# after emb.to(cpu) driver_free=1024 alloc=4270 reserved=6050 +# after empty_cache driver_free=1154 alloc=4270 reserved=5920 +# +# Read straight after the load that says "130 MiB, not worth it". It is the wrong +# place to read. The 909 MiB stays as REUSABLE ARENA: generation allocates its KV +# cache and activations inside it instead of asking the driver for new segments, +# so the card ends a generation with room on it. Three loads each way: +# +# free after generation generation +# embed=cuda 119 / 97 / 117 MiB 25.6 / 17.8 / 21.6 s +# embed=cpu 770 / 692 / 678 MiB 14.4 / 13.2 / 13.3 s +# +# ~600 MiB more headroom and generation 37% faster, because generation time swings +# ~5x with what is left on the card and this is what leaves something on it. Output +# is byte-identical across both placements -- one sha256 over six runs -- which is +# what makes the swap free rather than a quality trade. +# +# ⚠️ It does NOT move the load PEAK, so it does not lower the VRAM gate: the table +# is on the card while `from_pretrained` runs whatever happens to it afterwards. +LLM_EMBED_DEVICE = os.getenv("PM_LLM_EMBED_DEVICE", "cpu") # cpu | cuda +#: Free VRAM the 7B needs, from the DRIVER, before a load may start. One +#: definition, two readers: `s19_generate`'s capability check and the demo +#: service's `explanation.generator()`. It lived in the demo service alone until +#: 2026-09-07, which left the pipeline stage asserting `vram_free_gb > 5.5` -- +#: a figure that had drifted below a level already known to segfault. +#: +#: MEASURED, not bracketed. `tools/vram_probe.py` samples the driver at 5 Hz +#: across the load; three consecutive runs peaked at 6059, 6171 and 6239 MiB, +#: spread 180. 6420 = max peak + spread. The full table, the accepted risk and +#: the reason the peak does not move with LLM_EMBED_DEVICE are on +#: `MIN_FREE_VRAM_MIB` in `pulsemind_demo/back-end/pythonService/explanation.py`. +LLM_MIN_FREE_VRAM_MIB = 6420 LLM_MAX_NEW_TOKENS = 220 # Greedy, fixed seed. Non-determinism would make a prompt change unattributable, # and attributing changes is the entire purpose of the grounding checker. diff --git a/pipeline/core/generate.py b/pipeline/core/generate.py index 3e8b68b..37d4082 100644 --- a/pipeline/core/generate.py +++ b/pipeline/core/generate.py @@ -69,11 +69,103 @@ def _quant_config(quant: str): bnb_4bit_use_double_quant=True) +def _host_embedding(model, compute_device) -> None: + """Move the embedding table into host RAM and keep it there. + + ⚠️ NOT VIA `device_map`, and the reason matters. + + `{"model.embed_tokens": "cpu", "": 0}` resolves correctly -- accelerate walks + a parameter name from the longest prefix down (`utils/modeling.py:1991`) -- so + it looks exactly like the right answer. It is not. In a map whose main device + is a GPU, `cpu` means OFFLOAD, not "run there": + + offloaded_devices = ["disk"] if main_device == "cpu" ... else ["cpu", "disk"] + offload = {name: device in offloaded_devices ...} # big_modeling.py:405 + + The module is left on `meta`, its execution device is set to the GPU, and + `AlignDevicesHook` streams the weight ONTO the card for every forward and + frees it after. For a 1040 MiB table across a prefill and 220 decode steps + that is the wrong trade twice over: the copies dominate generation, and the + table is resident again at exactly the moment the card is fullest. Measured + here it did not finish loading a 0.5B in seven minutes. + + So: load normally, then move the module and let two hooks handle the crossing. + Public torch API, no accelerate internals, and the table crosses the bus once + at startup instead of once per token. + + What travels instead is the ACTIVATIONS -- `seq_len x 3584 x 2` bytes on the + prefill, 7 KB per decode step. Three orders of magnitude less. + + ⚠️ This lowers what the model HOLDS, not what the load PEAKS at. The table is + on the card while `from_pretrained` runs, so the VRAM gate -- which protects a + load -- cannot come down on the strength of this alone. + + ⚠️ AND THE DRIVER BARELY NOTICES, WHICH IS NOT THE SAME AS IT NOT WORKING. + Measured: torch's `allocated` falls by 1039 MiB and the driver gets 130 back, + because a freed block inside a partially-used segment needs + `expandable_segments` to be returned and that is a no-op on Windows. Read + there and this looks worthless. The other 909 MiB is REUSABLE ARENA: + generation puts its KV cache and activations inside it instead of asking the + driver for new segments, so the card ends a generation with room on it. + Three loads each way -- `embed=cuda` finished with 119/97/117 MiB free in + 25.6/17.8/21.6 s; `embed=cpu` with 770/692/678 MiB free in 14.4/13.2/13.3 s. + Generation swings ~5x with what is left on the card, and this is what leaves + something on it. + + ⚠️ AFTER THIS RUNS, `model.device` IS A LIE. `PreTrainedModel.device` reports + the device of the FIRST parameter, and `embed_tokens` is the first parameter, + so a model whose compute lives entirely on the GPU starts answering "cpu". + Anything that sends inputs to `model.device` then sends them to the host: the + ids land on the CPU, `cache_position` and `position_ids` are derived from + THEIR device, and the rotary embedding dies on `mat2 is on cpu`. Measured -- + the load succeeded and generation failed on the first forward. + + Hence `compute_device`, threaded explicitly and carried on `Generator`. The + load knows the answer before the move makes the question ambiguous. + """ + import torch + + emb = model.get_input_embeddings() + out = model.get_output_embeddings() + + # ⚠️ TIED WEIGHTS SHARE ONE STORAGE, so moving the input embedding moves + # `lm_head` with it -- silently, because they are the same Parameter object. + # `lm_head` runs a 152,064-way matmul every decode step and would then run it + # on the CPU. This model unties (`tie_word_embeddings: false`), but the + # documented fallback in config is "a 3B model", and every smaller Qwen2.5 + # ties. That is a reachable path, so it refuses rather than degrades. + if out is not None and out.weight is emb.weight: + # Named from the LOADED model, not from config: they agree in production + # and diverge exactly when someone is testing an override, which is the + # one moment a wrong model name in the message costs real time. + raise RuntimeError( + f"{getattr(model, 'name_or_path', None) or C.LLM_MODEL_ID} ties its " + f"input and output embeddings, so moving the table would put lm_head " + f"on the CPU as well. Set PM_LLM_EMBED_DEVICE=cuda for this model.") + + emb.to("cpu") + # The ids are built on the GPU by `generate`; the gather has to happen beside + # the table, and the activations have to come back. + emb.register_forward_pre_hook(lambda _m, args: (args[0].to("cpu"),) + args[1:]) + emb.register_forward_hook( + lambda _m, _args, out: out.to(compute_device, non_blocking=True)) + # Without this the allocator keeps the 1040 MiB and the driver still reports + # it used -- the saving would be real in torch's books and invisible in the + # only figure the gate reads. + torch.cuda.empty_cache() + + @dataclass(slots=True) class Generator: model: object tokenizer: object provenance: dict + #: Where the compute lives. NOT `model.device` -- with the embedding table on + #: the host that property reports the first parameter's device, which is the + #: host, and every input would follow it there. Defaults to None so an older + #: caller constructing a Generator positionally still works; `__call__` then + #: falls back to the old behaviour, which is correct whenever nothing moved. + device: object = None max_new_tokens: int = C.LLM_MAX_NEW_TOKENS latencies: list = field(default_factory=list) @@ -82,7 +174,8 @@ def __call__(self, payload: dict) -> str: msgs = E.render_prompt(payload) enc = self.tokenizer.apply_chat_template( msgs, add_generation_prompt=True, tokenize=True, - return_dict=True, return_tensors="pt").to(self.model.device) + return_dict=True, return_tensors="pt").to( + self.device if self.device is not None else self.model.device) n_in = enc["input_ids"].shape[1] t0 = time.time() with torch.inference_mode(): @@ -96,6 +189,49 @@ def __call__(self, payload: dict) -> str: return self.tokenizer.decode(out[0][n_in:], skip_special_tokens=True).strip() +def vram_status() -> dict: + """Free and total VRAM, from the driver rather than from CUDA's view of it. + + ⚠️ `torch.cuda.mem_get_info()` reports what the CUDA runtime believes a new + allocation could get, which on this WDDM setup is not what the device + actually has free. Measured 2026-08-19 with the 7B already resident: + + nvidia-smi 260 MiB free of 8151 + torch 6759 MiB free of 8151 + + A 6.3 GB disagreement, and torch is the optimistic one. That is precisely how + a preflight passes and the load then segfaults -- `s19_generate` gates on + `vram_free_gb > 5.5`, which torch would have cleared with a quarter of a + gigabyte actually available. NVML talks to the driver; CUDA talks to its own + bookkeeping. Prefer the driver, and say which one answered. + + Totals agree exactly and always did: 8151 MiB is 7.96 GiB is 8.55 decimal GB. + The card is an 8 GB card; every other figure in the notes was that same + number with its units mangled. + """ + import shutil + import subprocess + + exe = shutil.which("nvidia-smi") + if exe: + try: + raw = subprocess.run( + [exe, "--query-gpu=memory.total,memory.free", + "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=10, check=True).stdout + total_mib, free_mib = (int(v) for v in raw.splitlines()[0].split(",")) + return {"free_mib": free_mib, "total_mib": total_mib, "source": "nvidia-smi"} + except Exception: # noqa: BLE001 + pass # fall through -- an unreadable driver is not a reason to stop + + import torch + free, total = torch.cuda.mem_get_info() + return {"free_mib": round(free / 1024**2), "total_mib": round(total / 1024**2), + # Named so it shows up in the report: this number is the optimistic + # one, and a reader should know the driver was not available. + "source": "torch.cuda.mem_get_info (nvidia-smi unavailable)"} + + def capability_check() -> dict: """The cheap check the stage runs every time: is the toolchain present and is there room? @@ -130,9 +266,15 @@ def capability_check() -> dict: f"HF_HUB_CACHE) explicitly before importing transformers.") p = torch.cuda.get_device_properties(0) - free, total = torch.cuda.mem_get_info() + # From the driver, not from CUDA's bookkeeping -- see `vram_status`. The + # `_gb` keys keep their names and decimal-GB units because `s19_generate` + # gates on `vram_free_gb`; what changed is that they are now true. + vram = vram_status() out = {"gpu": p.name, "compute_capability": f"sm_{p.major}{p.minor}", - "vram_free_gb": round(free / 1e9, 2), "vram_total_gb": round(total / 1e9, 2), + "vram_free_gb": round(vram["free_mib"] * 1024**2 / 1e9, 2), + "vram_total_gb": round(vram["total_mib"] * 1024**2 / 1e9, 2), + "vram_free_mib": vram["free_mib"], "vram_total_mib": vram["total_mib"], + "vram_source": vram["source"], "hub_cache": str(hub), "quantisation": C.LLM_QUANT, "torch": torch.__version__} if C.LLM_QUANT != "none": @@ -187,6 +329,28 @@ def load_generator() -> Generator: C.LLM_MODEL_ID, revision=C.LLM_REVISION, quantization_config=_quant_config(C.LLM_QUANT), device_map={"": 0}) model.eval() + + # Read BEFORE anything moves: every parameter is still on the card here, so + # this is unambiguous. After `_host_embedding` it would not be. + compute_device = model.device + + if C.LLM_EMBED_DEVICE == "cpu": + _host_embedding(model, compute_device) + elif C.LLM_EMBED_DEVICE != "cuda": + raise ValueError( + f"unknown LLM_EMBED_DEVICE {C.LLM_EMBED_DEVICE!r}; expected 'cpu' or 'cuda'") + + # WHERE THE EMBEDDING ACTUALLY LANDED, asserted rather than assumed. + # + # A move that silently failed produces a model that loads, generates + # correctly, and quietly holds the 1040 MiB this setting exists to free. A + # saving that did not happen must not look like one that did. + embed = model.get_input_embeddings().weight + want = "cpu" if C.LLM_EMBED_DEVICE == "cpu" else "cuda" + if embed.device.type != want: + raise RuntimeError( + f"LLM_EMBED_DEVICE={C.LLM_EMBED_DEVICE} but embed_tokens is on " + f"{embed.device}; the move did not take.") gpu = torch.cuda.get_device_properties(0) prov = { @@ -194,7 +358,18 @@ def load_generator() -> Generator: "revision": C.LLM_REVISION, "quantisation": C.LLM_QUANT, "dtype": str(model.dtype), - "device": str(model.device), + # The COMPUTE device, read before the embedding moved. `model.device` + # would now name the host and misreport where this model actually runs. + "device": str(compute_device), + # Part of model identity for the same reason `quantisation` is: it says + # which machine configuration wrote this line. It is deliberately NOT in + # `FP_GENERATE` -- an embedding lookup is a gather, and bf16 round-trips + # through the host exactly, so moving it is provably output-neutral and + # fingerprinting it would force a re-run that changes nothing. That + # argument rests on a hash: `tools/vram_probe.py` reports one per run and + # the two placements must agree. If they ever stop agreeing, this is no + # longer a placement detail and belongs in the fingerprint. + "embed_device": str(model.get_input_embeddings().weight.device), "compute_capability": f"sm_{gpu.major}{gpu.minor}", "max_new_tokens": C.LLM_MAX_NEW_TOKENS, "seed": C.LLM_SEED, @@ -205,4 +380,5 @@ def load_generator() -> Generator: if C.LLM_QUANT != "none": import bitsandbytes prov["bitsandbytes"] = bitsandbytes.__version__ - return Generator(model=model, tokenizer=tok, provenance=prov) + return Generator(model=model, tokenizer=tok, provenance=prov, + device=compute_device) diff --git a/pipeline/stages/s19_generate.py b/pipeline/stages/s19_generate.py index b758f5f..fc5b1a2 100644 --- a/pipeline/stages/s19_generate.py +++ b/pipeline/stages/s19_generate.py @@ -145,10 +145,19 @@ def _run(sample: int | None = None, with_evidence: bool = True) -> None: # context and fragmented segments behind, and the 7B load then fails on # fragmentation rather than capacity -- which is exactly what happened. # The full probe is `--preflight`, a separate command and process. - assert pf["vram_free_gb"] > 5.5, ( - f"only {pf['vram_free_gb']:.2f} GB VRAM free; a 7B in NF4 needs a " - f"~4.8 GB block plus activations. Close whatever is holding the GPU, " - f"or set PM_LLM_QUANT=none with a 3B model.") + # + # MEASURED, in the same unit the driver reports. The old form asserted + # `vram_free_gb > 5.5`, written when the figure came from + # `torch.cuda.mem_get_info()` and was optimistic by gigabytes; once + # `vram_status()` started telling the truth, 5.5 GB (5245 MiB) sat well + # BELOW a level that had already segfaulted, so the gate protected + # nothing. The peak is now 6239 MiB across three runs -- see + # `tools/vram_probe.py` and `reports/tool_vram_probe.json`. + assert pf["vram_free_mib"] >= C.LLM_MIN_FREE_VRAM_MIB, ( + f"only {pf['vram_free_mib']} MiB VRAM free of " + f"{pf['vram_total_mib']}; the 7B peaks at ~6239 MiB and this stage " + f"needs {C.LLM_MIN_FREE_VRAM_MIB}. Close whatever is holding the GPU, or " + f"set PM_LLM_QUANT=none with a 3B model.") with stage(f"Load {C.LLM_MODEL_ID}"): gen = G.load_generator() diff --git a/pipeline/tools/check_env.py b/pipeline/tools/check_env.py index c90fa5a..42de025 100644 --- a/pipeline/tools/check_env.py +++ b/pipeline/tools/check_env.py @@ -54,9 +54,12 @@ def main() -> None: if ok: name = torch.cuda.get_device_name(0) cap = torch.cuda.get_device_capability(0) - free, total = torch.cuda.mem_get_info(0) + # Driver figures, not CUDA's -- torch over-reports free VRAM on this + # WDDM setup by gigabytes. See pipeline/core/generate.vram_status. + from pipeline.core.generate import vram_status + v = vram_status() log(f"[green]PyTorch CUDA : OK[/green] {name} sm_{cap[0]}{cap[1]} | " - f"{free/1e9:.1f} GB free of {total/1e9:.1f} GB") + f"{v['free_mib']} MiB free of {v['total_mib']} ({v['source']})") t = torch.randn(4096, 4096, device="cuda") torch.cuda.synchronize() log(f" matmul check: {float((t @ t).sum()):.1f}") diff --git a/pipeline/tools/vram_probe.py b/pipeline/tools/vram_probe.py new file mode 100644 index 0000000..e869f00 --- /dev/null +++ b/pipeline/tools/vram_probe.py @@ -0,0 +1,306 @@ +"""Measure what a 7B load actually PEAKS at, instead of bracketing it. + +WHY THIS EXISTS +--------------- +`explanation.MIN_FREE_VRAM_MIB` was set from two data points -- a segfault at +6561 MiB free and a success at 6721 -- and the constant's own comment says the +value chosen inside that band is "slightly optimistic". Nobody had watched the +card DURING a load, so the peak had never been measured, only inferred from +whether the process survived. + +That is an expensive way to learn a number. Each observation costs a 40 s load +and roughly half of them are crashes. + +WHAT IT MEASURES, AND WHY IN THAT UNIT +-------------------------------------- +`peak = free_before - min(free) observed across the run`, sampled from +`nvidia-smi` at 5 Hz. Deliberately the SAME source and the SAME unit the gate +compares against, because that is the only way the answer is directly usable: a +peak in torch's units would need converting through a bookkeeping layer that +`generate.vram_status()` documents as wrong by gigabytes on this WDDM setup. + +`torch.cuda.max_memory_allocated/reserved()` are recorded beside it as a +DECOMPOSITION, never as the answer. They see only torch's own allocations, so +they miss the CUDA context entirely. + +⚠️ THE SAMPLER RUNS IN THE PARENT +--------------------------------- +A load that does not fit does not raise, it SEGFAULTS -- no exception, no +unwinding, nothing in the child runs afterwards. A sampler started by the child +is therefore orphaned by exactly the outcome most worth recording, and keeps +writing to a file nobody will close. Measured: three leaked `nvidia-smi` +processes across three failed runs. + +So the parent owns the sampler and the trace file. The child reports its own +phase boundaries as epoch timestamps when it survives; when it does not, the +whole span is the load, which is the only window a crash has anyway. **A crash +is a measurement here**, and this is what makes it readable. + +⚠️ ONE LOAD PER PROCESS +----------------------- +`load_generator()` is never unloaded -- `explanation.py` caches it in a module +global and `model_runtime` documents that releasing it dies 0xC0000409. N loads +in one process would mean N models on an 8 GB card, so every repeat is a fresh +subprocess and `--once` is the inner half of that handoff. + +WHAT IT CHANGES +--------------- +Nothing. It writes one report and loads the model the way the service would. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics +import subprocess +import sys +import time +from datetime import datetime + +from .. import config as C +from ..common import log + +REPORT_JSON = C.RPT_TOOL_VRAM_PROBE + +#: Sampling period for the driver poll. 5 Hz against a load that takes 28-45 s +#: is 150-225 samples -- dense enough to catch a transient lasting a fraction of +#: a second, and cheap enough that the sampler is not itself load. One +#: long-running `nvidia-smi`, NOT a spawn per sample: a spawn costs ~50-100 ms +#: here, which would sit inside the window it is trying to measure. +SAMPLE_MS = 200 + +#: nvidia-smi's own timestamp format, so samples can be bucketed into the load +#: window and the generation window rather than reported as one blur. +TS_FMT = "%Y/%m/%d %H:%M:%S.%f" + + +def _start_sampler(path): + """`nvidia-smi` streaming timestamped free-VRAM to `path`.""" + path.parent.mkdir(parents=True, exist_ok=True) + handle = path.open("w", encoding="utf-8") + proc = subprocess.Popen( + ["nvidia-smi", "--query-gpu=timestamp,memory.free", + "--format=csv,noheader,nounits", "-lms", str(SAMPLE_MS)], + stdout=handle, stderr=subprocess.DEVNULL, text=True) + proc._pm_handle = handle + return proc + + +def _stop_sampler(proc, path) -> list[tuple[float, int]]: + """Stop it and read back `(epoch_seconds, free_mib)`. Idempotent.""" + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + if not proc._pm_handle.closed: + proc._pm_handle.close() + + out = [] + for line in path.read_text(encoding="utf-8").splitlines(): + ts, _, free = line.partition(",") + if not free.strip(): + continue + try: + out.append((datetime.strptime(ts.strip(), TS_FMT).timestamp(), int(free))) + except ValueError: + continue # a partially written final line is not a failure + return out + + +def _window(samples, t0: float | None, t1: float | None) -> dict: + """Summarise the samples inside a phase. Reports `n` so a window that caught + nothing is visible as such rather than as a missing key.""" + if t0 is None or t1 is None: + return {"n": 0, "min_free_mib": None, "max_free_mib": None} + got = [mib for when, mib in samples if t0 <= when <= t1] + if not got: + return {"n": 0, "min_free_mib": None, "max_free_mib": None} + return {"n": len(got), "min_free_mib": min(got), "max_free_mib": max(got)} + + +# ---------------------------------------------------------------- the child + + +def once(sample: int, record_index: int, with_evidence: bool) -> dict: + """One load and one generation, in a process that has never touched CUDA. + + Imports are function-local on purpose: `free_before` has to be read before + torch initialises a context, or the context is inside the figure the peak is + being measured against. + + Reports epoch timestamps rather than durations so the parent can bucket its + own samples into these phases. + """ + from ..core import generate as G + + before = G.vram_status() + + from ..core import explain as E + from ..stages.s18_explain import policy + from ..stages.s19_generate import sample_records + + pol = policy(with_evidence=with_evidence) + recs = sample_records(pol, sample) + if not recs: + raise RuntimeError("no records cleared the sufficiency floor; nothing to generate") + rec = recs[record_index % len(recs)] + + t_load0 = time.time() + gen = G.load_generator() + t_load1 = time.time() + after_load = G.vram_status() + + import torch + torch_peak = {"max_allocated_mib": round(torch.cuda.max_memory_allocated() / 1024**2), + "max_reserved_mib": round(torch.cuda.max_memory_reserved() / 1024**2)} + + emb = gen.model.get_input_embeddings().weight + placement = {"embed_device": str(emb.device), "embed_dtype": str(emb.dtype), + "embed_mib": round(emb.numel() * emb.element_size() / 1024**2)} + + t_gen0 = time.time() + blk = E.explain(rec, pol, generator=gen, generator_name=C.LLM_MODEL_ID) + t_gen1 = time.time() + after_gen = G.vram_status() + + # PM-LOG-003: the prose never leaves this process. The hash is what makes + # "byte-identical after the change" checkable without storing patient text. + text = blk["text"] + return { + "free_before_mib": before["free_mib"], + "free_after_load_mib": after_load["free_mib"], + "free_after_generate_mib": after_gen["free_mib"], + "total_mib": before["total_mib"], + "vram_source": before["source"], + "resident_mib": before["free_mib"] - after_load["free_mib"], + "t_load": [t_load0, t_load1], + "t_generate": [t_gen0, t_gen1], + "load_seconds": round(t_load1 - t_load0, 1), + "generate_seconds": round(t_gen1 - t_gen0, 1), + "torch": torch_peak, + "placement": placement, + "text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), + "text_chars": len(text), + "provenance": gen.provenance, + } + + +# ---------------------------------------------------------------- the parent + + +def _one_run(i: int, sample: int, record_index: int, with_evidence: bool) -> dict: + """Sample the card in this process while a child does the load.""" + trace = C.BUILD / "scratch" / f"vram_probe_{int(time.time() * 1000)}.csv" + sampler = _start_sampler(trace) + time.sleep(1.0) # a baseline before the child touches anything + try: + proc = subprocess.run( + [sys.executable, "-m", "pipeline.tools.vram_probe", "--once", + "--sample", str(sample), "--record-index", str(record_index)] + + ([] if with_evidence else ["--no-evidence"]), + capture_output=True, text=True) + samples = _stop_sampler(sampler, trace) + finally: + _stop_sampler(sampler, trace) + + span_min = min((m for _, m in samples), default=None) + span_max = max((m for _, m in samples), default=None) + + if proc.returncode != 0 or not proc.stdout.strip(): + # A CRASH IS A MEASUREMENT. It is the only evidence that narrows the + # requirement from below, so it is recorded with its trace rather than + # raised. `Segmentation fault` arrives on the shell's stderr, not the + # child's, so returncode is the only reliable signal. + trace.unlink(missing_ok=True) + return {"run": i, "ok": False, + "error": f"exit {proc.returncode}", + "start_free_mib": span_max, "min_free_mib": span_min, + "consumed_before_death_mib": (span_max - span_min) + if span_max is not None else None, + "n_samples": len(samples), + "stderr_tail": proc.stderr.strip()[-600:]} + + row = json.loads(proc.stdout.strip().splitlines()[-1]) + row["run"], row["ok"] = i, True + row["load_window"] = _window(samples, *row.pop("t_load")) + row["generate_window"] = _window(samples, *row.pop("t_generate")) + row["n_samples"] = len(samples) + lo = row["load_window"]["min_free_mib"] + row["peak_mib"] = (row["free_before_mib"] - lo) if lo is not None else None + row["peak_over_resident_mib"] = ((row["peak_mib"] - row["resident_mib"]) + if row["peak_mib"] is not None else None) + trace.unlink(missing_ok=True) + return row + + +def main(runs: int = 5, sample: int = 4, record_index: int = 0, + with_evidence: bool = True, tag: str = "") -> None: + rows = [] + for i in range(1, runs + 1): + log(f"run {i}/{runs} -- fresh process, one load") + row = _one_run(i, sample, record_index, with_evidence) + rows.append(row) + if row["ok"]: + log(f" peak {row['peak_mib']} MiB resident {row['resident_mib']} MiB " + f"load {row['load_seconds']}s gen {row['generate_seconds']}s " + f"embed on {row['placement']['embed_device']}") + else: + log(f" [red]{row['error']}[/red] -- consumed " + f"{row['consumed_before_death_mib']} MiB before dying, " + f"floor {row['min_free_mib']} MiB") + + ok = [r for r in rows if r["ok"] and r.get("peak_mib") is not None] + peaks = [r["peak_mib"] for r in ok] + residents = [r["resident_mib"] for r in ok] + hashes = sorted({r["text_sha256"] for r in ok}) + spread = (max(peaks) - min(peaks)) if len(peaks) > 1 else 0 + + report = { + "tag": tag or "untagged", + "embed_device": C.LLM_EMBED_DEVICE, + "model": C.LLM_MODEL_ID, "quantisation": C.LLM_QUANT, + "n_ok": len(ok), "n_failed": len(rows) - len(ok), + "peak_mib": {"max": max(peaks) if peaks else None, + "min": min(peaks) if peaks else None, + "median": statistics.median(peaks) if peaks else None, + "spread": spread}, + "resident_mib": {"max": max(residents) if residents else None, + "min": min(residents) if residents else None, + "spread": (max(residents) - min(residents)) + if len(residents) > 1 else 0}, + # One hash across every run is the greedy-decoding claim, measured. More + # than one means the generator is not deterministic and every + # before/after text comparison is void. + "text_sha256": hashes, + "deterministic": len(hashes) == 1 if hashes else None, + "gate_now_mib": 6700, + "runs": rows, + } + if peaks: + # The gate has to cover the peak AND the drift between runs: background + # software moves this card by hundreds of MiB during a 40 s load, and + # that drift is what made 6561 both a crash and a success. + report["suggested_gate_mib"] = max(peaks) + max(spread, 128) + REPORT_JSON.parent.mkdir(parents=True, exist_ok=True) + REPORT_JSON.write_text(json.dumps(report, indent=2, default=str)) + log(f"wrote {REPORT_JSON}") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--once", action="store_true", + help="inner half of the subprocess handoff: one load, JSON to stdout") + ap.add_argument("-n", "--runs", type=int, default=5) + ap.add_argument("--sample", type=int, default=4, + help="passed to sample_records; seeded, so a fixed value fixes the record") + ap.add_argument("--record-index", type=int, default=0) + ap.add_argument("--no-evidence", action="store_true") + ap.add_argument("--tag", default="", help="names the run in the report") + a = ap.parse_args() + if a.once: + print(json.dumps(once(a.sample, a.record_index, not a.no_evidence))) + else: + main(a.runs, a.sample, a.record_index, not a.no_evidence, a.tag) diff --git a/reports/tool_vram_probe.json b/reports/tool_vram_probe.json new file mode 100644 index 0000000..65cd1fc --- /dev/null +++ b/reports/tool_vram_probe.json @@ -0,0 +1,421 @@ +{ + "measured": "2026-09-07", + "what": "Free VRAM sampled from nvidia-smi at 5 Hz across a 7B NF4 load and one generation. peak = free_before - min(free) during the load window.", + "model": "Qwen/Qwen2.5-7B-Instruct", + "quantisation": "nf4", + "arms": { + "embed_cpu": { + "tag": "embed_cpu", + "embed_device": "cpu", + "model": "Qwen/Qwen2.5-7B-Instruct", + "quantisation": "nf4", + "n_ok": 3, + "n_failed": 0, + "peak_mib": { + "max": 6239, + "min": 6059, + "median": 6171, + "spread": 180 + }, + "resident_mib": { + "max": 6109, + "min": 5929, + "spread": 180 + }, + "text_sha256": [ + "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a" + ], + "deterministic": true, + "gate_now_mib": 6700, + "runs": [ + { + "free_before_mib": 6795, + "free_after_load_mib": 866, + "free_after_generate_mib": 770, + "total_mib": 8151, + "vram_source": "nvidia-smi", + "resident_mib": 5929, + "load_seconds": 16.7, + "generate_seconds": 14.4, + "torch": { + "max_allocated_mib": 5739, + "max_reserved_mib": 6050 + }, + "placement": { + "embed_device": "cpu", + "embed_dtype": "torch.float16", + "embed_mib": 1040 + }, + "text_sha256": "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a", + "text_chars": 723, + "provenance": { + "model": "Qwen/Qwen2.5-7B-Instruct", + "revision": "main", + "quantisation": "nf4", + "dtype": "torch.float16", + "device": "cuda:0", + "embed_device": "cpu", + "compute_capability": "sm_120", + "max_new_tokens": 220, + "seed": 20260808, + "decoding": "greedy", + "transformers": "4.57.6", + "torch": "2.11.0+cu128", + "bitsandbytes": "0.50.0" + }, + "run": 1, + "ok": true, + "load_window": { + "n": 79, + "min_free_mib": 736, + "max_free_mib": 6795 + }, + "generate_window": { + "n": 70, + "min_free_mib": 672, + "max_free_mib": 1400 + }, + "n_samples": 168, + "peak_mib": 6059, + "peak_over_resident_mib": 130 + }, + { + "free_before_mib": 7501, + "free_after_load_mib": 1392, + "free_after_generate_mib": 692, + "total_mib": 8151, + "vram_source": "nvidia-smi", + "resident_mib": 6109, + "load_seconds": 17.4, + "generate_seconds": 13.2, + "torch": { + "max_allocated_mib": 5739, + "max_reserved_mib": 6050 + }, + "placement": { + "embed_device": "cpu", + "embed_dtype": "torch.float16", + "embed_mib": 1040 + }, + "text_sha256": "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a", + "text_chars": 723, + "provenance": { + "model": "Qwen/Qwen2.5-7B-Instruct", + "revision": "main", + "quantisation": "nf4", + "dtype": "torch.float16", + "device": "cuda:0", + "embed_device": "cpu", + "compute_capability": "sm_120", + "max_new_tokens": 220, + "seed": 20260808, + "decoding": "greedy", + "transformers": "4.57.6", + "torch": "2.11.0+cu128", + "bitsandbytes": "0.50.0" + }, + "run": 2, + "ok": true, + "load_window": { + "n": 84, + "min_free_mib": 1262, + "max_free_mib": 7501 + }, + "generate_window": { + "n": 65, + "min_free_mib": 692, + "max_free_mib": 1392 + }, + "n_samples": 167, + "peak_mib": 6239, + "peak_over_resident_mib": 130 + }, + { + "free_before_mib": 7423, + "free_after_load_mib": 1382, + "free_after_generate_mib": 678, + "total_mib": 8151, + "vram_source": "nvidia-smi", + "resident_mib": 6041, + "load_seconds": 15.8, + "generate_seconds": 13.3, + "torch": { + "max_allocated_mib": 5739, + "max_reserved_mib": 6050 + }, + "placement": { + "embed_device": "cpu", + "embed_dtype": "torch.float16", + "embed_mib": 1040 + }, + "text_sha256": "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a", + "text_chars": 723, + "provenance": { + "model": "Qwen/Qwen2.5-7B-Instruct", + "revision": "main", + "quantisation": "nf4", + "dtype": "torch.float16", + "device": "cuda:0", + "embed_device": "cpu", + "compute_capability": "sm_120", + "max_new_tokens": 220, + "seed": 20260808, + "decoding": "greedy", + "transformers": "4.57.6", + "torch": "2.11.0+cu128", + "bitsandbytes": "0.50.0" + }, + "run": 3, + "ok": true, + "load_window": { + "n": 76, + "min_free_mib": 1252, + "max_free_mib": 7423 + }, + "generate_window": { + "n": 65, + "min_free_mib": 678, + "max_free_mib": 1382 + }, + "n_samples": 159, + "peak_mib": 6171, + "peak_over_resident_mib": 130 + } + ], + "suggested_gate_mib": 6419 + }, + "embed_cuda": { + "tag": "embed_cuda", + "embed_device": "cuda", + "model": "Qwen/Qwen2.5-7B-Instruct", + "quantisation": "nf4", + "n_ok": 3, + "n_failed": 0, + "peak_mib": { + "max": 6161, + "min": 6056, + "median": 6104, + "spread": 105 + }, + "resident_mib": { + "max": 6161, + "min": 6056, + "spread": 105 + }, + "text_sha256": [ + "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a" + ], + "deterministic": true, + "gate_now_mib": 6700, + "runs": [ + { + "free_before_mib": 7033, + "free_after_load_mib": 977, + "free_after_generate_mib": 119, + "total_mib": 8151, + "vram_source": "nvidia-smi", + "resident_mib": 6056, + "load_seconds": 29.1, + "generate_seconds": 25.6, + "torch": { + "max_allocated_mib": 5739, + "max_reserved_mib": 6050 + }, + "placement": { + "embed_device": "cuda:0", + "embed_dtype": "torch.float16", + "embed_mib": 1040 + }, + "text_sha256": "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a", + "text_chars": 723, + "provenance": { + "model": "Qwen/Qwen2.5-7B-Instruct", + "revision": "main", + "quantisation": "nf4", + "dtype": "torch.float16", + "device": "cuda:0", + "embed_device": "cuda:0", + "compute_capability": "sm_120", + "max_new_tokens": 220, + "seed": 20260808, + "decoding": "greedy", + "transformers": "4.57.6", + "torch": "2.11.0+cu128", + "bitsandbytes": "0.50.0" + }, + "run": 1, + "ok": true, + "load_window": { + "n": 140, + "min_free_mib": 977, + "max_free_mib": 7118 + }, + "generate_window": { + "n": 123, + "min_free_mib": 94, + "max_free_mib": 976 + }, + "n_samples": 283, + "peak_mib": 6056, + "peak_over_resident_mib": 0 + }, + { + "free_before_mib": 7176, + "free_after_load_mib": 1072, + "free_after_generate_mib": 97, + "total_mib": 8151, + "vram_source": "nvidia-smi", + "resident_mib": 6104, + "load_seconds": 17.7, + "generate_seconds": 17.8, + "torch": { + "max_allocated_mib": 5739, + "max_reserved_mib": 6050 + }, + "placement": { + "embed_device": "cuda:0", + "embed_dtype": "torch.float16", + "embed_mib": 1040 + }, + "text_sha256": "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a", + "text_chars": 723, + "provenance": { + "model": "Qwen/Qwen2.5-7B-Instruct", + "revision": "main", + "quantisation": "nf4", + "dtype": "torch.float16", + "device": "cuda:0", + "embed_device": "cuda:0", + "compute_capability": "sm_120", + "max_new_tokens": 220, + "seed": 20260808, + "decoding": "greedy", + "transformers": "4.57.6", + "torch": "2.11.0+cu128", + "bitsandbytes": "0.50.0" + }, + "run": 2, + "ok": true, + "load_window": { + "n": 85, + "min_free_mib": 1072, + "max_free_mib": 7233 + }, + "generate_window": { + "n": 87, + "min_free_mib": 97, + "max_free_mib": 1062 + }, + "n_samples": 190, + "peak_mib": 6104, + "peak_over_resident_mib": 0 + }, + { + "free_before_mib": 7242, + "free_after_load_mib": 1081, + "free_after_generate_mib": 117, + "total_mib": 8151, + "vram_source": "nvidia-smi", + "resident_mib": 6161, + "load_seconds": 18.1, + "generate_seconds": 21.6, + "torch": { + "max_allocated_mib": 5739, + "max_reserved_mib": 6050 + }, + "placement": { + "embed_device": "cuda:0", + "embed_dtype": "torch.float16", + "embed_mib": 1040 + }, + "text_sha256": "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a", + "text_chars": 723, + "provenance": { + "model": "Qwen/Qwen2.5-7B-Instruct", + "revision": "main", + "quantisation": "nf4", + "dtype": "torch.float16", + "device": "cuda:0", + "embed_device": "cuda:0", + "compute_capability": "sm_120", + "max_new_tokens": 220, + "seed": 20260808, + "decoding": "greedy", + "transformers": "4.57.6", + "torch": "2.11.0+cu128", + "bitsandbytes": "0.50.0" + }, + "run": 3, + "ok": true, + "load_window": { + "n": 87, + "min_free_mib": 1081, + "max_free_mib": 7242 + }, + "generate_window": { + "n": 106, + "min_free_mib": 74, + "max_free_mib": 1081 + }, + "n_samples": 211, + "peak_mib": 6161, + "peak_over_resident_mib": 0 + } + ], + "suggested_gate_mib": 6289 + } + }, + "peak_mib_all_runs": { + "values": [ + 6056, + 6059, + 6104, + 6161, + 6171, + 6239 + ], + "max": 6239, + "min": 6056, + "spread": 183, + "n": 6 + }, + "text_sha256": [ + "357abf7bae1264961e09a99ae49c69c838ce7195667b40411d057bf7b4080f8a" + ], + "byte_identical_across_placements": true, + "gate": { + "was": 6700, + "now": 6420, + "rule": "max peak across all runs + observed spread", + "note": "lowest-that-loads, chosen 2026-09-07; sits below the 6561 that segfaulted once, so warm the explainer before a demo" + }, + "embed_placement_finding": { + "torch_allocated_freed_mib": 1039, + "driver_returned_mib": 130, + "why": "a freed block inside a partially-used segment needs expandable_segments, which is a no-op on Windows; the remainder stays as reusable arena", + "free_after_generation_mib": { + "embed_cuda": [ + 119, + 97, + 117 + ], + "embed_cpu": [ + 770, + 692, + 678 + ] + }, + "generate_seconds": { + "embed_cuda": [ + 25.6, + 17.8, + 21.6 + ], + "embed_cpu": [ + 14.4, + 13.2, + 13.3 + ] + } + } +} \ No newline at end of file