From 017553ce80dd1eaa9c2c7a93cc9c63f6a381f128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BD=98=E8=B6=85?= Date: Tue, 11 Aug 2026 13:19:27 +0800 Subject: [PATCH] Add J-Space Contrast Lab MVP --- README.md | 14 + apps/serve/README.md | 6 + apps/serve/j7scope_serve/app.py | 56 + apps/serve/j7scope_serve/backends.py | 40 + apps/site/assets/jspace.css | 63 + apps/site/compare.html | 2 + apps/site/contrast.html | 260 ++++ apps/site/index.html | 6 +- apps/web/package-lock.json | 6 +- experiments/build_contrast.py | 79 ++ experiments/build_demo_trace.py | 20 +- j7scope/contrast.py | 313 +++++ j7scope/rigor.py | 12 +- j7scope/trace.py | 35 + .../traces/demo-narrative-en/manifest.json | 7 +- results/traces/demo-parallel-en/align.json | 1059 +++++++++++++++++ results/traces/demo-parallel-en/manifest.json | 7 +- results/traces/demo-parallel-zh/align.json | 1059 +++++++++++++++++ results/traces/demo-parallel-zh/manifest.json | 7 +- tests/test_contrast.py | 82 ++ tests/test_recorder.py | 35 + tests/test_trace_gallery.py | 16 + 22 files changed, 3167 insertions(+), 17 deletions(-) create mode 100644 apps/site/contrast.html create mode 100644 experiments/build_contrast.py create mode 100644 j7scope/contrast.py create mode 100644 tests/test_contrast.py diff --git a/README.md b/README.md index 353df04..df864eb 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,18 @@ python -m j7scope_serve --backend hf \

Compare view: cross-trace rigor strip + aligned zh/en sessions

+**Contrast Lab · 成对证据显微镜** —— 在 Compare 之上加入整对 permutation +null、同语言上限、bootstrap CI、单概念 rank 轨迹和答案身份混杂检查。指标由 +[`j7scope/contrast.py`](j7scope/contrast.py) 离线写入 `align.json`,浏览器只展示;连接 +`hf` GPU sidecar 后,还可从同一页面发起一次 activation patch,并把当前证据、干预 +结果与完整 provenance 导出为 Claim Card。 + +```bash +# 为任意两条已有 Trace v1 生成/更新 Contrast Lab 成对证据 +python experiments/build_contrast.py trace-a trace-b \ + --trace-root results/traces --mode cross_language +``` + **Gallery** —— 所有 trace 的入口(读 `traces/index.json`,平行对自动归为一张对比卡)。

J-Space Gallery landing page

@@ -185,12 +197,14 @@ j7scope/ │ ├── fitting.py # J-lens 拟合 + 残差捕获(sidecar 复用) │ ├── patching.py # M2 的跨语言 activation patching │ ├── metrics.py # CKA / SVCCA / overlap 计算 +│ ├── contrast.py # 成对 null / CI / 轨迹 / 混杂检查 + provenance │ └── viz.py # 双语对照的读出可视化页面 ├── apps/ │ ├── serve/ # ★ 实时 J-Space 旁路(sidecar) │ │ ├── j7scope_serve/ # OpenAI 兼容代理 + SSE 侧信道(纯标准库) │ │ ├── viewer/index.html # 自包含的实时视图 │ │ └── integrations/opencode/ # provider 配置 + 插件 +│ ├── site/contrast.html # Contrast Lab(离线证据 + 可选 GPU 干预) │ └── web/ # React/Vite J-Space Explorer(离线 artifact 浏览) ├── experiments/build_demo_run.py # 生成前端开发用 demo artifact(非真实实验) ├── notebooks/walkthrough_zh_en.ipynb diff --git a/apps/serve/README.md b/apps/serve/README.md index f8a5924..b7ed691 100644 --- a/apps/serve/README.md +++ b/apps/serve/README.md @@ -109,10 +109,16 @@ trace at capture time; the viewer only displays it. Trace Schema v1 lives in | `/v1/chat/completions` | POST | OpenAI chat, streaming or not; drives generation | | `/v1/models` | GET | one model entry (harnesses probe this) | | `/jspace/stream` | GET | SSE: one J-space event per generated token | +| `/jspace/interventions` | POST | one activation-patching probe (`hf` backend only) | | `/`, `/replay.html`, `/compare.html`, `/live.html`, `/assets/*` | GET | the static platform ([`../site`](../site)) | | `/traces/*` | GET | recorded Trace v1 files (when `--traces`/`--record` set) | | `/health` | GET | backend / model / layer / viewer count | +`/health.capabilities.activation_patch` tells Contrast Lab whether the connected +sidecar can run a causal A/B probe. The intervention endpoint accepts +`source_prompt`, `target_prompt`, `patch_layer`, optional source/target positions, +and `topk`; it is deliberately unavailable on the synthetic mock backend. + ### Side-channel event Each token emits (see [`protocol.py`](j7scope_serve/protocol.py)): diff --git a/apps/serve/j7scope_serve/app.py b/apps/serve/j7scope_serve/app.py index 175839b..9798c61 100644 --- a/apps/serve/j7scope_serve/app.py +++ b/apps/serve/j7scope_serve/app.py @@ -128,6 +128,11 @@ def do_GET(self): # noqa: N802 "layer": self.backend.layer, "is_demo": getattr(self.backend, "is_demo", False), "viewers": self.bus.subscriber_count(), + "capabilities": { + "activation_patch": bool( + getattr(self.backend, "supports_intervention", False) + ), + }, }) if path == "/v1/models": return self._json({ @@ -150,6 +155,8 @@ def do_POST(self): # noqa: N802 path = self.path.split("?", 1)[0] if path == "/v1/chat/completions": return self._chat_completions() + if path == "/jspace/interventions": + return self._run_intervention() return self._json({"error": "not found"}, status=404) # ---- handlers --------------------------------------------------------- @@ -287,6 +294,55 @@ def _chat_completions(self) -> None: "total_tokens": seq}, }) + def _run_intervention(self) -> None: + if not getattr(self.backend, "supports_intervention", False): + return self._json({ + "error": "activation patching requires the hf backend on a GPU sidecar", + "capability": "activation_patch", + }, status=409) + body = self._read_body() + source_prompt = body.get("source_prompt") + target_prompt = body.get("target_prompt") + if not isinstance(source_prompt, str) or not source_prompt.strip(): + return self._json({"error": "source_prompt is required"}, status=400) + if not isinstance(target_prompt, str) or not target_prompt.strip(): + return self._json({"error": "target_prompt is required"}, status=400) + if len(source_prompt) > 20_000 or len(target_prompt) > 20_000: + return self._json({"error": "prompt exceeds 20,000 characters"}, status=400) + try: + patch_layer = int(body.get("patch_layer")) + source_position = int(body.get("source_position", -1)) + target_position = int(body.get("target_position", -1)) + topk = min(100, max(1, int(body.get("topk", 20)))) + except (TypeError, ValueError): + return self._json({"error": "intervention positions and layers must be integers"}, status=400) + if patch_layer < 0 or patch_layer >= self.backend.layer: + return self._json({ + "error": f"patch_layer must be between 0 and {self.backend.layer - 1}" + }, status=400) + try: + result = self.backend.intervene( + source_prompt=source_prompt, + target_prompt=target_prompt, + patch_layer=patch_layer, + source_position=source_position, + target_position=target_position, + topk=topk, + ) + except (ValueError, IndexError) as exc: + return self._json({"error": str(exc)}, status=400) + except Exception as exc: + print(f" [intervention] failed: {exc!r}") + return self._json({ + "error": "activation patching failed; inspect the sidecar log" + }, status=500) + raw_readout = result.pop("readout") + result["readout"] = protocol.bucket_readout( + raw_readout, + per_lang=self.server.per_lang, # type: ignore[attr-defined] + ) + result["is_demo"] = False + return self._json(result) def _maybe_record(self, buffered, messages) -> None: record_dir = self.server.record_dir # type: ignore[attr-defined] diff --git a/apps/serve/j7scope_serve/backends.py b/apps/serve/j7scope_serve/backends.py index 6993343..79769cf 100644 --- a/apps/serve/j7scope_serve/backends.py +++ b/apps/serve/j7scope_serve/backends.py @@ -37,6 +37,7 @@ class Backend: layer: int is_demo: bool = False jacobian_estimator: str = "not_applicable" + supports_intervention: bool = False def generate(self, messages: Sequence[dict], **params) -> Iterator[Step]: raise NotImplementedError @@ -153,6 +154,8 @@ class HFBackend(Backend): the model weights available. Structured to mirror MockBackend's Step stream. """ + supports_intervention = True + def __init__(self, model_name: str = "Qwen/Qwen2.5-7B-Instruct", layer: int = 18, topk: int = 24, max_new_tokens: int = 256, device: str = None, jacobian_corpus: Sequence[str] = None, n_probes: int = 16, @@ -391,6 +394,43 @@ def generate(self, messages: Sequence[dict], **params) -> Iterator[Step]: cur = next_id.view(1, 1) + def intervene( + self, + *, + source_prompt: str, + target_prompt: str, + patch_layer: int, + source_position: int = -1, + target_position: int = -1, + topk: int = 20, + ) -> dict: + """Run one activation-patching A/B probe on the loaded GPU model.""" + if self._model is None: + self.load() + from j7scope.patching import patch_and_readout + + result = patch_and_readout( + self._jlens, + source_prompt, + target_prompt, + patch_layer=patch_layer, + src_pos=source_position, + tgt_pos=target_position, + k=topk, + ) + return { + "kind": "activation_patch", + "model": self.model_name, + "model_revision": self.model_revision_resolved, + "lens_layer": self.layer, + "patch_layer": patch_layer, + "source_position": source_position, + "target_position": target_position, + "jacobian_sha1": self.jacobian_sha1, + "readout": result["readout"], + "next_token": result["next_token"], + } + def make_backend(kind: str, **kw) -> Backend: if kind == "mock": diff --git a/apps/site/assets/jspace.css b/apps/site/assets/jspace.css index e0c4cd2..1190e9c 100644 --- a/apps/site/assets/jspace.css +++ b/apps/site/assets/jspace.css @@ -105,3 +105,66 @@ button:hover { border-color: var(--accent); } .export { display: flex; gap: 8px; flex-wrap: wrap; } footer { margin-top: 22px; color: var(--muted); font-size: 11px; line-height: 1.6; } + +/* Contrast Lab */ +.lab-toolbar { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; + padding: 10px 12px; margin-bottom: 12px; background: var(--panel); border: 1px solid var(--line); border-radius: 10px; } +.lab-toolbar label { color: var(--muted); font-size: 12px; } +.lab-toolbar .spacer { flex: 1; } +.lab-tabs { display: flex; gap: 3px; padding: 3px; background: var(--panel-2); border-radius: 8px; } +.lab-tab { border-color: transparent; background: transparent; color: var(--muted); } +.lab-tab.active { background: var(--panel); border-color: var(--line); color: var(--ink); } +.lab-summary { display: flex; align-items: center; gap: 8px; color: var(--muted); font-size: 11px; margin-bottom: 8px; flex-wrap: wrap; } +.lab-pair { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; } +.lab-trace { min-width: 0; padding: 13px; background: var(--panel); border: 1px solid var(--line); border-radius: 12px; } +.lab-trace-head { display: flex; justify-content: space-between; gap: 8px; align-items: baseline; } +.lab-trace-head span { color: var(--muted); font-size: 10px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.lab-trace .stream { max-height: 92px; overflow: auto; margin: 10px 0; background: var(--panel-2); } +.lab-token-readout { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; } +.lab-token-readout .col { padding: 10px; background: var(--panel-2); } +.lab-token-readout .rows { gap: 4px; } +.lab-token-readout .row { grid-template-columns: minmax(50px, auto) 1fr 34px; gap: 6px; } +.lab-token-readout .row:nth-child(n+5) { display: none; } +.lab-evidence { display: grid; grid-template-columns: minmax(0, 1.6fr) minmax(230px, .7fr); gap: 12px; margin-top: 12px; } +.lab-evidence h2, .lab-mode-panel h2, .lab-trajectory h2 { font-size: 14px; margin: 0 0 8px; } +.lab-evidence .rigor { margin: 0; } +.lab-calibration, .lab-mode-panel, .lab-trajectory, .lab-provenance { padding: 14px 16px; background: var(--panel); border: 1px solid var(--line); border-radius: 12px; } +.lab-calibration dl { margin: 0; display: grid; gap: 10px; } +.lab-calibration dl div { display: flex; justify-content: space-between; gap: 10px; padding-bottom: 8px; border-bottom: 1px solid var(--line); } +.lab-calibration dl div:last-child { border: 0; padding: 0; } +.lab-calibration dt { color: var(--muted); font-size: 11px; } +.lab-calibration dd { margin: 0; font-weight: 650; font-variant-numeric: tabular-nums; } +.lab-mode-panel { margin-top: 12px; } +.lab-mode-panel .caption, .lab-section-head span { color: var(--muted); font-size: 11px; } +.intervention-form { display: flex; gap: 10px; align-items: end; flex-wrap: wrap; margin-top: 12px; } +.intervention-form label { display: grid; gap: 4px; color: var(--muted); font-size: 11px; } +.intervention-form input { width: 94px; padding: 5px 7px; color: var(--ink); background: var(--panel-2); border: 1px solid var(--line); border-radius: 6px; } +.intervention-result { margin-top: 12px; color: var(--muted); font-size: 12px; } +.intervention-result pre, .lab-provenance pre { overflow: auto; max-height: 280px; color: var(--ink); background: var(--panel-2); padding: 10px; border-radius: 8px; } +.confound { display: flex; align-items: center; gap: 12px; padding: 12px; border: 1px solid var(--line); border-radius: 9px; } +.confound span { color: var(--muted); } +.confound.controlled { border-color: #29513c; }.confound.controlled b { color: var(--good); } +.confound.confounded { border-color: #6a4720; }.confound.confounded b { color: var(--warn); } +.lab-trajectory { margin-top: 12px; } +.lab-section-head { display: flex; justify-content: space-between; gap: 12px; align-items: start; } +.trajectory-chart { min-height: 220px; margin-top: 8px; } +.trajectory-chart svg { display: block; width: 100%; height: auto; overflow: visible; } +.trajectory-chart text { fill: var(--muted); font-size: 10px; } +.trajectory-chart .grid { stroke: var(--line); stroke-dasharray: 3 4; } +.trajectory-chart .cursor { stroke: var(--ink); stroke-width: 1; opacity: .35; } +.trajectory-chart line.series-a { stroke: var(--en); stroke-width: 2.5; } +.trajectory-chart circle.series-a { fill: var(--en); } +.trajectory-chart line.series-b { stroke: var(--zh); stroke-width: 2.5; } +.trajectory-chart circle.series-b { fill: var(--zh); } +.lab-provenance { margin-top: 12px; color: var(--muted); } +.lab-provenance summary { cursor: pointer; } +@media (max-width: 760px) { + .lab-pair, .lab-evidence { grid-template-columns: 1fr; } + .lab-token-readout { grid-template-columns: 1fr 1fr; } + .lab-toolbar .spacer { display: none; } +} +@media (max-width: 440px) { + .lab-token-readout { grid-template-columns: 1fr; } + .lab-tabs { width: 100%; } + .lab-tab { flex: 1; padding-inline: 4px; } +} diff --git a/apps/site/compare.html b/apps/site/compare.html index 31e4456..fb3ae10 100644 --- a/apps/site/compare.html +++ b/apps/site/compare.html @@ -11,6 +11,7 @@

J7Scope Compare · 双生对齐

@@ -51,6 +52,7 @@

跨会话 · Cross-trace rigor + + + + +J7Scope · Contrast Lab + + + +
+

J7Scope Contrast Lab · 成对证据显微镜

+ +
+ +

Compare matched traces, inspect one concept family at a time, + calibrate the observation against a paired permutation null, check answer-identity + confounds, and optionally run one causal activation patch on a connected GPU sidecar.

+ +
+ +
+ + + +
+ + GPU sidecar · 未连接 + +
+ +

Loading paired traces…

+ +
+ All calibration values are baked into align.json by + j7scope.contrast. The browser renders them but does not recompute + sharedness. A green result is evidence above the configured null, not proof of a + complete or universal representation. +
+ + + + + diff --git a/apps/site/index.html b/apps/site/index.html index 6a20655..5988557 100644 --- a/apps/site/index.html +++ b/apps/site/index.html @@ -72,10 +72,10 @@

J7Scope J-Space Gallery · 双生工作 } function groupCard(g, members) { var langs = members.map(function (m) { return m.language; }).join(" ↔ "); - return '' + - '

' + J.esc(g) + ' side-by-side

' + + return '
' + + '

' + J.esc(g) + ' Contrast Lab

' + '
' + members.length + ' aligned traces' + J.esc(langs) + '
' + - '
' + tag("compare zh ↔ en") + (members[0].is_demo ? tag("DEMO", "demo") : "") + '
'; + '
' + tag("paired evidence") + tag("null + ceiling") + (members[0].is_demo ? tag("DEMO", "demo") : "") + '
'; } })(); diff --git a/apps/web/package-lock.json b/apps/web/package-lock.json index dc3bd04..4a24563 100644 --- a/apps/web/package-lock.json +++ b/apps/web/package-lock.json @@ -3480,9 +3480,9 @@ "peer": true }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { diff --git a/experiments/build_contrast.py b/experiments/build_contrast.py new file mode 100644 index 0000000..b0f0c3c --- /dev/null +++ b/experiments/build_contrast.py @@ -0,0 +1,79 @@ +"""Build a Contrast Lab record for two existing Trace v1 directories.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from j7scope.artifacts import write_json # noqa: E402 +from j7scope.contrast import build_contrast # noqa: E402 +from j7scope.data import load_parallel_pairs # noqa: E402 +from j7scope.rigor import build_lexicon # noqa: E402 +from j7scope.trace import read_trace # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("trace_a") + parser.add_argument("trace_b") + parser.add_argument("--trace-root", type=Path, default=ROOT / "results" / "traces") + parser.add_argument("--data-dir", type=Path, default=ROOT / "data") + parser.add_argument( + "--mode", + choices=("cross_language", "intervention", "correctness"), + default="cross_language", + ) + parser.add_argument("--position-map", type=Path) + parser.add_argument("--permutations", type=int, default=1000) + parser.add_argument("--bootstrap", type=int, default=1000) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args() + + trace_a = read_trace(args.trace_root / args.trace_a) + trace_b = read_trace(args.trace_root / args.trace_b) + position_map = None + if args.position_map: + raw = json.loads(args.position_map.read_text(encoding="utf-8")) + position_map = raw.get("position_map", raw) if isinstance(raw, dict) else raw + + pairs = load_parallel_pairs(args.data_dir) if args.data_dir.exists() else None + lexicon = build_lexicon(pairs=pairs) + contrast = build_contrast( + trace_a, + trace_b, + lexicon, + position_map=position_map, + mode=args.mode, + n_permutations=args.permutations, + n_boot=args.bootstrap, + seed=args.seed, + ) + align = { + "parallel_group": ( + trace_a["manifest"].get("parallel_group") + or trace_b["manifest"].get("parallel_group") + or f"{args.trace_a}--{args.trace_b}" + ), + "members": {"a": args.trace_a, "b": args.trace_b}, + "position_map": contrast["position_map"], + "pair_rigor": contrast["pair_rigor"], + "contrast": contrast, + } + for trace_id in (args.trace_a, args.trace_b): + write_json(args.trace_root / trace_id / "align.json", align) + print(json.dumps({ + "trace_a": args.trace_a, + "trace_b": args.trace_b, + "observed": contrast["calibration"]["observed"], + "sharedness": contrast["calibration"]["sharedness"], + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/build_demo_trace.py b/experiments/build_demo_trace.py index 19c657e..18281d1 100644 --- a/experiments/build_demo_trace.py +++ b/experiments/build_demo_trace.py @@ -18,7 +18,6 @@ import argparse import sys -from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -26,6 +25,7 @@ sys.path.insert(0, str(ROOT)) from j7scope import rigor +from j7scope.contrast import build_contrast from j7scope.artifacts import write_json from j7scope.data import load_parallel_pairs from j7scope.trace import write_trace @@ -40,6 +40,7 @@ } FILLER_ZH = ["的", "一种", "行为", "这", "属于"] FILLER_EN = ["a", "kind", "of", "this", "is"] +DEMO_CREATED_AT = "2026-07-22T00:00:00+00:00" # A session narrative: (surface_token, concept_or_None). Abstract concepts are # interspersed with fillers; the concrete entity shows weaker cross-lingual tie. @@ -144,7 +145,7 @@ def build_trace(narrative, language: str, lexicon, *, trace_id: str, "sha1": None, }, "capture": {"tool": "build_demo_trace.py", - "created_at": datetime.now(timezone.utc).isoformat()}, + "created_at": DEMO_CREATED_AT}, "is_demo": True, "doi": None, "parallel_group": parallel_group, @@ -187,13 +188,22 @@ def main() -> None: m_zh, t_zh, mt_zh = build_trace(ZH_NARRATIVE, "zh", lexicon, trace_id=f"{group}-zh", parallel_group=group) position_map = [[i, i] for i in range(len(NARRATIVE))] # identical structure - # Cross-trace rigor: A = en session, B = zh session, at aligned positions. - pair_rigor = rigor.compute_pair_rigor(t_en, t_zh, position_map, lexicon) + # Cross-trace rigor + pair-level Contrast Lab evidence. The aggregate + # permutation null and trajectories are baked here; the browser only renders. + contrast = build_contrast( + {"manifest": m_en, "tokens": t_en, "metrics": mt_en}, + {"manifest": m_zh, "tokens": t_zh, "metrics": mt_zh}, + lexicon, + position_map=position_map, + n_permutations=1000, + n_boot=1000, + ) align = { "parallel_group": group, "members": {"en": m_en["trace_id"], "zh": m_zh["trace_id"]}, "position_map": position_map, - "pair_rigor": pair_rigor, + "pair_rigor": contrast["pair_rigor"], + "contrast": contrast, } write_trace(args.out / m_en["trace_id"], manifest=m_en, tokens=t_en, metrics=mt_en, align=align) diff --git a/j7scope/contrast.py b/j7scope/contrast.py new file mode 100644 index 0000000..7709225 --- /dev/null +++ b/j7scope/contrast.py @@ -0,0 +1,313 @@ +"""Research-grade paired evidence for the J-Space Contrast Lab. + +The browser is intentionally a renderer. This module turns two Trace v1 +sessions into a precomputed contrast record containing a paired permutation +null, a same-language ceiling, a bootstrap interval, concept-rank trajectories, +an answer-identity confound check, and exact provenance. +""" + +from __future__ import annotations + +from typing import Dict, List, Mapping, Optional, Sequence + +import numpy as np + +from .rigor import ( + SHAREDNESS_DEFINITION, + compute_pair_rigor, + concepts_of, + estimate_same_lang_baseline, + overlap_coef, +) + +CONTRAST_SCHEMA_VERSION = 1 +PAIR_SHAREDNESS_DEFINITION = ( + "(paired_observed_mean - permutation_null.mean) / " + "(same_lang_ceiling - permutation_null.mean)" +) +_EPS = 1e-9 + + +def _row_concept(row: Mapping, lexicon: Mapping[str, str]) -> Optional[str]: + concept = row.get("concept") + if isinstance(concept, str) and concept: + return concept + token = row.get("token") + if isinstance(token, str): + return lexicon.get(token) + return None + + +def token_concepts(token: Mapping, lexicon: Mapping[str, str]) -> List[str]: + """Ordered concepts in a token's complete bilingual readout. + + Captured traces can either bake ``row.concept`` into each readout or rely on + the corpus lexicon. Supporting both makes imported JSONL traces portable. + """ + out: List[str] = [] + seen = set() + readout = token.get("readout", {}) + for bucket in ("zh", "en", "other"): + for row in readout.get(bucket, []): + concept = _row_concept(row, lexicon) + if concept and concept not in seen: + seen.add(concept) + out.append(concept) + return out + + +def _rank_for_concept( + token: Mapping, concept: str, lexicon: Mapping[str, str] +) -> Optional[int]: + ranks = [] + for bucket in ("zh", "en", "other"): + for fallback_rank, row in enumerate(token.get("readout", {}).get(bucket, [])): + if _row_concept(row, lexicon) == concept: + ranks.append(int(row.get("rank", fallback_rank)) + 1) + return min(ranks) if ranks else None + + +def concept_rank_trajectories( + tokens_a: Sequence[Mapping], + tokens_b: Sequence[Mapping], + position_map: Sequence[Sequence[int]], + lexicon: Mapping[str, str], +) -> List[dict]: + concepts = sorted({ + concept + for token in (*tokens_a, *tokens_b) + for concept in token_concepts(token, lexicon) + }) + rows = [] + for concept in concepts: + rank_a = [_rank_for_concept(tokens_a[ia], concept, lexicon) + for ia, _ in position_map] + rank_b = [_rank_for_concept(tokens_b[ib], concept, lexicon) + for _, ib in position_map] + coverage = sum(rank is not None for rank in rank_a + rank_b) + if coverage: + rows.append({ + "concept": concept, + "rank_a": rank_a, + "rank_b": rank_b, + "coverage": round(coverage / max(1, 2 * len(position_map)), 4), + }) + return sorted(rows, key=lambda row: (-row["coverage"], row["concept"])) + + +def _permutation_null( + concepts_a: Sequence[Sequence[str]], + concepts_b: Sequence[Sequence[str]], + *, + n_permutations: int, + seed: int, +) -> tuple[float, dict]: + observed_values = [overlap_coef(a, b) for a, b in zip(concepts_a, concepts_b)] + observed = float(np.mean(observed_values)) if observed_values else 0.0 + if len(concepts_a) < 2 or n_permutations < 1: + return observed, { + "metric": "paired_mean_concept_overlap", + "mean": 0.0, + "p05": 0.0, + "p95": 0.0, + "n": 0, + "seed": seed, + } + + rng = np.random.default_rng(seed) + values = np.empty(n_permutations, dtype=np.float64) + base = np.arange(len(concepts_b)) + for i in range(n_permutations): + permuted = rng.permutation(base) + values[i] = np.mean([ + overlap_coef(concepts_a[j], concepts_b[int(permuted[j])]) + for j in range(len(concepts_a)) + ]) + return observed, { + "metric": "paired_mean_concept_overlap", + "mean": round(float(values.mean()), 4), + "p05": round(float(np.percentile(values, 5)), 4), + "p95": round(float(np.percentile(values, 95)), 4), + "n": int(n_permutations), + "seed": int(seed), + } + + +def _paired_bootstrap_ci( + overlaps: Sequence[float], + *, + null_mean: float, + ceiling: float, + n_boot: int, + seed: int, +) -> List[float]: + if not overlaps or n_boot < 1: + return [0.0, 0.0] + values = np.asarray(overlaps, dtype=np.float64) + rng = np.random.default_rng(seed) + denominator = max(_EPS, ceiling - null_mean) + samples = np.empty(n_boot, dtype=np.float64) + for i in range(n_boot): + picked = values[rng.integers(0, len(values), len(values))] + samples[i] = (float(picked.mean()) - null_mean) / denominator + lo, hi = np.percentile(samples, [2.5, 97.5]) + return [round(float(lo), 4), round(float(hi), 4)] + + +def detect_answer_identity_confound( + manifest_a: Mapping, manifest_b: Mapping +) -> dict: + """Classify whether a correctness contrast controls answer identity. + + This is deliberately conservative: absent structured ``correct`` and + ``answer_identity`` metadata, the tool reports ``not_assessable`` instead + of inferring correctness from free text. + """ + correct_a, correct_b = manifest_a.get("correct"), manifest_b.get("correct") + answer_a, answer_b = manifest_a.get("answer_identity"), manifest_b.get("answer_identity") + if not isinstance(correct_a, bool) or not isinstance(correct_b, bool): + return { + "status": "not_assessable", + "controlled": None, + "reason": "manifests need boolean correct and string answer_identity fields", + } + if correct_a == correct_b: + return { + "status": "not_applicable", + "controlled": None, + "reason": "the pair does not contrast correctness", + } + if not isinstance(answer_a, str) or not isinstance(answer_b, str): + return { + "status": "not_assessable", + "controlled": None, + "reason": "answer_identity is missing from one or both manifests", + } + controlled = answer_a == answer_b + return { + "status": "controlled" if controlled else "confounded", + "controlled": controlled, + "answer_identity_a": answer_a, + "answer_identity_b": answer_b, + "reason": ( + "correctness differs while answer identity is held constant" + if controlled else + "correctness and answer identity both change in this pair" + ), + } + + +def _trace_provenance(trace: Mapping) -> dict: + manifest = trace["manifest"] + jacobian = manifest.get("jacobian", {}) + capture = manifest.get("capture", {}) + return { + "trace_id": manifest.get("trace_id"), + "model": manifest.get("model"), + "model_revision": manifest.get("revision"), + "layer": manifest.get("layer"), + "language": manifest.get("language"), + "lens_sha1": jacobian.get("sha1"), + "corpus_id": jacobian.get("corpus_id"), + "estimator": jacobian.get("estimator"), + "capture_tool": capture.get("tool"), + "created_at": capture.get("created_at"), + "is_demo": bool(manifest.get("is_demo", False)), + } + + +def build_contrast( + trace_a: Mapping, + trace_b: Mapping, + lexicon: Mapping[str, str], + *, + position_map: Optional[Sequence[Sequence[int]]] = None, + mode: str = "cross_language", + n_permutations: int = 1000, + n_boot: int = 1000, + seed: int = 0, +) -> dict: + """Build a complete, serializable Contrast Lab evidence record.""" + tokens_a, tokens_b = trace_a["tokens"], trace_b["tokens"] + if position_map is None: + position_map = [[i, i] for i in range(min(len(tokens_a), len(tokens_b)))] + position_map = [[int(ia), int(ib)] for ia, ib in position_map] + if not position_map: + raise ValueError("position_map must contain at least one aligned position") + if any(ia < 0 or ia >= len(tokens_a) or ib < 0 or ib >= len(tokens_b) + for ia, ib in position_map): + raise IndexError("position_map references a token outside one of the traces") + + aligned_a = [token_concepts(tokens_a[ia], lexicon) for ia, _ in position_map] + aligned_b = [token_concepts(tokens_b[ib], lexicon) for _, ib in position_map] + # Empty/empty filler positions contain no hypothesis-bearing observation. + # Including them would dilute both observed overlap and the permutation null + # according to narrative length rather than representation quality. + informative = [ + i for i, (concepts_a, concepts_b) in enumerate(zip(aligned_a, aligned_b)) + if concepts_a or concepts_b + ] + evidence_a = [aligned_a[i] for i in informative] + evidence_b = [aligned_b[i] for i in informative] + if not evidence_a: + raise ValueError("the aligned pair contains no concepts in either trace") + observed, null = _permutation_null( + evidence_a, evidence_b, n_permutations=n_permutations, seed=seed + ) + ceiling = float(np.mean([ + estimate_same_lang_baseline(evidence_a), + estimate_same_lang_baseline(evidence_b), + ])) + denominator = max(_EPS, ceiling - null["mean"]) + sharedness = (observed - null["mean"]) / denominator + overlaps = [overlap_coef(a, b) for a, b in zip(evidence_a, evidence_b)] + ci = _paired_bootstrap_ci( + overlaps, + null_mean=null["mean"], + ceiling=ceiling, + n_boot=n_boot, + seed=seed + 17, + ) + pair_rigor = compute_pair_rigor( + tokens_a, tokens_b, position_map, dict(lexicon), seed=seed + ) + + return { + "schema_version": CONTRAST_SCHEMA_VERSION, + "mode": mode, + "trace_a": trace_a["manifest"].get("trace_id"), + "trace_b": trace_b["manifest"].get("trace_id"), + "position_map": position_map, + "pair_rigor": pair_rigor, + "calibration": { + "n_aligned_positions": len(position_map), + "n_informative_positions": len(informative), + "informative_positions": informative, + "observed": round(observed, 4), + "null": null, + "same_lang_ceiling": round(ceiling, 4), + "sharedness": { + "value": round(float(sharedness), 4), + "ci95": ci, + "definition": PAIR_SHAREDNESS_DEFINITION, + }, + "token_sharedness_definition": SHAREDNESS_DEFINITION, + }, + "concept_trajectories": concept_rank_trajectories( + tokens_a, tokens_b, position_map, lexicon + ), + "answer_identity_check": detect_answer_identity_confound( + trace_a["manifest"], trace_b["manifest"] + ), + "provenance": { + "trace_a": _trace_provenance(trace_a), + "trace_b": _trace_provenance(trace_b), + "analysis": { + "tool": "j7scope.contrast.build_contrast", + "schema_version": CONTRAST_SCHEMA_VERSION, + "n_permutations": int(n_permutations), + "n_boot": int(n_boot), + "seed": int(seed), + }, + }, + } diff --git a/j7scope/rigor.py b/j7scope/rigor.py index 634ea5e..463b15b 100644 --- a/j7scope/rigor.py +++ b/j7scope/rigor.py @@ -201,9 +201,15 @@ def _combined_concepts(tok: dict, lexicon: Dict[str, str]) -> List[str]: The cross-trace metric asks whether two sessions reach the same concept, so it pools zh and en read-out tokens into one concept set per token. """ - toks = ([t["token"] for t in tok["readout"].get("zh", [])] - + [t["token"] for t in tok["readout"].get("en", [])]) - return concepts_of(toks, lexicon) + out: List[str] = [] + seen = set() + for bucket in ("zh", "en", "other"): + for row in tok["readout"].get(bucket, []): + concept = row.get("concept") or lexicon.get(row.get("token")) + if concept is not None and concept not in seen: + seen.add(concept) + out.append(concept) + return out def compute_pair_rigor( diff --git a/j7scope/trace.py b/j7scope/trace.py index b47e463..db414e9 100644 --- a/j7scope/trace.py +++ b/j7scope/trace.py @@ -276,4 +276,39 @@ def validate_trace_gallery(trace_root: PathLike) -> List[str]: problems.append( f"{trace_id}: align.position_map[{pair_index}] is invalid" ) + contrast = align.get("contrast") + if contrast is not None: + from .contrast import ( + CONTRAST_SCHEMA_VERSION, + PAIR_SHAREDNESS_DEFINITION, + ) + + if contrast.get("schema_version") != CONTRAST_SCHEMA_VERSION: + problems.append( + f"{trace_id}: contrast schema_version must be " + f"{CONTRAST_SCHEMA_VERSION}" + ) + calibration = contrast.get("calibration", {}) + for key in ("observed", "null", "same_lang_ceiling", "sharedness"): + if key not in calibration: + problems.append( + f"{trace_id}: contrast.calibration missing {key}" + ) + if calibration.get("sharedness", {}).get("definition") != PAIR_SHAREDNESS_DEFINITION: + problems.append( + f"{trace_id}: contrast sharedness definition drifted" + ) + if contrast.get("position_map") != align.get("position_map"): + problems.append( + f"{trace_id}: contrast.position_map differs from align" + ) + if not isinstance(contrast.get("concept_trajectories"), list): + problems.append( + f"{trace_id}: contrast.concept_trajectories must be a list" + ) + provenance = contrast.get("provenance", {}) + if not all(key in provenance for key in ("trace_a", "trace_b", "analysis")): + problems.append( + f"{trace_id}: contrast provenance is incomplete" + ) return problems diff --git a/results/traces/demo-narrative-en/manifest.json b/results/traces/demo-narrative-en/manifest.json index 46d68c7..cec68d5 100644 --- a/results/traces/demo-narrative-en/manifest.json +++ b/results/traces/demo-narrative-en/manifest.json @@ -1,6 +1,6 @@ { "capture": { - "created_at": "2026-07-22T12:31:46.828887+00:00", + "created_at": "2026-07-22T00:00:00+00:00", "tool": "build_demo_trace.py" }, "concept": null, @@ -8,6 +8,11 @@ "is_demo": true, "jacobian": { "corpus_id": "demo-synthetic", + "estimator": "synthetic", + "n_probes": null, + "n_prompts": 0, + "position": null, + "seed": 0, "sha1": null }, "kind": "single", diff --git a/results/traces/demo-parallel-en/align.json b/results/traces/demo-parallel-en/align.json index 4f17905..1a5677c 100644 --- a/results/traces/demo-parallel-en/align.json +++ b/results/traces/demo-parallel-en/align.json @@ -1,4 +1,1063 @@ { + "contrast": { + "answer_identity_check": { + "controlled": null, + "reason": "manifests need boolean correct and string answer_identity fields", + "status": "not_assessable" + }, + "calibration": { + "informative_positions": [ + 3, + 6, + 13, + 18, + 23 + ], + "n_aligned_positions": 25, + "n_informative_positions": 5, + "null": { + "mean": 0.3708, + "metric": "paired_mean_concept_overlap", + "n": 1000, + "p05": 0.0, + "p95": 0.8, + "seed": 0 + }, + "observed": 1.0, + "same_lang_ceiling": 1.0, + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(paired_observed_mean - permutation_null.mean) / (same_lang_ceiling - permutation_null.mean)", + "value": 1.0 + }, + "token_sharedness_definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)" + }, + "concept_trajectories": [ + { + "concept": "deception", + "coverage": 0.08, + "rank_a": [ + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ], + "rank_b": [ + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ] + }, + { + "concept": "emotion", + "coverage": 0.08, + "rank_a": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ] + }, + { + "concept": "concession", + "coverage": 0.04, + "rank_a": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null + ] + }, + { + "concept": "entity", + "coverage": 0.04, + "rank_a": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null + ] + }, + { + "concept": "manipulation", + "coverage": 0.04, + "rank_a": [ + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + } + ], + "mode": "cross_language", + "pair_rigor": [ + { + "cross_lang_overlap": 0.0, + "ia": 0, + "ib": 0, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 0, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 1, + "ib": 1, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 1, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 2, + "ib": 2, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 2, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 3, + "ib": 3, + "null": { + "mean": 0.0417, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 3, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "deception" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 4, + "ib": 4, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 4, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 5, + "ib": 5, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 5, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 6, + "ib": 6, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 6, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "manipulation" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 7, + "ib": 7, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 7, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 8, + "ib": 8, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 8, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 9, + "ib": 9, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 9, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 10, + "ib": 10, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 10, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 11, + "ib": 11, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 11, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 12, + "ib": 12, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 12, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 13, + "ib": 13, + "null": { + "mean": 0.0417, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 13, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "emotion" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 14, + "ib": 14, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 14, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 15, + "ib": 15, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 15, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 16, + "ib": 16, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 16, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 17, + "ib": 17, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 17, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 18, + "ib": 18, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 18, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "concession" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 19, + "ib": 19, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 19, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 20, + "ib": 20, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 20, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 21, + "ib": 21, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 21, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 22, + "ib": 22, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 22, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 23, + "ib": 23, + "null": { + "mean": 0.0833, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.85 + }, + "position": 23, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "deception", + "emotion", + "entity" + ], + "sharedness": { + "ci95": [ + -0.0909, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 24, + "ib": 24, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 24, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + } + ], + "position_map": [ + [ + 0, + 0 + ], + [ + 1, + 1 + ], + [ + 2, + 2 + ], + [ + 3, + 3 + ], + [ + 4, + 4 + ], + [ + 5, + 5 + ], + [ + 6, + 6 + ], + [ + 7, + 7 + ], + [ + 8, + 8 + ], + [ + 9, + 9 + ], + [ + 10, + 10 + ], + [ + 11, + 11 + ], + [ + 12, + 12 + ], + [ + 13, + 13 + ], + [ + 14, + 14 + ], + [ + 15, + 15 + ], + [ + 16, + 16 + ], + [ + 17, + 17 + ], + [ + 18, + 18 + ], + [ + 19, + 19 + ], + [ + 20, + 20 + ], + [ + 21, + 21 + ], + [ + 22, + 22 + ], + [ + 23, + 23 + ], + [ + 24, + 24 + ] + ], + "provenance": { + "analysis": { + "n_boot": 1000, + "n_permutations": 1000, + "schema_version": 1, + "seed": 0, + "tool": "j7scope.contrast.build_contrast" + }, + "trace_a": { + "capture_tool": "build_demo_trace.py", + "corpus_id": "demo-synthetic", + "created_at": "2026-07-22T00:00:00+00:00", + "estimator": "synthetic", + "is_demo": true, + "language": "en", + "layer": 18, + "lens_sha1": null, + "model": "demo-synthetic", + "model_revision": null, + "trace_id": "demo-parallel-en" + }, + "trace_b": { + "capture_tool": "build_demo_trace.py", + "corpus_id": "demo-synthetic", + "created_at": "2026-07-22T00:00:00+00:00", + "estimator": "synthetic", + "is_demo": true, + "language": "zh", + "layer": 18, + "lens_sha1": null, + "model": "demo-synthetic", + "model_revision": null, + "trace_id": "demo-parallel-zh" + } + }, + "schema_version": 1, + "trace_a": "demo-parallel-en", + "trace_b": "demo-parallel-zh" + }, "members": { "en": "demo-parallel-en", "zh": "demo-parallel-zh" diff --git a/results/traces/demo-parallel-en/manifest.json b/results/traces/demo-parallel-en/manifest.json index 61d6af3..9a8ef25 100644 --- a/results/traces/demo-parallel-en/manifest.json +++ b/results/traces/demo-parallel-en/manifest.json @@ -1,6 +1,6 @@ { "capture": { - "created_at": "2026-07-22T12:31:46.837569+00:00", + "created_at": "2026-07-22T00:00:00+00:00", "tool": "build_demo_trace.py" }, "concept": null, @@ -8,6 +8,11 @@ "is_demo": true, "jacobian": { "corpus_id": "demo-synthetic", + "estimator": "synthetic", + "n_probes": null, + "n_prompts": 0, + "position": null, + "seed": 0, "sha1": null }, "kind": "parallel_member", diff --git a/results/traces/demo-parallel-zh/align.json b/results/traces/demo-parallel-zh/align.json index 4f17905..1a5677c 100644 --- a/results/traces/demo-parallel-zh/align.json +++ b/results/traces/demo-parallel-zh/align.json @@ -1,4 +1,1063 @@ { + "contrast": { + "answer_identity_check": { + "controlled": null, + "reason": "manifests need boolean correct and string answer_identity fields", + "status": "not_assessable" + }, + "calibration": { + "informative_positions": [ + 3, + 6, + 13, + 18, + 23 + ], + "n_aligned_positions": 25, + "n_informative_positions": 5, + "null": { + "mean": 0.3708, + "metric": "paired_mean_concept_overlap", + "n": 1000, + "p05": 0.0, + "p95": 0.8, + "seed": 0 + }, + "observed": 1.0, + "same_lang_ceiling": 1.0, + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(paired_observed_mean - permutation_null.mean) / (same_lang_ceiling - permutation_null.mean)", + "value": 1.0 + }, + "token_sharedness_definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)" + }, + "concept_trajectories": [ + { + "concept": "deception", + "coverage": 0.08, + "rank_a": [ + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ], + "rank_b": [ + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ] + }, + { + "concept": "emotion", + "coverage": 0.08, + "rank_a": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 3, + null + ] + }, + { + "concept": "concession", + "coverage": 0.04, + "rank_a": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null + ] + }, + { + "concept": "entity", + "coverage": 0.04, + "rank_a": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null + ] + }, + { + "concept": "manipulation", + "coverage": 0.04, + "rank_a": [ + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "rank_b": [ + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + } + ], + "mode": "cross_language", + "pair_rigor": [ + { + "cross_lang_overlap": 0.0, + "ia": 0, + "ib": 0, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 0, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 1, + "ib": 1, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 1, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 2, + "ib": 2, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 2, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 3, + "ib": 3, + "null": { + "mean": 0.0417, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 3, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "deception" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 4, + "ib": 4, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 4, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 5, + "ib": 5, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 5, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 6, + "ib": 6, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 6, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "manipulation" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 7, + "ib": 7, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 7, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 8, + "ib": 8, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 8, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 9, + "ib": 9, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 9, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 10, + "ib": 10, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 10, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 11, + "ib": 11, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 11, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 12, + "ib": 12, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 12, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 13, + "ib": 13, + "null": { + "mean": 0.0417, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 13, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "emotion" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 14, + "ib": 14, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 14, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 15, + "ib": 15, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 15, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 16, + "ib": 16, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 16, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 17, + "ib": 17, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 17, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 18, + "ib": 18, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 18, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "concession" + ], + "sharedness": { + "ci95": [ + 1.0, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 19, + "ib": 19, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 19, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 20, + "ib": 20, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 20, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 21, + "ib": 21, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 21, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 22, + "ib": 22, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 22, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + }, + { + "cross_lang_overlap": 1.0, + "ia": 23, + "ib": 23, + "null": { + "mean": 0.0833, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.85 + }, + "position": 23, + "same_lang_baseline": 1.0, + "shared_concepts": [ + "deception", + "emotion", + "entity" + ], + "sharedness": { + "ci95": [ + -0.0909, + 1.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 1.0 + } + }, + { + "cross_lang_overlap": 0.0, + "ia": 24, + "ib": 24, + "null": { + "mean": 0.0, + "metric": "concept_overlap", + "n": 24, + "p05": 0.0, + "p95": 0.0 + }, + "position": 24, + "same_lang_baseline": 1.0, + "shared_concepts": [], + "sharedness": { + "ci95": [ + 0.0, + 0.0 + ], + "definition": "(cross_lang_overlap - null.mean) / (same_lang_baseline - null.mean)", + "value": 0.0 + } + } + ], + "position_map": [ + [ + 0, + 0 + ], + [ + 1, + 1 + ], + [ + 2, + 2 + ], + [ + 3, + 3 + ], + [ + 4, + 4 + ], + [ + 5, + 5 + ], + [ + 6, + 6 + ], + [ + 7, + 7 + ], + [ + 8, + 8 + ], + [ + 9, + 9 + ], + [ + 10, + 10 + ], + [ + 11, + 11 + ], + [ + 12, + 12 + ], + [ + 13, + 13 + ], + [ + 14, + 14 + ], + [ + 15, + 15 + ], + [ + 16, + 16 + ], + [ + 17, + 17 + ], + [ + 18, + 18 + ], + [ + 19, + 19 + ], + [ + 20, + 20 + ], + [ + 21, + 21 + ], + [ + 22, + 22 + ], + [ + 23, + 23 + ], + [ + 24, + 24 + ] + ], + "provenance": { + "analysis": { + "n_boot": 1000, + "n_permutations": 1000, + "schema_version": 1, + "seed": 0, + "tool": "j7scope.contrast.build_contrast" + }, + "trace_a": { + "capture_tool": "build_demo_trace.py", + "corpus_id": "demo-synthetic", + "created_at": "2026-07-22T00:00:00+00:00", + "estimator": "synthetic", + "is_demo": true, + "language": "en", + "layer": 18, + "lens_sha1": null, + "model": "demo-synthetic", + "model_revision": null, + "trace_id": "demo-parallel-en" + }, + "trace_b": { + "capture_tool": "build_demo_trace.py", + "corpus_id": "demo-synthetic", + "created_at": "2026-07-22T00:00:00+00:00", + "estimator": "synthetic", + "is_demo": true, + "language": "zh", + "layer": 18, + "lens_sha1": null, + "model": "demo-synthetic", + "model_revision": null, + "trace_id": "demo-parallel-zh" + } + }, + "schema_version": 1, + "trace_a": "demo-parallel-en", + "trace_b": "demo-parallel-zh" + }, "members": { "en": "demo-parallel-en", "zh": "demo-parallel-zh" diff --git a/results/traces/demo-parallel-zh/manifest.json b/results/traces/demo-parallel-zh/manifest.json index 1de791d..c6c1009 100644 --- a/results/traces/demo-parallel-zh/manifest.json +++ b/results/traces/demo-parallel-zh/manifest.json @@ -1,6 +1,6 @@ { "capture": { - "created_at": "2026-07-22T12:31:46.844261+00:00", + "created_at": "2026-07-22T00:00:00+00:00", "tool": "build_demo_trace.py" }, "concept": null, @@ -8,6 +8,11 @@ "is_demo": true, "jacobian": { "corpus_id": "demo-synthetic", + "estimator": "synthetic", + "n_probes": null, + "n_prompts": 0, + "position": null, + "seed": 0, "sha1": null }, "kind": "parallel_member", diff --git a/tests/test_contrast.py b/tests/test_contrast.py new file mode 100644 index 0000000..9c83081 --- /dev/null +++ b/tests/test_contrast.py @@ -0,0 +1,82 @@ +from copy import deepcopy + +from j7scope.contrast import ( + PAIR_SHAREDNESS_DEFINITION, + build_contrast, + detect_answer_identity_confound, +) + + +def _token(seq, concept, surface): + return { + "seq": seq, + "token": surface, + "readout": { + "zh": [{"token": f"中-{surface}", "rank": 0, "score": 4.0, + "concept": concept}], + "en": [{"token": f"en-{surface}", "rank": 1, "score": 3.0, + "concept": concept}], + "other": [], + }, + } + + +def _trace(trace_id, language, concepts, **manifest_fields): + manifest = { + "trace_id": trace_id, + "model": "test-model", + "revision": "abc123", + "layer": 12, + "language": language, + "jacobian": {"sha1": "lens-hash", "corpus_id": "corpus-hash", + "estimator": "test"}, + "capture": {"tool": "pytest", "created_at": "2026-01-01T00:00:00Z"}, + "is_demo": False, + } + manifest.update(manifest_fields) + return { + "manifest": manifest, + "tokens": [_token(i, concept, f"t{i}") for i, concept in enumerate(concepts)], + "metrics": {}, + } + + +def test_build_contrast_is_deterministic_and_bakes_research_controls(): + trace_a = _trace("a", "en", ["alpha", "beta", "gamma", "delta"]) + trace_b = _trace("b", "zh", ["alpha", "beta", "gamma", "delta"]) + + first = build_contrast( + trace_a, trace_b, {}, n_permutations=200, n_boot=200, seed=9 + ) + second = build_contrast( + trace_a, trace_b, {}, n_permutations=200, n_boot=200, seed=9 + ) + + assert first == second + assert first["calibration"]["observed"] == 1.0 + assert first["calibration"]["null"]["n"] == 200 + assert first["calibration"]["sharedness"]["definition"] == PAIR_SHAREDNESS_DEFINITION + assert first["concept_trajectories"][0]["rank_a"][0] in (1, None) + assert first["provenance"]["trace_a"]["lens_sha1"] == "lens-hash" + assert len(first["pair_rigor"]) == 4 + + +def test_answer_identity_check_flags_and_clears_confound(): + base = _trace("a", "en", ["alpha"], correct=True, answer_identity="A") + changed_answer = _trace("b", "en", ["alpha"], correct=False, answer_identity="B") + same_answer = deepcopy(changed_answer) + same_answer["manifest"]["answer_identity"] = "A" + + confounded = detect_answer_identity_confound(base["manifest"], changed_answer["manifest"]) + controlled = detect_answer_identity_confound(base["manifest"], same_answer["manifest"]) + + assert confounded["status"] == "confounded" + assert confounded["controlled"] is False + assert controlled["status"] == "controlled" + assert controlled["controlled"] is True + + +def test_answer_identity_check_never_guesses_from_free_text(): + result = detect_answer_identity_confound({"prompt": "correct"}, {"prompt": "wrong"}) + assert result["status"] == "not_assessable" + assert result["controlled"] is None diff --git a/tests/test_recorder.py b/tests/test_recorder.py index 465cd0c..91807fb 100644 --- a/tests/test_recorder.py +++ b/tests/test_recorder.py @@ -115,3 +115,38 @@ def test_hf_backend_rejects_precomputed_jacobian_without_metadata(tmp_path): with pytest.raises(FileNotFoundError, match="metadata"): backend._load_precomputed_jacobian() + + +def test_hf_backend_exposes_activation_patch_with_provenance(monkeypatch): + import j7scope.patching + + backend = HFBackend(model_name="test/model", layer=12) + backend._model = object() + backend._jlens = object() + backend.model_revision_resolved = "revision-1" + backend.jacobian_sha1 = "lens-sha" + + def fake_patch(jlens, source, target, patch_layer, src_pos, tgt_pos, k): + assert jlens is backend._jlens + assert (source, target, patch_layer, src_pos, tgt_pos, k) == ( + "source", "target", 6, 2, 3, 5 + ) + return { + "readout": [("concept", 4.2)], + "next_token": [("answer", 3.1)], + } + + monkeypatch.setattr(j7scope.patching, "patch_and_readout", fake_patch) + result = backend.intervene( + source_prompt="source", + target_prompt="target", + patch_layer=6, + source_position=2, + target_position=3, + topk=5, + ) + + assert backend.supports_intervention is True + assert result["readout"] == [("concept", 4.2)] + assert result["jacobian_sha1"] == "lens-sha" + assert result["model_revision"] == "revision-1" diff --git a/tests/test_trace_gallery.py b/tests/test_trace_gallery.py index 97d1e28..c3ea87e 100644 --- a/tests/test_trace_gallery.py +++ b/tests/test_trace_gallery.py @@ -24,6 +24,22 @@ def test_demo_builder_produces_valid_gallery(tmp_path): ) assert validate_trace_gallery(tmp_path) == [] + align = json.loads( + (tmp_path / "demo-parallel-en" / "align.json").read_text(encoding="utf-8") + ) + assert align["contrast"]["calibration"]["null"]["n"] == 1000 + assert align["contrast"]["concept_trajectories"] + assert align["contrast"]["answer_identity_check"]["status"] == "not_assessable" + + +def test_contrast_lab_page_keeps_research_statistics_precomputed(): + page = (ROOT / "apps" / "site" / "contrast.html").read_text(encoding="utf-8") + + assert "Contrast Lab" in page + assert "C.calibration" in page + assert "jspace/interventions" in page + assert "j7scope.claim_card" in page + assert "cross_lang_overlap -" not in page def test_gallery_validator_detects_index_drift(tmp_path):