From eec7f285509c33c9b880f80d9775a4a76c2feca9 Mon Sep 17 00:00:00 2001 From: fbarez Date: Fri, 6 Nov 2026 10:08:44 +0000 Subject: [PATCH 1/2] =?UTF-8?q?Orientations=20page:=20six=20orientations?= =?UTF-8?q?=20from=20the=20arXiv=20paper=20(=C2=A712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page quoted the 13 Jul draft, which had five orientations, so Interpretability Across Time and Introspective Interpretability were empty and Agentic appeared as an extra card. It now quotes §12 of arXiv:2611.08438v1 verbatim: all six orientations, the six-column comparison table (Table 13), the decision framework and the tensions. The data is generated from the arXiv LaTeX source by scripts/data/build_orientations_arxiv.py; every passage was checked against the arXiv PDF text. The old #agentic anchor lands on Introspective. The rest of the handbook is unchanged. --- scripts/data/build_orientations_arxiv.py | 250 ++++++++++++++++ src/data/orientations-arxiv.json | 353 +++++++++++++++++++++++ src/pages/orientations.astro | 125 +++----- 3 files changed, 645 insertions(+), 83 deletions(-) create mode 100644 scripts/data/build_orientations_arxiv.py create mode 100644 src/data/orientations-arxiv.json diff --git a/scripts/data/build_orientations_arxiv.py b/scripts/data/build_orientations_arxiv.py new file mode 100644 index 0000000000..38d79f91b1 --- /dev/null +++ b/scripts/data/build_orientations_arxiv.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Build src/data/orientations-arxiv.json from the arXiv LaTeX source of the paper. + +The orientations page quotes §12 "Six Research Orientations" of the paper as +published on arXiv, while the rest of the handbook still shows the ingested +13 Jul draft. Everything written here is verbatim manuscript text: LaTeX markup +is removed, natbib citations are rendered from the build's .bbl exactly as +natbib prints author labels and years, and cross-references are resolved from +REF_NUMBERS (read off the arXiv PDF). + +Usage: + curl -L -o src.tar.gz https://arxiv.org/e-print/2611.08438 && mkdir src && tar -xzf src.tar.gz -C src + python3 scripts/data/build_orientations_arxiv.py --source src +""" +import argparse, json, re, sys +from pathlib import Path + +ARXIV = {"id": "2611.08438", "version": "v1", "abs": "https://arxiv.org/abs/2611.08438v1", + "pdf": "https://arxiv.org/pdf/2611.08438v1"} +CHAPTER_FILE = "chapters/12_six_research_orientations.tex" +CHAPTER_NUM = "12" + +# \label -> printed number, from the arXiv v1 PDF. +REF_NUMBERS = { + "sec:architectural-interventions": "6.10", + "sec:ld-localization": "10.2.1", + "sec:training_phase_why": "10.3.4", + "subsec: iat": "12.4", + "subsec:privileged-access": "5.3", + "subsec:valproce model self-reports": "12.5.2", + "subsec:choosing orientation": "12.8", + "fig:sixresearchorientation": "17", + "tab:orientations": "13", + "tab:human-automated-workflow": "12", +} + +# Card order and URL ids. The four ids that already existed keep their anchors. +ORDER = [ + ("pragmatic", "Pragmatic Interpretability", "Proxy-Task-Oriented Safety Interventions"), + ("ambitious", "Ambitious Interpretability", "Towards Complete Mechanistic Understanding"), + ("developmental", "Developmental Interpretability", "Characterising How Models Learn"), + ("interpretability-across-time", "Interpretability Across Time", "Understanding the Causes of Model Behaviours"), + ("introspective-interpretability", "Introspective Interpretability", "Models Interpreting Themselves"), + ("automated", "Automated Interpretability", "Scaling Discovery via Automation"), +] + + +def brace_arg(s, i): + """Return (content, end) of the {...} group starting at s[i] == '{'.""" + assert s[i] == "{", s[i:i + 30] + depth = 0 + for j in range(i, len(s)): + if s[j] == "{": depth += 1 + elif s[j] == "}": + depth -= 1 + if depth == 0: return s[i + 1:j], j + 1 + raise ValueError("unbalanced braces") + + +def load_bbl(path): + txt = path.read_text() + out = {} + for m in re.finditer(r"\\bibitem\[", txt): + i, depth = m.end(), 1 + j = i + while depth: + if txt[j] == "[": depth += 1 + elif txt[j] == "]": depth -= 1 + j += 1 + label = txt[i:j - 1] + key, _ = brace_arg(txt, txt.index("{", j - 1)) + lm = re.match(r"(.*?)\((.*?)\)", label, re.S) + authors = latex_to_text(lm.group(1)).strip() + year = re.sub(r"\{?\\natexlab\{(\w+)\}\}?", r"\1", lm.group(2)).strip() + out[key.strip()] = (authors, latex_to_text(year)) + return out + + +def latex_to_text(s): + s = s.replace("~", " ").replace("\\&", "&").replace("{\\'e}", "é").replace("{\\'a}", "á") + s = re.sub(r"\{\\(['`\"^~])\{?(\w)\}?\}", lambda m: ACCENTS.get(m.group(1) + m.group(2), m.group(2)), s) + s = re.sub(r"\\(['`\"^~])\{?(\w)\}?", lambda m: ACCENTS.get(m.group(1) + m.group(2), m.group(2)), s) + return re.sub(r"[{}]", "", s) + + +ACCENTS = {"'e": "é", "'a": "á", "'i": "í", "'o": "ó", "'u": "ú", '"o': "ö", '"u': "ü", '"a': "ä", "`e": "è", "`a": "à", "~n": "ñ", "^o": "ô", "'c": "ć", "'s": "ś"} + + +class Converter: + def __init__(self, bbl): + self.bbl = bbl + self.missing = set() + self.unresolved = set() + + def cite(self, keys, form): + parts, prev = [], None + for k in [k.strip() for k in keys.split(",") if k.strip()]: + if k not in self.bbl: + self.missing.add(k); parts.append(f"[{k}]"); continue + a, y = self.bbl[k] + if a == prev and form != "t": # natbib compresses repeated authors: "Biderman et al., 2023b; 2026" + parts.append(y); continue + prev = a + parts.append(f"{a} ({y})" if form == "t" else f"{a}, {y}") + if form == "t": return ", ".join(parts) + if form == "alp": return "; ".join(parts) + return "(" + "; ".join(parts) + ")" + + def convert(self, s): + s = re.sub(r"(? body, and the chapter intro.""" + tex = re.sub(r"\\begin\{(figure|table|warningbox|practicebox)\}.*?\\end\{\1\}", "", tex, flags=re.S) + tex = re.sub(r"\\label\{[^}]*\}", "", tex) + parts = re.split(r"\\subsection\{(.*?)\}\n", tex) + intro = parts[0].split("\n", 1)[1] + subs = {parts[i]: parts[i + 1] for i in range(1, len(parts), 2)} + return intro, subs + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--source", required=True, type=Path) + ap.add_argument("--out", type=Path, default=Path(__file__).resolve().parents[2] / "src/data/orientations-arxiv.json") + args = ap.parse_args() + bbl = load_bbl(next(args.source.glob("*.bbl"))) + cv = Converter(bbl) + tex = (args.source / CHAPTER_FILE).read_text() + intro, subs = sections(tex) + titles = list(subs) + + def num(i): return f"{CHAPTER_NUM}.{i + 1}" + + orientations = [] + for (oid, name, sub), title in zip(ORDER, titles[:6]): + assert title.startswith(name), (title, name) + i = titles.index(title) + body = subs[title] + # Main text only: the subsubsections are failure modes, guardrails and checklists. + main_part, *subsubs = re.split(r"\\subsubsection\{(.*?)\}\n", body) + blocks = split_blocks(main_part) + body = [] + for k, b in blocks: + if k == "p": + body.append({"p": cv.convert(b)}) + else: + items = [] + for it in b: + tm = re.match(r"\\textbf\{(.*?)\}", it) + items.append({"term": cv.convert(tm.group(1)).rstrip(",:") if tm else None, "text": cv.convert(it)}) + body.append({"ul": items}) + for j in range(0, len(subsubs), 2): + st, sb = subsubs[j], subsubs[j + 1] + if st.startswith("When to Use"): + body.append({"h": cv.convert(st), "section": f"{num(i)}.{j // 2 + 1}"}) + body += [{"p": cv.convert(b)} for k, b in split_blocks(sb) if k == "p"] + # Definition = the first two sentences; the rest of that paragraph stays in the body. + first = re.split(r"(?<=[.!?])\s+(?=[A-Z])", body[0]["p"]) + body = ([{"p": " ".join(first[2:])}] if len(first) > 2 else []) + body[1:] + terms = [it["term"] for blk in body if "ul" in blk for it in blk["ul"] if it["term"]] + lead = next((body[k - 1].get("p", "") for k, blk in enumerate(body) if "ul" in blk and k), "") + list_label = "Example metrics" if "metrics" in lead else "Methods" + orientations.append({ + "id": oid, "name": name, "subtitle": sub, "section": num(i), "section_title": title, + "definition": " ".join(first[:2]), "list_label": list_label, "list_terms": terms, "body": body, + }) + + def sec_text(prefix): + t = next(t for t in titles if t.startswith(prefix)) + return titles.index(t), [cv.convert(b) for k, b in split_blocks(subs[t]) if k == "p"] + + ci, comparison_intro = sec_text("Comparative Analysis") + di, decision = sec_text("Choosing an Orientation") + ti, tensions = sec_text("Critical Concerns") + + # Comparison table (tab:orientations), cell by cell. + tab = re.search(r"\\label\{tab:orientations\}.*?\\midrule(.*?)\\bottomrule", tex, re.S).group(1) + cap = re.search(r"\\caption\{((?:(?!\\caption).)*?)\}\s*\\label\{tab:orientations\}", tex, re.S).group(1) + cols = ["Dimension", "Pragmatic", "Ambitious", "Developmental", "Across Time", "Introspective", "Automated"] + rows = [] + for r in re.split(r"\\\\", tab): + r = re.sub(r"\\rowcolor\{[^}]*\}", "", r).strip() + if not r: continue + cells = [cv.convert(c) for c in r.split("&")] + assert len(cells) == 7, cells + rows.append(dict(zip(cols, cells))) + + if cv.missing or cv.unresolved: + sys.exit(f"missing citations: {sorted(cv.missing)}; unresolved refs: {sorted(cv.unresolved)}") + + data = { + "about": "Generated by scripts/data/build_orientations_arxiv.py from the arXiv v1 LaTeX source. Verbatim; do not edit by hand.", + "source": {**ARXIV, "chapter": CHAPTER_NUM, "chapter_title": "Six Research Orientations"}, + "intro": [cv.convert(b) for k, b in split_blocks(intro) if k == "p"], + "orientations": orientations, + "comparison": {"section": num(ci), "intro": comparison_intro, "caption": f"Table {REF_NUMBERS['tab:orientations']}: " + cv.convert(cap), "columns": cols, "rows": rows}, + "decision": {"section": num(di), "paragraphs": decision}, + "tensions": {"section": num(ti), "paragraphs": tensions}, + } + args.out.write_text(json.dumps(data, indent=1, ensure_ascii=False) + "\n") + print(f"wrote {args.out}: {len(orientations)} orientations, {len(rows)} table rows") + + +if __name__ == "__main__": + main() diff --git a/src/data/orientations-arxiv.json b/src/data/orientations-arxiv.json new file mode 100644 index 0000000000..91373ba443 --- /dev/null +++ b/src/data/orientations-arxiv.json @@ -0,0 +1,353 @@ +{ + "about": "Generated by scripts/data/build_orientations_arxiv.py from the arXiv v1 LaTeX source. Verbatim; do not edit by hand.", + "source": { + "id": "2611.08438", + "version": "v1", + "abs": "https://arxiv.org/abs/2611.08438v1", + "pdf": "https://arxiv.org/pdf/2611.08438v1", + "chapter": "12", + "chapter_title": "Six Research Orientations" + }, + "intro": [ + "An orientation is a coherent research programme defined by its goals, assumptions, and trade-offs. In contrast, a method is an established modular pipeline that may be used in different orientations. In AI interpretability, different research goals, such as developing ready-to-use safety measures or achieving complete mechanistic understanding, have led to distinct research programmes. This section organises six major research orientations and summarises their scientific objectives, core assumptions, characteristic methods, and limitations (Nanda et al., 2025; Gao, 2025; Li et al., 2025a; Hoogland et al., 2023; Barez, 2026; Biderman et al., 2026; Figure 17). Note that no single orientation is universally preferable. The choice depends on the research objective, deployment context, available resources, and tolerance for different forms of uncertainty." + ], + "orientations": [ + { + "id": "pragmatic", + "name": "Pragmatic Interpretability", + "subtitle": "Proxy-Task-Oriented Safety Interventions", + "section": "12.1", + "section_title": "Pragmatic Interpretability: Proxy-Task-Oriented Safety Interventions", + "definition": "Pragmatic Interpretability focuses on doing the research most helpful for longer-term alignment and interpretability goals, such as helping AGI go well (Nanda et al., 2025). Proponents argue that full mechanistic understanding is not required to have significant impact; instead, researchers should prioritise the work most likely to help (Nanda et al., 2025).", + "list_label": "Methods", + "list_terms": [ + "Steering vectors", + "Probes", + "Behavioural monitoring and anomaly detection", + "Prompting" + ], + "body": [ + { + "p": "The central idea is to work on a set of operationally meaningful proxy tasks that can be empirically studied on models today to help make powerful future models safe. Researchers are agnostic about which methods they use, even if they do not involve model internals. Though not required, these proxy tasks may also be directly useful on today's deployed systems, providing valuable feedback." + }, + { + "p": "A representative example is the study of Claude Sonnet 4.5’s evaluation awareness (Anthropic, 2025a). The system was shown to condition its behaviour on the presence of evaluation signals, making results from safety evaluations used for previous models harder to interpret. Researchers identified and applied a steering vector that suppresses this awareness, allowing evaluation of its actual behaviour. While this study does not fully explain the internal mechanism of evaluation awareness, it is sufficient to produce a concrete and safety-relevant behavioural change, illustrating the pragmatic philosophy." + }, + { + "p": "Another example is Marks et al. (2025b). In this case, researchers constructed a proxy task: they fine-tuned a model with a hidden goal, then blinded teams of researchers audited the model to discover it. This served as a proxy task for detecting a naturally emerging hidden goal in a future model. The different auditing teams used different techniques, allowing researchers to compare their efficacy." + }, + { + "p": "Common techniques in this paradigm include:" + }, + { + "ul": [ + { + "term": "Steering vectors", + "text": "Steering vectors, which shift models to targeted behaviours." + }, + { + "term": "Probes", + "text": "Probes, which detect latent safety-relevant properties in activations." + }, + { + "term": "Behavioural monitoring and anomaly detection", + "text": "Behavioural monitoring and anomaly detection, which track signals indicative of emerging risks, such as monitoring the model's verbalised reasoning." + }, + { + "term": "Prompting", + "text": "Prompting, an effective black-box baseline method for steering model behaviour." + } + ] + }, + { + "p": "Recent work following this pragmatic trajectory has demonstrated effective control of refusal behaviour (Arditi et al., 2024) and evaluation awareness (Anthropic, 2025a). These interventions provide direct gains to contemporary systems. However, they may not (and are not designed to) fully explain how models implement their behaviours." + } + ] + }, + { + "id": "ambitious", + "name": "Ambitious Interpretability", + "subtitle": "Towards Complete Mechanistic Understanding", + "section": "12.2", + "section_title": "Ambitious Interpretability: Towards Complete Mechanistic Understanding", + "definition": "In contrast to pragmatic approaches, Ambitious Interpretability seeks a comprehensive mechanistic understanding of how models implement their computations (Gao, 2025). Proponents argue that partial or surface-level understanding cannot guarantee safety in next-generation systems.", + "list_label": "Example metrics", + "list_terms": [ + "Feature quality", + "Circuit faithfulness" + ], + "body": [ + { + "p": "Full understanding is needed to rule out hidden failure modes or deceptive internal dynamics. Specifically, Ambitious Interpretability does not treat interpretability as a collection of useful heuristics. Instead, it treats interpretability as a scientific objective: to reverse-engineer the model and develop a transparent, predictive theory of its internal algorithms." + }, + { + "p": "Ambitious Interpretability particularly emphasises metric-driven progress. Motivated by the fact that existing mechanistic approaches still struggle with stringent metrics, it proposes starting with looser ones, tracking and improving methods’ performance on them, and adopting stricter metrics when suitable. Two example metrics by Gao (2025) are:" + }, + { + "ul": [ + { + "term": "Feature quality", + "text": "Feature quality: demonstrating that extracted features and their natural-language explanations are faithful and simulatable." + }, + { + "term": "Circuit faithfulness", + "text": "Circuit faithfulness: demonstrating circuits are necessary and sufficient for explaining behaviour via causal intervention (e.g., causal scrubbing (Chan et al., 2022a)), without performance degradation." + } + ] + }, + { + "p": "The disadvantage of this ambition is its cost. Fine-grained mechanistic analysis of frontier large models is computationally expensive and sometimes intractable. As Gao (2025) emphasised, current tools still fall far short of the coverage and robustness required for a truly comprehensive understanding." + }, + { + "p": "Interpretability-by-design as a Synthesis. The emergence of interpretability-by-design models (Section 6.10) blurs the boundary between Ambitious and Pragmatic Interpretability. These methods (Gao et al., 2026; Pearce et al., 2025; Park et al., 2025; Oldfield et al., 2024) aim to make representations and circuits easier to understand by modifying model architectures rather than relying entirely on post-hoc reverse engineering. This practical change to model design pursues the ambitious goal of mechanistic understanding. However, it remains unclear whether these methods can ultimately achieve complete mechanistic understanding. In the short term, they may simply represent a separate “Engineering” orientation in which interpretability is treated as an optimisation objective." + } + ] + }, + { + "id": "developmental", + "name": "Developmental Interpretability", + "subtitle": "Characterising How Models Learn", + "section": "12.3", + "section_title": "Developmental Interpretability: Characterising How Models Learn", + "definition": "Complementing post-hoc analysis of trained models, Developmental Interpretability (Hoogland et al., 2023; Hoogland et al., 2025) treats the training process itself as an object of scientific inquiry. Taking a learning-theory perspective, it aims to mathematically formalise the emergence and evolution of model representations, effective complexity, and capabilities throughout training.", + "list_label": "Methods", + "list_terms": [ + "Singular Learning Theory (SLT)", + "Checkpoint-based analysis" + ], + "body": [ + { + "p": "Its central claim is that many neural network properties which are hard to analyse through a fixed model checkpoint can be understood through their developmental process. This process contains measurable, modelable signatures that can serve as proxies for the model’s properties of interest, such as developmental stages, phase transitions, and other organising principles that help mathematically explain how complex behaviours emerge and evolve throughout training. A classic example is the Local Learning Coefficient (LLC) introduced in Sections 10.2.1 and 10.3.4. It successfully serves as a signature of the model’s effective complexity, allowing us to observe developmental transitions and compare two models with the same training loss." + }, + { + "p": "Methods in this paradigm include:" + }, + { + "ul": [ + { + "term": "Singular Learning Theory (SLT)", + "text": "Singular Learning Theory (SLT) and related mathematical frameworks for characterising developmental stages, symmetry breaking, and changes in the geometry of the loss landscape during training." + }, + { + "term": "Checkpoint-based analysis", + "text": "Checkpoint-based analysis to track the emergence of representations and mechanisms by applying specialised probes, feature discovery methods, and circuit analyses across a model's training checkpoints." + } + ] + }, + { + "p": "More specifically, Developmental Interpretability aims to answer questions such as: Can we characterise the development of model capabilities or behaviours as phase transitions during training? What developmental stages precede the formation of a certain mechanism? How does the optimisation process transform distributed representations into structured mechanisms? Unlike Interpretability Across Time (detailed in Section 12.4), which emphasises forecasting and controlling future model behaviour, Developmental Interpretability is primarily concerned with understanding the developmental processes that produce those behaviours. Its primary goal is rigorous scientific explanation rather than predicting or intervening on capabilities." + } + ] + }, + { + "id": "interpretability-across-time", + "name": "Interpretability Across Time", + "subtitle": "Understanding the Causes of Model Behaviours", + "section": "12.4", + "section_title": "Interpretability Across Time: Understanding the Causes of Model Behaviours", + "definition": "Like Developmental Interpretability, Interpretability Across Time (Biderman et al., 2023b; 2026) treats a model as a time-evolving process shaped by its training data, loss functions, architectures, and optimisation dynamics. However, the two orientations differ in their primary goals.", + "list_label": "Methods", + "list_terms": [ + "Longitudinal analysis of model internals", + "Monitoring training runs and forecasting future behaviour", + "Training-process interventions and design" + ], + "body": [ + { + "p": "Developmental Interpretability mainly focuses on characterising the developmental processes that give rise to the model's internal features, mechanisms, and capabilities. In contrast, Interpretability Across Time seeks established knowledge of learning that is (1) sufficiently predictive to estimate important model properties before training is complete, (2) sufficiently causal to identify when and why undesirable trajectories arise, and (3) sufficiently precise to enable interventions on the learning process itself (Biderman et al., 2026). From a safety perspective, this orientation aims to understand and ultimately shape the training process to prevent the model from developing undesirable behaviours in the first place rather than repairing them after they develop." + }, + { + "p": "Interpretability Across Time builds on three areas of research. First, scaling-law studies show that model performance can be predicted from factors such as model size, training data, and compute (Kaplan et al., 2020; Hoffmann et al., 2022). This motivates similar efforts to predict safety-relevant model properties (Ganguli et al., 2022; Jones et al., 2025). Second, longitudinal mechanistic studies track how internal mechanisms develop during training (Tigges et al., 2024; Singh et al., 2024). Third, studies of training data show that the training distribution can affect downstream model behaviour. For example, Razeghi et al. (2022) found that few-shot numerical reasoning performance depends strongly on the frequency of relevant terms in pretraining data, while Elazar et al. (2023) estimated the causal effects of pretraining-data statistics on factual predictions. Interpretability Across Time builds on these findings to predict and influence safety-relevant model behaviour during training." + }, + { + "p": "Methods in this paradigm include:" + }, + { + "ul": [ + { + "term": "Longitudinal analysis of model internals", + "text": "Longitudinal analysis of model internals, tracking representations, features, or implemented mechanisms across checkpoints to determine when and how they form and evolve." + }, + { + "term": "Monitoring training runs and forecasting future behaviour", + "text": "Monitoring training runs and forecasting future behaviour, using early behavioural or internal signals to predict later capabilities, biases, or unsafe behaviours before they are fully expressed." + }, + { + "term": "Training-process interventions and design", + "text": "Training-process interventions and design, manipulating data, activations, objectives, architectures, or other aspects of optimisation to encourage desirable developmental trajectories, prevent undesirable capabilities or behaviours from arising, or ensure that learned information is represented in ways that make it easier to control." + } + ] + }, + { + "p": "Several studies have examined whether safety risks can be identified before or during training. Zhang et al. (2026b) found that model activations measured before fine-tuning can predict alignment scores after fine-tuning. Similarly, Wang et al. (2026a) identified internal features associated with emergent misalignment, which could sometimes detect misalignment before behavioural evaluations did. Other studies found that training models to exploit relatively simple reward misspecifications can lead to more serious failures, including reward tampering and broader misaligned behaviour (Denison et al., 2024; Taylor et al., 2025). These findings motivate monitoring early signs of undesirable behaviour during training rather than relying solely on the evaluation of the final model." + }, + { + "p": "Another line of work under this paradigm attempts to directly shape models by preventing them from learning unsafe behaviours. Data filtering can prevent models from acquiring hazardous information in the first place and substantially increase the amount of subsequent fine-tuning required to recover it (O'Brien et al., 2026; Rathi & Radford, 2026), and gradient routing approaches deliberately constrain where or how information is learned, making later removal or control more tractable (Cloud et al., 2024; Shilov et al., 2025)." + }, + { + "p": "Common confusion. Although Biderman et al. (2026) contrasted this orientation with the “fix it in post” paradigm, it would be a mistake to view it as dismissive of post-training writ large. Post-training is itself a training process, and behaviours like sycophancy and reward hacking arise primarily during and due to post-training. This orientation's criticism of post-training safety focuses on cases where, instead of understanding why a behaviour developed, post-training patches undesirable behaviour. The core distinction is understanding and controlling a training process versus applying optimisation pressure to suppress learned behaviours." + }, + { + "p": "Interpretability Across Time is not mutually exclusive with the other research orientations. It can be Pragmatic, by using a simple probe as an early-warning signal, or Ambitious, by seeking a mechanistic theory of why a circuit forms under gradient descent. Its defining goal is using insights into training dynamics to explain and predict how model behaviour develops, and to intervene before undesirable behaviours emerge." + } + ] + }, + { + "id": "introspective-interpretability", + "name": "Introspective Interpretability", + "subtitle": "Models Interpreting Themselves", + "section": "12.5", + "section_title": "Introspective Interpretability: Models Interpreting Themselves", + "definition": "Introspective Interpretability (Binder et al., 2025; Plunkett et al., 2025; Li et al., 2025a; Guo et al., 2026; Li, 2026) uses the model being explained as its own explainer. Its main practical advantage is its unique accessibility.", + "list_label": "Methods", + "list_terms": [], + "body": [ + { + "p": "For a given introspective model, it requires no white-box access or specialised tooling to explain its decisions; anyone can get the self-explanation (or interchangeably self-report) at deployment time by asking the model a follow-up question in an ordinary chat interface." + }, + { + "p": "This research direction is supported by the privileged access hypothesis discussed in Section 5.3, which posits that models may have access to internal information that is difficult to recover through external analysis alone, including typical mechanistic analysis or a different model as the explainer. Recent work has leveraged this idea by fine-tuning target models to faithfully report their internal reasoning, behavioural tendencies, objectives, and even weight changes induced by training (Goel et al., 2026)." + }, + { + "p": "A related line of work fine-tunes metamodels (external explainers) on the internal representations of a target model (Pan et al., 2026; Karvonen et al., 2026a; Bauer et al., 2026; Huang et al., 2026). These methods are generally not considered instances of Introspective Interpretability because the explainer is technically a separate model. However, the distinction can be blurry in practice, as many of these explainers are initialised from the target model and can be viewed as specialised, fine-tuned descendants of the original model." + }, + { + "h": "When to Use Introspective Methods", + "section": "12.5.3" + }, + { + "p": "Introspective methods are best suited for three scenarios: (1) Hypothesis generation, where introspection provides candidate explanations to be validated by other methods; (2) Rapid triage, where introspection is used to quickly identify which input–output pairs or model components are worth further investigation; (3) Human–AI collaboration, where introspective models provide human-readable explanations tailored to users' levels of expertise and understanding." + }, + { + "p": "Introspective methods are insufficient for: (1) Claims of absence: claims about the absence of deceptive reasoning or hidden objectives cannot be established by self-report alone; (2) Formal guarantees: self-reports do not provide the causal evidence needed for formal verification; (3) Adversarial settings: in scenarios where the model has strong incentives to misreport, introspection may be less reliable and require additional safeguards." + }, + { + "p": "Implications. Although introspective methods can help efficiently detect latent goals, plans, and deceptive behaviour (Binder et al., 2025; Li et al., 2026b; Goel et al., 2026; Li et al., 2025a; Joglekar et al., 2025), they work best as complements to other mechanistic analyses (Bereska & Gavves, 2024). Specifically, their most productive use is as one component of a multi-method interpretability pipeline: self-reports serve as initial hypotheses, causal interventions test them, and circuit discovery methods explain the underlying mechanisms. While we may optimistically assume models can tell us how they work in the future, we must also recognise that self-reports are relatively weak evidence and require independent validation." + } + ] + }, + { + "id": "automated", + "name": "Automated Interpretability", + "subtitle": "Scaling Discovery via Automation", + "section": "12.6", + "section_title": "Automated Interpretability: Scaling Discovery via Automation", + "definition": "Automated Interpretability seeks to scale mechanistic analysis to large models by replacing human analysis with automated pipelines. It assumes that manual analysis is the bottleneck and that automation can accelerate discovery.", + "list_label": "Methods", + "list_terms": [ + "Automated circuit discovery", + "LLM-based feature or circuit explanations" + ], + "body": [ + { + "p": "Methods include:" + }, + { + "ul": [ + { + "term": "Automated circuit discovery", + "text": "Automated circuit discovery to automatically find the critical computational graph inside the model that processes a given input prompt or implements a specific capability." + }, + { + "term": "LLM-based feature or circuit explanations", + "text": "LLM-based feature or circuit explanations to automatically discover, summarise, and evaluate representational units and internal computations and give human-readable interpretations or reports." + } + ] + }, + { + "p": "Automated interpretability is not limited to any particular type of internal representation or mechanism. Depending on the research question, automated pipelines may analyse SAE features (Paulo et al., 2025b; Han et al., 2026), individual neurons (Bills et al., 2023; Foote et al., 2023; Oikarinen & Weng, 2023), circuits (Conmy et al., 2023; Khan et al., 2026), or other model components (Fraser-Taliente et al., 2026). This approach distinguishes itself by automatically discovering, explaining, or evaluating these objects rather than relying on manual analysis. Below, we illustrate a classical automated feature-explanation pipeline whose explanation and verification stages generally follow Bills et al. (2023)." + }, + { + "p": "Such a pipeline enables analysis of a model with billions of parameters with minimal human intervention (Bills et al., 2023), a prerequisite for comprehensive model oversight (Foote et al., 2023)." + }, + { + "p": "Recent research has substantially expanded the scope of Automated Interpretability. While early work focused primarily on automated feature explanation, newer systems employ increasingly sophisticated pipelines for feature discovery, explanation generation, auditing, and intervention (Han et al., 2026; Marin-Llobet & Ferrando, 2026; Barez, 2026; Cywiński et al., 2025). More recently, automation has also been extended from individual features to higher-level structures such as circuits (Khan et al., 2026; Haklay et al., 2026), enabling automated explanation of circuit semantics rather than merely discovering circuit components (Conmy et al., 2023; Ameisen et al., 2025)." + } + ] + } + ], + "comparison": { + "section": "12.7", + "intro": [ + "Table 13 compares the six research orientations across several dimensions, including their problem choices, primary techniques, and practical scalability. None of them is universally preferable. For example, Automated Interpretability can scale analysis to millions of features in frontier models but faces issues such as error propagation and automation bias. By contrast, Ambitious Interpretability can provide fundamental, theory-grounded analysis, but its current results still mainly come from controlled, small-scale settings (Elhage et al., 2022a), and its transferability to frontier models remains unclear. Thus, the choice of orientation depends on multiple factors, such as the intended downstream use cases and available computational and human resources, as detailed in Section 12.8." + ], + "caption": "Table 13: Comparison of six interpretability orientations across their core theses, evaluation criteria, methodological approaches, research time horizons, scalability, and robustness to model deception.", + "columns": [ + "Dimension", + "Pragmatic", + "Ambitious", + "Developmental", + "Across Time", + "Introspective", + "Automated" + ], + "rows": [ + { + "Dimension": "Main Focus", + "Pragmatic": "Immediate safety interventions without complete understanding", + "Ambitious": "Complete causal understanding regardless of cost", + "Developmental": "Developmental processes of representations, mechanisms, and capabilities", + "Across Time": "Prediction and control of future model behaviour through training dynamics", + "Introspective": "Introspective models to explain themselves", + "Automated": "Scalable discovery via automated pipelines" + }, + { + "Dimension": "Key Metrics", + "Pragmatic": "Performance in proxy tasks", + "Ambitious": "Faithfulness and simulatability", + "Developmental": "Phase-transition signatures (e.g., LLC) and their correspondence to mechanism formation", + "Across Time": "Forecasting accuracy and intervention effectiveness", + "Introspective": "Alignment between self-reports and actual mechanisms", + "Automated": "Coverage, scalability, robustness, and faithfulness of analysis" + }, + { + "Dimension": "Primary Tools", + "Pragmatic": "Steering vectors, probes, behavioural evaluation", + "Ambitious": "Circuit discovery, loose-to-strict verification", + "Developmental": "SLT, loss landscape geometry, representation analysis", + "Across Time": "Longitudinal analysis, behavioural forecasting, training interventions", + "Introspective": "Introspective training, activation verbalisers", + "Automated": "Circuit discovery, SAEs, LLM agents" + }, + { + "Dimension": "Expected Time to Impact", + "Pragmatic": "Short-term", + "Ambitious": "Long-term", + "Developmental": "Long-term", + "Across Time": "Medium", + "Introspective": "Medium", + "Automated": "Short-term" + }, + { + "Dimension": "Scalability", + "Pragmatic": "High", + "Ambitious": "Low", + "Developmental": "Low", + "Across Time": "Medium", + "Introspective": "High", + "Automated": "Medium" + }, + { + "Dimension": "Robustness to Deception", + "Pragmatic": "Medium", + "Ambitious": "High", + "Developmental": "Unknown", + "Across Time": "Unknown", + "Introspective": "Low", + "Automated": "Low" + } + ] + }, + "decision": { + "section": "12.8", + "paragraphs": [ + "The investigator's priorities and constraints determine the research orientation to select. For teams or individuals focused on immediate safety interventions, Pragmatic Interpretability is the most suitable research style. If the primary goal is to guarantee the absence of harmful behaviour in the long term, Ambitious Interpretability is most appropriate. Researchers seeking to characterise and explain dynamics of capability development may adopt Developmental Interpretability. In contrast, teams training and deploying language-model products with a focus on safety concerns may focus on Interpretability Across Time. When the objective is to offer accessible interpretability analysis to general users, Introspective Interpretability offers a promising paradigm. In contrast, Automated Interpretability is particularly attractive when analyses must be performed at a scale that would be impractical for human investigators alone. Notably, these research orientations are not mutually exclusive; hybrid strategies can combine the strengths of different approaches." + ] + }, + "tensions": { + "section": "12.9", + "paragraphs": [ + "Several concerns and tensions arise within and between these research orientations. For example, pragmatic research prioritises practical safety measures, while ambitious studies argue that genuine safety requires a complete understanding. Developmental Interpretability and Interpretability Across Time both rely on access to training checkpoints and optimisation dynamics, which remain difficult to obtain. Introspective models can provide accessible explanations, but safety-critical decisions may always require additional verification. Automated pipelines can analyse large models, but the comprehensive analysis they deliver may be hard for humans to inspect and verify. Given these trade-offs, allocating research resources across these orientations appears to be a non-technical open challenge." + ] + } +} diff --git a/src/pages/orientations.astro b/src/pages/orientations.astro index d7529828b1..edca14d24a 100644 --- a/src/pages/orientations.astro +++ b/src/pages/orientations.astro @@ -1,120 +1,79 @@ --- import Base from '../layouts/Base.astro'; -import DraftBanner from '../components/DraftBanner.astro'; import Excerpt from '../components/Excerpt.astro'; import { excerptHtml } from '../lib/manuscript'; -import SectionLinks from '../components/SectionLinks.astro'; -import ManuscriptNote from '../components/ManuscriptNote.astro'; -import { framework, sections, resolveMethod } from '../lib/data'; -import { isDraft } from '../lib/manuscript'; +import { resolveMethod } from '../lib/data'; import { url } from '../lib/site'; -const o = framework.orientations; -const items: any[] = o.orientations; -const byId = new Map(items.map((x) => [x.id, x])); -// The six orientations named on the public project page / abstract, in order, -// with the subtitles from that page. `draft` links to the entry in the -// ingested manuscript when one exists (see public_name_mapping in framework.json). -const PUBLIC = [ - { name: 'Pragmatic Interpretability', sub: 'Proxy-task-oriented safety interventions', draft: 'pragmatic' }, - { name: 'Ambitious Interpretability', sub: 'Towards complete mechanistic understanding', draft: 'ambitious' }, - { name: 'Developmental Interpretability', sub: 'Characterising how models learn', draft: 'developmental' }, - { name: 'Interpretability Across Time', sub: 'Understanding the causes of model behaviours', draft: null }, - { name: 'Introspective Interpretability', sub: 'Models interpreting themselves', draft: null }, - { name: 'Automated Interpretability', sub: 'Scaling discovery via automation', draft: 'automated' }, -]; -const mapping = new Map((o.public_name_mapping as any[]).map((m) => [m.public_name, m])); -const unmatched = items.filter((x) => !PUBLIC.some((p) => p.draft === x.id)); -const table = o.comparison_table; -const fields: [string, string][] = [['goals', 'Goals'], ['assumptions', 'Assumptions'], ['strengths', 'Strengths'], ['limitations', 'Limitations'], ['safety_relevance', 'Safety relevance']]; -const sec = (id: string) => sections([id])[0]; +import data from '../data/orientations-arxiv.json'; +// Quoted from §12 of the paper as published on arXiv (generated by +// scripts/data/build_orientations_arxiv.py). The handbook reader still shows the +// ingested working draft, which had five orientations; this page follows the paper. +const src = data.source; +const ref = (num: string, title: string) => ({ num, title, href: src.abs }); +const orientations: any[] = data.orientations; +const table = data.comparison; import { draftHidden as __hide } from '../lib/manuscript'; if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/')}held/`); --- - +

Research directions

Research orientations

An orientation is a research agenda, defined by its goals, assumptions and trade-offs. It is not a method: several orientations can use the same tool for different ends.

- - - - {isDraft && ( -
- Six on the project page, five in this draft. The abstract lists six orientations. The draft text ingested here has five. It has no orientation called Interpretability Across Time, and it has Agentic Interpretability where the abstract says Introspective Interpretability. - The two missing orientations are shown with their project-page subtitles only, and they will fill in once the final manuscript is ingested. We have not assumed that “Agentic” was renamed “Introspective”. -
- )} +

Quoted from §{src.chapter} {src.chapter_title} of the paper, arXiv:{src.id}{src.version}.

+ {data.intro.map((t: string) => )}
- {PUBLIC.map((p, i) => { - const d = p.draft ? byId.get(p.draft) : null; - const map = mapping.get(p.name.replace(' Interpretability', '')) ?? mapping.get(p.name); - return ( -
- {i + 1} -

{p.name}

-

{p.sub}

- {d ? (<> - - {d.methods_used?.length > 0 &&

Methods the survey associates with it:{' '} - {d.methods_used.map((n: string, j: number) => { const m = resolveMethod(n); return <>{j > 0 && ', '}{m ? {n} : n}; })}

} -
- Goals, assumptions, strengths and limitations - {fields.map(([k, label]) =>

{label}

{d[k]?.length ? d[k].map((t: string) => ) :

Not stated in the survey.

}
)} -
- ) : ( -

Not in the draft manuscript ingested here. {map?.status && <>Closest material: {map.draft_sections?.filter((s: string) => sec(s)?.href).slice(0, 3).map((s: string, j: number) => <>{j > 0 && ', '}§{sec(s)!.num})}.}

- )} -
- ); - })} - {unmatched.map((d) => ( -
- · -

{d.name}

-

{d.subtitle} · in the draft, not on the project page

- + {orientations.map((o, i) => ( +
+ {o.id === 'introspective-interpretability' && } + {i + 1} +

{o.name}

+

{o.subtitle}

+ + {o.list_terms.length > 0 &&

{o.list_label}:{' '} + {o.list_terms.map((n: string, j: number) => { const m = o.list_label === 'Methods' ? resolveMethod(n) : null; return <>{j > 0 && ', '}{m ? {n} : n}; })}

}
- Goals, assumptions, strengths and limitations - {fields.map(([k, label]) =>

{label}

{d[k]?.length ? d[k].map((t: string) => ) :

Not stated in the survey.

}
)} + Read §{o.section} in full + {o.body.map((b: any) => b.p ? + : b.h ?

§{b.section} {b.h}

+ :
    {b.ul.map((it: any) =>
  • )}
)}
))}
- {table && ( -
-

Side-by-side comparison

-

{table.caption}

-
- {table.columns.slice(1).map((c: string) => )} -
-
- {table.columns.map((c: string) => )} - {table.rows.map((r: any) => {table.columns.map((c: string, j: number) => j === 0 ? )} -
{c}
: )}
- -
- )} +
+

Side-by-side comparison

+ {table.intro.map((t: string) => )} +

{table.caption}

+
+ {table.columns.slice(1).map((c: string) => )} +
+
+ {table.columns.map((c: string) => )} + {table.rows.map((r: any) => {table.columns.map((c: string, j: number) => j === 0 ? )} +
{c}
: )}
+

In the survey: §{table.section}

+

Choosing an orientation

- {o.decision_framework.map((t: string) => )} - + {data.decision.paragraphs.map((t: string) => )}

Critical tensions

- {o.tensions.map((t: string) => )} - + {data.tensions.paragraphs.map((t: string) => )}