publications.json

{
"preprints": [],
    "publications": [
      {
      "title": "Base Models Look Human To AI Detectors",
      "url": "https://arxiv.org/abs/2605.19516",
      "authors": [
        "Yixuan Even Xu",
        "Ziqian Zhong",
        "Aditi Raghunathan",
        "Fei Fang",
        "J. Zico Kolter"
      ],
      "venue": "arXiv",
      "awards": [],
      "year": "2026",
      "code": "",
      "enabled": true,
      "abstract": "As AI-generated text enters the real-world at scale, institutions increasingly use commercial AI-text detectors, especially in education and academic-integrity workflows. We report a surprising empirical finding about such systems: when evaluated by GPTZero and Pangram, generated text from base models is often judged overwhelmingly human, whereas text generated by their instruction-tuned counterparts is not. Building on this observation, we propose Humanization by Iterative Paraphrasing (HIP), a detector-agnostic pipeline that minimally fine-tunes a base model into a paraphraser and applies it iteratively. Compared with the baselines we test, HIP yields a stronger trade-off between semantic preservation and detector evasion on commercial detectors. Across Llama-3 and Qwen-3 families, spanning model sizes from 0.6B to 70B, HIP consistently improves detector human-likeness. Our findings suggest that current detectors are tracking artifacts of instruction tuning and local context more than any invariant notion of machine-generated text. This, in turn, calls for detector designs that model these factors more explicitly."
      },

      {
      "title": "The Finetuner's Fallacy: When to Pretrain with Your Finetuning Data",
      "url": "https://arxiv.org/abs/2603.16177",
      "authors": [
        "Christina Baek",
        "...",
        "Aditi Raghunathan",
        "J. Zico Kolter",
        "DatologyAI",
        "...",
        "Pratyush Maini"
      ],
      "venue": "COLM",
      "awards": [],
      "year": "2026",
      "code": "",
      "enabled": true,
      "abstract": "Real-world model deployments demand strong performance on narrow domains where data is often scarce. Typically, practitioners finetune models to specialize them, but this risks overfitting to the domain and forgetting general knowledge. We study a simple strategy, specialized pretraining (SPT), where a small domain dataset, typically reserved for finetuning, is repeated starting from pretraining as a fraction of the total tokens. Across three specialized domains (ChemPile, MusicPile, and ProofPile), SPT improves domain performance and preserves general capabilities after finetuning compared to standard pretraining. In our experiments, SPT reduces the pretraining tokens needed to reach a given domain performance by up to 1.75x. These gains grow when the target domain is underrepresented in the pretraining corpus: on domains far from web text, a 1B SPT model outperforms a 3B standard pretrained model. Beyond these empirical gains, we derive overfitting scaling laws to guide practitioners in selecting the optimal domain-data repetition for a given pretraining compute budget. Our observations reveal the finetuner's fallacy: while finetuning may appear to be the cheapest path to domain adaptation, introducing specialized domain data during pretraining stretches its utility. SPT yields better specialized domain performance (via reduced overfitting across repeated exposures) and better general domain performance (via reduced forgetting during finetuning), ultimately achieving stronger results with fewer parameters and less total compute when amortized over inference. To get the most out of domain data, incorporate it as early in training as possible."
      },

      {
      "title": "Early Data Exposure Improves Robustness to Subsequent Fine-Tuning",
      "url": "https://arxiv.org/abs/2605.12705",
      "authors": [
        "Lawrence Feng",
        "Gaurav R. Ghosal",
        "Jacob Mitchell Springer",
        "Ziqian Zhong",
        "Aditi Raghunathan"
      ],
      "venue": "COLM",
      "awards": [],
      "year": "2026",
      "code": "",
      "enabled": true,
      "abstract": "How can we train models whose post-trained capabilities survive subsequent fine-tuning? Rather than focusing on downstream interventions to mitigate forgetting of upstream capabilities, we study how upstream training choices - that is, the manner in which a capability is acquired - shape how robustly that capability is retained. We investigate this question in a controlled three-stage language-model pipeline: pretraining, post-training to acquire a target capability, and downstream fine-tuning on a new objective. Across 135M and 1B models, two post-training domains, and two downstream fine-tuning tasks, we find that immediate post-training performance does not reliably predict retention after subsequent fine-tuning: training recipes that look equivalent immediately after post-training can retain the target capability very differently after subsequent fine-tuning. In particular, early exposure - mixing post-training data into pretraining - consistently improves the frontier between retained upstream performance and downstream performance. In compute-matched experiments, where the target data must be allocated between pretraining and post-training, we find that the optimum lies at neither extreme. Together with our other empirical and theoretical findings, this supports the view that post-training drives immediate specialization while early exposure improves robustness to later forgetting. Replay and dropout, typically used to mitigate forgetting as it occurs during fine-tuning, provide complementary gains to early exposure when applied during post-training. Our findings suggest that robustness to subsequent fine-tuning should be treated as a first-class objective of upstream training, addressed preventatively through choices like early exposure rather than reactively during fine-tuning itself."
      },

      {
      "title": "Hardening Agent Benchmarks with Adversarial Hacker-Fixer Loops",
      "url": "https://arxiv.org/abs/2606.08960",
      "authors": [
        "Ziqian Zhong",
        "Ivgeni Segal",
        "Ivan Bercovich",
        "Shashwat Saxena",
        "Kexun Zhang",
        "Aditi Raghunathan"
      ],
      "venue": "arXiv",
      "awards": [],
      "year": "2026",
      "code": "",
      "enabled": true,
      "abstract": "Agent benchmarks score submissions with outcome verifiers that are typically hand-written and brittle, leaving them open to reward hacking. We audit 1,968 tasks across five terminal-agent benchmarks and find 323 (16%) hackable by frontier models given only the task description. This corrupts both leaderboard rankings and RL training signal, yet the standard response is manual and reactive.\n\nWe introduce the hacker-fixer loop, a method for building exploit-resistant verifiers without per-task manual patching. The loop alternates three LLM agents: a hacker tries to pass the verifier without solving the task, a fixer patches the verifier to reject each discovered exploit, and a solver confirms the patched verifier still admits legitimate solutions. The loop iterates: each patch reshapes what the verifier rewards, surfacing the next exploit. We further add verifier access, and let patches transfer across tasks, to broaden the exploits the loop discovers.\n\nOn KernelBench, the loop drives the attack success rate from 62% to 0% on a held-out corpus of publicly reported exploits. We also find that weaker agents in the loop can defend against much stronger hackers: Gemini 3 Flash's loop drives the stronger Gemini 3.1 Pro and Claude Opus 4.7's attack success rate from 76% and 61% to 0% on KernelBench, and Gemini 3.1 Pro's from 39% to 17% on Terminal Bench across 77 tasks. We release Terminal Wrench (323 hackable environments, 3,632 hack trajectories) as a snapshot of the current attack surface, our patched verifiers, the exploits the loop discovered, and our implementation as a basis for future work."
      },

      {
      "title": "Self-Trained Verification for Training- and Test-Time Self-Improvement",
      "url": "https://arxiv.org/abs/2605.30290",
      "authors": [
        "Chen Henry Wu",
        "Aditi Raghunathan"
      ],
      "venue": "arXiv",
      "awards": [],
      "year": "2026",
      "code": "",
      "website": "https://ar-forum.github.io/stv-webpage",
      "enabled": true,
      "abstract": "Self-improvement at scale has been a longstanding goal for reasoning models, and there are two natural places to do it: at test time, through verification-refinement (V-R) loops; and at training time, through self-training methods. Both are gated by the same bottleneck: the verifier. V-R loops stall when verifier scores inflate while accuracy stagnates, and when feedback is too generic to act on; self-training fails similarly when bad self-generated data are added to training. Better verification would unlock both, but the capability we want to train, i.e., catching self-generated errors, lacks training signal. To address this challenge, we propose self-trained verification (STV). Our key observation is that, while a model cannot catch these errors alone, it can when shown the reference solution. We turn this asymmetry into a supervision target and train the verifier to imitate a more informed version of itself. At test time, STV substantially improves V-R loops on hard problems, while alternatives (e.g., SFT, RL on verifier scores, and even meta-verifiers) do not. STV roughly doubles accuracy on hard math and lifts it 14x on scientific reasoning tasks (1.5% to 21%). At training time, we additionally train the generator using RL with STV verifier's feedback inside the V-R loop - a procedure we call verifier-in-the-loop training (ViL). Starting from an RL-converged generator, ViL yields a further 33% gain in pass@1. More notably, the generator's standalone pass@1, with no verifier at test time, climbs 30% relative past where standard RL had converged. Hence, the next frontier in reasoning on hard problems may lie in how we train for and with verification."
      },

      {
      "title": "Understanding and Mitigating Premature Confidence for Better LLM Reasoning",
      "url": "https://arxiv.org/abs/2605.24396",
      "authors": [
        "Jingchu Gai",
        "Guanning Zeng",
        "Christina Baek",
        "Chen Wu",
        "J. Zico Kolter",
        "Andrej Risteski",
        "Aditi Raghunathan"
      ],
      "venue": "COLM",
      "awards": [],
      "year": "2026",
      "code": "",
      "enabled": true,
      "abstract": "Long chains of thought (CoT) from current language models frequently contain logical gaps and unjustified leaps, limiting the gains from additional test-time compute. Improving reasoning quality directly would require process reward models, but the step-level annotations needed to train them are expensive and scarce. We find such a signal in how the model's confidence evolves during reasoning: premature confidence, the tendency to commit to an answer early and use the remaining tokens to rationalize it, strongly predicts flawed reasoning across tasks and model scales. We exploit this in progressive confidence shaping, a reinforcement learning objective that trains models to update their confidence as they reason rather than commit early -- rewarding gradual confidence growth and penalizing early commitment, with no external labels or reward models. The method improves accuracy and reasoning quality from 1.5B to 8B parameters across arithmetic (Countdown), math (DAPO, AIME), and science (ScienceQA): on Countdown, accuracy improves 3.2x (+42.0pp) and flawed reasoning drops 48pp; on AIME, Pass@64 improves 6.6pp. Consistent with this mechanism, the method also improves faithfulness: on a safety benchmark, our models more transparently surface misleading content in their reasoning traces rather than concealing it. Controlled experiments reveal that the problem and its remedy scale together: premature confidence grows with model size and task difficulty, and so do the gains from addressing it."
      },

      {
      "title": "Natively Unlearnable Large Language Models",
      "url": "https://ar-forum.github.io/assets/nulls_preprint.pdf",
      "authors": [
        "Gaurav R. Ghosal", 
        "Pratyush Maini", 
        "Aditi Raghunathan"
      ],
      "venue": "arXiv",
      "awards":[], 
      "year": "2026",
      "code": "",
      "awards": [],
      "enabled": true,
      "abstract": "Post-hoc unlearning for LLMs is fundamentally limited: it either degrades the model beyond the intended target or fails to fully remove the targeted information. This difficulty primarily arises from how standard models store information. We propose NULLs (Natively Unlearnable LLMs), which impose an architectural separation between sources during training. The central challenge is routing source-specific memorization into removable components without fragmenting the representations that enable generalization. NULLs addresses this by partitioning each layer's neurons into a shared subset that learns transferable structure across all sources, and a pool of memorization sink neurons over which each source is assigned a known sparse mask. At deployment time, unlearning reduces to simply disabling a source's mask over the sinks. Because these masks are combinatorial, a fixed pool of 8,000 neurons supports millions of independently controllable sources without increasing model size. On Wikipedia (6M articles), disabling an article's sink sharply suppresses article-specific knowledge while preserving semantically related facts, closely matching the gold standard of retraining from scratch. Gradient-based unlearning, by contrast, degrades shared and unique knowledge at comparable rates. In a Harry Potter case study, NULLs removes topic-level knowledge and remains resistant to adversarial extraction and fine-tuning-based relearning attacks that rapidly undo post-hoc methods. These results demonstrate that natively unlearnable language models are feasible via memorization sinks with minimal degradation of general capabilities." },
      
      {
      "title": "Pando: Do Interpretability Methods Work When Models Won't Explain Themselves?",
      "url": "https://arxiv.org/pdf/2604.11061",
      "authors": [
        "Ziqian Zhong", 
        "Aashiq Muhamed",
        "Mona T. Diab", 
        "Virginia Smith", 
        "Aditi Raghunathan"
      ],
      "venue": "COLM",
      "awards":[""], 
      "year": "2026",
      "code": "",
      "awards": [],
      "enabled": true,
      "abstract": "Mechanistic interpretability is often motivated for alignment auditing, where a model's verbal explanations can be absent, incomplete, or misleading. Yet many evaluations do not control whether black-box prompting alone can recover the target behavior, so apparent gains from white-box tools may reflect elicitation rather than internal signal; we call this the elicitation confounder. We introduce Pando, a model-organism benchmark that breaks this confound via an explanation axis: models are trained to produce either faithful explanations of the true rule, no explanation, or confident but unfaithful explanations of a disjoint distractor rule. Across 720 finetuned models implementing hidden decision-tree rules, agents predict held-out model decisions from 10 labeled query-response pairs, optionally augmented with one interpretability tool output. When explanations are faithful, black-box elicitation matches or exceeds all white-box methods; when explanations are absent or misleading, gradient-based attribution improves accuracy by 3-5 percentage points, and relevance patching, RelP, gives the largest gains, while logit lens, sparse autoencoders, and circuit tracing provide no reliable benefit. Variance decomposition suggests gradients track decision computation, which fields causally drive the output, whereas other readouts are dominated by task representation, biases toward field identity and value. We release all models, code, and evaluation infrastructure."}, 

      {
      "title": "Hodoscope: Unsupervised Monitoring for AI Misbehaviors",
      "url": "https://arxiv.org/pdf/2604.11072",
      "authors": [
        "Ziqian Zhong", 
        "Shashwat Saxena", 
        "Aditi Raghunathan"
      ],
      "venue": "arXiv",
      "awards":[""], 
      "year": "2026",
      "code": "",
      "awards": [],
      "enabled": true,
      "abstract":"Existing approaches to monitoring AI agents rely on supervised evaluation: human-written rules or LLM-based judges that check for known failure modes. However, novel misbehaviors may fall outside predefined categories entirely and LLM-based judges can be unreliable. To address this, we formulate unsupervised monitoring, drawing an analogy to unsupervised learning. Rather than checking for specific misbehaviors, an unsupervised monitor assists humans in discovering problematic agent behaviors without prior assumptions about what counts as problematic, leaving that determination to the human. We observe that problematic behaviors are often distinctive: a model exploiting a benchmark loophole exhibits actions absent from well-behaved baselines, and a vulnerability unique to one evaluation manifests as behavioral anomalies when the same model runs across multiple benchmarks. This motivates using group-wise behavioral differences as the primary signal for unsupervised monitoring. We introduce Hodoscope, a tool that operationalizes this insight. Hodoscope compares behavior distributions across groups and highlights distinctive and potentially suspicious action patterns for human review. Using Hodoscope, we discover a previously unknown vulnerability in the Commit0 benchmark (unsquashed git history allowing ground-truth recovery, inflating scores for at least five models) and independently recover known exploits on ImpossibleBench and SWE-bench. Quantitative evaluation estimates that our method reduces review effort by 6-23× compared to naive uniform sampling. Finally, we show that behavior descriptions discovered through Hodoscope could improve the detection accuracy of LLM-based judges, demonstrating a path from unsupervised to supervised monitoring."}, 
      
      {
      "title": "Sharpness-Aware Pretraining Mitigates Catastrophic Forgetting",
      "url": "https://arxiv.org/pdf/2605.02105",
      "authors": [
        "Ishaan Watts", 
        "Catherine Li",
        "Sachin Goyal", 
        "Jacob Mitchell Springer", 
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "awards":["Spotlight at ICBINB Workshop at ICLR 2026"], 
      "year": "2026",
      "code": "",
      "enabled": true,
      "abstract": "Pretraining optimizers are tuned to produce the strongest possible base model, on the assumption that a stronger starting point yields a stronger model after subsequent changes like post-training and quantization. This overlooks the geometry of the base model which controls how much of the base model's capabilities survive subsequent parameter updates. We study three pretraining optimization approaches that bias optimization toward flatter minima: Sharpness-Aware Minimization (SAM), large learning rates, and shortened learning rate annealing periods. Across model sizes ranging from 20M to 150M parameters, we find that these interventions consistently improve downstream performance after post-training on five common datasets with up to 80% less forgetting. These principles hold at scale: a short SAM mid-training phase applied to an existing OLMo-2-1B checkpoint reduces forgetting by 31% after MetaMath post-training and by 40% after 4-bit quantization." },
      
      {
      "title": "Annotations Mitigate Post-training Mode Collapse",
      "url": "",
      "authors": [
        "Jacob Mitchell Springer", 
        "Madhu Advani",
        "Lukas Aichberger", 
        "Arwen Bradley", 
        "Eran Malach", 
        "Omid Saremi", 
        "Sinead Williamson", 
        "Preetum Nakkiran", 
        "Etai Littwin", 
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "awards":[], 
      "year": "2026",
      "code": "",
      "awards": [],
      "enabled": true,
      "abstract": "" },
      {
      "title": "Demystifying Entropy Control in LLM RL Training: Theoretical Analysis and Dynamic Scheduling",
      "url": "",
      "authors": [
        "Jingchu Gai", 
        "Guanning Zeng", 
        "Huaqing Zhang",
        "Han Zhong", 
        "Yige Hong", 
        "Andrej Risteski", 
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "awards":[], 
      "year": "2026",
      "code": "",
      "awards": ["Spotlight"],
      "enabled": true,
      "abstract": "" },     
      {
      "title": "Differential Smoothing Mitigates Sharpening and Improves LLM Reasoning",
      "url": "https://arxiv.org/pdf/2511.19942",
      "authors": [
        "Jingchu Gai", 
        "Guanning Zeng", 
        "Huaqing Zhang", 
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "awards":[], 
      "year": "2026",
      "code": "",
      "awards": [],
      "enabled": true,
      "abstract": "It is widely recognized that reinforcement learning (RL) fine-tuning of large language models often leads to diversity collapse, where outputs lack variety. Prior work has proposed a range of heuristics to counteract this effect, but these methods are ad hoc: they frequently trade off correctness for diversity, their effectiveness varies across tasks, and in some cases they even contradict one another. In this work, we place these observations on a rigorous foundation. We first provide a formal proof of why RL fine-tuning exhibits diversity collapse via a selection and reinforcement bias. Next, we make a key observation that any reward modification to address diversity collapse only needs to be applied on the correct trajectories. Building directly on this analysis, we introduce a principled method -- differential smoothing -- that provably improves both correctness and diversity, outperforming vanilla RL as well as widely used entropy-based heuristics. Our theory precisely characterizes when existing heuristics help and why they fail, while showing that differential smoothing is universally superior. Extensive experiments with models from 1B to 7B parameters, across domains including CountDown and real-world mathematical reasoning, demonstrate consistent gains. Differential smoothing improves both Pass@1 and Pass@k, with up to 6.7% improvements on AIME24 dataset."   },
      {
      "title": "Mode-Conditioning Unlocks Superior Test-Time Scaling",
      "url": "https://www.arxiv.org/pdf/2512.01127",
      "authors": [
        "Chen Henry Wu", 
        "Sachin Goyal", 
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "awards":[], 
      "year": "2026",
      "website": "https://ar-forum.github.io/mod-c/",
      "code": "",
      "awards": [],
      "enabled": true,
      "abstract": "Parallel sampling promises substantial gains in test-time scaling, but its effectiveness is sharply limited by diversity collapse, where models concentrate on a few modes and repeated samples produce the same mistakes. We propose the mode-conditioning (ModC) framework, which explicitly allocates test-time compute across reasoning modes using either specialist models or mode-specific prefixes. ModC consistently improves scaling across controlled graph-search tasks and large-scale reasoning benchmarks, spanning model families and sizes from 0.5B to 7B. On OpenThoughts, fine-tuning Qwen2.5-7B with ModC achieves a 4x efficiency gain over standard training while also improving the maximum attainable Pass@k. We further show that gradient clustering enables ModC without explicit mode labels, yielding up to 10% gains on datasets such as NuminaMath. Finally, we show that ModC improves reinforcement learning (RL) and can further boost diversity-inducing RL methods. These results demonstrate that standard training underutilizes the diversity in data, and that ModC provides a simple, effective remedy for unlocking the full benefits of diversity in test-time scaling."
    },	
      {
      "title": "ImpossibleBench: Measuring LLMs' Propensity of Exploiting Test Cases",
      "url": "https://arxiv.org/pdf/2510.20270",
      "authors": [
        "Ziqian Zhong", 
        "Aditi Raghunathan", 
        "Nicholas Carlini"
      ],
      "venue": "ICLR",
      "awards":[], 
      "year": "2026",
      "code": "https://github.com/safety-research/impossiblebench",
      "awards": [],
      "enabled": true,
      "abstract": "The tendency to find and exploit \\\"shortcuts\\\" to complete tasks poses significant risks for reliable assessment and deployment of large language models (LLMs). For example, an LLM agent with access to unit tests may delete failing tests rather than fix the underlying bug. Such behavior undermines both the validity of benchmark results and the reliability of real-world LLM coding assistant deployments.\\nTo quantify, study, and mitigate such behavior, we introduce ImpossibleBench, a benchmark framework that systematically measures LLM agents' propensity to exploit test cases. ImpossibleBench creates \\\"impossible\\\" variants of tasks from existing benchmarks like LiveCodeBench and SWE-bench by introducing direct conflicts between the natural-language specification and the unit tests. We measure an agent's \\\"cheating rate\\\" as its pass rate on these impossible tasks, where any pass necessarily implies a specification-violating shortcut.\\nAs a practical framework, ImpossibleBench is not just an evaluation but a versatile tool. We demonstrate its utility for: (1) studying model behaviors, revealing more fine-grained details of cheating behaviors from simple test modification to complex operator overloading; (2) context engineering, showing how prompt, test access and feedback loop affect cheating rates; and (3) developing monitoring tools, providing a testbed with verified deceptive solutions. We hope ImpossibleBench serves as a useful framework for building more robust and reliable LLM systems."
    },
      {
      "title": "Jailbreaking in the Haystack",
      "url": "https://arxiv.org/pdf/2511.04707",
      "authors": [
        "Rishi Rajesh Shah",
        "Chen Henry Wu",
        "Shashwat Saxena",
        "Ziqian Zhong",
        "Alexander Robey",
        "Aditi Raghunathan"
      ],
      "venue": "arXiv",
      "awards": [],
      "year": "2025",
      "code": "https://github.com/AR-FORUM/NINJA_Attack",
      "website": "https://ar-forum.github.io/ninjaattackweb/",
      "enabled": true,
      "abstract": "Recent advances in long-context language models (LMs) have enabled million-token inputs, expanding their capabilities across complex tasks like computer-use agents. Yet, the safety implications of these extended contexts remain unclear. To bridge this gap, we introduce NINJA (short for Needle-in-haystack jailbreak attack), a method that jailbreaks aligned LMs by appending benign, model-generated content to harmful user goals. Critical to our method is the observation that the position of harmful goals play an important role in safety. Experiments on standard safety benchmark, HarmBench, show that NINJA significantly increases attack success rates across state-of-the-art open and proprietary models, including LLaMA, Qwen, Mistral, and Gemini. Unlike prior jailbreaking methods, our approach is low-resource, transferable, and less detectable. Moreover, we show that NINJA is compute-optimal -- under a fixed compute budget, increasing context length can outperform increasing the number of trials in best-of-N jailbreak. These findings reveal that even benign long contexts -- when crafted with careful goal positioning -- introduce fundamental vulnerabilities in modern LMs."
    },
    {
      "title": "Watch the Weights: Unsupervised monitoring and control of fine-tuned LLMs",
      "url": "https://arxiv.org/pdf/2508.00161",
      "authors": [
        "Ziqian Zhong", 
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "awards":["Runner-up best paper award at Reliable ML from Unreliable Data at NeurIPS 2025"], 
      "year": "2026",
      "code": "https://github.com/fjzzq2002/WeightWatch", 
      "website": "https://fjzzq2002.github.io/WeightWatch/", 
      "enabled": true,
      "abstract": "The releases of powerful open-weight large language models (LLMs) are often not accompanied by access to their full training data. Existing interpretability methods, particularly those based on activations, often require or assume distributionally similar data. This is a significant limitation when detecting and defending against novel potential threats like backdoors, which are by definition out-of-distribution.\\nIn this work, we introduce a new method for understanding, monitoring and controlling fine-tuned LLMs that interprets weights, rather than activations, thereby side stepping the need for data that is distributionally similar to the unknown training data. We demonstrate that the top singular vectors of the weight difference between a fine-tuned model and its base model correspond to newly acquired behaviors. By monitoring the cosine similarity of activations along these directions, we can detect salient behaviors introduced during fine-tuning with high precision.\\nFor backdoored models that bypasses safety mechanisms when a secret trigger is present, our method stops up to 100% of attacks with a false positive rate below 1.2%. For models that have undergone unlearning, we detect inference on erased topics with accuracy up to 95.42% and can even steer the model to recover \"unlearned\" information. Besides monitoring, our method also shows potential for pre-deployment model auditing: by analyzing commercial instruction-tuned models (OLMo, Llama, Qwen), we are able to uncover model-specific fine-tuning focus including marketing strategies and Midjourney prompt generation."
    },
    {
      "title": "Roll the dice & look before you leap: Going beyond the creative limits of next-token prediction",
      "url": "https://arxiv.org/pdf/2504.15266",
      "authors": [
        "Vaishnavh Nagarajan",
        "Chen Henry Wu",
        "Charles Ding",
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "year": "2025",
      "awards": [
        "Outstanding Paper Award"
      ],
      "enabled": true,
      "abstract": "We design a suite of minimal algorithmic tasks that are a loose abstraction of open-ended real-world tasks. This allows us to cleanly and controllably quantify the creative limits of the present-day language model. Much like real-world tasks that require a creative, far-sighted leap of thought, our tasks require an implicit, open-ended stochastic planning step that either (a) discovers new connections in an abstract knowledge graph (like in wordplay, drawing analogies, or research) or (b) constructs new patterns (like in designing math problems or new proteins). In these tasks, we empirically and conceptually argue how next-token learning is myopic; multi-token approaches, namely teacherless training and diffusion models, comparatively excel in producing diverse and original output. Secondly, to elicit randomness without hurting coherence, we find that injecting noise at the input layer (dubbed seed-conditioning) works surprisingly as well as (and in some conditions, better than) temperature sampling from the output layer. Thus, our work offers a principled, minimal test-bed for analyzing open-ended creative skills, and offers new arguments for going beyond next-token learning and temperature sampling. We make part of the code available under this https URL"
    },
    {
      "title": "Overtrained Language Models Are Harder to Fine-Tune",
      "url": "https://arxiv.org/pdf/2503.19206",
      "authors": [
        "Jacob Mitchell Springer",
        "Sachin Goyal",
        "Kaiyue Wen",
        "Tanishq Kumar",
        "Xiang Yue",
        "Sadhika Malladi",
        "Graham Neubig",
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "year": "2025",
      "awards": [
        "Outstanding Paper Award at SCOPE workshop at ICLR 2025",
        "Entropic Paper Award at ICBINB workshop at ICLR 2025"
      ],
      "enabled": true,
      "abstract": "Large language models are pre-trained on ever-growing token budgets under the assumption that better pre-training performance translates to improved downstream models. In this work, we challenge this assumption and show that extended pre-training can make models harder to fine-tune, leading to degraded final performance. We term this phenomenon catastrophic overtraining. For example, the instruction-tuned OLMo-1B model pre-trained on 3T tokens leads to over 2% worse performance on multiple standard LLM benchmarks than its 2.3T token counterpart. Through controlled experiments and theoretical analysis, we show that catastrophic overtraining arises from a systematic increase in the broad sensitivity of pre-trained parameters to modifications, including but not limited to fine-tuning. Our findings call for a critical reassessment of pre-training design that considers the downstream adaptability of the model."
    },
    {
      "title": "Memorization Sinks: Isolating memorization during LLM training",
      "url": "https://arxiv.org/pdf/2507.09937",
      "authors": [
        "Gaurav Ghosal",
        "Pratyush Maini",
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "year": "2025",
      "website":"https://ar-forum.github.io/memsinksweb/",
      "code": "https://github.com/AR-FORUM/MemSinks",
      "awards": [],
      "enabled": true,
      "abstract": "Large language models are susceptible to memorizing repeated sequences, posing privacy and copyright concerns. A popular mitigation strategy is to remove memorized information from specific neurons post-hoc. However, such approaches have shown limited success so far. In a controlled setting, we show that the memorization of natural sequences (those that resemble linguistically plausible text) become mechanistically entangled with general language abilities, thereby becoming challenging to remove post-hoc. In this work, we put forward a new paradigm of MemSinks that promotes isolation of memorization by design. We leverage a sequence identifier that activates a unique set of memorization neurons for each sequence across repetitions. By analyzing the dynamics of learning and forgetting, we argue that MemSinks facilitates isolation of memorized content, making it easier to remove without compromising general language capabilities. We implement MemSinks at the billion-parameter and billion-token scale, and observe both effective isolation and strong generalization. To our knowledge, this is the first proof-of-concept on real data demonstrating that simultaneous generalization and isolation is achievable. We open-source our code atthis http URL."
    },
    {
      "title": "Scaling Laws for Precision",
      "url": "https://arxiv.org/pdf/2411.04330",
      "authors": [
        "Tanishq Kumar",
        "Zachary Ankner",
        "Benjamin F. Spector",
        "Blake Bordelon",
        "Niklas Muennighoff",
        "Mansheej Paul",
        "Cengiz Pehlevan",
        "Christopher Ré",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2025",
      "awards": [
        "Oral"
      ],
      "enabled": true,
      "abstract": "Low precision training and inference affect both the quality and cost of language models, but current scaling laws do not account for this. In this work, we devise \"precision-aware\" scaling laws for both training and inference. We propose that training in lower precision reduces the model's \"effective parameter count,\" allowing us to predict the additional loss incurred from training in low precision and post-train quantization. For inference, we find that the degradation introduced by post-training quantization increases as models are trained on more data, eventually making additional pretraining data actively harmful. For training, our scaling laws allow us to predict the loss of a model with different parts in different precisions, and suggest that training larger models in lower precision may be compute optimal. We unify the scaling laws for post and pretraining quantization to arrive at a single functional form that predicts degradation from training and inference in varied precisions. We fit on over 465 pretraining runs and validate our predictions on model sizes up to 1.7B parameters trained on up to 26B tokens."
    },
    {
      "title": "Mitigating Modal Imbalance in Multimodal Reasoning",
      "url": "https://arxiv.org/pdf/2510.02608",
      "authors": [
        "Chen Wu",
        "Neil Kale",
        "Aditi Raghunathan"
      ],
      "venue": "COLM",
      "year": "2025",
      "awards": [],
      "enabled": true, 
      "website": "https://ar-forum.github.io/modalimbalanceweb/", 
      "abstract": "Foundation models (FMs) deployed in real-world tasks such as computer-use agents must integrate diverse modalities. How good are FMs at performing joint reasoning, simultaneously reasoning over multiple modalities, especially when the modalities interact and relate to each other to form cross-modal context? To better understand this problem, we study FMs on cross-modal conflicts: scenarios where conflicting evidence is presented across modalities. This allows us to examine whether FMs prioritize one modality over another or reason jointly to reconcile the conflict. Our experiments reveal that FMs can recognize conflicts in unimodal contexts, composed of a single modality, 90% of the time, but the ratio falls as low as 3% when evidence is split across modalities– similar observations hold in cross-lingual contexts, composed of multiple languages. We trace this failure to cross-modal attention imbalance, showing that FMs exhibit extreme asymmetry in attention scores, disproportionately prioritizing certain modalities. Weshowthat cross-modal attention imbalance does not go away by simply scaling up multimodal or multilingual datasets blindly, since they lack training examplesthatexplicitly require cross-modal reasoning. We demonstrate that even a simple and scalable method of explicitly combining multiple modalities within each training instance significantly reduces attention imbalance. Reduced attention imbalance directly translates to improved downstream performance on several vision-language benchmarks. Our f indings underscore the importance of systematically addressing crossmodal contexts to build reliable foundation models."
    },
    {
      "title": "Weight Ensembling Improves Reasoning in Language Models",
      "url": "https://arxiv.org/pdf/2504.10478",
      "authors": [
        "Xingyu Dang",
        "Christina Baek",
        "Kaiyue Wen",
        "Zico Kolter",
        "Aditi Raghunathan"
      ],
      "venue": "COLM",
      "year": "2025",
      "awards": [],
      "enabled": true,
      "abstract": "We investigate a failure mode that arises during the training of reasoning models, where the diversity of generations begins to collapse, leading to suboptimal test-time scaling. Notably, the Pass@1 rate reliably improves during supervised finetuning (SFT), but Pass@k rapidly deteriorates. Surprisingly, a simple intervention of interpolating the weights of the latest SFT checkpoint with an early checkpoint, otherwise known as WiSE-FT, almost completely recovers Pass@k while also improving Pass@1. The WiSE-FT variant achieves better test-time scaling (Best@k, majority vote) and achieves superior results with less data when tuned further by reinforcement learning. Finally, we find that WiSE-FT provides complementary performance gains that cannot be achieved only through diversity-inducing decoding strategies, like temperature scaling. We formalize a bias-variance tradeoff of Pass@k with respect to the expectation and variance of Pass@1 over the test distribution. We find that WiSE-FT can reduce bias and variance simultaneously, while temperature scaling inherently trades off between bias and variance."
    },
    {
      "title": "On the Feasibility of In-Context Probing for Data Attribution",
      "url": "https://arxiv.org/pdf/2407.12259",
      "authors": [
        "Cathy Jiao",
        "Weizhen Gao",
        "Aditi Raghunathan",
        "Chenyan Xiong"
      ],
      "venue": "NAACL Findings",
      "year": "2025",
      "awards": [],
      "enabled": true,
      "abstract": "Data attribution methods are used to measure the contribution of training data towards model outputs, and have several important applications in areas such as dataset curation and model interpretability. However, many standard data attribution methods, such as influence functions, utilize model gradients and are computationally expensive. In our paper, we show in-context probing (ICP) -- prompting a LLM -- can serve as a fast proxy for gradient-based data attribution for data selection under conditions contingent on data similarity. We study this connection empirically on standard NLP tasks, and show that ICP and gradient-based data attribution are well-correlated in identifying influential training data for tasks that share similar task type and content as the training data. Additionally, fine-tuning models on influential data selected by both methods achieves comparable downstream performance, further emphasizing their similarities. We also examine the connection between ICP and gradient-based data attribution using synthetic data on linear regression tasks. Our synthetic data experiments show similar results with those from NLP tasks, suggesting that this connection can be isolated in simpler settings, which offers a pathway to bridging their differences."
    },
    {
      "title": "Exact Unlearning of Finetuning Data via Model Merging at Scale",
      "url": "https://arxiv.org/pdf/2504.04626",
      "authors": [
        "Kevin Kuo",
        "Amrith Setlur",
        "Kartik Srinivas",
        "Aditi Raghunathan",
        "Virginia Smith"
      ],
      "venue": "SATML",
      "year": "2026",
      "awards": [],
      "enabled": true,
      "abstract": "Approximate unlearning has gained popularity as an approach to efficiently update an LLM so that it behaves (roughly) as if it was not trained on a subset of data to begin with. However, existing methods are brittle in practice and can easily be attacked to reveal supposedly unlearned information. To alleviate issues with approximate unlearning, we instead propose SIFT-Masks (SIgn-Fixed Tuning-Masks), an exact unlearning method based on model merging. SIFT-Masks addresses two key limitations of standard model merging: (1) merging a large number of tasks can severely harm utility; and (2) methods that boost utility by sharing extra information across tasks make exact unlearning prohibitively expensive. SIFT-Masks solves these issues by (1) applying local masks to recover task-specific performance; and (2) constraining finetuning to align with a global sign vector as a lightweight approach to determine masks independently before merging. Across four settings where we merge up to 500 models, SIFT-Masks improves accuracy by 5-80% over naive merging and uses up to 250x less compute for exact unlearning compared to other merging baselines."
    },
    {
      "title": "Not-Just-Scaling Laws: Towards a Better Understanding of the Downstream Impact of Language Model Design Decisions",
      "url": "https://arxiv.org/pdf/2503.03862v1",
      "authors": [
        "Emmy Liu",
        "Amanda Bertsch",
        "Lintang Sutawika",
        "Lindia Tjuatja",
        "Patric Fernandes",
        "Lara Marinov",
        "Michael Chen",
        "Shreya Singhal",
        "Carolin Lawrence",
        "Aditi Raghunathan",
        "Kiril Gashteovski",
        "Graham Neubig"
      ],
      "venue": "EMNLP",
      "year": "2025",
      "awards": [],
      "enabled": true,
      "abstract": "Improvements in language model capabilities are often attributed to increasing model size or training data, but in some cases smaller models trained on curated data or with different architectural decisions can outperform larger ones trained on more tokens. What accounts for this? To quantify the impact of these design choices, we meta-analyze 92 open-source pretrained models across a wide array of scales, including state-of-the-art open-weights models as well as less performant models and those with less conventional design decisions. We find that by incorporating features besides model size and number of training tokens, we can achieve a relative 3-28% increase in ability to predict downstream performance compared with using scale alone. Analysis of model design decisions reveal insights into data composition, such as the trade-off between language and code tasks at 15-25\\% code, as well as the better performance of some architectural decisions such as choosing rotary over learned embeddings. Broadly, our framework lays a foundation for more systematic investigation of how model development choices shape final capabilities."
    },
    {
      "title": "Reasoning as an Adaptive Defense for Safety",
      "url": "https://arxiv.org/pdf/2507.00971",
      "authors": [
        "Taeyoun Kim", 
        "Fahim Tajwar", 
        "Aditi Raghunathan", 
        "Aviral Kumar"
      ],
      "venue": "NeurIPS",
      "awards": [],
      "year": "2025",
      "code": "",
      "website": "",
      "enabled": true,
      "abstract": "Reasoning methods that adaptively allocate test-time compute have advanced LLM performance on easy to verify domains such as math and code. In this work, we study how to utilize this approach to train models that exhibit a degree of robustness to safety vulnerabilities, and show that doing so can provide benefits. We build a recipe called TARS (Training Adaptive Reasoners for Safety), a reinforcement learning (RL) approach that trains models to reason about safety using chain-of-thought traces and a reward signal that balances safety with task completion. To build TARS, we identify three critical design choices: (1) a ``lightweight'' warmstart SFT stage, (2) a mix of harmful, harmless, and ambiguous prompts to prevent shortcut behaviors such as too many refusals, and (3) a reward function to prevent degeneration of reasoning capabilities during training. Models trained with TARS exhibit adaptive behaviors by spending more compute on ambiguous queries, leading to better safety-refusal trade-offs. They also internally learn to better distinguish between safe and unsafe prompts and attain greater robustness to both white-box (e.g., GCG) and black-box attacks (e.g., PAIR). Overall, our work provides an effective, open recipe for training LLMs against jailbreaks and harmful requests by reasoning per prompt."
    },
    {
      "title": "Vulnerability of Text-Matching in ML/AI Conference Reviewer Assignments to Collusions",
      "url": "https://arxiv.org/pdf/2412.06606",
      "authors": [
        "Jhih-Yi (Janet) Hsieh",
        "Aditi Raghunathan",
        "Nihar Shah"
      ],
      "venue": "USENIX",
      "year": "2025",
      "awards": [],
      "enabled": true,
      "abstract": "In the peer review process of top-tier machine learning (ML) and artificial intelligence (AI) conferences, reviewers are assigned to papers through automated methods. These assignment algorithms consider two main factors: (1) reviewers' expressed interests indicated by their bids for papers, and (2) reviewers' domain expertise inferred from the similarity between the text of their previously published papers and the submitted manuscripts. A significant challenge these conferences face is the existence of collusion rings, where groups of researchers manipulate the assignment process to review each other's papers, providing positive evaluations regardless of their actual quality. Most efforts to combat collusion rings have focused on preventing bid manipulation, under the assumption that the text similarity component is secure. In this paper, we demonstrate that even in the absence of bidding, colluding reviewers and authors can exploit the machine learning based text-matching component of reviewer assignment used at top ML/AI venues to get assigned their target paper. We also highlight specific vulnerabilities within this system and offer suggestions to enhance its robustness."
    },
    {
      "title": "Context-Parametric Inversion: Why Instruction Finetuning May Not Actually Improve Context Reliance",
      "url": "https://arxiv.org/pdf/2410.10796",
      "authors": [
        "Sachin Goyal",
        "Christina Baek",
        "Zico Kolter",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2025",
      "awards": [
        "Oral"
      ],
      "enabled": true,
      "abstract": "A standard practice when using large language models is for users to supplement their instruction with an input context containing new information for the model to process. However, models struggle to reliably follow the input context, especially when it conflicts with their parametric knowledge from pretraining. In-principle, one would expect models to adapt to the user context better after instruction finetuning, particularly when handling knowledge conflicts. However, we observe a surprising failure mode: during instruction tuning, the context reliance under knowledge conflicts initially increases as expected, but then gradually decreases as instruction finetuning progresses. This happens while the performance on standard benchmarks keeps on increasing far after this drop. We call this phenomenon context-parametric inversion and observe it across multiple general purpose instruction tuning datasets such as TULU, Alpaca and Ultrachat, across different model families like Llama, Mistral, and Pythia. We perform various controlled studies and theoretical analysis to show that context-parametric inversion occurs due to examples in the instruction finetuning data where the input context provides information that aligns with model's parametric knowledge. Our analysis suggests some natural mitigation strategies with limited but insightful gains, and serves as a useful starting point in addressing this deficiency in instruction finetuning."
    },
    {
      "title": "Dissecting Adversarial Robustness of Multimodal Agents",
      "url": "https://arxiv.org/pdf/2406.12814",
      "authors": [
        "Chen Henry Wu",
        "Rishi Shah",
        "Jing Yu Koh",
        "Ruslan Salakhutdinov",
        "Daniel Fried",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2025",
      "awards": [
        "Oral at Open World Agents Workshop at NeurIPS 2024"
      ],
      "enabled": true,
      "abstract": "As language models (LMs) are used to build autonomous agents in real environments, ensuring their adversarial robustness becomes a critical challenge. Unlike chatbots, agents are compound systems with multiple components taking actions, which existing LMs safety evaluations do not adequately address. To bridge this gap, we manually create 200 targeted adversarial tasks and evaluation scripts in a realistic threat model on top of VisualWebArena, a real environment for web agents. To systematically examine the robustness of agents, we propose the Agent Robustness Evaluation (ARE) framework. ARE views the agent as a graph showing the flow of intermediate outputs between components and decomposes robustness as the flow of adversarial information on the graph. We find that we can successfully break latest agents that use black-box frontier LMs, including those that perform reflection and tree search. With imperceptible perturbations to a single image (less than 5% of total web page pixels), an attacker can hijack these agents to execute targeted adversarial goals with success rates up to 67%. We also use ARE to rigorously evaluate how the robustness changes as new components are added. We find that inference-time compute that typically improves benign performance can open up new vulnerabilities and harm robustness. An attacker can compromise the evaluator used by the reflexion agent and the value function of the tree search agent, which increases the attack success relatively by 15% and 20%. Our data and code for attacks, defenses, and evaluation are atthis https URL"
    },
    {
      "title": "Repetition Improves Language Model Embeddings",
      "url": "https://arxiv.org/pdf/2402.15449.pdf",
      "authors": [
        "Jacob Mitchell Springer",
        "Suhas Kotha",
        "Daniel Fried",
        "Graham Neubig",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2025",
      "awards": [],
      "enabled": true,
      "abstract": "Bidirectional models are considered essential for strong text embeddings. Recent approaches to adapt autoregressive language models (LMs) into strong text embedding models have largely had the requirement to modify the LM architecture to be bidirectional. We challenge this premise by introducing \"echo embeddings\" which converts autoregressive LMs into high quality text embedding models without changing the architecture or requiring fine-tuning. By repeating the input and extracting embeddings from the repeated tokens -- which have access to all original tokens -- echo embeddings improve over classical LM embeddings by over 5% in zero-shot settings. Our zero-shot embeddings nearly match those obtained by bidirectionally-converted LMs that undergo additional masked-language modeling training. Echo embeddings are also compatible with supervised fine-tuning, matching or outperforming bidirectionally-converted LMs in an apples-to-apples comparison, even with an identical compute budget during training and inference. Overall, repetition is a simple and effective strategy to circumvent the need for bidirectional attention in embedding models, paving the way towards a unified architecture for all NLP tasks."
    },
    {
      "title": "Understanding the Influence of Synthetic Data for Text Embedders",
      "url": null,
      "authors": [
        "Jacob Mitchell Springer",
        "Vaibhav Adlakha",
        "Siva Reddy",
        "Aditi Raghunathan",
        "Marius Mosbach"
      ],
      "venue": "ACL Findings",
      "year": 2025,
      "awards": [],
      "enabled": true,
      "abstract": "Recent progress in developing general purpose text embedders has been driven by training on ever-growing corpora of synthetic LLM-generated data. Nonetheless, no publicly available synthetic dataset exists, posing a barrier to studying its role for generalization. To address this issue, we first reproduce and publicly release the synthetic data proposed by Wang et al. (2024) (Mistral-E5). Our synthetic data is high quality and leads to consistent improvements in performance. Next, we critically examine where exactly synthetic data improves model generalization. Our analysis reveals that benefits from synthetic data are sparse and highly localized to individual datasets. Moreover, we observe trade-offs between the performance on different categories and data that benefits one task, degrades performance on another. Our findings highlight the limitations of current synthetic data approaches for building general-purpose embedders and challenge the notion that training on synthetic data leads to more robust embedding models across tasks."
    },
    {
      "title": "Mitigating Bias in RAG: Controlling the Embedder",
      "url": "https://arxiv.org/pdf/2502.17390",
      "authors": [
        "Taeyoun Kim",
        "Jacob Mitchell Springer",
        "Aditi Raghunathan",
        "Maarten Sap"
      ],
      "venue": "ACL Findings",
      "year": 2025,
      "awards": [],
      "enabled": true,
      "abstract": "In retrieval augmented generation (RAG) systems, each individual component -- the LLM, embedder, and corpus -- could introduce biases in the form of skews towards outputting certain perspectives or identities. In this work, we study the conflict between biases of each component and their relationship to the overall bias of the RAG system, which we call bias conflict. Examining both gender and political biases as case studies, we show that bias conflict can be characterized through a linear relationship among components despite its complexity in 6 different LLMs. Through comprehensive fine-tuning experiments creating 120 differently biased embedders, we demonstrate how to control bias while maintaining utility and reveal the importance of reverse-biasing the embedder to mitigate bias in the overall system. Additionally, we find that LLMs and tasks exhibit varying sensitivities to the embedder bias, a crucial factor to consider for debiasing. Our results underscore that a fair RAG system can be better achieved by carefully controlling the bias of the embedder rather than increasing its fairness."
    },
    {
      "title": "Theory of Agreement-on-the-Line in Linear Models and Gaussian Data",
      "url": "https://openreview.net/attachment?id=cUTzrd9mIA&name=pdf",
      "authors": [
        "Christina Baek",
        "Aditi Raghunathan",
        "Zico Kolter"
      ],
      "venue": "AISTATS",
      "year": "2025",
      "awards": [],
      "enabled": true,
      "abstract": "Under distribution shifts, deep networks exhibit a surprising phenomenon: in-distribution (ID) versus out-of-distribution (OOD) accuracy is often strongly linearly correlated across architectures and hyperparameters, accompanied by the same linear trend in ID versus OOD agreement between the predictions of any pair of such independently trained networks. The latter phenomenon called \"agreement-on-the-line\" enables precise unlabeled OOD performance estimation of models. In this work, we discover that agreement-on-the-line emerges even in linear classifiers over Gaussian class conditional distributions. We provide theoretical guarantees for this phenomenon in classifiers optimized via randomly initialized gradient descent, approximated by linear interpolations between random vectors and the Bayes-optimal classifier. Next, we prove a lower bound on the residual of the correlation between ID versus OOD agreement that grows proportionally with the residual of accuracy. Real-world experiments on CIFAR10C shifts, validate our findings and the broader relevance of our theoretical framework."
    },
    {
      "title": "Testing the Limits of Jailbreaking Defenses via the Purple Problem",
      "url": "https://arxiv.org/pdf/2403.14725.pdf",
      "authors": [
        "Taeyoun Kim",
        "Suhas Kotha",
        "Aditi Raghunathan"
      ],
      "venue": "NeurIPS Safe Generative AI Workshop",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "The rise of \"jailbreak\" attacks on language models has led to a flurry of defenses aimed at preventing undesirable responses. We critically examine the two stages of the defense pipeline: (i) defining what constitutes unsafe outputs, and (ii) enforcing the definition via methods such as input processing or fine-tuning. To test the efficacy of existing enforcement mechanisms, we consider a simple and well-specified definition of unsafe outputs--outputs that contain the word \"purple\". Surprisingly, existing fine-tuning and input defenses fail on this simple problem, casting doubt on whether enforcement algorithms can be robust for more complicated definitions. We find that real safety benchmarks similarly test enforcement for a fixed definition. We hope that future research can lead to effective/fast enforcement as well as high quality definitions used for enforcement and evaluation."
    },
    {
      "title": "Predicting the Performance of Foundation Models via Agreement-on-the-Line",
      "url": "https://arxiv.org/pdf/2404.01542.pdf",
      "authors": [
        "Rahul Saxena",
        "Taeyoun Kim",
        "Aman Mehra",
        "Christina Baek",
        "Zico Kolter",
        "Aditi Raghunathan"
      ],
      "venue": "NeurIPS",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "Estimating the out-of-distribution performance in regimes where labels are scarce is critical to safely deploy foundation models. Recently, it was shown that ensembles of neural networks observe the phenomena \"agreement-on-the-line\", which can be leveraged to reliably predict OOD performance without labels. However, in contrast to classical neural networks that are trained on in-distribution data from scratch for numerous epochs, foundation models undergo minimal finetuning from heavily pretrained weights, which may reduce the ensemble diversity needed to observe agreement-on-the-line. In our work, we demonstrate that when lightly finetuning multiple runs from a single foundation model, the choice of randomness during training (linear head initialization, data ordering, and data subsetting) can lead to drastically different levels of agreement-on-the-line in the resulting ensemble. Surprisingly, only random head initialization is able to reliably induce agreement-on-the-line in finetuned foundation models across vision and language benchmarks. Second, we demonstrate that ensembles of multiple foundation models pretrained on different datasets but finetuned on the same task can also show agreement-on-the-line. In total, by careful construction of a diverse ensemble, we can utilize agreement-on-the-line-based methods to predict the OOD performance of foundation models with high precision."
    },
    {
      "title": "Test-Time Adaptation Induces Stronger Accuracy and Agreement-on-the-Line",
      "url": "https://arxiv.org/pdf/2310.04941.pdf",
      "authors": [
        "Eungyeup Kim",
        "Mingjie Sun",
        "Christina Baek",
        "Aditi Raghunathan",
        "Zico Kolter"
      ],
      "venue": "NeurIPS",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "Recently, Miller et al. (2021) and Baek et al. (2022) empirically demonstrated strong linear correlations between in-distribution (ID) versus out-of-distribution (OOD) accuracy and agreement. These trends, coined accuracy-on-the-line (ACL) and agreement-on-the-line (AGL), enable OOD model selection and performance estimation without labeled data. However, these phenomena also break for certain shifts, such as CIFAR10-C Gaussian Noise, posing a critical bottleneck. In this paper, we make a key finding that recent test-time adaptation (TTA) methods not only improve OOD performance, but drastically strengthen the ACL and AGL trends in models, even in shifts where models showed very weak correlations before. To analyze this, we revisit the theoretical conditions from Miller et al. (2021) that outline the types of distribution shifts needed for perfect ACL in linear models. Surprisingly, these conditions are satisfied after applying TTA to deep models in the penultimate feature embedding space. In particular, TTA causes the data distribution to collapse complex shifts into those can be expressed by a singular scaling variable in the feature space. Our results show that by combining TTA with AGL-based estimation methods, we can estimate the OOD performance of models with high precision for a broader set of distribution shifts. This lends us a simple system for selecting the best hyperparameters and adaptation strategy without any OOD labeled data."
    },
    {
      "title": "Understanding Finetuning for Factual Knowledge Extraction",
      "url": "https://arxiv.org/abs/2406.14785",
      "authors": [
        "Gaurav Ghosal",
        "Tatsunori Hashimoto",
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "In this work, we study the impact of QA fine-tuning data on downstream factuality. We show that fine-tuning on lesser-known facts that are poorly stored during pretraining yields significantly worse factuality than fine-tuning on well-known facts, even when all facts are seen during pretraining. We prove this phenomenon theoretically, showing that training on lesser-known facts can lead the model to ignore subject entity names and instead output a generic plausible response even when the relevant factual knowledge is encoded in the model. On three question answering benchmarks (PopQA, Entity Questions, and MMLU) and two language models (Llama-2-7B and Mistral-7B), we find that (i) finetuning on a completely factual but lesser-known subset of the data deteriorates downstream factuality (5-10%) and (ii) finetuning on a subset of better-known examples matches or outperforms finetuning on the entire dataset. Ultimately, our results shed light on the interaction between pretrained knowledge and finetuning data and demonstrate the importance of taking into account how facts are stored in the pretrained model when fine-tuning for knowledge-intensive tasks."
    },
    {
      "title": "Scaling Laws for Data Filtering: Data Curation Cannot Be Compute Agnostic",
      "url": "https://arxiv.org/pdf/2404.07177.pdf",
      "authors": [
        "Sachin Goyal",
        "Pratyush Maini",
        "Zachary Chase Lipton",
        "Aditi Raghunathan",
        "Zico Kolter"
      ],
      "venue": "CVPR",
      "year": "2024",
      "awards": [
        "Best Paper Award at DPFM Workshop at ICLR 2024"
      ],
      "enabled": true,
      "abstract": "Vision-language models (VLMs) are trained for thousands of GPU hours on carefully curated web datasets. In recent times, data curation has gained prominence with several works developing strategies to retain 'high-quality' subsets of 'raw' scraped data. For instance, the LAION public dataset retained only 10% of the total crawled data. However, these strategies are typically developed agnostic of the available compute for training. In this paper, we first demonstrate that making filtering decisions independent of training compute is often suboptimal: the limited high-quality data rapidly loses its utility when repeated, eventually requiring the inclusion of 'unseen' but 'lower-quality' data. To address this quality-quantity tradeoff ($\\texttt{QQT}$), we introduce neural scaling laws that account for the non-homogeneous nature of web data, an angle ignored in existing literature. Our scaling laws (i) characterize the $\\textit{differing}$ 'utility' of various quality subsets of web data; (ii) account for how utility diminishes for a data point at its 'nth' repetition; and (iii) formulate the mutual interaction of various data pools when combined, enabling the estimation of model performance on a combination of multiple data pools without ever jointly training on them. Our key message is that data curation $\\textit{cannot}$ be agnostic of the total compute that a model will be trained for. Our scaling laws allow us to curate the best possible pool for achieving top performance on Datacomp at various compute budgets, carving out a pareto-frontier for data curation. Code is available atthis https URL."
    },
    {
      "title": "Multitask Learning Can Improve Worst-Group Outcomes",
      "url": "https://arxiv.org/pdf/2312.03151.pdf",
      "authors": [
        "Atharva Kulkarni",
        "Lucio M. Dery",
        "Amrith Setlur",
        "Aditi Raghunathan",
        "Ameet Talwalkar",
        "Graham Neubig"
      ],
      "venue": "TMLR",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "In order to create machine learning systems that serve a variety of users well, it is vital to not only achieve high average performance but also ensure equitable outcomes across diverse groups. However, most machine learning methods are designed to improve a model's average performance on a chosen end task without consideration for their impact on worst group error. Multitask learning (MTL) is one such widely used technique. In this paper, we seek not only to understand the impact of MTL on worst-group accuracy but also to explore its potential as a tool to address the challenge of group-wise fairness. We primarily consider the standard setting of fine-tuning a pre-trained model, where, following recent work \\citep{gururangan2020don, dery2023aang}, we multitask the end task with the pre-training objective constructed from the end task data itself. In settings with few or no group annotations, we find that multitasking often, but not consistently, achieves better worst-group accuracy than Just-Train-Twice (JTT; \\citet{pmlr-v139-liu21f}) -- a representative distributionally robust optimization (DRO) method. Leveraging insights from synthetic data experiments, we propose to modify standard MTL by regularizing the joint multitask representation space. We run a large number of fine-tuning experiments across computer vision and natural language processing datasets and find that our regularized MTL approach \\emph{consistently} outperforms JTT on both average and worst-group outcomes. Our official code can be found here: \\href{this https URL}{\\url{this https URL}}."
    },
    {
      "title": "Understanding Catastrophic Forgetting in Language Models via Implicit Inference",
      "url": "https://arxiv.org/pdf/2309.10105.pdf",
      "authors": [
        "Suhas Kotha",
        "Jacob Mitchell Springer",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "We lack a systematic understanding of the effects of fine-tuning (via methods such as instruction-tuning or reinforcement learning from human feedback), particularly on tasks outside the narrow fine-tuning distribution. In a simplified scenario, we demonstrate that improving performance on tasks within the fine-tuning data distribution comes at the expense of capabilities on other tasks. We hypothesize that language models implicitly infer the task of the prompt and that fine-tuning skews this inference towards tasks in the fine-tuning distribution. To test this, we propose Conjugate Prompting, which artificially makes the task look farther from the fine-tuning distribution while requiring the same capability, and we find that this recovers some of the pretraining capabilities in our synthetic setup. Since real-world fine-tuning distributions are predominantly English, we apply conjugate prompting to recover pretrained capabilities in LLMs by simply translating the prompts to different languages. This allows us to recover in-context learning abilities lost via instruction tuning, natural reasoning capability lost during code fine-tuning, and, more concerningly, harmful content generation suppressed by safety fine-tuning in chatbots like ChatGPT."
    },
    {
      "title": "Sharpness-Aware Minimization Enhances Feature Quality via Balanced Learning",
      "url": "https://arxiv.org/abs/2405.20439",
      "authors": [
        "Jacob Springer",
        "Vaishnavh Nagarajan",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "Sharpness-Aware Minimization (SAM) has emerged as a promising alternative optimizer to stochastic gradient descent (SGD). The originally-proposed motivation behind SAM was to bias neural networks towards flatter minima that are believed to generalize better. However, recent studies have shown conflicting evidence on the relationship between flatness and generalization, suggesting that flatness does fully explain SAM's success. Sidestepping this debate, we identify an orthogonal effect of SAM that is beneficial out-of-distribution: we argue that SAM implicitly balances the quality of diverse features. SAM achieves this effect by adaptively suppressing well-learned features which gives remaining features opportunity to be learned. We show that this mechanism is beneficial in datasets that contain redundant or spurious features where SGD falls for the simplicity bias and would not otherwise learn all available features. Our insights are supported by experiments on real data: we demonstrate that SAM improves the quality of features in datasets containing redundant or spurious features, including CelebA, Waterbirds, CIFAR-MNIST, and DomainBed."

    },
    {
      "title": "T-MARS: Improving Visual Representations by Circumventing Text Feature Learning",
      "url": "https://arxiv.org/pdf/2307.03132.pdf",
      "authors": [
        "Pratyush Maini",
        "Sachin Goyal",
        "Zachary Lipton",
        "Zico Kolter",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2024",
      "awards": [
        "Oral at ICCV 2023 Datacomp Workshop"
      ],
      "enabled": true,
      "abstract": "Large web-sourced multimodal datasets have powered a slew of new methods for learning general-purpose visual representations, advancing the state of the art in computer vision and revolutionizing zero- and few-shot recognition. One crucial decision facing practitioners is how, if at all, to curate these ever-larger datasets. For example, the creators of the LAION-5B dataset chose to retain only image-caption pairs whose CLIP similarity score exceeded a designated threshold. In this paper, we propose a new state-of-the-art data filtering approach motivated by our observation that nearly 40% of LAION's images contain text that overlaps significantly with the caption. Intuitively, such data could be wasteful as it incentivizes models to perform optical character recognition rather than learning visual features. However, naively removing all such data could also be wasteful, as it throws away images that contain visual features (in addition to overlapping text). Our simple and scalable approach, T-MARS (Text Masking and Re-Scoring), filters out only those pairs where the text dominates the remaining visual features -- by first masking out the text and then filtering out those with a low CLIP similarity score of the masked image. Experimentally, T-MARS outperforms the top-ranked method on the \"medium scale\" of DataComp (a data filtering benchmark) by a margin of 6.5% on ImageNet and 4.7% on VTAB. Additionally, our systematic evaluation on various data pool sizes from 2M to 64M shows that the accuracy gains enjoyed by T-MARS linearly increase as data and compute are scaled exponentially. Code is available atthis https URL."
    },
    {
      "title": "Why is Sharpness-Aware Minimization Robust to Label Noise?",
      "url": "https://arxiv.org/pdf/2405.03676",
      "authors": [
        "Christina Baek",
        "Zico Kolter",
        "Aditi Raghunathan"
      ],
      "venue": "ICLR",
      "year": "2024",
      "awards": [],
      "enabled": true,
      "abstract": "Sharpness-Aware Minimization (SAM) is most known for achieving state-of the-art performances on natural image and language tasks. However, its most pronounced improvements (of tens of percent) is rather in the presence of label noise. Understanding SAM's label noise robustness requires a departure from characterizing the robustness of minimas lying in \"flatter\" regions of the loss landscape. In particular, the peak performance under label noise occurs with early stopping, far before the loss converges. We decompose SAM's robustness into two effects: one induced by changes to the logit term and the other induced by changes to the network Jacobian. The first can be observed in linear logistic regression where SAM provably up-weights the gradient contribution from clean examples. Although this explicit up-weighting is also observable in neural networks, when we intervene and modify SAM to remove this effect, surprisingly, we see no visible degradation in performance. We infer that SAM's effect in deeper networks is instead explained entirely by the effect SAM has on the network Jacobian. We theoretically derive the implicit regularization induced by this Jacobian effect in two layer linear networks. Motivated by our analysis, we see that cheaper alternatives to SAM that explicitly induce these regularization effects largely recover the benefits in deep networks trained on real-world datasets."
    },
    {
      "title": "AutoFT: Learning an Objective for Robust Fine-Tuning",
      "url": "https://arxiv.org/pdf/2401.10220.pdf",
      "authors": [
        "Caroline Choi",
        "Yoonho Lee",
        "Annie Chen",
        "Allan Zhou",
        "Aditi Raghunathan",
        "Chelsea Finn"
      ],
      "venue": "arXiv",
      "year": "2024",
      "awards": [],
      "enabled": false,
      "abstract": "Foundation models encode rich representations that can be adapted to downstream tasks by fine-tuning. However, fine-tuning a model on one data distribution often degrades performance under distribution shifts. Current approaches to robust fine-tuning use hand-crafted regularization techniques to constrain the fine-tuning process towards the pretrained model. Yet, it is hard to specify how to adapt relevant characteristics of the foundation model during fine-tuning, as this depends on how the pre-training, fine-tuning, and test data distributions relate to each other. We propose AutoFT, a data-driven approach for robust fine-tuning. Given a task, AutoFT searches for a fine-tuning procedure that enhances out-of-distribution (OOD) generalization. Specifically, AutoFT uses bi-level optimization to search for an objective function and hyperparameters that maximize post-adaptation performance on a small OOD validation set. We evaluate AutoFT on nine natural distribution shifts. Our experiments show that AutoFT significantly improves generalization to OOD inputs, outperforming existing robust fine-tuning methods. Notably, AutoFT achieves a new state-of-the-art on the WILDS iWildCam and FMoW benchmarks, outperforming the previous best methods by $6.0\\%$ and $1.5\\%$, respectively."
    },
    {
      "title": "Complementary Benefits of Contrastive Learning and Self-Training Under Distribution Shift",
      "url": "https://arxiv.org/pdf/2312.03318.pdf",
      "authors": [
        "Saurabh Garg",
        "Amrith Setlur",
        "Zachary Lipton",
        "Sivaraman Balakrishnan",
        "Virginia Smith",
        "Aditi Raghunathan"
      ],
      "venue": "NeurIPS",
      "year": "2023",
      "awards": [],
      "enabled": true,
      "abstract": "Self-training and contrastive learning have emerged as leading techniques for incorporating unlabeled data, both under distribution shift (unsupervised domain adaptation) and when it is absent (semi-supervised learning). However, despite the popularity and compatibility of these techniques, their efficacy in combination remains unexplored. In this paper, we undertake a systematic empirical investigation of this combination, finding that (i) in domain adaptation settings, self-training and contrastive learning offer significant complementary gains; and (ii) in semi-supervised learning settings, surprisingly, the benefits are not synergistic. Across eight distribution shift datasets (e.g., BREEDs, WILDS), we demonstrate that the combined method obtains 3--8% higher accuracy than either approach independently. We then theoretically analyze these techniques in a simplified model of distribution shift, demonstrating scenarios under which the features produced by contrastive learning can yield a good initialization for self-training to further amplify gains and achieve optimal performance, even when either method alone would fail."
    },
    {
      "title": "Contextual Reliability: When Different Features Matter in Different Contexts",
      "url": "https://openreview.net/pdf?id=s1hrcLUcld",
      "authors": [
        "Gaurav Rohit Ghosal",
        "Amrith Setlur",
        "Daniel S. Brown",
        "Anca Dragan",
        "Aditi Raghunathan"
      ],
      "venue": "ICML",
      "year": "2023",
      "awards": [],
      "enabled": true,
      "abstract": "Deep neural networks often fail catastrophically by relying on spurious correlations. Most prior work assumes a clear dichotomy into spurious and reliable features; however, this is often unrealistic. For example, most of the time we do not want an autonomous car to simply copy the speed of surrounding cars -- we don't want our car to run a red light if a neighboring car does so. However, we cannot simply enforce invariance to next-lane speed, since it could provide valuable information about an unobservable pedestrian at a crosswalk. Thus, universally ignoring features that are sometimes (but not always) reliable can lead to non-robust performance. We formalize a new setting called contextual reliability which accounts for the fact that the \"right\" features to use may vary depending on the context. We propose and analyze a two-stage framework called Explicit Non-spurious feature Prediction (ENP) which first identifies the relevant features to use for a given context, then trains a model to rely exclusively on these features. Our work theoretically and empirically demonstrates the advantages of ENP over existing methods and provides new benchmarks for contextual reliability."
    },
    {
      "title": "Automatically Auditing Large Language Models via Discrete Optimization",
      "url": "https://arxiv.org/pdf/2303.04381.pdf",
      "authors": [
        "Erik Jones",
        "Anca Dragan",
        "Aditi Raghunathan",
        "Jacob Steinhardt"
      ],
      "venue": "ICML",
      "year": "2023",
      "awards": [],
      "enabled": true,
      "abstract": "Auditing large language models for unexpected behaviors is critical to preempt catastrophic deployments, yet remains challenging. In this work, we cast auditing as an optimization problem, where we automatically search for input-output pairs that match a desired target behavior. For example, we might aim to find a non-toxic input that starts with \"Barack Obama\" that a model maps to a toxic output. This optimization problem is difficult to solve as the set of feasible points is sparse, the space is discrete, and the language models we audit are non-linear and high-dimensional. To combat these challenges, we introduce a discrete optimization algorithm, ARCA, that jointly and efficiently optimizes over inputs and outputs. Our approach automatically uncovers derogatory completions about celebrities (e.g. \"Barack Obama is a legalized unborn\" -> \"child murderer\"), produces French inputs that complete to English outputs, and finds inputs that generate a specific name. Our work offers a promising new tool to uncover models' failure-modes before deployment."
    },
    {
      "title": "Finetune Like You Pretrain: Improved Finetuning of Zero-Shot Vision Models",
      "url": "https://arxiv.org/pdf/2212.00638.pdf",
      "authors": [
        "Sachin Goyal",
        "Ananya Kumar",
        "Sankalp Garg",
        "Zico Kolter",
        "Aditi Raghunathan"
      ],
      "venue": "CVPR",
      "year": "2023",
      "awards": [],
      "enabled": true,
      "abstract": "Finetuning image-text models such as CLIP achieves state-of-the-art accuracies on a variety of benchmarks. However, recent works like WiseFT (Wortsman et al., 2021) and LP-FT (Kumar et al., 2022) have shown that even subtle differences in the finetuning process can lead to surprisingly large differences in the final performance, both for in-distribution (ID) and out-of-distribution (OOD) data. In this work, we show that a natural and simple approach of mimicking contrastive pretraining consistently outperforms alternative finetuning approaches. Specifically, we cast downstream class labels as text prompts and continue optimizing the contrastive loss between image embeddings and class-descriptive prompt embeddings (contrastive finetuning).Our method consistently outperforms baselines across 7 distribution shifts, 6 transfer learning, and 3 few-shot learning benchmarks. On WILDS-iWILDCam, our proposed approach FLYP outperforms the top of the leaderboard by $2.3\\%$ ID and $2.7\\%$ OOD, giving the highest reported accuracy. Averaged across 7 OOD datasets (2 WILDS and 5 ImageNet associated shifts), FLYP gives gains of $4.2\\%$ OOD over standard finetuning and outperforms the current state of the art (LP-FT) by more than $1\\%$ both ID and OOD. Similarly, on 3 few-shot learning benchmarks, our approach gives gains up to $4.6\\%$ over standard finetuning and $4.4\\%$ over the state of the art. In total, these benchmarks establish contrastive finetuning as a simple, intuitive, and state-of-the-art approach for supervised finetuning of image-text models like CLIP. Code is available atthis https URL."
    },
    {
      "title": "Using Language to Extend to Unseen Domains",
      "url": "https://arxiv.org/pdf/2210.09520.pdf",
      "authors": [
        "Lisa Dunlap",
        "Clara Mohri",
        "Devin Guillory",
        "Han Zhang",
        "Trevor Darrell",
        "Joseph E. Gonzalez",
        "Aditi Raghunathan",
        "Anja Rohrbach"
      ],
      "venue": "ICLR",
      "year": "2023",
      "awards": [
        "Spotlight"
      ],
      "enabled": true,
      "abstract": "It is expensive to collect training data for every possible domain that a vision model may encounter when deployed. We instead consider how simply verbalizing the training domain (e.g. \"photos of birds\") as well as domains we want to extend to but do not have data for (e.g. \"paintings of birds\") can improve robustness. Using a multimodal model with a joint image and language embedding space, our method LADS learns a transformation of the image embeddings from the training domain to each unseen test domain, while preserving task relevant information. Without using any images from the unseen test domain, we show that over the extended domain containing both training and unseen test domains, LADS outperforms standard fine-tuning and ensemble approaches over a suite of four benchmarks targeting domain adaptation and dataset bias."
    },
    {
      "title": "Bitrate-Constrained DRO: Beyond Worst Case Robustness To Unknown Group Shifts",
      "url": "#",
      "authors": [
        "Amrith Setlur",
        "Don Dennis",
        "Benjamin Eysenbach",
        "Aditi Raghunathan",
        "Chelsea Finn",
        "Virginia Smith",
        "Sergey Levine"
      ],
      "venue": "ICLR",
      "year": "2023",
      "awards": [],
      "enabled": true,
      "abstract": "Training machine learning models robust to distribution shifts is critical for real-world applications. Some robust training algorithms (e.g., Group DRO) specialize to group shifts and require group information on all training points. Other methods (e.g., CVaR DRO) that do not need group annotations can be overly conservative, since they naively upweight high loss points which may form a contrived set that does not correspond to any meaningful group in the real world (e.g., when the high loss points are randomly mislabeled training points). In this work, we address limitations in prior approaches by assuming a more nuanced form of group shift: conditioned on the label, we assume that the true group function (indicator over group) is simple. For example, we may expect that group shifts occur along low bitrate features (e.g., image background, lighting). Thus, we aim to learn a model that maintains high accuracy on simple group functions realized by these low bitrate features, that need not spend valuable model capacity achieving high accuracy on contrived groups of examples. Based on this, we consider the two-player game formulation of DRO where the adversary's capacity is bitrate-constrained. Our resulting practical algorithm, Bitrate-Constrained DRO (BR-DRO), does not require group information on training samples yet matches the performance of Group DRO on datasets that have training group annotations and that of CVaR DRO on long-tailed distributions. Our theoretical analysis reveals that in some settings BR-DRO objective can provably yield statistically efficient and less conservative solutions than unconstrained CVaR DRO."

    },
    {
      "title": "Agreement-on-the-Line: Predicting the Performance of Neural Networks under Distribution Shift",
      "url": "https://arxiv.org/pdf/2206.13089.pdf",
      "authors": [
        "Christina Baek",
        "Yiding Jiang",
        "Aditi Raghunathan",
        "Zico Kolter"
      ],
      "venue": "NeurIPS",
      "year": "2022",
      "awards": [
        "Oral"
      ],
      "enabled": true,
      "abstract": "Recently, Miller et al. showed that a model's in-distribution (ID) accuracy has a strong linear correlation with its out-of-distribution (OOD) accuracy on several OOD benchmarks -- a phenomenon they dubbed ''accuracy-on-the-line''. While a useful tool for model selection (i.e., the model most likely to perform the best OOD is the one with highest ID accuracy), this fact does not help estimate the actual OOD performance of models without access to a labeled OOD validation set. In this paper, we show a similar but surprising phenomenon also holds for the agreement between pairs of neural network classifiers: whenever accuracy-on-the-line holds, we observe that the OOD agreement between the predictions of any two pairs of neural networks (with potentially different architectures) also observes a strong linear correlation with their ID agreement. Furthermore, we observe that the slope and bias of OOD vs ID agreement closely matches that of OOD vs ID accuracy. This phenomenon, which we call agreement-on-the-line, has important practical applications: without any labeled data, we can predict the OOD accuracy of classifiers}, since OOD agreement can be estimated with just unlabeled data. Our prediction algorithm outperforms previous methods both in shifts where agreement-on-the-line holds and, surprisingly, when accuracy is not on the line. This phenomenon also provides new insights into deep neural networks: unlike accuracy-on-the-line, agreement-on-the-line appears to only hold for neural network classifiers."
    },
    {
      "title": "Test-Time Adaptation via Conjugate Pseudo-Labels",
      "url": "https://arxiv.org/pdf/2207.09640.pdf",
      "authors": [
        "Sachin Goyal",
        "Mingjie Sun",
        "Aditi Raghunathan",
        "Zico Kolter"
      ],
      "venue": "NeurIPS",
      "year": "2022",
      "awards": [],
      "enabled": true,
      "abstract": "Test-time adaptation (TTA) refers to adapting neural networks to distribution shifts, with access to only the unlabeled test samples from the new domain at test-time. Prior TTA methods optimize over unsupervised objectives such as the entropy of model predictions in TENT [Wang et al., 2021], but it is unclear what exactly makes a good TTA loss. In this paper, we start by presenting a surprising phenomenon: if we attempt to meta-learn the best possible TTA loss over a wide class of functions, then we recover a function that is remarkably similar to (a temperature-scaled version of) the softmax-entropy employed by TENT. This only holds, however, if the classifier we are adapting is trained via cross-entropy; if trained via squared loss, a different best TTA loss emerges. To explain this phenomenon, we analyze TTA through the lens of the training losses's convex conjugate. We show that under natural conditions, this (unsupervised) conjugate function can be viewed as a good local approximation to the original supervised loss and indeed, it recovers the best losses found by meta-learning. This leads to a generic recipe that can be used to find a good TTA loss for any given supervised training loss function of a general class. Empirically, our approach consistently dominates other baselines over a wide range of benchmarks. Our approach is particularly of interest when applied to classifiers trained with novel loss functions, e.g., the recently-proposed PolyLoss, where it differs substantially from (and outperforms) an entropy-based loss. Further, we show that our approach can also be interpreted as a kind of self-training using a very specific soft label, which we refer to as the conjugate pseudolabel. Overall, our method provides a broad framework for better understanding and improving test-time adaptation. Code is available atthis https URL."
    },
    {
      "title": "Learning Representations that Enable Generalization in Assistive Tasks",
      "url": "#",
      "authors": [
        "Jerry Zhi-yang He",
        "Zackory Erickson",
        "Daniel S. Brown",
        "Aditi Raghunathan",
        "Anca Dragan"
      ],
      "venue": "CoRL",
      "year": "2022",
      "awards": [],
      "enabled": true,
      "abstract": "Recent work in sim2real has successfully enabled robots to act in physical environments by training in simulation with a diverse ''population'' of environments (i.e. domain randomization). In this work, we focus on enabling generalization in assistive tasks: tasks in which the robot is acting to assist a user (e.g. helping someone with motor impairments with bathing or with scratching an itch). Such tasks are particularly interesting relative to prior sim2real successes because the environment now contains a human who is also acting. This complicates the problem because the diversity of human users (instead of merely physical environment parameters) is more difficult to capture in a population, thus increasing the likelihood of encountering out-of-distribution (OOD) human policies at test time. We advocate that generalization to such OOD policies benefits from (1) learning a good latent representation for human policies that test-time humans can accurately be mapped to, and (2) making that representation adaptable with test-time interaction data, instead of relying on it to perfectly capture the space of human policies based on the simulated population only. We study how to best learn such a representation by evaluating on purposefully constructed OOD test policies. We find that sim2real methods that encode environment (or population) parameters and work well in tasks that robots do in isolation, do not work well in assistance. In assistance, it seems crucial to train the representation based on the history of interaction directly, because that is what the robot will have access to at test time. Further, training these representations to then predict human actions not only gives them better structure, but also enables them to be fine-tuned at test-time, when the robot observes the partner act. this https URL."
    },
    {
      "title": "Calibrated Ensembles Can Mitigate Accuracy Tradeoffs Under Distribution Shift",
      "url": "https://arxiv.org/pdf/2207.08977.pdf",
      "authors": [
        "Ananya Kumar",
        "Tengyu Ma",
        "Percy Liang",
        "Aditi Raghunathan"
      ],
      "venue": "UAI",
      "year": "2022",
      "awards": [],
      "enabled": true,
      "abstract": "We often see undesirable tradeoffs in robust machine learning where out-of-distribution (OOD) accuracy is at odds with in-distribution (ID) accuracy: a robust classifier obtained via specialized techniques such as removing spurious features often has better OOD but worse ID accuracy compared to a standard classifier trained via ERM. In this paper, we find that ID-calibrated ensembles -- where we simply ensemble the standard and robust models after calibrating on only ID data -- outperforms prior state-of-the-art (based on self-training) on both ID and OOD accuracy. On eleven natural distribution shift datasets, ID-calibrated ensembles obtain the best of both worlds: strong ID accuracy and OOD accuracy. We analyze this method in stylized settings, and identify two important conditions for ensembles to perform well both ID and OOD: (1) we need to calibrate the standard and robust models (on ID data, because OOD data is unavailable), (2) OOD has no anticorrelated spurious features."
    },
    {
      "title": "Fine-Tuning Can Distort Pre-Trained Features and Underperforms Out-of-Distribution",
      "url": "https://openreview.net/pdf?id=UYneFzXSJWh",
      "authors": [
        "Ananya Kumar",
        "Aditi Raghunathan",
        "Robbie Jones",
        "Tengyu Ma",
        "Percy Liang"
      ],
      "venue": "ICLR",
      "year": "2022",
      "awards": [
        "Oral"
      ],
      "enabled": true,
      "abstract": "When transferring a pretrained model to a downstream task, two popular methods are full fine-tuning (updating all the model parameters) and linear probing (updating only the last linear layer -- the \"head\"). It is well known that fine-tuning leads to better accuracy in-distribution (ID). However, in this paper, we find that fine-tuning can achieve worse accuracy than linear probing out-of-distribution (OOD) when the pretrained features are good and the distribution shift is large. On 10 distribution shift datasets (Breeds-Living17, Breeds-Entity30, DomainNet, CIFAR → STL, CIFAR10.1, FMoW, ImageNetV2, ImageNet-R, ImageNet-A, ImageNet-Sketch), fine-tuning obtains on average 2% higher accuracy ID but 7% lower accuracy OOD than linear probing. We show theoretically that this tradeoff between ID and OOD accuracy arises even in a simple setting: fine-tuning overparameterized two-layer linear networks. We prove that the OOD error of fine-tuning is high when we initialize with a fixed or random head -- this is because while fine-tuning learns the head, the lower layers of the neural network change simultaneously and distort the pretrained features. Our analysis suggests that the easy two-step strategy of linear probing then full fine-tuning (LP-FT), sometimes used as a fine-tuning heuristic, combines the benefits of both fine-tuning and linear probing. Empirically, LP-FT outperforms both fine-tuning and linear probing on the above datasets (1% better ID, 10% better OOD than full fine-tuning)."
    },
    {
      "title": "An Explanation of In-Context Learning as Implicit Bayesian Inference",
      "url": "https://arxiv.org/pdf/2111.02080.pdf",
      "authors": [
        "Sang Michael Xie",
        "Aditi Raghunathan",
        "Percy Liang",
        "Tengyu Ma"
      ],
      "venue": "ICLR",
      "year": "2022",
      "awards": [],
      "enabled": true,
      "abstract": "Large language models (LMs) such as GPT-3 have the surprising ability to do in-context learning, where the model learns to do a downstream task simply by conditioning on a prompt consisting of input-output examples. The LM learns from these examples without being explicitly pretrained to learn. Thus, it is unclear what enables in-context learning. In this paper, we study how in-context learning can emerge when pretraining documents have long-range coherence. Here, the LM must infer a latent document-level concept to generate coherent next tokens during pretraining. At test time, in-context learning occurs when the LM also infers a shared latent concept between examples in a prompt. We prove when this occurs despite a distribution mismatch between prompts and pretraining data in a setting where the pretraining distribution is a mixture of HMMs. In contrast to messy large-scale datasets used to train LMs capable of in-context learning, we generate a small-scale synthetic dataset (GINC) where Transformers and LSTMs both exhibit in-context learning. Beyond the theory, experiments on GINC exhibit large-scale real-world phenomena including improved in-context performance with model scaling (despite the same pretraining loss), sensitivity to example order, and instances where zero-shot is better than few-shot in-context learning."
    },
    {
      "title": "Accuracy on the Line: On the Strong Correlation Between Out-of-Distribution and In-Distribution Generalization",
      "url": "https://arxiv.org/pdf/2107.04649.pdf",
      "authors": [
        "John Miller",
        "Rohan Taori",
        "Aditi Raghunathan",
        "Shiori Sagawa",
        "Pang Wei Koh",
        "Vaishaal Shankar",
        "Percy Liang",
        "Yair Carmon",
        "Ludwig Schmidt"
      ],
      "venue": "ICML",
      "year": "2021",
      "awards": [],
      "enabled": true,
      "abstract": "For machine learning systems to be reliable, we must understand their performance in unseen, out-of-distribution environments. In this paper, we empirically show that out-of-distribution performance is strongly correlated with in-distribution performance for a wide range of models and distribution shifts. Specifically, we demonstrate strong correlations between in-distribution and out-of-distribution performance on variants of CIFAR-10 & ImageNet, a synthetic pose estimation task derived from YCB objects, satellite imagery classification in FMoW-WILDS, and wildlife classification in iWildCam-WILDS. The strong correlations hold across model architectures, hyperparameters, training set size, and training duration, and are more precise than what is expected from existing domain adaptation theory. To complete the picture, we also investigate cases where the correlation is weaker, for instance some synthetic distribution shifts from CIFAR-10-C and the tissue classification dataset Camelyon17-WILDS. Finally, we provide a candidate theory based on a Gaussian data model that shows how changes in the data covariance arising from distribution shift can affect the observed correlations."
    },
    {
      "title": "Just Train Twice: Improving Group Robustness without Training Group Information",
      "url": "https://arxiv.org/pdf/2107.09044.pdf",
      "authors": [
        "Evan Liu",
        "Behzaad Haghgoo",
        "Annie Chen",
        "Aditi Raghunathan",
        "Pang Wei Koh",
        "Shiori Sagawa",
        "Percy Liang",
        "Chelsea Finn"
      ],
      "venue": "ICML",
      "year": "2021",
      "awards": [],
      "enabled": true,
      "abstract": "Standard training via empirical risk minimization (ERM) can produce models that achieve high accuracy on average but low accuracy on certain groups, especially in the presence of spurious correlations between the input and label. Prior approaches that achieve high worst-group accuracy, like group distributionally robust optimization (group DRO) require expensive group annotations for each training point, whereas approaches that do not use such group annotations typically achieve unsatisfactory worst-group accuracy. In this paper, we propose a simple two-stage approach, JTT, that first trains a standard ERM model for several epochs, and then trains a second model that upweights the training examples that the first model misclassified. Intuitively, this upweights examples from groups on which standard ERM models perform poorly, leading to improved worst-group performance. Averaged over four image classification and natural language processing tasks with spurious correlations, JTT closes 75% of the gap in worst-group accuracy between standard ERM and group DRO, while only requiring group annotations on a small validation set in order to tune hyperparameters."
    },
    {
      "title": "Explore then Execute: Adapting without Rewards via Factorized Meta-Reinforcement Learning",
      "url": "https://arxiv.org/pdf/2008.02790.pdf",
      "authors": [
        "Evan Liu",
        "Aditi Raghunathan",
        "Percy Liang",
        "Chelsea Finn"
      ],
      "venue": "ICML",
      "year": "2021",
      "awards": [],
      "enabled": true,
      "abstract": "The goal of meta-reinforcement learning (meta-RL) is to build agents that can quickly learn new tasks by leveraging prior experience on related tasks. Learning a new task often requires both exploring to gather task-relevant information and exploiting this information to solve the task. In principle, optimal exploration and exploitation can be learned end-to-end by simply maximizing task performance. However, such meta-RL approaches struggle with local optima due to a chicken-and-egg problem: learning to explore requires good exploitation to gauge the exploration's utility, but learning to exploit requires information gathered via exploration. Optimizing separate objectives for exploration and exploitation can avoid this problem, but prior meta-RL exploration objectives yield suboptimal policies that gather information irrelevant to the task. We alleviate both concerns by constructing an exploitation objective that automatically identifies task-relevant information and an exploration objective to recover only this information. This avoids local optima in end-to-end training, without sacrificing optimal exploration. Empirically, DREAM substantially outperforms existing approaches on complex meta-RL problems, such as sparse-reward 3D visual navigation. Videos of DREAM:this https URL"
    },
    {
      "title": "Enabling Certification of Verification-Agnostic Networks via Memory-Efficient Semidefinite Programming",
      "url": "https://arxiv.org/pdf/2010.11645.pdf",
      "authors": [
        "Sumanth Dathathri*",
        "Krishnamurthy Dvijotham*",
        "Alexey Kurakin*",
        "Aditi Raghunathan*",
        "Jonathan Uesato",
        "Rudy Bunel",
        "Shreya Shankar",
        "Jacob Steinhardt",
        "Ian Goodfellow",
        "Percy Liang",
        "Pushmeet Kohli"
      ],
      "venue": "NeurIPS",
      "year": "2020",
      "awards": [],
      "enabled": true,
      "abstract": "Convex relaxations have emerged as a promising approach for verifying desirable properties of neural networks like robustness to adversarial perturbations. Widely used Linear Programming (LP) relaxations only work well when networks are trained to facilitate verification. This precludes applications that involve verification-agnostic networks, i.e., networks not specially trained for verification. On the other hand, semidefinite programming (SDP) relaxations have successfully be applied to verification-agnostic networks, but do not currently scale beyond small networks due to poor time and space asymptotics. In this work, we propose a first-order dual SDP algorithm that (1) requires memory only linear in the total number of network activations, (2) only requires a fixed number of forward/backward passes through the network per iteration. By exploiting iterative eigenvector methods, we express all solver operations in terms of forward and backward passes through the network, enabling efficient use of hardware like GPUs/TPUs. For two verification-agnostic networks on MNIST and CIFAR-10, we significantly improve L-inf verified robust accuracy from 1% to 88% and 6% to 40% respectively. We also demonstrate tight verification of a quadratic stability specification for the decoder of a variational autoencoder."
    },
    {
      "title": "The Pitfalls of Simplicity Bias in Neural Networks",
      "url": "https://arxiv.org/pdf/2006.07710.pdf",
      "authors": [
        "Harshay Shah",
        "Kaustav Tamuly",
        "Aditi Raghunathan",
        "Prateek Jain",
        "Praneeth Netrapalli"
      ],
      "venue": "NeurIPS",
      "year": "2020",
      "awards": [],
      "enabled": true,
      "abstract": "Several works have proposed Simplicity Bias (SB)---the tendency of standard training procedures such as Stochastic Gradient Descent (SGD) to find simple models---to justify why neural networks generalize well [Arpit et al. 2017, Nakkiran et al. 2019, Soudry et al. 2018]. However, the precise notion of simplicity remains vague. Furthermore, previous settings that use SB to theoretically justify why neural networks generalize well do not simultaneously capture the non-robustness of neural networks---a widely observed phenomenon in practice [Goodfellow et al. 2014, Jo and Bengio 2017]. We attempt to reconcile SB and the superior standard generalization of neural networks with the non-robustness observed in practice by designing datasets that (a) incorporate a precise notion of simplicity, (b) comprise multiple predictive features with varying levels of simplicity, and (c) capture the non-robustness of neural networks trained on real data. Through theory and empirics on these datasets, we make four observations: (i) SB of SGD and variants can be extreme: neural networks can exclusively rely on the simplest feature and remain invariant to all predictive complex features. (ii) The extreme aspect of SB could explain why seemingly benign distribution shifts and small adversarial perturbations significantly degrade model performance. (iii) Contrary to conventional wisdom, SB can also hurt generalization on the same data distribution, as SB persists even when the simplest feature has less predictive power than the more complex features. (iv) Common approaches to improve generalization and robustness---ensembles and adversarial training---can fail in mitigating SB and its pitfalls. Given the role of SB in training neural networks, we hope that the proposed datasets and methods serve as an effective testbed to evaluate novel algorithmic approaches aimed at avoiding the pitfalls of SB."
    },
    {
      "title": "DROCC: Deep Robust One-Class Classification",
      "url": "https://arxiv.org/pdf/2002.12718.pdf",
      "authors": [
        "Sachin Goyal",
        "Aditi Raghunathan",
        "Moksh Jain",
        "Harsha Vardhan Simhadri",
        "Prateek Jain"
      ],
      "venue": "ICML",
      "year": "2020",
      "awards": [],
      "enabled": true,
      "abstract": "Classical approaches for one-class problems such as one-class SVM and isolation forest require careful feature engineering when applied to structured domains like images. State-of-the-art methods aim to leverage deep learning to learn appropriate features via two main approaches. The first approach based on predicting transformations (Golan & El-Yaniv, 2018; Hendrycks et al., 2019a) while successful in some domains, crucially depends on an appropriate domain-specific set of transformations that are hard to obtain in general. The second approach of minimizing a classical one-class loss on the learned final layer representations, e.g., DeepSVDD (Ruff et al., 2018) suffers from the fundamental drawback of representation collapse. In this work, we propose Deep Robust One-Class Classification (DROCC) that is both applicable to most standard domains without requiring any side-information and robust to representation collapse. DROCC is based on the assumption that the points from the class of interest lie on a well-sampled, locally linear low dimensional manifold. Empirical evaluation demonstrates that DROCC is highly effective in two different one-class problem settings and on a range of real-world datasets across different domains: tabular data, images (CIFAR and ImageNet), audio, and time-series, offering up to 20% increase in accuracy over the state-of-the-art in anomaly detection. Code is available atthis https URL."
    },
    {
      "title": "An Investigation of Why Overparameterization Exacerbates Spurious Correlations",
      "url": "https://arxiv.org/pdf/2005.04345.pdf",
      "authors": [
        "Shiori Sagawa",
        "Aditi Raghunathan",
        "Pang Wei Koh",
        "Percy Liang"
      ],
      "venue": "ICML",
      "year": "2020",
      "awards": [],
      "enabled": true,
      "abstract": "We study why overparameterization -- increasing model size well beyond the point of zero training error -- can hurt test error on minority groups despite improving average test error when there are spurious correlations in the data. Through simulations and experiments on two image datasets, we identify two key properties of the training data that drive this behavior: the proportions of majority versus minority groups, and the signal-to-noise ratio of the spurious correlations. We then analyze a linear setting and theoretically show how the inductive bias of models towards \"memorizing\" fewer examples can cause overparameterization to hurt. Our analysis leads to a counterintuitive approach of subsampling the majority group, which empirically achieves low minority error in the overparameterized regime, even though the standard approach of upweighting the minority fails. Overall, our results suggest a tension between using overparameterized models versus using all the training data for achieving low worst-group error."
    },
    {
      "title": "Understanding and Mitigating the Tradeoff Between Robustness and Accuracy",
      "url": "https://arxiv.org/pdf/2002.10716.pdf",
      "authors": [
        "Aditi Raghunathan",
        "Sang Michael Xie",
        "Fanny Yang",
        "John Duchi",
        "Percy Liang"
      ],
      "venue": "ICML",
      "year": "2020",
      "awards": [],
      "enabled": true,
      "abstract": "Adversarial training augments the training set with perturbations to improve the robust error (over worst-case perturbations), but it often leads to an increase in the standard error (on unperturbed test inputs). Previous explanations for this tradeoff rely on the assumption that no predictor in the hypothesis class has low standard and robust error. In this work, we precisely characterize the effect of augmentation on the standard error in linear regression when the optimal linear predictor has zero standard and robust error. In particular, we show that the standard error could increase even when the augmented perturbations have noiseless observations from the optimal linear predictor. We then prove that the recently proposed robust self-training (RST) estimator improves robust error without sacrificing standard error for noiseless linear regression. Empirically, for neural networks, we find that RST with different adversarial training methods improves both standard and robust error for random and adversarial rotations and adversarial $\\ell_\\infty$ perturbations in CIFAR-10."
    },
    {
      "title": "Robust Encodings: A Framework for Combating Adversarial Typos",
      "url": "https://arxiv.org/pdf/2005.01229.pdf",
      "authors": [
        "Erik Jones",
        "Robin Jia",
        "Aditi Raghunathan",
        "Percy Liang"
      ],
      "venue": "ACL",
      "year": "2020",
      "awards": [],
      "enabled": true,
      "abstract": "Despite excellent performance on many tasks, NLP systems are easily fooled by small adversarial perturbations of inputs. Existing procedures to defend against such perturbations are either (i) heuristic in nature and susceptible to stronger attacks or (ii) provide guaranteed robustness to worst-case attacks, but are incompatible with state-of-the-art models like BERT. In this work, we introduce robust encodings (RobEn): a simple framework that confers guaranteed robustness, without making compromises on model architecture. The core component of RobEn is an encoding function, which maps sentences to a smaller, discrete space of encodings. Systems using these encodings as a bottleneck confer guaranteed robustness with standard training, and the same encodings can be used across multiple tasks. We identify two desiderata to construct robust encoding functions: perturbations of a sentence should map to a small set of encodings (stability), and models using encodings should still perform well (fidelity). We instantiate RobEn to defend against a large family of adversarial typos. Across six tasks from GLUE, our instantiation of RobEn paired with BERT achieves an average robust accuracy of 71.3% against all adversarial typos in the family considered, while previous work using a typo-corrector achieves only 35.3% accuracy against a simple greedy attack."
    },
    {
      "title": "Adversarial Training Can Hurt Generalization",
      "url": "https://arxiv.org/abs/1906.06032",
      "authors": [
        "Aditi Raghunathan",
        "Sang Michael Xie",
        "Fanny Yang",
        "John Duchi",
        "Percy Liang"
      ],
      "venue": "ICML 2019 Workshop: Identifying and Understanding Deep Learning",
      "year": "2019",
      "awards": [],
      "enabled": true,
      "abstract": "While adversarial training can improve robust accuracy (against an adversary), it sometimes hurts standard accuracy (when there is no adversary). Previous work has studied this tradeoff between standard and robust accuracy, but only in the setting where no predictor performs well on both objectives in the infinite data limit. In this paper, we show that even when the optimal predictor with infinite data performs well on both objectives, a tradeoff can still manifest itself with finite data. Furthermore, since our construction is based on a convex learning problem, we rule out optimization concerns, thus laying bare a fundamental tension between robustness and generalization. Finally, we show that robust self-training mostly eliminates this tradeoff by leveraging unlabeled data."

    },
    {
      "title": "Unlabeled Data Improves Adversarial Robustness",
      "url": "https://arxiv.org/pdf/1905.13736.pdf",
      "authors": [
        "Yair Carmon",
        "Aditi Raghunathan*",
        "Ludwig Schmidt",
        "Percy Liang",
        "John Duchi"
      ],
      "venue": "NeurIPS",
      "year": "2019",
      "awards": [],
      "enabled": true,
      "abstract": "We demonstrate, theoretically and empirically, that adversarial robustness can significantly benefit from semisupervised learning. Theoretically, we revisit the simple Gaussian model of Schmidt et al. that shows a sample complexity gap between standard and robust classification. We prove that unlabeled data bridges this gap: a simple semisupervised learning procedure (self-training) achieves high robust accuracy using the same number of labels required for achieving high standard accuracy. Empirically, we augment CIFAR-10 with 500K unlabeled images sourced from 80 Million Tiny Images and use robust self-training to outperform state-of-the-art robust accuracies by over 5 points in (i) $\\ell_\\infty$ robustness against several strong attacks via adversarial training and (ii) certified $\\ell_2$ and $\\ell_\\infty$ robustness via randomized smoothing. On SVHN, adding the dataset's own extra training set with the labels removed provides gains of 4 to 10 points, within 1 point of the gain from using the extra labels."
    },
    {
      "title": "Certified Robustness to Adversarial Word Substitutions",
      "url": "https://arxiv.org/pdf/1909.00986.pdf",
      "authors": [
        "Robin Jia",
        "Aditi Raghunathan",
        "Kerem Göksel",
        "Percy Liang"
      ],
      "venue": "EMNLP",
      "year": "2019",
      "awards": [],
      "enabled": true,
      "abstract": "State-of-the-art NLP models can often be fooled by adversaries that apply seemingly innocuous label-preserving transformations (e.g., paraphrasing) to input text. The number of possible transformations scales exponentially with text length, so data augmentation cannot cover all transformations of an input. This paper considers one exponentially large family of label-preserving transformations, in which every word in the input can be replaced with a similar word. We train the first models that are provably robust to all word substitutions in this family. Our training procedure uses Interval Bound Propagation (IBP) to minimize an upper bound on the worst-case loss that any combination of word substitutions can induce. To evaluate models' robustness to these transformations, we measure accuracy on adversarially chosen word substitutions applied to test examples. Our IBP-trained models attain $75\\%$ adversarial accuracy on both sentiment analysis on IMDB and natural language inference on SNLI. In comparison, on IMDB, models trained normally and ones trained with data augmentation achieve adversarial accuracy of only $8\\%$ and $35\\%$, respectively."
    },
    {
      "title": "Maximum Weighted Loss Discrepancy",
      "url": "https://arxiv.org/pdf/1906.03518.pdf",
      "authors": [
        "Fereshte Khani",
        "Aditi Raghunathan",
        "Percy Liang"
      ],
      "venue": "ICLR 2019 SafeML Workshop",
      "year": "2019",
      "awards": [],
      "enabled": true,
      "abstract": "Though machine learning algorithms excel at minimizing the average loss over a population, this might lead to large discrepancies between the losses across groups within the population. To capture this inequality, we introduce and study a notion we call maximum weighted loss discrepancy (MWLD), the maximum (weighted) difference between the loss of a group and the loss of the population. We relate MWLD to group fairness notions and robustness to demographic shifts. We then show MWLD satisfies the following three properties: 1) It is statistically impossible to estimate MWLD when all groups have equal weights. 2) For a particular family of weighting functions, we can estimate MWLD efficiently. 3) MWLD is related to loss variance, a quantity that arises in generalization bounds. We estimate MWLD with different weighting functions on four common datasets from the fairness literature. We finally show that loss variance regularization can halve the loss variance of a classifier and hence reduce MWLD without suffering a significant drop in accuracy."
    },
    {
      "title": "Semidefinite Relaxations for Certifying Robustness to Adversarial Examples",
      "url": "https://arxiv.org/pdf/1811.01057.pdf",
      "authors": [
        "Aditi Raghunathan",
        "Jacob Steinhardt",
        "Percy Liang"
      ],
      "venue": "NeurIPS",
      "year": "2018",
      "awards": [],
      "enabled": true,
      "abstract": "Despite their impressive performance on diverse tasks, neural networks fail catastrophically in the presence of adversarial inputs---imperceptibly but adversarially perturbed versions of natural inputs. We have witnessed an arms race between defenders who attempt to train robust networks and attackers who try to construct adversarial examples. One promise of ending the arms race is developing certified defenses, ones which are provably robust against all attackers in some family. These certified defenses are based on convex relaxations which construct an upper bound on the worst case loss over all attackers in the family. Previous relaxations are loose on networks that are not trained against the respective relaxation. In this paper, we propose a new semidefinite relaxation for certifying robustness that applies to arbitrary ReLU networks. We show that our proposed relaxation is tighter than previous relaxations and produces meaningful robustness guarantees on three different \"foreign networks\" whose training objectives are agnostic to our proposed relaxation."
    },
    {
      "title": "Certified Defenses Against Adversarial Examples",
      "url": "https://arxiv.org/abs/1801.09344",
      "authors": [
        "Aditi Raghunathan",
        "Jacob Steinhardt",
        "Percy Liang"
      ],
      "venue": "ICLR",
      "year": "2018",
      "awards": [],
      "enabled": true,
      "abstract": "While neural networks have achieved high accuracy on standard image classification benchmarks, their accuracy drops to nearly zero in the presence of small adversarial perturbations to test inputs. Defenses based on regularization and adversarial training have been proposed, but often followed by new, stronger attacks that defeat these defenses. Can we somehow end this arms race? In this work, we study this problem for neural networks with one hidden layer. We first propose a method based on a semidefinite relaxation that outputs a certificate that for a given network and test input, no attack can force the error to exceed a certain value. Second, as this certificate is differentiable, we jointly optimize it with the network parameters, providing an adaptive regularizer that encourages robustness against all attacks. On MNIST, our approach produces a network and a certificate that no attack that perturbs each pixel by at most \\epsilon = 0.1 can cause more than 35% test error."

    },
    {
      "title": "Learning Mixture of Gaussians with Streaming Data",
      "url": "https://arxiv.org/pdf/1707.02391.pdf",
      "authors": [
        "Aditi Raghunathan",
        "Prateek Jain",
        "Ravishankar Krishnaswamy"
      ],
      "venue": "NeurIPS",
      "year": "2017",
      "awards": [],
      "enabled": true,
      "abstract": "In this paper, we study the problem of learning a mixture of Gaussians with streaming data: given a stream of $N$ points in $d$ dimensions generated by an unknown mixture of $k$ spherical Gaussians, the goal is to estimate the model parameters using a single pass over the data stream. We analyze a streaming version of the popular Lloyd's heuristic and show that the algorithm estimates all the unknown centers of the component Gaussians accurately if they are sufficiently separated. Assuming each pair of centers are $C\\sigma$ distant with $C=\\Omega((k\\log k)^{1/4}\\sigma)$ and where $\\sigma^2$ is the maximum variance of any Gaussian component, we show that asymptotically the algorithm estimates the centers optimally (up to constants); our center separation requirement matches the best known result for spherical Gaussians \\citep{vempalawang}. For finite samples, we show that a bias term based on the initial estimate decreases at $O(1/{\\rm poly}(N))$ rate while variance decreases at nearly optimal rate of $\\sigma^2 d/N$.Our analysis requires seeding the algorithm with a good initial estimate of the true cluster centers for which we provide an online PCA based clustering algorithm. Indeed, the asymptotic per-step time complexity of our algorithm is the optimal $d\\cdot k$ while space complexity of our algorithm is $O(dk\\log k)$.In addition to the bias and variance terms which tend to $0$, the hard-thresholding based updates of streaming Lloyd's algorithm is agnostic to the data distribution and hence incurs an approximation error that cannot be avoided. However, by using a streaming version of the classical (soft-thresholding-based) EM method that exploits the Gaussian distribution explicitly, we show that for a mixture of two Gaussians the true means can be estimated consistently, with estimation error decreasing at nearly optimal rate, and tending to $0$ for $N\\rightarrow \\infty$."
    },
    {
      "title": "Estimating the Unseen from Multiple Populations",
      "url": "https://arxiv.org/pdf/1707.03854.pdf",
      "authors": [
        "Aditi Raghunathan",
        "Greg Valiant",
        "James Zou"
      ],
      "venue": "ICML",
      "year": "2017",
      "awards": [],
      "enabled": true,
      "abstract": "Given samples from a distribution, how many new elements should we expect to find if we continue sampling this distribution? This is an important and actively studied problem, with many applications ranging from unseen species estimation to genomics. We generalize this extrapolation and related unseen estimation problems to the multiple population setting, where population $j$ has an unknown distribution $D_j$ from which we observe $n_j$ samples. We derive an optimal estimator for the total number of elements we expect to find among new samples across the populations. Surprisingly, we prove that our estimator's accuracy is independent of the number of populations. We also develop an efficient optimization algorithm to solve the more general problem of estimating multi-population frequency distributions. We validate our methods and theory through extensive experiments. Finally, on a real dataset of human genomes across multiple ancestries, we demonstrate how our approach for unseen estimation can enable cohort designs that can discover interesting mutations with greater efficiency."
    },
    {
      "title": "Estimation from Indirect Supervision with Linear Moments",
      "url": "https://arxiv.org/pdf/1608.03100.pdf",
      "authors": [
        "Aditi Raghunathan",
        "Roy Frostig",
        "John Duchi",
        "Percy Liang"
      ],
      "venue": "ICML",
      "year": "2016",
      "awards": [],
      "enabled": true,
      "abstract": "In structured prediction problems where we have indirect supervision of the output, maximum marginal likelihood faces two computational obstacles: non-convexity of the objective and intractability of even a single gradient computation. In this paper, we bypass both obstacles for a class of what we call linear indirectly-supervised problems. Our approach is simple: we solve a linear system to estimate sufficient statistics of the model, which we then use to estimate parameters via convex optimization. We analyze the statistical properties of our approach and show empirically that it is effective in two settings: learning with local privacy constraints and learning from low-cost count-based annotations."
    },
    {
      "title": "A Reinforcement Learning Approach to Online Learning of Decision Trees",
      "url": "https://arxiv.org/pdf/1507.06923.pdf",
      "authors": [
        "Abhinav Garlapati",
        "Aditi Raghunathan",
        "Vaishnavh Nagarajan",
        "Balaraman Ravindran"
      ],
      "venue": "EWRL",
      "year": "2015",
      "awards": [],
      "enabled": true,
      "abstract": "Online decision tree learning algorithms typically examine all features of a new data point to update model parameters. We propose a novel alternative, Reinforcement Learning- based Decision Trees (RLDT), that uses Reinforcement Learning (RL) to actively examine a minimal number of features of a data point to classify it with high accuracy. Furthermore, RLDT optimizes a long term return, providing a better alternative to the traditional myopic greedy approach to growing decision trees. We demonstrate that this approach performs as well as batch learning algorithms and other online decision tree learning algorithms, while making significantly fewer queries about the features of the data points. We also show that RLDT can effectively handle concept drift."
    },
    {
      "title": "Probabilistic Dependency Networks for Prediction and Diagnostics",
      "url": "https://arxiv.org/pdf/1508.03130.pdf",
      "authors": [
        "Narayanan U. Edakunni",
        "Aditi Raghunathan",
        "Abhishek Tripathi",
        "John Handley",
        "Fredric Roulland"
      ],
      "venue": "TRB Annual Meeting",
      "year": "2014",
      "awards": [],
      "enabled": true,
      "abstract": "Research in transportation frequently involve modelling and predicting attributes of events that occur at regular intervals. The event could be arrival of a bus at a bus stop, the volume of a traffic at a particular point, the demand at a particular bus stop etc. In this work, we propose a specific implementation of probabilistic graphical models to learn the probabilistic dependency between the events that occur in a network. A dependency graph is built from the past observed instances of the event and we use the graph to understand the causal effects of some events on others in the system. The dependency graph is also used to predict the attributes of future events and is shown to have a good prediction accuracy compared to the state of the art."
    }
  ]
}