diff --git a/.gitignore b/.gitignore index 2609f2b..6919a6d 100644 --- a/.gitignore +++ b/.gitignore @@ -605,3 +605,6 @@ TSWLatexianTemp* # Other /figures/ + +# Mac +.DS_Store diff --git a/paper/ModernmolBERT.bib b/paper/ModernmolBERT.bib deleted file mode 100644 index 4a9ae0c..0000000 --- a/paper/ModernmolBERT.bib +++ /dev/null @@ -1,315 +0,0 @@ -@online{ahmadChemBERTa2ChemicalFoundation2022, - title = {{{ChemBERTa-2}}: {{Towards Chemical Foundation Models}}}, - shorttitle = {{{ChemBERTa-2}}}, - author = {Ahmad, Walid and Simon, Elana and Chithrananda, Seyone and Grand, Gabriel and Ramsundar, Bharath}, - date = {2022-09-05}, - eprint = {2209.01712}, - eprinttype = {arXiv}, - eprintclass = {cs.LG}, - doi = {10.48550/arXiv.2209.01712}, - url = {http://arxiv.org/abs/2209.01712}, - urldate = {2026-05-15}, - abstract = {Large pretrained models such as GPT-3 have had tremendous impact on modern natural language processing by leveraging self-supervised learning to learn salient representations that can be used to readily finetune on a wide variety of downstream tasks [1]. We investigate the possibility of transferring such advances to molecular machine learning by building a chemical foundation model, ChemBERTa-2, using the “language" of SMILES. While labeled data for molecular prediction tasks is typically scarce, libraries of SMILES strings are readily available.}, - langid = {english}, - pubstate = {prepublished}, - keywords = {Computer Science - Artificial Intelligence,Computer Science - Machine Learning,Quantitative Biology - Biomolecules}, - file = {/Users/skn506/Zotero/storage/RK4B4L3L/Ahmad et al. - 2022 - ChemBERTa-2 Towards Chemical Foundation Models.pdf} -} - -@article{darrinWhenEmbedderMore, - title = {When Is an {{Embedder More Promising}} than {{Another}}?}, - author = {Darrin, Maxime and Formont, Philippe and Ayed, Ismail BEN and Cheung, Jackie Chi Kit and Piantanida, Pablo}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/H9EG3U9U/Darrin et al. - When is an Embedder More Promising than Another.pdf} -} - -@article{dengSystematicStudyKey2023, - title = {A Systematic Study of Key Elements Underlying Molecular Property Prediction}, - author = {Deng, Jianyuan and Yang, Zhibo and Wang, Hehe and Ojima, Iwao and Samaras, Dimitris and Wang, Fusheng}, - date = {2023-10-13}, - journaltitle = {Nature Communications}, - shortjournal = {Nat Commun}, - volume = {14}, - number = {1}, - pages = {6395}, - issn = {2041-1723}, - doi = {10.1038/s41467-023-41948-6}, - url = {https://www.nature.com/articles/s41467-023-41948-6}, - urldate = {2026-05-22}, - abstract = {Abstract Artificial intelligence (AI) has been widely applied in drug discovery with a major task as molecular property prediction. Despite booming techniques in molecular representation learning, key elements underlying molecular property prediction remain largely unexplored, which impedes further advancements in this field. Herein, we conduct an extensive evaluation of representative models using various representations on the MoleculeNet datasets, a suite of opioids-related datasets and two additional activity datasets from the literature. To investigate the predictive power in low-data and high-data space, a series of descriptors datasets of varying sizes are also assembled to evaluate the models. In total, we have trained 62,820 models, including 50,220 models on fixed representations, 4200 models on SMILES sequences and 8400 models on molecular graphs. Based on extensive experimentation and rigorous comparison, we show that representation learning models exhibit limited performance in molecular property prediction in most datasets. Besides, multiple key elements underlying molecular property prediction can affect the evaluation results. Furthermore, we show that activity cliffs can significantly impact model prediction. Finally, we explore into potential causes why representation learning models can fail and show that dataset size is essential for representation learning models to excel.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/6NF95RLT/Deng et al. - 2023 - A systematic study of key elements underlying molecular property prediction.pdf} -} - -@article{huangTherapeuticsDataCommons, - title = {Therapeutics {{Data Commons}}: {{Machine Learning Datasets}} and {{Tasks}} for {{Drug Discovery}} and {{Development}}}, - author = {Huang, Kexin and Fu, Tianfan and Gao, Wenhao and Zhao, Yue and Roohani, Yusuf and Leskovec, Jure and Coley, Connor W and Xiao, Cao and Sun, Jimeng and Zitnik, Marinka}, - abstract = {Therapeutics machine learning is an emerging field with incredible opportunities for innovation and impact. However, advancement in this field requires formulation of meaningful tasks and careful curation of datasets. Here, we introduce Therapeutics Data Commons (TDC), the first unifying platform to systematically access and evaluate machine learning across the entire range of therapeutics. To date, TDC includes 66 AI-ready datasets spread across 22 learning tasks and spanning the discovery and development of safe and effective medicines. TDC also provides an ecosystem of tools and community resources, including 33 data functions and diverse types of data splits, 23 strategies for systematic model evaluation, 17 molecule generation oracles, and 29 public leaderboards. All resources are integrated and accessible via an open Python library. We carry out extensive experiments on selected datasets, demonstrating that even the strongest algorithms fall short of solving key therapeutics challenges, including distributional shifts, multi-scale and multi-modal learning, and robust generalization to novel data points. We envision that TDC can facilitate algorithmic advances and considerably accelerate machine-learning model development, validation and transition into biomedical and clinical implementation. TDC is available at https://tdcommons.ai.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/7EABB9SD/Huang et al. - Therapeutics Data Commons Machine Learning Datasets and Tasks for Drug Discovery and Development.pdf} -} - -@article{irwinChemformerPretrainedTransformer2022, - title = {Chemformer: A Pre-Trained Transformer for Computational Chemistry}, - shorttitle = {Chemformer}, - author = {Irwin, Ross and Dimitriadis, Spyridon and He, Jiazhen and Bjerrum, Esben Jannik}, - date = {2022-03-01}, - journaltitle = {Machine Learning: Science and Technology}, - shortjournal = {Mach. Learn.: Sci. Technol.}, - volume = {3}, - number = {1}, - pages = {015022}, - issn = {2632-2153}, - doi = {10.1088/2632-2153/ac3ffb}, - url = {https://iopscience.iop.org/article/10.1088/2632-2153/ac3ffb}, - urldate = {2026-05-22}, - abstract = {Transformer models coupled with a simplified molecular line entry system (SMILES) have recently proven to be a powerful combination for solving challenges in cheminformatics. These models, however, are often developed specifically for a single application and can be very resource-intensive to train. In this work we present the Chemformer model—a Transformer-based model which can be quickly applied to both sequence-to-sequence and discriminative cheminformatics tasks. Additionally, we show that self-supervised pre-training can improve performance and significantly speed up convergence on downstream tasks. On direct synthesis and retrosynthesis prediction benchmark datasets we publish state-of-the-art results for top-1 accuracy. We also improve on existing approaches for a molecular optimisation task and show that Chemformer can optimise on multiple discriminative tasks simultaneously. Models, datasets and code will be made available after publication.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/W8RVM5M2/Irwin et al. - 2022 - Chemformer a pre-trained transformer for computational chemistry.pdf} -} - -@online{jaegerMol2vecUnsupervisedMachine2017, - title = {Mol2vec: {{Unsupervised Machine Learning Approach}} with {{Chemical Intuition}}}, - shorttitle = {Mol2vec}, - author = {Jaeger, Sabrina and Fulle, Simone and Turk, Samo}, - date = {2017-10-23}, - doi = {10.26434/chemrxiv.5513581.v1}, - url = {https://chemrxiv.org/doi/full/10.26434/chemrxiv.5513581.v1}, - urldate = {2026-05-15}, - abstract = {Inspired by natural language processing techniques we here introduce Mol2vec which is an unsupervised machine learning approach to learn vector representations of molecular substructures. Similarly, to the Word2vec models where vectors of closely related words are in close proximity in the vector space, Mol2vec learns vector representations of molecular substructures that are pointing in similar directions for chemically related substructures. Compounds can finally be encoded as vectors by summing up vectors of the individual substructures and, for instance, feed into supervised machine learning approaches to predict compound properties. The underlying substructure vector embeddings are obtained by training an unsupervised machine learning approach on a so-called corpus of compounds that consists of all available chemical matter. The resulting Mol2vec model is pre-trained once, yields dense vector representations and overcomes drawbacks of common compound feature representations such as sparseness and bit collisions. The prediction capabilities are demonstrated on several compound property and bioactivity data sets and compared with results obtained for Morgan fingerprints as reference compound representation. Mol2vec can be easily combined with ProtVec, which employs the same Word2vec concept on protein sequences, resulting in a proteochemometric approach that is alignment independent and can be thus also easily used for proteins with low sequence similarities.}, - langid = {english}, - pubstate = {prepublished}, - file = {/Users/skn506/Zotero/storage/VICMPWGP/Jaeger et al. - 2017 - Mol2vec Unsupervised Machine Learning Approach with Chemical Intuition.pdf} -} - -@article{joshiSpanBERTImprovingPretraining2020, - title = {{{SpanBERT}}: {{Improving Pre-training}} by {{Representing}} and {{Predicting Spans}}}, - shorttitle = {{{SpanBERT}}}, - author = {Joshi, Mandar and Chen, Danqi and Liu, Yinhan and Weld, Daniel S. and Zettlemoyer, Luke and Levy, Omer}, - date = {2020-12}, - journaltitle = {Transactions of the Association for Computational Linguistics}, - shortjournal = {Transactions of the Association for Computational Linguistics}, - volume = {8}, - pages = {64--77}, - issn = {2307-387X}, - doi = {10.1162/tacl_a_00300}, - url = {https://direct.mit.edu/tacl/article/43539}, - urldate = {2026-05-20}, - abstract = {We present SpanBERT, a pre-training method that is designed to better represent and predict spans of text. Our approach extends BERT by (1) masking contiguous random spans, rather than random tokens, and (2) training the span boundary representations to predict the entire content of the masked span, without relying on the individual token representations within it. SpanBERT consistently outperforms BERT and our better-tuned baselines, with substantial gains on span selection tasks such as question answering and coreference resolution. In particular, with the same training data and model size as BERT large , our single model obtains 94.6\% and 88.7\% F1 on SQuAD 1.1 and 2.0 respectively. We also achieve a new state of the art on the OntoNotes coreference resolution task (79.6\% F1), strong performance on the TACRED relation extraction benchmark, and even gains on GLUE. 1}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/W9ZPAYHB/Joshi et al. - 2020 - SpanBERT Improving Pre-training by Representing and Predicting Spans.pdf} -} - -@article{krennSelfreferencingEmbeddedStrings2020, - title = {Self-Referencing Embedded Strings ({{SELFIES}}): {{A}} 100\% Robust Molecular String Representation}, - shorttitle = {Self-Referencing Embedded Strings ({{SELFIES}})}, - author = {Krenn, Mario and Häse, Florian and Nigam, AkshatKumar and Friederich, Pascal and Aspuru-Guzik, Alan}, - date = {2020-12-01}, - journaltitle = {Machine Learning: Science and Technology}, - shortjournal = {Mach. Learn.: Sci. Technol.}, - volume = {1}, - number = {4}, - pages = {045024}, - issn = {2632-2153}, - doi = {10.1088/2632-2153/aba947}, - url = {https://iopscience.iop.org/article/10.1088/2632-2153/aba947}, - urldate = {2026-05-20}, - abstract = {The discovery of novel materials and functional molecules can help to solve some of society’s most urgent challenges, ranging from efficient energy harvesting and storage to uncovering novel pharmaceutical drug candidates. Traditionally matter engineering–generally denoted as inverse design–was based massively on human intuition and high-throughput virtual screening. The last few years have seen the emergence of significant interest in computer-inspired designs based on evolutionary or deep learning methods. The major challenge here is that the standard strings molecular representation SMILES shows substantial weaknesses in that task because large fractions of strings do not correspond to valid molecules. Here, we solve this problem at a fundamental level and introduce SELFIES (SELF-referencIng Embedded Strings), a string-based representation of molecules which is 100\% robust. Every SELFIES string corresponds to a valid molecule, and SELFIES can represent every molecule. SELFIES can be directly applied in arbitrary machine learning models without the adaptation of the models; each of the generated molecule candidates is valid. In our experiments, the model’s internal memory stores two orders of magnitude more diverse molecules than a similar test with SMILES. Furthermore, as all molecules are valid, it allows for explanation and interpretation of the internal working of the generative models.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/L2JRKPR3/Krenn et al. - 2020 - Self-referencing embedded strings (SELFIES) A 100% robust molecular string representation.pdf} -} - -@article{krugerMolEncoderOptimalMasked2025, - title = {{{MolEncoder}}: Towards Optimal Masked Language Modeling for Molecules}, - shorttitle = {{{MolEncoder}}}, - author = {Krüger, Fabian P. and Österbacka, Nicklas and Kabeshov, Mikhail and Engkvist, Ola and Tetko, Igor}, - date = {2025}, - journaltitle = {Digital Discovery}, - shortjournal = {Digital Discovery}, - volume = {4}, - number = {12}, - pages = {3552--3566}, - issn = {2635-098X}, - doi = {10.1039/D5DD00369E}, - url = {https://xlink.rsc.org/?DOI=D5DD00369E}, - urldate = {2026-05-15}, - abstract = {Predicting molecular properties is a key challenge in drug discovery. , Predicting molecular properties is a key challenge in drug discovery. Machine learning models, especially those based on transformer architectures, are increasingly used to make these predictions from chemical structures. Inspired by recent progress in natural language processing, many studies have adopted encoder-only transformer architectures similar to BERT (Bidirectional Encoder Representations from Transformers) for this task. These models are pretrained using masked language modeling, where parts of the input are hidden and the model learns to recover them before fine-tuning on downstream tasks. In this work, we systematically investigate whether core assumptions from natural language processing, which are commonly adopted in molecular BERT-based models, actually hold when applied to molecules represented using the Simplified Molecular Input Line Entry System (SMILES). Specifically, we examine how masking ratio, pretraining dataset size, and model size affect performance in molecular property prediction. We find that higher masking ratios than commonly used significantly improve performance. In contrast, increasing model or pretraining dataset size quickly leads to diminishing returns, offering no consistent benefit while incurring significantly higher computational cost. Based on these insights, we develop MolEncoder, a BERT-based model that outperforms existing approaches on drug discovery tasks while being more computationally efficient. Our results highlight key differences between molecular pretraining and natural language processing, showing that they require different design choices. This enables more efficient model development and lowers barriers for researchers with limited computational resources. We release MolEncoder publicly to support future work and hope our findings help make molecular representation learning more accessible and cost-effective in drug discovery.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/GPYK6BMT/Krüger et al. - 2025 - MolEncoder towards optimal masked language modeling for molecules.pdf} -} - -@article{leonComparingSMILESSELFIES2024, - title = {Comparing {{SMILES}} and {{SELFIES}} Tokenization for Enhanced Chemical Language Modeling}, - author = {Leon, Miguelangel and Perezhohin, Yuriy and Peres, Fernando and Popovič, Aleš and Castelli, Mauro}, - date = {2024-10-23}, - journaltitle = {Scientific Reports}, - shortjournal = {Sci Rep}, - volume = {14}, - number = {1}, - pages = {25016}, - issn = {2045-2322}, - doi = {10.1038/s41598-024-76440-8}, - url = {https://www.nature.com/articles/s41598-024-76440-8}, - urldate = {2026-05-15}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/TTKBVIME/Leon et al. - 2024 - Comparing SMILES and SELFIES tokenization for enhanced chemical language modeling.pdf} -} - -@article{liuRoBERTaRobustlyOptimized, - title = {{{RoBERTa}}: {{A Robustly Optimized BERT Pretraining Approach}}}, - author = {Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/9STZXESA/Liu et al. - RoBERTa A Robustly Optimized BERT Pretraining Approach.pdf} -} - -@online{nussbaumNomicEmbedTraining2025, - title = {Nomic {{Embed}}: {{Training}} a {{Reproducible Long Context Text Embedder}}}, - shorttitle = {Nomic {{Embed}}}, - author = {Nussbaum, Zach and Morris, John X. and Duderstadt, Brandon and Mulyar, Andriy}, - date = {2025-02-03}, - eprint = {2402.01613}, - eprinttype = {arXiv}, - eprintclass = {cs.CL}, - doi = {10.48550/arXiv.2402.01613}, - url = {http://arxiv.org/abs/2402.01613}, - urldate = {2026-05-15}, - abstract = {This technical report describes the training of nomic-embed-text-v1, the first fully reproducible, open-source, open-weights, open-data, 8192 context length English text embedding model that outperforms both OpenAI Ada-002 and OpenAI text-embedding-3-small on the short-context MTEB benchmark and the long context LoCo benchmark. We release the training code and model weights under an Apache 2.0 license. In contrast with other open-source models, we release the full curated training data and code that allows for full replication of nomic-embed-text-v1. You can find code and data to replicate the model at https://github.com/nomic-ai/contrastors.}, - langid = {english}, - pubstate = {prepublished}, - keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language}, - file = {/Users/skn506/Zotero/storage/YPLEWBNH/Nussbaum et al. - 2025 - Nomic Embed Training a Reproducible Long Context Text Embedder.pdf} -} - -@article{pengPretrainedMolecularLanguage2025, - title = {Pre-Trained Molecular Language Models with Random Functional Group Masking}, - author = {Peng, Tianhao and Li, Yuchen and Li, Xuhong and Bian, Jiang and Xie, Zeke and Sui, Ning and Mumtaz, Shahid and Xu, Yanwu and Kong, Linghe and Xiong, Haoyi}, - date = {2025-10-01}, - journaltitle = {npj Artificial Intelligence}, - shortjournal = {npj Artif. Intell.}, - volume = {1}, - number = {1}, - pages = {28}, - issn = {3005-1460}, - doi = {10.1038/s44387-025-00029-3}, - url = {https://www.nature.com/articles/s44387-025-00029-3}, - urldate = {2026-05-15}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/52ZGH2LR/Peng et al. - 2025 - Pre-trained molecular language models with random functional group masking.pdf} -} - -@article{pengPretrainedMolecularLanguage2025a, - title = {Pre-Trained Molecular Language Models with Random Functional Group Masking}, - author = {Peng, Tianhao and Li, Yuchen and Li, Xuhong and Bian, Jiang and Xie, Zeke and Sui, Ning and Mumtaz, Shahid and Xu, Yanwu and Kong, Linghe and Xiong, Haoyi}, - date = {2025-10-01}, - journaltitle = {npj Artificial Intelligence}, - shortjournal = {npj Artif. Intell.}, - volume = {1}, - number = {1}, - pages = {28}, - issn = {3005-1460}, - doi = {10.1038/s44387-025-00029-3}, - url = {https://www.nature.com/articles/s44387-025-00029-3}, - urldate = {2026-05-20}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/ZEK634TI/Peng et al. - 2025 - Pre-trained molecular language models with random functional group masking.pdf} -} - -@online{praskiBenchmarkingPretrainedMolecular2026, - title = {Benchmarking {{Pretrained Molecular Embedding Models For Molecular Representation Learning}}}, - author = {Praski, Mateusz and Adamczyk, Jakub and Czech, Wojciech}, - date = {2026-02-23}, - eprint = {2508.06199}, - eprinttype = {arXiv}, - eprintclass = {cs.LG}, - doi = {10.48550/arXiv.2508.06199}, - url = {http://arxiv.org/abs/2508.06199}, - urldate = {2026-05-15}, - abstract = {Pretrained neural networks have attracted significant interest in chemistry and small molecule drug design. Embeddings from these models are widely used for molecular property prediction, virtual screening, and small data learning in molecular chemistry. This study presents the most extensive comparison of such models to date, evaluating 25 models across 25 datasets. Under a fair comparison framework, we assess models spanning various modalities, architectures, and pretraining strategies. Using a dedicated hierarchical Bayesian statistical testing model, we arrive at a surprising result: nearly all neural models show negligible or no improvement over the baseline ECFP molecular fingerprint. Only the CLAMP model, which is also based on molecular fingerprints, performs statistically significantly better than the alternatives. These findings raise concerns about the evaluation rigor in existing studies. We discuss potential causes, propose solutions, and offer practical recommendations.}, - langid = {english}, - pubstate = {prepublished}, - keywords = {Computer Science - Artificial Intelligence,Computer Science - Machine Learning}, - file = {/Users/skn506/Zotero/storage/EYTEA25G/Praski et al. - 2026 - Benchmarking Pretrained Molecular Embedding Models For Molecular Representation Learning.pdf} -} - -@online{singhChemBERTa3OpenSource2025, - title = {{{ChemBERTa-3}}: {{An Open Source Training Framework}} for {{Chemical Foundation Models}}}, - shorttitle = {{{ChemBERTa-3}}}, - author = {Singh, Riya and Barsainyan, Aryan Amit and Irfan, Rida and Amorin, Connor Joseph and He, Stewart and Davis, Tony and Thiagarajan, Arun and Sankaran, Shiva and Chithrananda, Seyone and Ahmad, Walid and Jones, Derek and McLoughlin, Kevin and Kim, Hyojin and Bhutani, Anoushka and Sathyanarayana, Shreyas Vinaya and Viswanathan, Venkat and Allen, Jonathan E. and Ramsundar, Bharath}, - date = {2025-08-03}, - doi = {10.26434/chemrxiv-2025-4glrl-v2}, - url = {https://chemrxiv.org/doi/full/10.26434/chemrxiv-2025-4glrl-v2}, - urldate = {2026-05-15}, - abstract = {The rapid advancement of machine learning in computational chemistry has opened new doors for designing molecules, predicting molecular properties, and discovering novel materials. However, building scalable and robust models for molecular property prediction remains a significant challenge due to the vast size and complexity of chemical space. In this paper, we introduce ChemBERTa-3, an open-source training framework designed to train and fine-tune largescale chemical foundation models. We explore the potential of multiple model architectures by evaluating their performance across various molecular datasets from the MoleculeNet suite. Our experiments demonstrated that pre-training on the expansive ZINC20 dataset yields models capable of performing well on both classification and regression tasks, providing valuable insights into drug discovery and materials science. For scalability, we leveraged both AWS-based Ray deployments and on-premise high-performance computing clusters to support the processing power required to train on billions of molecules. In support of reproducible and extensible science, we have open-sourced all ChemBERTa3 models.}, - langid = {english}, - pubstate = {prepublished}, - file = {/Users/skn506/Zotero/storage/MSU7RX5W/Singh et al. - 2025 - ChemBERTa-3 An Open Source Training Framework for Chemical Foundation Models.pdf} -} - -@article{wangUnderstandingHowDimension, - title = {Understanding {{How Dimension Reduction Tools Work}}: {{An Empirical Approach}} to {{Deciphering}} t-{{SNE}}, {{UMAP}}, {{TriMap}}, and {{PaCMAP}} for {{Data Visualization}}}, - author = {Wang, Yingfan and Huang, Haiyang and Rudin, Cynthia and Shaposhnik, Yaron}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/TQMR53S3/Wang et al. - Understanding How Dimension Reduction Tools Work An Empirical Approach to Deciphering t-SNE, UMAP,.pdf} -} - -@article{warnerSmarterBetterFaster, - title = {Smarter, {{Better}}, {{Faster}}, {{Longer}}: {{A Modern Bidirectional Encoder}} for {{Fast}}, {{Memory Efficient}}, and {{Long Context Finetuning}} and {{Inference}}}, - author = {Warner, Benjamin and Chaffin, Antoine and Clavié, Benjamin and Weller, Orion and Hallström, Oskar and Taghadouini, Said and Gallagher, Alexis and Biswas, Raja and Ladhak, Faisal and Aarsen, Tom and Cooper, Nathan and Adams, Griffin and Howard, Jeremy and Poli, Iacopo}, - abstract = {Encoder-only transformer models such as BERT offer a great performance-size tradeoff for retrieval and classification tasks with respect to larger decoder-only models. Despite being the workhorse of numerous production pipelines, there have been limited Pareto improvements to BERT since its release. In this paper, we introduce ModernBERT, bringing modern model optimizations to encoder-only models and representing a major Pareto improvement over older encoders. Trained on 2 trillion tokens with a native 8192 sequence length, ModernBERT models exhibit state-ofthe-art results on a large pool of evaluations encompassing diverse classification tasks and both single and multi-vector retrieval on different domains (including code). In addition to strong downstream performance, ModernBERT is also the most speed and memory efficient encoder and is designed for inference on common GPUs.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/CRVKE2QL/Warner et al. - Smarter, Better, Faster, Longer A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long.pdf} -} - -@online{wuMoleculeNetBenchmarkMolecular2018, - title = {{{MoleculeNet}}: {{A Benchmark}} for {{Molecular Machine Learning}}}, - shorttitle = {{{MoleculeNet}}}, - author = {Wu, Zhenqin and Ramsundar, Bharath and Feinberg, Evan N. and Gomes, Joseph and Geniesse, Caleb and Pappu, Aneesh S. and Leswing, Karl and Pande, Vijay}, - date = {2018-10-26}, - eprint = {1703.00564}, - eprinttype = {arXiv}, - eprintclass = {cs.LG}, - doi = {10.48550/arXiv.1703.00564}, - url = {http://arxiv.org/abs/1703.00564}, - urldate = {2026-05-16}, - abstract = {Molecular machine learning has been maturing rapidly over the last few years. Improved methods and the presence of larger datasets have enabled machine learning algorithms to make increasingly accurate predictions about molecular properties. However, algorithmic progress has been limited due to the lack of a standard benchmark to compare the efficacy of proposed methods; most new algorithms are benchmarked on different datasets making it challenging to gauge the quality of proposed methods. This work introduces MoleculeNet, a large scale benchmark for molecular machine learning. MoleculeNet curates multiple public datasets, establishes metrics for evaluation, and offers high quality open-source implementations of multiple previously proposed molecular featurization and learning algorithms (released as part of the DeepChem open source library). MoleculeNet benchmarks demonstrate that learnable representations are powerful tools for molecular machine learning and broadly offer the best performance. However, this result comes with caveats. Learnable representations still struggle to deal with complex tasks under data scarcity and highly imbalanced classification. For quantum mechanical and biophysical datasets, the use of physics-aware featurizations can be more important than choice of particular learning algorithm.}, - langid = {english}, - pubstate = {prepublished}, - keywords = {Computer Science - Machine Learning,Physics - Chemical Physics,Statistics - Machine Learning}, - file = {/Users/skn506/Zotero/storage/6U99LBSZ/Wu et al. - 2018 - MoleculeNet A Benchmark for Molecular Machine Learning.pdf} -} - -@article{yangAnalyzingLearnedMolecular2019, - title = {Analyzing {{Learned Molecular Representations}} for {{Property Prediction}}}, - author = {Yang, Kevin and Swanson, Kyle and Jin, Wengong and Coley, Connor and Eiden, Philipp and Gao, Hua and Guzman-Perez, Angel and Hopper, Timothy and Kelley, Brian and Mathea, Miriam and Palmer, Andrew and Settels, Volker and Jaakkola, Tommi and Jensen, Klavs and Barzilay, Regina}, - date = {2019-08-26}, - journaltitle = {Journal of Chemical Information and Modeling}, - shortjournal = {J. Chem. Inf. Model.}, - volume = {59}, - number = {8}, - pages = {3370--3388}, - issn = {1549-9596, 1549-960X}, - doi = {10.1021/acs.jcim.9b00237}, - url = {https://pubs.acs.org/doi/10.1021/acs.jcim.9b00237}, - urldate = {2026-05-22}, - abstract = {Advancements in neural machinery have led to a wide range of algorithmic solutions for molecular property prediction. Two classes of models in particular have yielded promising results: neural networks applied to computed molecular fingerprints or expert-crafted descriptors and graph convolutional neural networks that construct a learned molecular representation by operating on the graph structure of the molecule. However, recent literature has yet to clearly determine which of these two methods is superior when generalizing to new chemical space. Furthermore, prior research has rarely examined these new models in industry research settings in comparison to existing employed models. In this paper, we benchmark models extensively on 19 public and 16 proprietary industrial data sets spanning a wide variety of chemical end points. In addition, we introduce a graph convolutional model that consistently matches or outperforms models using fixed molecular descriptors as well as previous graph neural architectures on both public and proprietary data sets. Our empirical findings indicate that while approaches based on these representations have yet to reach the level of experimental reproducibility, our proposed model nevertheless offers significant improvements over models currently used in industrial workflows.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/TRTEJHEX/Yang et al. - 2019 - Analyzing Learned Molecular Representations for Property Prediction.pdf} -} - -@article{zdrazilChEMBLDatabase20232024, - title = {The {{ChEMBL Database}} in 2023: A Drug Discovery Platform Spanning Multiple Bioactivity Data Types and Time Periods}, - shorttitle = {The {{ChEMBL Database}} in 2023}, - author = {Zdrazil, Barbara and Felix, Eloy and Hunter, Fiona and Manners, Emma J and Blackshaw, James and Corbett, Sybilla and {de~Veij}, Marleen and Ioannidis, Harris and Lopez, David Mendez and Mosquera, Juan~F and Magarinos, Maria~Paula and Bosc, Nicolas and Arcila, Ricardo and Kizilören, Tevfik and Gaulton, Anna and Bento, A~Patrícia and Adasme, Melissa~F and Monecke, Peter and Landrum, Gregory~A and Leach, Andrew~R}, - date = {2024-01-05}, - journaltitle = {Nucleic Acids Research}, - volume = {52}, - number = {D1}, - pages = {D1180-D1192}, - issn = {0305-1048, 1362-4962}, - doi = {10.1093/nar/gkad1004}, - url = {https://academic.oup.com/nar/article/52/D1/D1180/7337608}, - urldate = {2026-05-15}, - abstract = {ChEMBL (https://www.ebi.ac.uk/chembl/) is a manually curated, high-quality, large-scale, open, FAIR and Global Core Biodata Resource of bioactive molecules with drug-like properties, previously described in the 2012, 2014, 2017 and 2019 Nucleic Acids Research Database Issues. Since its introduction in 2009, ChEMBL’s content has changed dramatically in size and diversity of data types. Through incorporation of multiple new datasets from depositors since the 2019 update, ChEMBL now contains slightly more bioactivity data from deposited data vs data extracted from literature. In collaboration with the EUbOPEN consortium, chemical probe data is now regularly deposited into ChEMBL. Release 27 made curated data available for compounds screened for potential anti-SARS-CoV-2 activity from several large-scale drug repurposing screens. In addition, new patent bioactivity data have been added to the latest ChEMBL releases, and various new features have been incorporated, including a Natural Product likeness score, updated flags for Natural Products, a new flag for Chemical Probes, and the initial annotation of the action type for ∼270 000 bioactivity measurements.}, - langid = {english}, - file = {/Users/skn506/Zotero/storage/T528UVIF/Zdrazil et al. - 2024 - The ChEMBL Database in 2023 a drug discovery platform spanning multiple bioactivity data types and.pdf} -} diff --git a/paper/Notes.docx b/paper/Notes.docx deleted file mode 100644 index 3a13533..0000000 Binary files a/paper/Notes.docx and /dev/null differ diff --git a/paper/figures/Fig1_A.pdf b/paper/figures/Fig1_A.pdf deleted file mode 100644 index 187623c..0000000 Binary files a/paper/figures/Fig1_A.pdf and /dev/null differ diff --git a/paper/figures/Fig1_B.pdf b/paper/figures/Fig1_B.pdf deleted file mode 100644 index 502d036..0000000 Binary files a/paper/figures/Fig1_B.pdf and /dev/null differ diff --git a/paper/figures/Fig1_C.pdf b/paper/figures/Fig1_C.pdf deleted file mode 100644 index 25f82d8..0000000 Binary files a/paper/figures/Fig1_C.pdf and /dev/null differ diff --git a/paper/figures/Fig1_New.pdf b/paper/figures/Fig1_New.pdf new file mode 100644 index 0000000..f8be18a Binary files /dev/null and b/paper/figures/Fig1_New.pdf differ diff --git a/paper/figures/Fig2_New.pdf b/paper/figures/Fig2_New.pdf new file mode 100644 index 0000000..4a55232 Binary files /dev/null and b/paper/figures/Fig2_New.pdf differ diff --git a/paper/figures/Fig_groupbars.pdf b/paper/figures/Fig_groupbars.pdf deleted file mode 100644 index 3674a6a..0000000 Binary files a/paper/figures/Fig_groupbars.pdf and /dev/null differ diff --git a/paper/figures/bootstrap_ci_forest.png b/paper/figures/bootstrap_ci_forest.png deleted file mode 100644 index 562a97e..0000000 Binary files a/paper/figures/bootstrap_ci_forest.png and /dev/null differ diff --git a/paper/main.tex b/paper/main.tex index e401c61..99e1f8d 100644 --- a/paper/main.tex +++ b/paper/main.tex @@ -1,8 +1,8 @@ -% main.tex — ModernMolBERT arXiv preprint [canonical; reconciled 2026-05-22] +% main.tex — ModernMolBERT ChemRxiv preprint [canonical; reconciled 2026-06-18] % Compile: pdflatex → bibtex → pdflatex → pdflatex % % Quick-start checklist before submission: -% [x] \date{} intentionally left empty for preprint (bioRxiv stamps its own date) +% [x] \date{} intentionally left empty for preprint (ChemRxiv stamps its own posting date/version) % [x] \usepackage{todonotes} disabled (commented out below) % [x] No \todo{} notes remain in the source % [x] bibtex run clean: 0 undefined citations, 0 multiply-defined keys @@ -69,11 +69,13 @@ linkcolor=MidnightBlue, citecolor=MidnightBlue, urlcolor=MidnightBlue, - pdftitle={ModernMolBERT: A ModernBERT Encoder Family for - SELFIES Molecular Language Modeling}, - pdfauthor={Jakob S. Madsen, Sara Angelucci, Alexander S. Hauser}, - pdfkeywords={molecular language model, SELFIES, ModernBERT, - representation learning, cheminformatics, APE tokeniser}, + pdftitle={ModernMolBERT: A Compact SELFIES Encoder for + Molecular Representation Learning}, + pdfauthor={Jakob S. Madsen, Sara Angelucci, Christopher L. Porterfield, Xiaoting Liu, Alexander S. Hauser}, + pdfkeywords={molecular representation learning, molecular embeddings, + chemical language models, SELFIES, masked language + modelling, frozen embeddings, drug-discovery benchmarks, + ModernBERT}, pdfsubject={Chemical Machine Learning}]{hyperref} % ---------- Smart cross-references (must come after hyperref) ---------- @@ -102,7 +104,7 @@ % ---------- Date ---------- \usepackage[en-GB]{datetime2} -\date{} % Suppressed for preprint; bioRxiv stamps its own date +\date{} % Suppressed for preprint; ChemRxiv stamps its own posting date/version % ---------- Custom macros ---------- \input{macros} @@ -110,14 +112,15 @@ % ============================================================ % TITLE AND AUTHORS % ============================================================ -\title{\textbf{ModernMolBERT}: A \textsc{ModernBERT} Encoder Family\\ - for \selfies{} Molecular Language Modeling} +\title{\textbf{ModernMolBERT}: A Compact \selfies{} Encoder\\ + for Molecular Representation Learning} % authblk style: each author lists affiliation by number; % shared affiliations are collapsed automatically. \author[1,$*$]{Jakob S.\ Madsen} \author[1]{Sara Angelucci} -\author[x]{Xiaoting Liu} +\author[1]{Christopher L. Porterfield} +\author[1]{Xiaoting Liu} \author[1]{Alexander S.\ Hauser} \affil[1]{% @@ -138,60 +141,51 @@ % ---- Abstract ----------------------------------------------- \begin{abstract} -We introduce \model{}, a family of encoder-only transformer models for -small-molecule representation learning. \model{} adapts the \modernbert{} -architecture to molecular language modelling and pairs it with a -\selfies{}-aware \emph{Atom Pair Encoding} (\ape{}) tokeniser. -Both released variants (\emph{small} and \emph{base}) are pre-trained with -masked language modelling on $\sim$\num{2.4}M unique \selfies{} strings from -\chembl{}~36 and evaluated as frozen molecular embedders on 25 binary -classification benchmarks. -\model{}-base attains a mean ROC-AUC of 77.9, -competitive with a strong fingerprint baseline and improving on comparable -pre-trained string encoders by roughly four to five points. -\model{} provides a compact, reproducible, open-source foundation for -\selfies{}-based molecular representation learning; we release the model -weights, the \ape{} tokeniser, and training code. +% Suppress line-break hyphenation so the abstract copies as clean, unbroken words. +\hyphenpenalty=10000\exhyphenpenalty=10000\sloppy +We introduce \model{}, an encoder-only transformer model for small-molecule representation learning. \model{} adapts the \modernbert{} architecture to molecular language modelling and pairs it with a \selfies{}-aware \emph{Atom Pair Encoding} (\ape{}) tokeniser. Both released variants (\emph{small} and \emph{base}) are pre-trained with masked language modelling on approximately {2.4}M unique \selfies{} strings from \chembl{}~36 and evaluated as frozen molecular embedders on 25 binary classification benchmarks. \model{}-base attains a mean ROC-AUC of 77.9 (reported on a 0 to 100 scale), competitive with a strong fingerprint baseline and improving on comparable pre-trained string encoders by roughly five points. \model{} provides a compact, reproducible, open-source foundation for \selfies{}-based molecular representation learning; we release the model weights, the \ape{} tokeniser, and training code. \end{abstract} +\vspace{0.4em} +\noindent\textbf{Keywords:} molecular representation learning; molecular +embeddings; chemical language models; \selfies{}; masked language modelling; +frozen embeddings; drug-discovery benchmarks; \modernbert{}. + % ============================================================ \section{Introduction}% \label{sec:introduction} % ============================================================ -Molecular embeddings are the numerical interface between chemical structure and machine learning. They are the vectors passed to property predictors, virtual screening pipelines, similarity searches, clustering analyses, analogue-retrieval workflows, and dataset-triage tools. Their quality determines which chemical differences are visible to a model, which structure–activity patterns can be learned from sparse assay labels, and how well a representation transfers from known to unexplored chemical space. +Molecular embeddings are the interface between molecular structure and modern machine learning methods. Embeddings are the vectors passed to property predictors, virtual-screening pipelines, similarity searches, clustering analyses, analogue-retrieval workflows, and dataset-triage tools. Their quality influences which chemical differences a model can see, which structure--activity patterns can be learned from sparse assay labels, and how well a representation transfers from known to unexplored chemical space. -Molecular representation learning is therefore an essential preprocessing step. A useful molecular embedder should preserve chemically meaningful structure, avoid artefacts introduced by molecular notation, and be efficient enough to apply across large compound collections. ModernMolBERT addresses all of these by combining chemically valid SELFIES strings, a SELFIES-aware Atom Pair Encoding tokeniser, and an efficient ModernBERT encoder trained as a reusable frozen molecular embedder (\cref{fig:overview}). +Molecular representation learning is therefore not just a preprocessing choice but plays a key role in downstream performance. A good molecular embedder must preserve chemically meaningful structure, it must avoid artefacts introduced by molecular notation, and it should stay efficient enough to apply across large-scale compound libraries. The three design choices behind \model{} aim to address these needs. We use chemically valid \selfies{} inputs, a \selfies{}-aware tokeniser, and an efficient \modernbert{} encoder. This introduction explains and motivates each of these. -For decades, the dominant answer was to map each molecule to a fixed-length -vector of hand-crafted descriptors, such as physicochemical properties and -substructure-based fingerprints. The most prominent examples are circular +A long-standing way to represent a molecule is to map it to a +fixed-length vector of hand-crafted descriptors, such as physicochemical +properties and substructure-based fingerprints. The most prominent examples are circular fingerprints such as the Extended Connectivity Fingerprint~(ECFP)~\citep{rogersExtendedConnectivityFingerprints2010}, which hash the presence of circular atomic neighbourhoods at increasing radii into a -fixed-length bit vector and have been a workhorse of quantitative +fixed-length bit vector. ECFP4 in particular has been a workhorse of quantitative structure--activity relationship (QSAR) modelling. -Their compactness and interpretability are desirable properties, but their -fixed dimensionality and bespoke design mean they cannot adapt to new -structural motifs, lose information through hashing collisions, and provide -no mechanism for learning richer structure--activity correlations from data. +These fingerprints are compact and interpretable. However, their fixed dimensionality and bespoke +design mean they cannot adapt to new structural motifs, lose information through +hashing collisions, and cannot learn richer representations of structure--activity correlations from data. An early bridge between fixed fingerprints and fully learned representations was Mol2Vec~\citep{jaegerMol2vecUnsupervisedMachine2017}, which applied -the Word2Vec algorithm to sequences of Morgan substructure identifiers, +the Word2Vec natural language algorithm to sequences of Morgan substructure identifiers, learning distributed embeddings for molecular fragments without any labelled data. -Learnable alternatives that operate directly on molecular graphs --- treating -atoms as nodes and bonds as edges --- have also more recently been proposed -\citep{gilmerNeuralMessagePassing2017,rongGroverTransformerSelfsupervised2020}, -but these require specialised aggregation schemes and sit outside the -mainstream transformer pretraining paradigm. +Learnable alternatives that operate directly on molecular graphs, treating +atoms as nodes and bonds as edges, have also been proposed +\citep{gilmerNeuralMessagePassing2017,rongGroverTransformerSelfsupervised2020}. However, these require specialised aggregation schemes and sit outside the mainstream transformer pretraining paradigm. The BERT~\citep{devlinBERTPreTrainingDeep2019} family of bidirectional -encoder models revolutionised how masked language model~(MLM) pretraining on -large unlabelled corpora could produce representations that transfer to many +encoder models showed that masked language model~(MLM) pretraining on +large unlabelled corpora produces representations that transfer to many downstream tasks with minimal fine-tuning. -Molecules are encoded as \smiles{} (Simplified Molecular-Input Line-Entry +Here, molecules are represented as \smiles{} (Simplified Molecular-Input Line-Entry System) strings~\citep{weininger1988smiles}, a linear notation that encodes atomic connectivity as a depth-first traversal of the molecular graph. This sequential form is a natural substrate for the same sequence models originally @@ -201,10 +195,10 @@ \section{Introduction}% ChemBERTa~\citep{chithranandaChemBERTaLargeScaleSelf2020}, MolBERT~\citep{fabianMolecularRepresentationLearning2020}, ChemBERTa-2~\citep{ahmadChemBERTa2TowardsChemical2022}, and -MoLFormer~\citep{rossLargescaleChemicalLanguage2022}. These collectively -established that MLM pretraining on \smiles{} yielded representations +MoLFormer~\citep{rossLargescaleChemicalLanguage2022}. These models all +established that MLM pretraining on \smiles{} yielded molecular representations competitive with fingerprint baselines on property-prediction benchmarks -without task-specific feature engineering, and that the approach scales to +without task-specific feature engineering, and that this approach scales to billion-molecule corpora. \Cref{tab:embedder-comparison} summarises their key design choices alongside \model{}; \cref{sec:related-work} discusses them in detail. @@ -243,12 +237,12 @@ \section{Introduction}% \end{tabularx} \caption{% Overview of representative molecular string encoder models. - \emph{Repr.}: molecular string representation used for pretraining. + \emph{Repr.} (Representation): molecular string representation used for pretraining. \emph{Tokeniser}: tokenisation strategy (BPE = Byte Pair Encoding; atom = atom-level; APE = Atom Pair Encoding). - \emph{Arch.}: encoder backbone. - \emph{Params}: trainable parameters in millions. - \emph{Size}: pretraining corpus size in millions of molecules. + \emph{Arch.} (Architecture): encoder backbone. + \emph{Params} (Parameters): trainable parameters in millions. + \emph{Size} (of dataset): pretraining corpus size in millions of molecules. Embedding dimensions for these and additional models are listed in \cref{tab:embedders-full} (\cref{app:embedders}). \textbf{Bold} rows indicate the models introduced here. @@ -259,33 +253,33 @@ \section{Introduction}% \label{tab:embedder-comparison} \end{table} -\smiles{}-based language models have two weaknesses as notation. +As a notation for language modelling, however, \smiles{} has two weaknesses. First, most perturbations of a \smiles{} string are chemically -invalid~\citep{krennSELFIESSelfReferencingEmbedded2020}, so a model must spend -capacity recognising ill-formed inputs, and a single molecule admits many -equivalent strings. This inflates the effective vocabulary and complicates -self-supervised objectives. +invalid~\citep{krennSELFIESSelfReferencingEmbedded2020}, so a model spends +capacity learning to recognise ill-formed inputs. In addition, a single molecule can be represented by +many equivalent strings, which inflates the effective vocabulary and +complicates self-supervised objectives. Second, subword tokenisers such as Byte Pair Encoding~(BPE)~\citep{sennrichNeuralMachineTranslation2016}, -applied na\"{i}vely to molecular strings, split multi-character atomic symbols +applied na\"{i}vely to molecular strings, can split multi-character atomic symbols (e.g.\ \texttt{Cl} into \texttt{C}~+~\texttt{l})~\citep{leonComparingSMILESSELFIES2024}, -destroying the chemically meaningful units a molecular tokeniser should +and destroy the chemically meaningful units a molecular tokeniser should preserve. SELFIES (Self-Referencing Embedded Strings)~\citep{krennSELFIESSelfReferencingEmbedded2020} -addresses the validity problem by construction. Its context-free grammar +addresses the validity problem by design. Its context-free grammar guarantees that every syntactically valid \selfies{} string decodes to a chemically valid molecule under any editing or sampling operation, a guarantee -\smiles{} cannot provide (\cref{fig:design}A). +\smiles{} cannot provide (\hyperref[fig:design]{\cref*{fig:design}A}). It removes invalid-string noise from the pretraining signal while preserving the sequential structure that makes transformer encoders effective. Yet \selfies{} has been adopted mainly for molecular \emph{generation}, while encoder-only \selfies{} representation learning remains comparatively -unexplored~\citep{leonComparingSMILESSELFIES2024}. To our knowledge the +less explored~\citep{leonComparingSMILESSELFIES2024}. To our knowledge the combination of \selfies{}, a \selfies{}-adapted atom-preserving tokeniser, and a \modernbert{}-style encoder has not been systematically evaluated as a molecular embedder. -Tokenisation, the mapping from a raw string to discrete vocabulary units, is almost as essential as the model itself. +Tokenisation, the mapping from a raw string to discrete vocabulary units, is a central modelling choice. Atom-level tokenisation is chemically principled but yields long sequences and misses recurring substructures, whereas BPE operates on raw character frequencies and produces entries that straddle atom boundaries. @@ -294,27 +288,25 @@ \section{Introduction}% restricting merges to adjacent complete molecular symbols. We adapt \ape{} to \selfies{} by merging only adjacent complete \selfies{} primitives, so every vocabulary entry is a valid concatenation of complete \selfies{} symbols rather -than a character-level artefact (\cref{fig:design}C). \ape{} thus occupies a +than a character-level artefact (\hyperref[fig:design]{\cref*{fig:design}C}). \ape{} thus occupies a useful middle ground: compact like subword tokenisation, chemically principled like atom-level tokenisation. Many molecular string encoders such as ChemBERTa, ChemBERTa-2, MolBERT, -and SELFormer are built on BERT- or RoBERTa-era +and SELFormer are built on the BERT- or RoBERTa-era backbones~\citep{devlinBERTPreTrainingDeep2019,liuRoBERTaRobustlyOptimized} with absolute positional embeddings and full quadratic attention. -Several post-BERT advances have been consolidated in \modernbert{}~\citep{warnerSmarterBetterFaster2024}: Rotary Position Embeddings (RoPE), Flash Attention, alternating -local/global attention, and GeGLU activations (\cref{sec:rel-modernbert}). To our knowledge, these +Several newer post-BERT advances have been combined in \modernbert{}~\citep{warnerSmarterBetterFaster2024}: Rotary Position Embeddings (RoPE), Flash Attention, alternating +local/global attention, and GeGLU activations (\cref{sec:rel-modernbert}). To our knowledge, these innovations together have not been applied to \selfies{}-based molecular language modelling. -Especially RoPE may be suited to molecular strings, where absolute sequence +RoPE in particular suits molecular strings, where absolute sequence position is partly an artefact of the chosen graph traversal rather than an intrinsic molecular property. Because RoPE conditions attention on the \emph{relative} offset between two tokens rather than their absolute indices, -the encoder becomes largely invariant to where a traversal happens to begin --- -precisely the component of \smiles{}/\selfies{} positioning that carries no -chemical meaning --- while still distinguishing the local ordering of -neighbouring symbols. +it may reduce the encoder's sensitivity to where a traversal happens to begin. This starting point is one part of \smiles{}/\selfies{} positioning that carries no +chemical meaning. -The MLM objective does not directly specify which tokens to mask. Standard BERT masks tokens +The masked language model (MLM) objective does not directly specify which tokens to mask. Standard BERT masks tokens independently at random~\citep{devlinBERTPreTrainingDeep2019}, whereas SpanBERT~\citep{joshiSpanBERTImprovingPreTraining2020} masks contiguous spans to encourage modelling of longer-range structure. Masking can also be made @@ -324,11 +316,12 @@ \section{Introduction}% structure-aware masking benefits a frozen \selfies{} embedder is an open question we examine in an ablation (\cref{sec:ablations}). -We introduce \model{}, a compact family of encoder-only molecular language -models combining three design choices not previously studied together: a \modernbert{} -backbone, chemically valid \selfies{} inputs, and a \selfies{}-adapted \ape{} -tokeniser. The models are trained from scratch with MLM on 2.4M curated -\chembl{}~36 molecules and released in small and base variants for molecular property prediction. We evaluate them on the +We introduce \model{}, a compact encoder-only molecular language +model that brings together three design choices that have not previously been +evaluated together: chemically valid \selfies{} inputs, a \selfies{}-adapted +\ape{} tokeniser, and a \modernbert{} backbone. The models are trained from +scratch with MLM on 2.4M curated \chembl{}~36 molecules and released in small +and base variants. We evaluate them as frozen molecular embedders on the 25-dataset benchmark of \citet{praskiBenchmarkingPretrainedMolecular2025} against a fingerprint baseline and representative pre-trained string encoders. \Cref{fig:overview} shows the pretraining and frozen-evaluation pipeline, and @@ -337,19 +330,22 @@ \section{Introduction}% \begin{figure}[htbp] \centering - \includegraphics[width=\linewidth]{figures/Fig1_C} + \includegraphics[width=\linewidth]{figures/Fig1_New} \caption{% \textbf{\model{} pretraining and frozen-evaluation pipeline.} - Approximately \num{2.4}M unique \chembl{}~36 small molecules are curated, - canonicalised, and converted to \selfies{} strings, then pre-tokenised with - the \selfies{}-adapted \ape{} tokeniser (\cref{fig:design}C). A - \modernbert{} encoder is trained from scratch on this corpus with the masked - language modelling (MLM) objective, yielding \model{}. For evaluation the - encoder is \emph{frozen}: mean-pooled final-layer token embeddings over - non-special \selfies{} tokens are extracted without any gradient updates - and passed to lightweight downstream classifiers (ridge, random forest, - $k$-nearest neighbours) on 25 binary molecular property-prediction - benchmarks, where \model{} is compared against fixed fingerprint and + \textbf{(A)}~Approximately \num{2.4}M unique \chembl{}~36 small molecules + are curated, canonicalised, and converted to \selfies{} strings. + \textbf{(B)}~The \selfies{} strings are pre-tokenised with the + \selfies{}-adapted \ape{} tokeniser (\hyperref[fig:design]{\cref*{fig:design}C}). + \textbf{(C)}~A \modernbert{} encoder is trained from scratch on this corpus + with the masked language modelling (MLM) objective, yielding \model{}. + \textbf{(D)}~For evaluation the encoder is \emph{frozen}: mean-pooled + final-layer token embeddings over non-special \selfies{} tokens are + extracted without any gradient updates and passed to lightweight downstream + classifiers (ridge, random forest, $k$-nearest neighbours) on 25 binary + molecular property-prediction benchmarks (7 biophysical endpoints from + MoleculeNet and 18 toxicity, ADME and HTS endpoints from the Therapeutics Data + Commons), where \model{} is compared against fixed fingerprint and pre-trained string-encoder baselines under an identical protocol. }% \label{fig:overview} @@ -357,16 +353,15 @@ \section{Introduction}% \begin{figure}[htbp] \centering - \includegraphics[width=\linewidth]{figures/Fig1_A}\\[0.6em] - \includegraphics[width=\linewidth]{figures/Fig1_B} + \includegraphics[width=\linewidth]{figures/Fig2_New} \caption{% \textbf{Method comparison of the two molecular design choices in \model{}: input representation and tokenisation.} \textbf{(A)}~\emph{Input representation: \smiles{} versus \selfies{}.} The same molecule is shown encoded as a \smiles{} string (top) and as a \selfies{} string (bottom). \smiles{} tokens can violate chemical syntax - under perturbation or naive tokenisation --- mismatched brackets and - ill-formed ring-closure indices are marked with red crosses --- whereas + under perturbation or naive tokenisation (mismatched brackets and + ill-formed ring-closure indices are marked with red crosses), whereas every syntactically valid \selfies{} \emph{string} decodes to a chemically valid molecule~\citep{krennSELFIESSelfReferencingEmbedded2020}. \textbf{(B)}~\emph{Byte Pair Encoding (BPE).} @@ -388,30 +383,30 @@ \section{Related Work}% The introduction motivated the three design choices behind \model{}; this section places them in the context of prior work along the four axes most -relevant to a frozen \selfies{} embedder --- molecular string representations, +relevant to a frozen \selfies{} embedder: molecular string representations, molecular language models, modern encoder architectures, and molecular -tokenisation --- and identifies the specific models we later use as baselines. +tokenisation. It also identifies the specific models we later use as baselines. \subsection{Molecular String Representations} \label{sec:rel-representations} -The chemical-validity guarantee of \selfies{} holds under arbitrary mutation: -as reported by \citet{krennSELFIESSelfReferencingEmbedded2020}, a single random +The chemical-validity guarantee of \selfies{} holds under arbitrary mutation. +As reported by \citet{krennSELFIESSelfReferencingEmbedded2020}, a single random bit-flip leaves a \smiles{} string valid only 26.6\% of the time and ten bit-flips reduce validity to 0.2\%, whereas \selfies{} remains 100\% valid in -both cases. The representation -is also complete, meaning that every molecule can be encoded, and it can serve as a drop-in -input to any model. In generative settings, VAEs and GANs trained on \selfies{} -populate their latent spaces with two orders of magnitude more diverse valid -molecules than equivalent \smiles{}-based -models~\citep{krennSELFIESSelfReferencingEmbedded2020}. +both cases. \selfies{} is also complete (every molecule can be encoded) +and serves as a drop-in input to any model. +\selfies{} thus offers a chemically attractive input space, yet the field +still lacks a compact, modern encoder that tests whether \selfies{} strings +can serve as strong off-the-shelf molecular embeddings under a broad frozen +benchmark. \subsection{Molecular Language Models} \label{sec:rel-lms} The fingerprint ECFP~\citep{rogersExtendedConnectivityFingerprints2010} remains the dominant deterministic baseline: a circular-substructure fingerprint -with no trainable parameters that continues to compete with learned +with no trainable parameters that competes handily with learned representations on many property prediction tasks. Among learned string encoders, ChemBERTa-2~\citep{ahmadChemBERTa2TowardsChemical2022} revisited the RoBERTa-on-\smiles{} recipe, pretraining on PubChem @@ -454,6 +449,9 @@ \subsection{Molecular Language Models} data available~\citep{yangAnalyzingLearnedMolecular2019,dengSystematicStudyKey2023}. This motivates treating ECFP4 and other fixed representations as serious baselines rather than as outdated reference points. +Collectively these models show the value of molecular pretraining, but leave +open whether a smaller, validity-preserving encoder can supply a useful default +representation without fine-tuning or specialised runtime dependencies. \subsection{Modern BERT Architectures} \label{sec:rel-modernbert} @@ -474,6 +472,9 @@ \subsection{Modern BERT Architectures} \modernbert{}-large (395M). They are trained with a 30\% MLM masking rate and achieve state-of-the-art results among encoder-only models on natural-language and code benchmarks. +\model{} tests whether these encoder improvements transfer to molecular +\selfies{} MLM without requiring a very large model or a non-standard +downstream workflow. \subsection{Molecular Tokenisation} \label{sec:rel-tokenisation} @@ -485,6 +486,9 @@ \subsection{Molecular Tokenisation} (ROC-AUC improvements of $+$0.059 on BBBP and HIV and $+$0.017 on Tox21; Table~6 of \citealt{leonComparingSMILESSELFIES2024}), a result attributed to its merges aligning with \selfies{}'s symbol-level structure. +We therefore treat \ape{} as a practical way to compress recurring \selfies{} +patterns while preserving a \selfies{}-based input space, and ask whether this +design helps empirically rather than assuming it is chemically optimal. % ============================================================ \section{Methods}% @@ -532,6 +536,12 @@ \subsection{Atom Pair Encoding Tokeniser} this step. It guarantees a zero unknown-token rate on evaluation molecules; all model weights were initialised and trained on the completed 631-token vocabulary from the outset. +This step prevents out-of-vocabulary failures rather than providing task +information: it mirrors the fixed, closed vocabulary that any deployed encoder +must define before use. No labels or train/test split assignments enter the +vocabulary, and the completed vocabulary is fixed before any model weight is +trained, so the injected primitives carry no predictive signal about the +downstream tasks. \subsection{Model Architecture} \label{sec:architecture} @@ -539,10 +549,10 @@ \subsection{Model Architecture} Both \model{} variants use the \modernbert{} encoder backbone (\cref{sec:rel-modernbert})~\citep{warnerSmarterBetterFaster2024} in its default configuration: RoPE, GeGLU feed-forward blocks, pre-normalisation, no -linear-layer biases. It uses unpadded Flash Attention with global attention -every 3 layers and a 128-token local sliding window. They are trained from -scratch on our \selfies{} \ape{} vocabulary; no weights are transferred from -the official \modernbert{} checkpoints~\citep{warnerModernBERTBlog2024}. +linear-layer biases. Both use unpadded Flash Attention, with global attention every three layers +and a 128-token local sliding window. The variants are trained from scratch on +our \selfies{} \ape{} vocabulary; no weights are transferred from the official +\modernbert{} checkpoints~\citep{warnerModernBERTBlog2024}. Per-variant hyper-parameters are listed in \cref{tab:model-config} (\cref{app:training-details}). \model{}-small (34.1M parameters, 512-dimensional hidden state) sits in the @@ -550,7 +560,7 @@ \subsection{Model Architecture} (45M), MoLFormer-XL~\citep{rossLargescaleChemicalLanguage2022} (46.8M), and Uni-Mol~\citep{zhouUniMolUniversal3D2023} (47.6M), and is notably smaller than the closest \selfies{} comparator SELFormer (86.7M). \model{}-base (114.3M -parameters, 768-dimensional) is significantly larger than the existing models considered here. +parameters, 768-dimensional) is substantially larger than the other models considered here. \subsection{Pretraining Data} \label{sec:pretraining-data} @@ -623,8 +633,8 @@ \subsection{Evaluation Protocol} representations are obtained by mean-pooling the final-layer token embeddings over non-special \selfies{} tokens; no model parameters are updated. These embeddings are evaluated with three lightweight downstream models chosen to -span complementary inductive biases --- a linear ridge classifier, ensemble -random forests, and instance-based $k$-nearest neighbours --- so that +span complementary inductive biases: a linear ridge classifier, ensemble +random forests, and instance-based $k$-nearest neighbours. This way, representation quality is not judged through a single decision boundary. We adopt these three heads, and their cross-validation search grids, directly from the benchmark protocol of \citet{praskiBenchmarkingPretrainedMolecular2025} to @@ -637,6 +647,25 @@ \subsection{Evaluation Protocol} finite labels and both classes present in the test split. The best downstream model per dataset is used in the main comparison. +\subsection{Design Rationale} +\label{sec:design-rationale} + +Several configuration choices admit reasonable alternatives; we record the +reasoning here rather than leave it implicit. We use mean pooling over +non-special token embeddings, rather than a dedicated \texttt{[CLS]} token, +because the evaluation is frozen: with no fine-tuning to adapt a special token +to each task, a simple architecture-agnostic pool over all non-special tokens +is a more robust molecule-level summary. The maximum sequence length of 128 +tokens was set because the \selfies{}+\ape{} representation covers essentially +the entire filtered \chembl{}~36 corpus, with fewer than $0.1\%$ of molecules +truncated at this length (\cref{app:tokeniser}). The \ape{} thresholds +(\texttt{min\_freq}~$=3000$, \texttt{max\_merge\_pieces}~$=2$) were chosen to +give moderate compression while avoiding long, sparse merges; they are +pragmatic engineering settings, not a claim that this tokenisation is chemically +optimal. The 30{,}000-step budget was chosen as a reproducible medium-scale +regime, with the validation curves (\cref{fig:loss-curves}) confirming that the +selected runs were stable and had reached a performance plateau. + % ============================================================ \section{Experiments}% \label{sec:experiments} @@ -706,7 +735,11 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin Throughout, ROC-AUC values are reported $\times100$, so one point corresponds to $0.01$ ROC-AUC. Direct paired comparisons use only jointly evaluated datasets, reported in \cref{fig:bootstrap-ci} and -\cref{tab:bootstrap-cis}. +\cref{tab:bootstrap-cis}. Because the comparison spans many benchmark tasks, we +treat individual task wins as descriptive and base inferential statements on +these paired aggregate differences; the bootstrap intervals quantify uncertainty +in the mean task-level difference under task resampling, not variation across +pretraining runs. \Cref{tab:main-results} reports mean ROC-AUC by task group; per-dataset results for all models are given in \cref{tab:pertask}. @@ -724,6 +757,9 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin lie entirely above zero, that for MoLFormer entirely below, and that for ECFP4 spans zero. Numerical values are given in \cref{tab:bootstrap-cis} and full per-dataset results in \cref{tab:pertask}. + Resampling is over benchmark tasks, preserving the paired model difference + within each task; the intervals therefore quantify uncertainty in the mean + task-level difference, not variation across pretraining runs. }% \label{fig:bootstrap-ci} \end{figure} @@ -731,13 +767,12 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin \input{tables/main_results_table} \model{}-base attains a mean ROC-AUC of 77.9 across all 25 benchmark -datasets, and -\model{}-small attains 77.4 across all 25 datasets +datasets, and \model{}-small attains 77.4 across all 25 datasets (per-dataset results in \cref{tab:pertask}). For comparison, ECFP4 reaches 78.9, MoLFormer 79.8, ChemBERTa-2 72.9, and SELFormer 72.1. Both \model{} variants therefore land between the strong fixed-fingerprint and large-scale \smiles{} baselines (ECFP4, MoLFormer) and the closest-scale pre-trained string encoders -(ChemBERTa-2, SELFormer), which they exceed by roughly four to five ROC-AUC +(ChemBERTa-2, SELFormer), which they exceed by roughly five ROC-AUC points. \Cref{fig:baseline-paired} shows the corresponding per-dataset paired comparisons against each baseline. @@ -758,10 +793,8 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin with a mean $\Delta$ ROC-AUC of $-1.0$ and a 95\% bootstrap confidence interval of $[-2.3,\;+0.4]$ (\cref{tab:bootstrap-cis}); the interval spans zero, consistent with neither a reliable advantage nor a reliable -deficit at this scale. \model{}-small wins on 6 of 25. As such, -\model{} is approaching, rather than surpassing, ECFP4 under the frozen-embedding -protocol, consistent with the continued strength of circular fingerprints in -low-data property prediction. +deficit at this scale. \model{}-small wins on 6 of 25. This is consistent with the continued strength +of circular fingerprints in low-data property prediction. The comparison most diagnostic of \model{}'s design is SELFormer, the closest prior \selfies{} encoder (a RoBERTa-era backbone on the same representation). @@ -782,7 +815,7 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin embeddings organise molecules along smooth, chemically interpretable gradients in physicochemical descriptors spanning both whole-molecule properties (lipophilicity, drug-likeness, polarity, size) and more local structural counts -(flexibility, aromaticity), indicating that the embedding space encodes +(flexibility, aromaticity). This indicates that the embedding space encodes chemically meaningful structure even though it is never given these properties as training signal. We read this only qualitatively: the smooth arrangement shows that neighbouring embeddings tend to correspond to chemically similar @@ -812,15 +845,21 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin \begin{figure}[htbp] \centering - \includegraphics[width=0.85\linewidth]{figures/Fig_groupbars} + \includegraphics[width=\linewidth]{figures/Fig5_task_group_distributions} \caption{% - \textbf{Mean test ROC-AUC by task group.} - Unweighted mean held-out ROC-AUC within each of the four task groups + \textbf{Task-level ROC-AUC distributions by benchmark group.} + Each point is one benchmark task evaluated under the frozen-embedding + protocol with the best cross-validated downstream head, grouped by task + family and model; the horizontal mark is the unweighted group mean (TDC-ADME, 12 tasks; TDC-Tox, 4; TDC-HTS, 2; MoleculeNet, 7) for ECFP4, ChemBERTa-2 (MLM), SELFormer, MoLFormer, and the two released \model{} - variants, all evaluated as frozen embedders. Group means for the - masking-strategy ablation variants are omitted here for clarity and reported - in \cref{tab:pertask}. + variants. For multi-endpoint datasets, endpoint ROC-AUCs are averaged before + task-level aggregation. Colour redundantly encodes the model already labelled + on the $x$-axis and follows a colourblind-safe palette used consistently + across figures. The plot is descriptive; paired aggregate differences and + their uncertainty are quantified in \cref{fig:bootstrap-ci} and + \cref{tab:bootstrap-cis}. Masking-strategy ablation variants are omitted here + and reported in \cref{tab:pertask}. }% \label{fig:group-bars} \end{figure} @@ -833,13 +872,15 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin PaCMAP projection~\citep{wangUnderstandingHowDimension} of frozen \model{} molecular embeddings, coloured by selected physicochemical properties from \chembl{}~36. + A random subsample of \num{100000} \chembl{}~36 molecules is shown. Each point represents one molecule; colour indicates the corresponding property value: ALogP, the quantitative estimate of drug-likeness (QED), polar surface area, rotatable bonds, aromatic rings, and heavy atom count. PaCMAP is used solely for two-dimensional visualisation of the high-dimensional embeddings; the absolute axis coordinates are not chemically meaningful, and only relative neighbourhoods and broad spatial - trends should be interpreted. + trends should be interpreted. All panels show the same two-dimensional + projection, recoloured by each descriptor. }% \label{fig:embedding-space} \end{figure} @@ -847,15 +888,16 @@ \subsection{\model{} Approaches Fingerprints and Improves on Closest-Scale Strin \subsection{Ablation Studies} \label{sec:ablations} -We ablate two pretraining axes --- the MLM masking strategy and model size --- +We ablate two pretraining axes (the MLM masking strategy and model size) on validation MLM metrics (\cref{fig:masking-sweep}; full heatmaps in \cref{fig:masking-heatmap}) and on downstream ROC-AUC (\cref{fig:four-model}). The three design choices shared by all released models (the \selfies{} representation, the \ape{} tokeniser, and the \modernbert{} backbone) are not isolated here by matched single-axis ablations; they are instead probed jointly -through the external baselines of \cref{sec:main-results}. The -SELFormer comparison, which shares the \selfies{} representation and -dedicated tokeniser and representation ablations are left for future work. +through the external baselines of \cref{sec:main-results}. In particular, the +SELFormer comparison shares the \selfies{} representation, so it probes the +\ape{} tokeniser and \modernbert{} backbone jointly; dedicated single-axis +tokeniser and representation ablations are left for future work. On the validation MLM objective, the choices separate more clearly: for the small model, span masking at $2\!\times\!10^{-4}$ is strongest and heteroatom-biased span @@ -909,14 +951,22 @@ \section{Discussion}% \label{sec:discussion} % ============================================================ +\model{} set out to test whether a compact encoder-only model can provide useful +frozen molecular embeddings from chemically valid \selfies{} strings and a +\selfies{}-aware \ape{} tokeniser. The results support this premise: under a +strictly frozen-embedding protocol, \model{}-base reaches a mean ROC-AUC of +77.9, approaching the strong +ECFP4 fingerprint baseline and improving on the closest prior \selfies{} and +\smiles{} encoders by roughly five points. The subsections below set the scope of +this claim, locate the gain among \model{}'s design choices, bound what the +frozen-feature evaluation can establish, and outline what follows from it. + \subsection{Scope and interpretation} \label{sec:scope} -\model{} is designed around a narrow but practical question: whether a -compact encoder-only model can provide useful frozen molecular embeddings -when trained on chemically valid \selfies{} strings with a \selfies{}-aware -\ape{} tokeniser. This scope differs from large generative chemical language models, -multimodal assay--molecule models, and graph or 3D molecular encoders. The +This question is deliberately narrow. Its scope differs from large generative +chemical language models, multimodal assay--molecule models, and graph or 3D +molecular encoders. The resulting comparison is therefore most informative when interpreted as a test of frozen, string-based molecular representation learning rather than as a claim about all molecular foundation models. \model{} is used here purely as an @@ -924,8 +974,8 @@ \subsection{Scope and interpretation} molecules from it. We note, however, that because both inputs and the MLM head operate over the \selfies{} vocabulary, any \selfies{} sequence the model predicts decodes to a chemically valid molecule by construction, so a -generative use of the architecture would inherit \selfies{}' validity guarantee ---- a direction we leave to future work. +generative use of the architecture would inherit \selfies{}' validity guarantee. +We leave this direction to future work. Every number in this paper should be read as a \emph{frozen-embedding} result. No encoder is fine-tuned downstream; each produces a fixed mean-pooled @@ -953,56 +1003,73 @@ \subsection{Advances over prior string models} \selfies{} encoder that approaches fixed-fingerprint quality while substantially improving on prior string-based language models at comparable scale. -A practical advantage of \model{} is accessibility. ModernMolBERT is released in the standard Hugging Face Transformers format and can be used with the current Python ecosystem without model-specific runtime dependencies, legacy Python versions, custom inference kernels, or external chemistry toolkits at embedding time once inputs are provided as SELFIES strings. Frozen embeddings can therefore be extracted with a few lines of standard Transformers code, making the model easier to incorporate into existing screening, retrieval, and benchmark pipelines than models requiring specialised environments. Likewise, the standalone tokeniser is straightforward to use in a similar fashion. +A practical advantage is accessibility. \model{} is released in the standard Hugging Face Transformers format and runs in the current Python ecosystem without model-specific inference kernels, legacy Python versions, or specialised runtime dependencies. Once molecules are provided as \selfies{} strings, frozen embeddings can be extracted with a few lines of standard Transformers code, making the model straightforward to incorporate into screening, retrieval, and benchmark pipelines. The standalone \ape{} tokeniser follows the same interface. -ModernMolBERT therefore occupies a practical trade-off point rather than a pure performance extreme. Larger or more specialised models remain preferable when maximum benchmark accuracy is the only objective, but many molecular-embedding workflows require representations that are adequate, reproducible, compact, and easy to deploy. In this setting, ModernMolBERT is a reasonable default: it approaches ECFP4, improves over comparable pretrained string encoders, and exposes frozen embeddings through the standard Hugging Face Transformers interface without specialised inference infrastructure. +ModernMolBERT therefore occupies a practical trade-off point rather than a pure performance extreme. Larger or more specialised models remain preferable when maximum benchmark accuracy is the only objective, but many molecular-embedding workflows require representations that are adequate, reproducible, compact, and easy to deploy. In this setting, ModernMolBERT is a reasonable default for frozen string-based molecular embedding workflows: it approaches ECFP4, improves over comparable pretrained string encoders, and exposes frozen embeddings through the standard Hugging Face Transformers interface without specialised inference infrastructure. \subsection{Limitations} \label{sec:limitations} -\model{} is pre-trained exclusively on \chembl{}~36, a corpus -dominated by drug-like, Lipinski-compliant compounds. -Compared with MoLFormer, which draws from 1.1B PubChem~\citep{kimPubChem2023Update2023} and ZINC~\citep{irwinZINC20FreeUltralargeScale2020} -molecules, \model{}'s training distribution is narrower; generalisation to +\model{} is pre-trained only on \chembl{}~36, a corpus dominated by drug-like, +Lipinski-compliant compounds. Its training distribution is therefore narrower +than MoLFormer's 1.1B PubChem~\citep{kimPubChem2023Update2023} and +ZINC~\citep{irwinZINC20FreeUltralargeScale2020} molecules, so generalisation to natural products, agrochemicals, peptides, macrocycles, or other -under-represented chemical spaces may be limited. -Pretraining uses canonical-\smiles{}-derived \selfies{} strings only; -we do not apply randomised \smiles{} augmentation or multiple-enumeration -strategies, which have been explored as a means of improving representation -diversity in prior molecular language +under-represented chemical spaces may be weaker. This limitation is less severe +for the drug-like benchmarks evaluated here, but broader pretraining +(\cref{sec:future-work}) would address it. Pretraining also uses canonical-\smiles{}-derived \selfies{} +strings only, with no randomised \smiles{} augmentation or multiple-enumeration, +which have improved representation diversity in prior molecular language models~\citep{winterLearningContinuousDataDriven2019,leeSimSonSimpleContrastive2025}. -Like all sequential string models, \model{} has no access to explicit -3D conformational information; graph- and geometry-based models such -as Uni-Mol~\citep{zhouUniMolUniversal3D2023} and -GEM~\citep{fangGeometryenhancedMolecularRepresentation2022} address -this at the cost of greater architectural complexity. -The \selfies{}-adapted \ape{} tokeniser forms merge candidates from -sequence-adjacent \selfies{} primitives rather than reconstructing -the molecular graph; merge boundaries therefore do not necessarily -align with graph-level bond adjacency. -Our evaluation is restricted to binary classification tasks scored -by ROC-AUC, following the benchmark of -\citet{praskiBenchmarkingPretrainedMolecular2025}; performance on -regression tasks, generative objectives, or out-of-distribution -scaffolds is not assessed here. -A further caveat concerns possible train--benchmark overlap: both the -\chembl{}~36 pretraining corpus and several of the benchmark datasets are -derived from overlapping medicinal-chemistry sources, and we do not explicitly -deduplicate benchmark structures against the pretraining set. Some benchmark -molecules or close analogues may therefore also appear during pretraining, -which could modestly inflate absolute scores for all pretrained encoders -evaluated here; the relative comparisons between encoders are less affected, -since every pretrained baseline is exposed to similar public chemical space. -Finally, all models are evaluated as frozen encoders, which may not -reflect relative performance under full fine-tuning; frozen evaluation -may favour representations with broader but shallower chemical -coverage. More generally, systematic comparisons of molecular property -prediction models show that representation learning gains are highly -context-dependent and are affected by data regime, split strategy, and -activity cliffs~\citep{yangAnalyzingLearnedMolecular2019,dengSystematicStudyKey2023}. -These considerations are especially relevant for frozen-feature evaluations, -where the representation must transfer without task-specific adaptation. + +Two further constraints are properties of the string-encoder design rather than +defects in this model. Like all sequential string models, \model{} has no +access to explicit 3D conformational information, which graph- and +geometry-based models such as Uni-Mol~\citep{zhouUniMolUniversal3D2023} and +GEM~\citep{fangGeometryenhancedMolecularRepresentation2022} capture at the cost +of greater architectural complexity; and the \selfies{}-adapted \ape{} +tokeniser forms merge candidates from sequence-adjacent primitives, so merge +boundaries need not align with graph-level bond adjacency. Neither affects the +\selfies{} validity guarantee or the frozen-embedding results reported here. + +The evaluation is deliberately confined to frozen embeddings on binary +classification tasks scored by ROC-AUC, following +\citet{praskiBenchmarkingPretrainedMolecular2025}. This isolates representation +quality as an off-the-shelf feature vector and mirrors fingerprint workflows, +but it does not measure the maximum performance reachable by supervised +fine-tuning, nor does it establish behaviour on regression, retrieval, +generative-conditioning, or structure-based tasks. A model with weaker frozen +embeddings may still fine-tune well, and frozen evaluation may favour +representations with broader but shallower chemical coverage. + +Each released checkpoint was pre-trained from a single seed. The paired +bootstrap intervals therefore quantify task-level variation under resampling of +benchmark tasks, not pretraining-run variance, and the reported ranking should +be read as a benchmark estimate for these released checkpoints rather than a +full distribution over training runs; repeated-seed pretraining +(\cref{sec:future-work}) would supply that distribution. + +The external baseline comparisons test the \model{} design package as a whole, +not isolated causes. The improvement over SELFormer is consistent with the +combined effect of the \modernbert{} architecture, \ape{} tokenisation, +training recipe, and implementation differences, but matched single-axis +ablations (\cref{sec:future-work}) would be required to attribute the gain to +any individual component. + +Finally, the \chembl{}~36 corpus and several benchmark datasets derive from +overlapping medicinal-chemistry sources, and we do not deduplicate benchmark +structures against the pretraining set. Some benchmark molecules or close +analogues may therefore appear during pretraining, which could modestly inflate +absolute scores for all pretrained encoders; the relative comparisons are less +affected, since every pretrained baseline draws on similar public chemical +space, and a structure-level overlap analysis (\cref{sec:future-work}) would +turn this caveat into a measured sensitivity check. More generally, systematic +comparisons show that representation-learning gains are context-dependent, +varying with data regime, split strategy, and activity +cliffs~\citep{yangAnalyzingLearnedMolecular2019,dengSystematicStudyKey2023}. +These considerations bear especially on frozen-feature evaluation, where the +representation must transfer without task-specific adaptation. \subsection{Future Work} \label{sec:future-work} @@ -1012,7 +1079,7 @@ \subsection{Future Work} pretraining beyond \chembl{}, including PubChem, ZINC, Enamine, or other sources that better cover natural products, fragments, reagents, and non-drug-like chemistry. Pretraining \model{} on the same corpora used by the -baselines---for example the PubChem/ZINC mixture behind MoLFormer---would +baselines (for example the PubChem/ZINC mixture behind MoLFormer) would further allow the architecture and tokeniser to be compared at matched data, separating the contribution of pretraining scale from that of model design. A closely related question is how \model{} scales: the present study fixes a @@ -1020,8 +1087,8 @@ \subsection{Future Work} increasing model size, corpus size, and training compute would establish whether the remaining gap to large-scale \smiles{} models such as MoLFormer narrows with scale. A second direction is matched single-axis ablations -of the tokeniser and backbone choices---substituting \ape{} with standard BPE -or the \modernbert{} backbone with a RoBERTa-era encoder---to isolate each +of the tokeniser and backbone choices (substituting \ape{} with standard BPE +or the \modernbert{} backbone with a RoBERTa-era encoder) to isolate each component's contribution to the gains over SELFormer. A third is to explore \selfies{} encoders as molecular indexing models for chemical search, analogue retrieval, and dataset auditing, where deterministic valid string encodings and @@ -1032,8 +1099,8 @@ \section{Conclusion}% \label{sec:conclusion} % ============================================================ -We presented \model{}, a family of compact encoder-only transformer models -for \selfies{} molecular language modelling. The models combine the +We presented \model{}, a compact encoder-only transformer model +for \selfies{} molecular language modelling. The released small and base variants combine the \modernbert{} encoder architecture with a \selfies{}-adapted \ape{} tokeniser and are pre-trained from scratch on a curated \chembl{}~36 corpus using the masked language modelling objective. @@ -1042,8 +1109,8 @@ \section{Conclusion}% protocol, \model{}-base achieves a mean ROC-AUC of 77.9. This is competitive with the strong ECFP4 fingerprint baseline (78.9) and trails the much larger MoLFormer (79.8), while clearly exceeding the closest prior \selfies{} and -\smiles{} string encoders, SELFormer (72.1) and ChemBERTa-2 (72.9), by four -to five ROC-AUC points. Internal ablations further show that downstream +\smiles{} string encoders, SELFormer (72.1) and ChemBERTa-2 (72.9), by roughly +five ROC-AUC points. Internal ablations further show that downstream embedding quality is largely insensitive to masking strategy and to the small-to-base size increase, so the gains over prior string models are consistent with the shared \selfies{}, \ape{}, and \modernbert{} design choices. @@ -1056,8 +1123,10 @@ \section{Conclusion}% modern efficient transformer components rather than inherited BERT-era backbones. By evaluating \model{} as a frozen embedder against ECFP4 and representative pre-trained molecular language models, we position -\selfies{}-based ModernBERT encoders as a reproducible and computationally -manageable foundation for molecular representation learning. +a \selfies{}-based ModernBERT encoder as a reproducible and computationally +manageable foundation for molecular representation learning. Broader pretraining +corpora and matched single-axis ablations are the natural next steps toward +closing the remaining gap to large-scale models. % ============================================================ @@ -1070,29 +1139,36 @@ \section*{Data and Code Availability} at \url{https://huggingface.co/HauserGroup}. \chembl{}~36 is publicly available at \url{https://www.ebi.ac.uk/chembl/}. +Source data for each quantitative figure are provided in \texttt{source\_data/} +in the code repository, with one machine-readable file per figure. % ============================================================ \section*{CRediT authorship contribution statement} % ============================================================ J.S.M.\ Writing – original draft, Methodology, Formal -analysis, Conceptualization +analysis, Conceptualization. S.A.\ Methodology. +C.L.P.\ Visualization, Writing – review \& editing. X.L.\ Methodology. A.S.H.\ Methodology, Funding acquisition, Conceptualization. % ============================================================ -\section*{Funding} +\section*{Competing Interests} % ============================================================ -This work was supported by the Independent Research Fund Denmark. +The authors declare no competing interests. % ============================================================ -\section*{Competing Interests} +\section*{Use of AI Tools} % ============================================================ -The authors declare no competing interests. +During manuscript preparation, the authors used a large language model to assist +with language editing, restructuring, and clarity improvements. All AI-assisted +text was reviewed and edited by the authors, who take full responsibility for the +final manuscript. No AI tool was used to generate, analyse, or interpret the +experimental results. % ============================================================ \section*{Acknowledgements} @@ -1111,6 +1187,7 @@ \section*{Acknowledgements} % ============================================================ \begin{appendices} +\clearpage \section{Training Hyper-parameters}% \label{app:training-details} @@ -1243,7 +1320,11 @@ \section{Additional Benchmark Results}% \textbf{(c)}~\model{}-small versus the heteroatom-biased span-masking variant. The dashed line is the identity $y=x$, points are coloured by task group, and each inset reports the number of tasks on which the $y$-axis - model wins. All three panels are evaluated on the full 25 datasets. + model wins. All three panels are evaluated on the full 25 datasets + (see \cref{tab:pertask}). + These ablations vary masking strategy and model size within the \model{} + setup; they do not isolate the effects of \selfies{}, \ape{} tokenisation, + or the \modernbert{} backbone relative to matched alternatives. }% \label{fig:four-model} \end{figure} diff --git a/paper/references.bib b/paper/references.bib index 3b82ced..69ed75e 100644 --- a/paper/references.bib +++ b/paper/references.bib @@ -87,7 +87,8 @@ @article{weininger1988smiles } @article{krennSELFIESSelfReferencingEmbedded2020, - title = {{SELFIES}: A Robust Molecular String Representation}, + title = {Self-Referencing Embedded Strings ({SELFIES}): + A 100\% Robust Molecular String Representation}, author = {Krenn, Mario and Häse, Florian and Nigam, AkshatKumar and Friederich, Pascal and Aspuru-Guzik, Alán}, journal = {Machine Learning: Science and Technology}, @@ -153,7 +154,7 @@ @article{singhChemBERTa3OpenSource2025 @article{fabianMolecularRepresentationLearning2020, title = {Molecular Representation Learning with Language Models and Domain-Relevant Auxiliary Tasks}, - author = {Fabian, Bernd and Edlich, Thomas and Gaspar, Héléna + author = {Fabian, Benedek and Edlich, Thomas and Gaspar, Héléna and Segler, Marwin and Meyers, Joshua and Fiscato, Marco and Ahmed, Mohamed}, journal = {arXiv preprint arXiv:2011.13230}, @@ -218,21 +219,24 @@ @inproceedings{seidlEnhancingActivityPrediction2023 } @article{axelrodCOATIMultimodalContrastive2024, - title = {{COATI}: Multi-Modal Contrastive Pre-Training for + title = {{COATI}: Multimodal Contrastive Pretraining for Representing and Traversing Chemical Space}, - author = {Axelrod, Simon and Gomez-Bombarelli, Rafael}, - journal = {Journal of Chemical Theory and Computation}, - volume = {20}, + author = {Kaufman, Benjamin and Williams, Edward C. + and Underkoffler, Carl and Pederson, Ryan + and Mardirossian, Narbe and Watson, Ian + and Parkhill, John}, + journal = {Journal of Chemical Information and Modeling}, + volume = {64}, number = {4}, - pages = {1457--1472}, + pages = {1145--1157}, year = {2024}, publisher = {ACS Publications}, - doi = {10.1021/acs.jctc.3c01015}, + doi = {10.1021/acs.jcim.3c01753}, } @inproceedings{rongGroverTransformerSelfsupervised2020, - title = {{GROVER}: Transformers for Molecular Graph Self-supervised - Learning}, + title = {Self-Supervised Graph Transformer on Large-Scale + Molecular Data}, author = {Rong, Yu and Bian, Yatao and Xu, Tingyang and Xie, Weiyang and Wei, Ying and Huang, Wenbing and Huang, Junzhou}, @@ -241,6 +245,7 @@ @inproceedings{rongGroverTransformerSelfsupervised2020 pages = {12559--12571}, year = {2020}, url = {https://proceedings.neurips.cc/paper/2020/hash/94aef38441efa3380a3bed3faf1f9d5d-Abstract.html}, + note = {GROVER}, } @article{krugerMolEncoderOptimalMasked2025, @@ -579,14 +584,14 @@ @inproceedings{zhouUniMolUniversal3D2023 and Zhang, Linfeng and Ke, Guolin}, booktitle = {International Conference on Learning Representations}, year = {2023}, - url = {https://arxiv.org/abs/2303.16982}, + url = {https://openreview.net/forum?id=6K2RM6wVqKu}, } @article{yukselSELFormerMolecularRepresentation2023, title = {{SELFormer}: Molecular Representation Learning via {SELFIES} Language Models}, - author = {Y{\"u}ksel, Asu and Ulusoy, Erva and Erzincan, - Abdulkadir and {\"O}zt{\"u}rk, Hakime}, + author = {Y{\"u}ksel, Atakan and Ulusoy, Erva and {\"U}nl{\"u}, + Atabey and Do{\u{g}}an, Tunca}, journal = {Machine Learning: Science and Technology}, volume = {4}, number = {2}, @@ -599,13 +604,14 @@ @article{yukselSELFormerMolecularRepresentation2023 % ADDITIONAL — NOT YET CITED (retain for future use) % ============================================================ -% [TODO] Incomplete stub — verify full bibliographic details before citing. -@article{darrinWhenEmbedderMore, +@inproceedings{darrinWhenEmbedderMore, title = {When Is an Embedder More Promising than Another?}, author = {Darrin, Maxime and Formont, Philippe and Ayed, Ismail Ben and Cheung, Jackie Chi Kit and Piantanida, Pablo}, - journal = {arXiv preprint}, + booktitle = {Advances in Neural Information Processing Systems}, + volume = {37}, year = {2024}, + url = {https://arxiv.org/abs/2406.07640}, } @article{wangUnderstandingHowDimension, diff --git a/paper/source_data/Fig_bootstrap_ci.csv b/paper/source_data/Fig_bootstrap_ci.csv new file mode 100644 index 0000000..089b94a --- /dev/null +++ b/paper/source_data/Fig_bootstrap_ci.csv @@ -0,0 +1,5 @@ +model_A,model_B,n,wins,ties,losses,mean_delta_x100,ci_low,ci_high +MMB-base,SELFormer,23,21,0,2,5.43,3.9,7.1 +MMB-base,ChemBERTa-2,23,17,0,6,4.55,1.6,7.9 +MMB-base,ECFP4,23,7,0,16,-1.02,-2.5,0.5 +MMB-base,MoLFormer,23,4,0,19,-1.67,-2.6,-0.7 diff --git a/paper/tables/Embedders_Table_OLD.tex b/paper/tables/Embedders_Table_OLD.tex deleted file mode 100644 index 98caabf..0000000 --- a/paper/tables/Embedders_Table_OLD.tex +++ /dev/null @@ -1,117 +0,0 @@ -\documentclass[11pt,a4paper]{article} - -% ── Packages ────────────────────────────────────────────────────────────────── -\usepackage[a4paper, landscape, top=2cm, bottom=2cm, left=2cm, right=2cm]{geometry} -\usepackage{booktabs} -\usepackage{longtable} -\usepackage{array} -\usepackage{ragged2e} -\usepackage{microtype} -\usepackage[table]{xcolor} -\usepackage{caption} -\usepackage{setspace} -\usepackage[T1]{fontenc} -\usepackage[utf8]{inputenc} -\usepackage{lmodern} - -% ── Colours ─────────────────────────────────────────────────────────────────── -\definecolor{headerblue}{RGB}{31,56,100} -\definecolor{rowgray}{RGB}{238,242,247} - -% ── Column types ────────────────────────────────────────────────────────────── -\newcolumntype{L}[1]{>{\RaggedRight\arraybackslash}p{#1}} -\newcolumntype{C}[1]{>{\centering\arraybackslash}p{#1}} - -% ── Table spacing ───────────────────────────────────────────────────────────── -\setlength{\tabcolsep}{4pt} -\renewcommand{\arraystretch}{1.25} - -% ── Caption formatting ──────────────────────────────────────────────────────── -\captionsetup[table]{labelfont=bf, labelsep=period, justification=justified, - singlelinecheck=false, skip=4pt} - -\begin{document} - -\begin{longtable}{% - L{2.3cm} % Model - L{3.0cm} % Architecture - C{1.6cm} % Parameters (M) - C{1.4cm} % Emb. dim. - L{15.9cm} % Description - } - -\caption{Overview of molecular embedding models: architecture, size, and description.} -\label{tab:embedders}\\ - -% ── First-page header ───────────────────────────────────────────────────────── -\toprule -\rowcolor{headerblue} -\textcolor{white}{\textbf{Model}} & -\textcolor{white}{\textbf{Architecture}} & -\textcolor{white}{\textbf{Parameters (M)}} & -\textcolor{white}{\textbf{Emb.\ dim.}} & -\textcolor{white}{\textbf{Description}} \tabularnewline -\midrule -\endfirsthead - -% ── Continuation header ─────────────────────────────────────────────────────── -\multicolumn{5}{c}{\tablename~\thetable{} \textit{(continued)}}\tabularnewline[2pt] -\toprule -\rowcolor{headerblue} -\textcolor{white}{\textbf{Model}} & -\textcolor{white}{\textbf{Architecture}} & -\textcolor{white}{\textbf{Parameters (M)}} & -\textcolor{white}{\textbf{Emb.\ dim.}} & -\textcolor{white}{\textbf{Description}} \tabularnewline -\midrule -\endhead - -% ── Foot (all pages except last) ───────────────────────────────────────────── -\midrule -\multicolumn{5}{r}{\footnotesize\textit{Continued on next page\ldots}}\tabularnewline -\endfoot - -% ── Last-page foot ──────────────────────────────────────────────────────────── -\bottomrule -\multicolumn{5}{p{25.2cm}}{% - \footnotesize\setstretch{1.15}% - \textbf{Note.} Parameters~(M)\,=\,trainable parameters in millions as reported - by the original publication; N/K\,=\,not known or not reported; - $0^{a}$\,=\,non-neural fingerprint with zero trainable parameters. - Where multiple model variants exist, the parameter count of the variant used - in the benchmark is reported (e.g.\ GROVER-Large, ChemFM-3\,B, ChemGPT-4.7\,M). - \par\medskip - \textbf{References.} $^{1}$\,Seidl P et al. Enhancing activity prediction models in drug discovery with the ability to understand human language. \textit{ICML} 2023.\newline $^{2}$\,Maziarka {\L} et al. Relative molecule self-attention transformer. \textit{J.~Cheminform.} 2022;14:2.\newline $^{3}$\,Fabian B et al. Molecular representation learning with language models and domain-relevant auxiliary tasks. \textit{arXiv} 2020:2011.03814.\newline $^{4}$\,Chithrananda S et al. ChemBERTa: Large-scale self-supervised pretraining for molecular property prediction. \textit{arXiv} 2020:2010.09885.\newline $^{5}$\,Rogers D, Hahn M. Extended-connectivity fingerprints. \textit{J.~Chem.~Inf.~Model.} 2010;50(5):742--754.\newline $^{6}$\,Winter R et al. Learning continuous and data-driven molecular descriptors by translating equivalent chemical representations. \textit{Chem.~Sci.} 2019;10:1692--1701.\newline $^{7}$\,Carhart RE et al. Atom pairs as molecular features in structure-activity studies. \textit{J.~Chem.~Inf.~Comput.~Sci.} 1985;25(2):64--69.\newline $^{8}$\,Maziarka {\L} et al. Molecule attention transformer. \textit{arXiv} 2020:2002.08264.\newline $^{9}$\,Ross J et al. Large-scale chemical language representations capture molecular structure and properties. \textit{Nat.~Mach.~Intell.} 2022;4:1256--1264.\newline $^{10}$\,Jaeger S et al. Mol2vec: Unsupervised machine learning approach with chemical intuition. \textit{J.~Chem.~Inf.~Model.} 2018;58(1):27--35.\newline $^{11}$\,Nilakantan R et al. Topological torsion: A new molecular descriptor for SAR applications. \textit{J.~Chem.~Inf.~Comput.~Sci.} 1987;27(2):82--85.\newline $^{12}$\,Bao Z et al. ChemFM: A versatile foundation model for chemistry. \textit{arXiv} 2024:2404.01196.\newline $^{13}$\,Zhou G et al. Uni-Mol2: Exploring molecular pretraining model at scale. \textit{arXiv} 2024:2406.14969.\newline $^{14}$\,Wang H et al. Chemical-reaction-aware molecule representation learning. \textit{ICLR} 2022.\newline $^{15}$\,Chilingaryan G et al. COATI: Multi-modal contrastive pre-training for representing and traversing chemical space. \textit{arXiv} 2023:2310.00981.\newline $^{16}$\,Liu S et al. Pre-training molecular graph representation with 3D geometry. \textit{ICLR} 2022.\newline $^{17}$\,Irwin R et al. Chemformer: A pre-trained transformer for computational chemistry. \textit{Mach.~Learn.:~Sci.~Technol.} 2022;3:015022.\newline $^{18}$\,Zhou G et al. Uni-Mol: A universal 3D molecular representation learning framework. \textit{ICLR} 2023.\newline $^{19}$\,Rong Y et al. Self-supervised graph transformer on large-scale molecular data. \textit{NeurIPS} 2020.\newline $^{20}$\,Hu W et al. Strategies for pre-training graph neural networks. \textit{ICLR} 2020.\newline $^{21}$\,Mukherjee S et al. SimSon: Simple contrastive learning of SMILES representations. \textit{arXiv} 2023.\newline $^{22}$\,Frey NC et al. Neural scaling of deep chemical models. \textit{Nat.~Mach.~Intell.} 2023;5:1288--1298.\newline $^{23}$\,Fang X et al. Geometry-enhanced molecular representation learning for property prediction. \textit{Nat.~Mach.~Intell.} 2022;4:127--134.\newline $^{24}$\,Y\"{u}ksel A et al. SELFormer: Molecular representation learning via SELFIES language models. \textit{Mach.~Learn.:~Sci.~Technol.} 2023;4:025035.\newline $^{25}$\,Shen T et al. Molecular property prediction: A multilevel quantum interactions modeling perspective. \textit{AAAI} 2023. -}\tabularnewline -\endlastfoot - -% ── Table body ──────────────────────────────────────────────────────────────── - \textbf{CLAMP}$^{1}$ & Fingerprint / multimodal & N/K & 768 & Contrastive assay--molecule model using concatenated molecular fingerprints passed through an MLP and aligned to assay text. \tabularnewline - \rowcolor{rowgray}\textbf{R-MAT}$^{2}$ & Graph & 4.0 & 3,072 & Relative Molecule Attention Transformer using molecular graph, distance, and neighbourhood-aware attention. \tabularnewline - \textbf{MolBERT}$^{3}$ & SMILES-based & 85.0 & 768 & BERT-style molecular language model trained on SMILES-derived molecular substructure sequences. \tabularnewline - \rowcolor{rowgray}\textbf{ChemBERTa}$^{4}$ & SMILES-based & N/K & 384 & RoBERTa-style transformer pretrained on large PubChem SMILES corpora with MLM or chemically guided objectives. \tabularnewline - \textbf{ECFP}$^{5}$ & Fingerprint & $0^{a}$ & 2,048 & Extended-connectivity circular fingerprint encoding local atom environments by hashing circular neighbourhoods. \tabularnewline - \rowcolor{rowgray}\textbf{CDDD}$^{6}$ & SMILES-based & N/K & 512 & Seq2seq translation model that learns fixed-size continuous descriptors from equivalent molecular string representations. \tabularnewline - \textbf{Atom Pair}$^{7}$ & Fingerprint & $0^{a}$ & 2,048 & Hashed atom-pair fingerprint based on shortest-path atom-pair features. \tabularnewline - \rowcolor{rowgray}\textbf{MAT}$^{8}$ & Graph & 101.0 & 1,024 & Molecule Attention Transformer augmenting self-attention with molecular graph adjacency and interatomic distances. \tabularnewline - \textbf{MoLFormer}$^{9}$ & SMILES-based & 46.8 & 768 & Large-scale SMILES transformer using linear attention and rotary positional encoding. \tabularnewline - \rowcolor{rowgray}\textbf{Mol2Vec}$^{10}$ & Fingerprint-like & N/K & 300 & Word2Vec-style model treating Morgan substructures as `words' and molecules as `sentences'. \tabularnewline - \textbf{TT}$^{11}$ & Fingerprint & $0^{a}$ & 2,048 & Topological torsion fingerprint encoding paths of four bonded atoms. \tabularnewline - \rowcolor{rowgray}\textbf{ChemFM}$^{12}$ & SMILES-based & 3,000 & 3,072 & Causal molecular foundation model pretrained on UniChem SMILES (3\textasciitilde{}B variant used in benchmark). \tabularnewline - \textbf{Uni-Mol2}$^{13}$ & Graph / 3D & 84.0 & 768 & Scaled two-track transformer integrating atom-level, graph-level, and 3D geometric information. \tabularnewline - \rowcolor{rowgray}\textbf{MolR}$^{14}$ & Graph & N/K & 1,024 & Reaction-aware GNN enforcing compositional consistency of representations across chemical reactions. \tabularnewline - \textbf{COATI}$^{15}$ & SMILES / multimodal & N/K & 256 & Contrastive model aligning molecular strings and 3D information into invertible latent embeddings. \tabularnewline - \rowcolor{rowgray}\textbf{GraphMVP}$^{16}$ & Graph & 1.8 & 300 & Self-supervised graph pretraining aligning 2D molecular topology with 3D conformer geometry. \tabularnewline - \textbf{Chemformer}$^{17}$ & SMILES-based & 45.0 & 512 & BART-style encoder--decoder transformer for molecular SMILES generation and representation learning. \tabularnewline - \rowcolor{rowgray}\textbf{Uni-Mol}$^{18}$ & Graph / 3D & 47.0 & 512 & Universal 3D molecular representation model using transformer encoders with spatial positional information. \tabularnewline - \textbf{GROVER}$^{19}$ & Graph & 100.0 & 2,400 & Graph transformer combining message passing with self-supervised node-, edge-, and graph-level objectives (Large variant). \tabularnewline - \rowcolor{rowgray}\textbf{GIN}$^{20}$ & Graph & N/K & 300 & Graph Isomorphism Network backbone with standard graph self-supervised pretraining objectives. \tabularnewline - \textbf{SimSon}$^{21}$ & SMILES-based & N/K & 512 & Contrastive model trained to map randomised SMILES of the same molecule to similar embeddings. \tabularnewline - \rowcolor{rowgray}\textbf{ChemGPT}$^{22}$ & SMILES / SELFIES-based & 4.7 & 128 & GPT-style autoregressive chemical language model for molecular generation and embeddings (4.7\textasciitilde{}M variant). \tabularnewline - \textbf{GEM}$^{23}$ & Graph / 3D & N/K & 32 & Geometry-enhanced GNN modelling atoms, bonds, and bond angles with geometry-level self-supervised tasks. \tabularnewline - \rowcolor{rowgray}\textbf{SELFormer}$^{24}$ & SMILES / SELFIES-based & 86.7 & 768 & Transformer language model using SELFIES rather than SMILES for robust molecular string representations. \tabularnewline - \textbf{GraphFP}$^{25}$ & Graph & N/K & 300 & Fragment-based graph pretraining using contrastive and predictive tasks over molecular and fragment graphs. \tabularnewline - -\end{longtable} - -\end{document} diff --git a/paper/writing_style/scientific_writing_reference.md b/paper/writing_style/scientific_writing_reference.md index 5e54aea..c82b5c8 100644 --- a/paper/writing_style/scientific_writing_reference.md +++ b/paper/writing_style/scientific_writing_reference.md @@ -1,6 +1,7 @@ --- title: Scientific Writing & Manuscript Preparation — Master Reference subtitle: The full arc of a manuscript, distilled for researchers who already know the basics +source: Synthesised from Nature Masterclasses materials context: University of Copenhagen — Practitioner Reference audience: Experienced researchers (practitioner-grade synthesis, not novice instruction) --- diff --git a/src/modernmolbert/tokenization_ape.py b/src/modernmolbert/tokenization_ape.py index 66717c8..cdac883 100644 --- a/src/modernmolbert/tokenization_ape.py +++ b/src/modernmolbert/tokenization_ape.py @@ -23,8 +23,14 @@ "smiles_vocab_file": "smiles_vocab.json", } SELFIES_RE = re.compile(r"\[[^\]]+\]") +# Only the organic subset (B C N O P S F Cl Br I) may appear unbracketed in +# canonical SMILES; two-letter metals (Si, Se, Na, Mg, Al, Ca, Fe, Zn, ...) are +# always bracketed and matched by the leading \[[^\]]+\] branch. The previous +# pattern listed those metals as optional-second-letter alternatives (Si?, Na?, +# ...), which could match bare invalid single letters (L, M, A, Z) and was dead +# weight for valid input. Keep only Br?/Cl? (B, C, Br, Cl all valid bare). SMILES_RE = re.compile( - r"(\[[^\]]+\]|Br?|Cl?|Si?|Se?|Li?|Na?|Mg?|Al?|Ca?|Fe?|Zn?|" + r"(\[[^\]]+\]|Br?|Cl?|" r"N|O|S|P|F|I|K|B|C|H|" r"b|c|n|o|s|p|" r"\%\d{2}|\d|" @@ -128,7 +134,24 @@ def ape_tokenize( unk_token: str = "", max_piece_span: int | None = None, ) -> list[str]: - pieces = pre_tokenize_molecule(text, representation) + """Segment a molecule against the APE vocabulary by greedy longest match. + + Note this is *not* a replay of the training merges in learned order: train() + learns which substrings become vocab entries, but decoding here just takes + the longest vocab token at each position (up to ``max_piece_span`` pieces). + The two can disagree on segmentation. That is fine and intended — both + pretraining and fine-tuning encode through this same function, so the model + only ever sees greedy-longest-match output and stays internally consistent. + The learned merge *order* is intentionally discarded; only the vocab set is + used at inference. + """ + # A single malformed SELFIES (stray text outside bracket tokens) must not + # crash encoding. Map the whole string to so it stays detectable via + # the validator's unk_rate gate instead of raising mid-batch. + try: + pieces = pre_tokenize_molecule(text, representation) + except ValueError: + return [unk_token] if not pieces: return [unk_token] @@ -572,9 +595,16 @@ def train( tokenized_corpus = [] vocabulary_frequency: defaultdict[str, int] = defaultdict(int) saw_tokens = False + skipped_malformed = 0 for sentence in corpus: - tokens = self.pre_tokenize(str(sentence)) + # One malformed row must not abort a multi-hour training run. Skip and + # count it; surface the total so a corrupt corpus is still visible. + try: + tokens = self.pre_tokenize(str(sentence)) + except ValueError: + skipped_malformed += 1 + continue if not tokens: continue saw_tokens = True @@ -582,6 +612,8 @@ def train( vocabulary_frequency[token] += 1 if len(tokens) > 1: tokenized_corpus.append(tokens) + if skipped_malformed: + print(f"Skipped {skipped_malformed} malformed sequences", flush=True) print( f"Pretokenization complete, found {len(vocabulary_frequency)} tokens", flush=True, @@ -620,7 +652,6 @@ def get_most_common_pair(tokenized): pair_counts[pair] += 1 - self.pair_counts = dict(pair_counts) if not pair_counts: return ("", ""), 0 @@ -686,14 +717,35 @@ def get_most_common_pair(tokenized): flush=True, ) merged_counter += 1 + # Each merged occurrence consumes one left + one right piece, so debit + # both constituents to keep vocabulary_frequency (the *_freq.json + # diagnostic) an accurate post-merge count. Keys are never removed — + # a primitive merged to zero must stay in vocab for coverage. vocabulary_frequency[merged_word] += freq + vocabulary_frequency[left_token] = max(0, vocabulary_frequency[left_token] - freq) + vocabulary_frequency[right_token] = max(0, vocabulary_frequency[right_token] - freq) new_tokenized_corpus = [] + append_seq = new_tokenized_corpus.append for tokens in tokenized_corpus: + token_count = len(tokens) + + # Fast path: a sequence with no adjacent (left, right) is + # unchanged by this merge. Keep the existing list by reference + # instead of reallocating + re-appending every token. Most + # sequences are untouched per merge, so this avoids the bulk of + # the per-iteration allocation without altering the output. + has_pair = any( + tokens[i] == left_token and tokens[i + 1] == right_token + for i in range(token_count - 1) + ) + if not has_pair: + append_seq(tokens) + continue + new_tokens = [] append_token = new_tokens.append i = 0 - token_count = len(tokens) while i < token_count: if ( i < token_count - 1 @@ -707,7 +759,7 @@ def get_most_common_pair(tokenized): i += 1 if len(new_tokens) > 1: - new_tokenized_corpus.append(new_tokens) + append_seq(new_tokens) tokenized_corpus = new_tokenized_corpus diff --git a/src/modernmolbert/train_ape_tokenizer.py b/src/modernmolbert/train_ape_tokenizer.py index 9134273..11f5496 100644 --- a/src/modernmolbert/train_ape_tokenizer.py +++ b/src/modernmolbert/train_ape_tokenizer.py @@ -165,9 +165,13 @@ # p95_len: <60 # unk_rate: 0 # truncation_rate@128: ~0 -# Red flag: inspect actual token strings — if tokens contain half-open -# parentheses like 'C(=O' or unclosed ring digits, max_merge_pieces is -# crossing structural character boundaries inappropriately. +# Note: SMILES APE tokens routinely straddle structural boundaries — +# half-open parens like 'C(=O', leading bonds like '#N)', or partial ring +# openers are EXPECTED, not a defect. APE merges adjacent SMILES symbol +# pieces by frequency (BPE-style), with no awareness of paren/ring balance; +# the paper's APE behaves the same. The shipped max6 vocab already contains +# such tokens. Watch instead for mean_len collapsing below ~8 (too few +# attention positions) or any nonzero unk_rate (coverage gap). # Validate diff --git a/tests/test_regen_groupfig.py b/tests/test_regen_groupfig.py index 08627c0..4b09e08 100644 --- a/tests/test_regen_groupfig.py +++ b/tests/test_regen_groupfig.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import cast import pandas as pd import pytest @@ -59,7 +60,7 @@ def test_generate_group_distribution_figure_accepts_external_csv(tmp_path: Path) ) assert out.exists() - assert means.loc["TDC-ADME", "MMB-small"] > 79 + assert cast(float, means.loc["TDC-ADME", "MMB-small"]) > 79 def test_validate_rejects_incomplete_coverage() -> None: diff --git a/tests/test_tokenizer_training.py b/tests/test_tokenizer_training.py index 5b0e864..005fd73 100644 --- a/tests/test_tokenizer_training.py +++ b/tests/test_tokenizer_training.py @@ -19,6 +19,39 @@ def test_ape_train_terminates_on_tiny_corpus(): assert len(tokenizer.vocabulary) > len(tokenizer.special_tokens) +def test_ape_train_skips_malformed_rows_without_crashing(): + tokenizer = APEPreTrainedTokenizer() + # A stray non-bracket row must be skipped, not abort the whole run. + corpus = ["[C][C][O]", "not selfies", "[C][O][C]", "[C][C][C]"] * 20 + + tokenizer.train(corpus=corpus, max_vocab_size=32, min_freq_for_merge=2) + + assert len(tokenizer.vocabulary) > len(tokenizer.special_tokens) + + +def test_ape_tokenize_maps_malformed_selfies_to_unk(): + tokenizer = APEPreTrainedTokenizer() + + ids = tokenizer.encode("totally not selfies", add_special_tokens=False) + + assert ids == [tokenizer.unk_token_id] + + +def test_ape_train_freq_debits_merged_constituents(): + tokenizer = APEPreTrainedTokenizer() + corpus = ["[C][C]"] * 10 + + tokenizer.train(corpus=corpus, max_vocab_size=32, min_freq_for_merge=2) + + freq = tokenizer.vocabulary_frequency + # The merge consumes every [C] (20 occurrences across 10 merges of [C]+[C]). + assert freq["[C][C]"] == 10 + assert freq["[C]"] == 0 + # Debited primitive stays in the vocab so single-atom coverage survives. + assert "[C]" in tokenizer.vocabulary + assert all(count >= 0 for count in freq.values()) + + def test_ape_train_rejects_empty_corpus() -> None: tokenizer = APEPreTrainedTokenizer()