From fc6efe93ea22f043c526af23578a3a4ce36de7d6 Mon Sep 17 00:00:00 2001
From: Angelos Katharopoulos <a_katharopoulos@apple.com>
Date: Tue, 30 Sep 2025 17:58:20 -0700
Subject: [PATCH] Add the rest of the tokenizer helpers to the docs

---
 docs/src/python/tokenizing.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/src/python/tokenizing.rst b/docs/src/python/tokenizing.rst
index a82b8a6..e69e956 100644
--- a/docs/src/python/tokenizing.rst
+++ b/docs/src/python/tokenizing.rst
@@ -52,3 +52,6 @@ or a plain text vocabulary file.
    core.CharTrie
    tokenizer_helpers.read_trie_from_vocab
    tokenizer_helpers.read_trie_from_spm
+   tokenizer_helpers.read_bpe_from_spm
+   tokenizer_helpers.read_bpe_from_hf
+   tokenizer_helpers.gpt2_byte_map