diff --git a/tools/langchain_text_splitters/.shed.yml b/tools/langchain_text_splitters/.shed.yml
index 9a514eaf8f..af0209a17c 100644
--- a/tools/langchain_text_splitters/.shed.yml
+++ b/tools/langchain_text_splitters/.shed.yml
@@ -4,8 +4,9 @@ description: Split text into chunks using LangChain text splitters.
long_description: |
Split plain-text-like datasets into chunks for LLM and RAG workflows using
langchain-text-splitters. The tool currently supports recursive character, character,
- and tiktoken-based token splitting, and reports per-chunk lengths and start
- offsets as both readable text and structured JSON.
+ sentence-based splitting with NLTK or spaCy, and tiktoken-based token splitting,
+ and reports per-chunk lengths and start offsets as both readable text and
+ structured JSON.
type: unrestricted
categories:
- Natural Language Processing
diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml
index afc2665cff..75524fe48b 100644
--- a/tools/langchain_text_splitters/langchain_text_splitters.xml
+++ b/tools/langchain_text_splitters/langchain_text_splitters.xml
@@ -181,7 +181,10 @@
English model (en_core_web_sm)
-
+
+
@@ -710,7 +713,9 @@
-
+
@@ -721,14 +726,14 @@
-
-
+
+
+
+
+
-
+
+
-
+
-
+
@@ -1283,8 +1297,26 @@ and other LLM workflows where long text needs to be bounded by a chunk size.
- `RecursiveCharacterTextSplitter`: Recommended for most use cases as it attempts to preserve larger text units by trying an ordered list of separators, from coarser boundaries such as paragraph breaks to finer boundaries such as line breaks, or spaces, down to individual characters.
- `CharacterTextSplitter`: Splits text using one defined separator or character sequence, such as a paragraph break, line break, space, or custom separator.
- **Token-based splitting:** The `TokenTextSplitter` ignores any potential text structure and simply creates chunks of a specified token length.
+- **Sentence-based splitting:** Detects sentence boundaries first and then fills the chunks with whole sentences, so a chunk never ends in the middle of a sentence. Pick this for prose. A single sentence that is already longer than the target chunk size becomes an oversized chunk of its own, because the sentence is never cut.
+ - `NLTKTextSplitter`: Uses NLTK's Punkt sentence tokenizer, which is trained per language and therefore handles language-specific abbreviations such as `Dr.` in English or `z.B.` in German. Select the language of the input.
+ - `SpacyTextSplitter`: Uses spaCy. The rule-based sentencizer is fast and needs no model, but it applies English tokenization rules. The `en_core_web_sm` model is more accurate on English text, and is roughly five times slower and needs about ten times more memory.
+
+Both sentence splitters keep the whitespace between the sentences, so a chunk
+starts with the line break that follows the previous sentence. The chunk overlap
+is applied in whole sentences: an overlapping sentence is only carried over if it
+fits into the next chunk together with the following sentence, otherwise the
+overlap is silently smaller than requested.
+
+The NLTK splitter drops the text after the last detected sentence, usually the
+trailing line break, so its chunks do not always add up to the complete input; a
+warning is written to the tool log when that happens. The spaCy splitter keeps
+everything.
-
+The NLTK splitter needs the `punkt_tab` resource for the selected language. It is
+not part of the `nltk` package, so a Galaxy instance has to provide it locally,
+for example below a directory listed in the `NLTK_DATA` environment variable.
+Without it, any run of the NLTK splitter fails. The spaCy splitter has no such
+requirement.
**Used Library for Token Counting**
diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py
index 93ee4fbb9d..6ed414f54e 100644
--- a/tools/langchain_text_splitters/split_text.py
+++ b/tools/langchain_text_splitters/split_text.py
@@ -283,6 +283,10 @@ def build_splitter(args, input_text, length_function, tiktoken_options):
}
if args.splitter_type == "nltk":
+ # separator="" together with strip_whitespace=False keeps the chunk text
+ # identical to the matching slice of the input, so the reported start
+ # indices stay usable. Note that the span based tokenizer drops whatever
+ # follows the last sentence, see the warning in main().
return NLTKTextSplitter(
**length_options,
language=args.sentence_language,
@@ -292,13 +296,23 @@ def build_splitter(args, input_text, length_function, tiktoken_options):
)
if args.splitter_type == "spacy":
- return SpacyTextSplitter(
+ splitter = SpacyTextSplitter(
**length_options,
pipeline=args.spacy_pipeline,
max_length=args.spacy_max_length,
separator="",
strip_whitespace=False,
)
+ # langchain only forwards max_length to the pipelines it loads with
+ # spacy.load(). The sentencizer is built from English() instead and
+ # silently keeps the spaCy default of one million characters, so it is
+ # applied here for both. Guarded because _tokenizer is private API.
+ tokenizer = getattr(splitter, "_tokenizer", None)
+
+ if hasattr(tokenizer, "max_length"):
+ tokenizer.max_length = args.spacy_max_length
+
+ return splitter
character_options = {
**length_options,
@@ -413,6 +427,17 @@ def main():
"There is nothing to split."
)
+ # Checked before the tokenizer is built so that a job that cannot run does
+ # not first pay for loading the tiktoken encoding.
+ if args.splitter_type == "spacy" and len(input_text) > args.spacy_max_length:
+ sys.exit(
+ f"The input dataset holds {len(input_text)} characters, which is "
+ "more than the configured spaCy maximum input length of "
+ f"{args.spacy_max_length}. Raise 'Maximum input length' or split "
+ "the dataset into smaller parts first. Mind the memory cost per "
+ "input character stated in the help of that setting."
+ )
+
length_mode = "token" if args.splitter_type == "token" else args.length_mode
tiktoken_options = get_tiktoken_options(args)
@@ -442,37 +467,57 @@ def main():
f"Original error: {error}"
)
raise
-
- try:
- documents = splitter.create_documents([input_text])
except LookupError as error:
- if args.splitter_type == "nltk":
- # nltk is searching for the punkt_tab data by default here:
- # -/usr/share/nltk_data
- # -/usr/local/share/nltk_data
- # -/usr/lib/nltk_data
- # -/usr/local/lib/nltk_data
- # alternative conda package:
- # https://anaconda.org/channels/conda-forge/packages/nltk_data/overview
- # but it is from 2022.05.27 so quite outdated
- # here is the feedstock https://github.com/conda-forge/nltk_data-feedstock
+ # NLTKTextSplitter loads the punkt_tab data in its constructor, so this
+ # has to be caught around build_splitter() and not around
+ # create_documents().
+ # nltk is searching for the punkt_tab data by default here:
+ # -/usr/share/nltk_data
+ # -/usr/local/share/nltk_data
+ # -/usr/lib/nltk_data
+ # -/usr/local/lib/nltk_data
+ # alternative conda package:
+ # https://anaconda.org/channels/conda-forge/packages/nltk_data/overview
+ # but it is from 2022.05.27 and ships 'punkt' instead of 'punkt_tab'
+ # here is the feedstock https://github.com/conda-forge/nltk_data-feedstock
+ # KeyError and IndexError also derive from LookupError, so they are
+ # excluded to keep an unrelated failure from being reported as missing
+ # Punkt data.
+ if args.splitter_type == "nltk" and not isinstance(
+ error, (KeyError, IndexError)
+ ):
sys.exit(
"The NLTK Punkt data required for the selected language is not "
"installed. The Galaxy tool environment must provide the "
- "'punkt_tab' NLTK resource.\n"
+ "'punkt_tab' NLTK resource, for example below a directory "
+ "listed in the NLTK_DATA environment variable. Use the spaCy "
+ "sentence splitter if the data cannot be installed.\n"
f"Original error: {error}"
)
raise
+
+ try:
+ documents = splitter.create_documents([input_text])
except ValueError as error:
fail_on_disallowed_special_token(error)
raise
chunks = []
start_index_warnings = []
+ empty_chunks = 0
for document in documents:
raw_chunk_text = document.page_content
start_index = document.metadata.get("start_index")
+
+ # A chunk that holds nothing but whitespace carries no content for a
+ # downstream step to work on. The sentence splitters produce one for the
+ # trailing line break of the input, because they keep the whitespace
+ # between the sentences and never strip a chunk.
+ if not raw_chunk_text.strip():
+ empty_chunks += 1
+ continue
+
chunk_number = len(chunks) + 1
if start_index is not None and start_index < 0:
@@ -512,6 +557,33 @@ def main():
flush=True,
)
+ # The NLTK splitter builds the chunks from the sentence spans reported by
+ # punkt, which end at the last sentence. Punkt trims trailing whitespace, so
+ # whatever follows is dropped and the chunks no longer add up to the input.
+ # The first span always starts at offset 0, so nothing is lost in front.
+ # The spaCy splitter keeps everything.
+ if empty_chunks:
+ print(
+ f"WARNING: {empty_chunks} chunk(s) held nothing but whitespace and "
+ "were left out of the outputs.",
+ flush=True,
+ )
+
+ if args.splitter_type == "nltk" and chunks:
+ last_text = chunks[-1]["text"]
+ last_start = input_text.rfind(last_text)
+ dropped = (
+ len(input_text) - (last_start + len(last_text)) if last_start >= 0 else 0
+ )
+
+ if dropped > 0:
+ print(
+ f"WARNING: The NLTK sentence splitter dropped the {dropped} "
+ "character(s) after the last sentence. Use the spaCy sentence "
+ "splitter to keep the complete input.",
+ flush=True,
+ )
+
metadata = {
"input_characters": len(input_text),
"input_length": length_function(input_text),