Skip to content

Fix the failing sentence splitter tests #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
IvoLeist merged 1 commit into from
Aug 6, 2026
Merged

Fix the failing sentence splitter tests #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions tools/langchain_text_splitters/.shed.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ description: Split text into chunks using LangChain text splitters.
long_description: |
Split plain-text-like datasets into chunks for LLM and RAG workflows using
langchain-text-splitters. The tool currently supports recursive character, character,
and tiktoken-based token splitting, and reports per-chunk lengths and start
offsets as both readable text and structured JSON.
sentence-based splitting with NLTK or spaCy, and tiktoken-based token splitting,
and reports per-chunk lengths and start offsets as both readable text and
structured JSON.
type: unrestricted
categories:
- Natural Language Processing
Expand Down
56 changes: 44 additions & 12 deletions tools/langchain_text_splitters/langchain_text_splitters.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,10 @@
English model (en_core_web_sm)
</option>
</param>
<param name="max_length" type="integer" value="1000000" min="1" label="Maximum input length" help="Maximum number of input characters accepted by the spaCy pipeline."/>
<!-- The upper bound keeps a single job from exhausting the
memory of a shared compute node, see the per-character
cost quoted in the help below. -->
<param name="max_length" type="integer" value="1000000" min="1" max="10000000" label="Maximum input length" help="Maximum number of input characters accepted by the spaCy pipeline. The tool stops with an error if the input is longer. Raising this costs memory: the English model needs roughly 3.5 kB per input character, the rule-based sentencizer roughly 0.2 kB."/>
</when>
</conditional>
<conditional name="length">
Expand Down Expand Up @@ -710,7 +713,9 @@
</element>
</output_collection>
</test>
<!-- NLTK overlap is applied at complete-sentence granularity. -->
<!-- NLTK overlap is applied at complete-sentence granularity. The chunk
size must fit the overlapping sentence plus the following one,
otherwise the overlap is dropped again to make room. -->
<test>
<param name="input" value="sentence_overlap.txt" ftype="txt"/>
<conditional name="splitter">
Expand All @@ -721,14 +726,14 @@
</conditional>
<conditional name="length">
<param name="length_mode" value="characters"/>
<param name="chunk_size" value="23"/>
<param name="chunk_overlap" value="12"/>
<param name="chunk_size" value="25"/>
<param name="chunk_overlap" value="13"/>
</conditional>
</conditional>
<output name="chunks_json">
<assert_contents>
<has_json_property_with_value property="chunk_size" value="23"/>
<has_json_property_with_value property="chunk_overlap" value="12"/>
<has_json_property_with_value property="chunk_size" value="25"/>
<has_json_property_with_value property="chunk_overlap" value="13"/>
<has_json_property_with_value property="number_of_chunks" value="3"/>
<not_has_text text="&quot;start_index&quot;: null"/>
</assert_contents>
Expand Down Expand Up @@ -787,12 +792,13 @@
</element>
<element name="chunk_0002">
<assert_contents>
<has_text_matching expression="\A&#10;ASecond sentence is also short\.\Z"/>
<has_text_matching expression="\A&#10;Second sentence is also short\.\Z"/>
</assert_contents>
</element>
<element name="chunk_0003">
<assert_contents>
<has_text_matching expression="\A&#10;Third sentence ends here\.\Z"/>
<!-- Galaxy's upload appends a trailing newline to the input file. -->
<has_text_matching expression="\A&#10;Third sentence ends here\.&#10;\Z"/>
</assert_contents>
</element>
</output_collection>
Expand Down Expand Up @@ -867,25 +873,33 @@
<has_json_property_with_text property="spacy_pipeline" text="sentencizer"/>
<has_json_property_with_value property="chunk_size" value="4"/>
<has_json_property_with_value property="chunk_overlap" value="0"/>
<!-- The sentencizer reports the trailing newline that Galaxy's
upload appends as a fourth sentence. It holds nothing but
whitespace and is left out of the outputs. -->
<has_json_property_with_value property="number_of_chunks" value="3"/>
<not_has_text text="&quot;start_index&quot;: null"/>
</assert_contents>
</output>
<assert_stdout>
<has_text text="WARNING: 1 chunk(s) held nothing but whitespace and were left out of the outputs."/>
</assert_stdout>
<expand macro="assert_chunks_tsv"/>
<output_collection name="chunks_collection" type="list" count="3">
<element name="chunk_0001">
<assert_contents>
<has_text_matching expression="\Aone two\.&#10;\Z"/>
<has_text_matching expression="\Aone two\.\Z"/>
</assert_contents>
</element>
<!-- The sentence separator stays at the start of the following
chunk because strip_whitespace is disabled. -->
<element name="chunk_0002">
<assert_contents>
<has_text_matching expression="\Athree four\.&#10;\Z"/>
<has_text_matching expression="\A&#10;three four\.\Z"/>
</assert_contents>
</element>
<element name="chunk_0003">
<assert_contents>
<has_text_matching expression="\Afive six\.&#10;\Z"/>
<has_text_matching expression="\A&#10;five six\.\Z"/>
</assert_contents>
</element>
</output_collection>
Expand Down Expand Up @@ -1283,8 +1297,26 @@ and other LLM workflows where long text needs to be bounded by a chunk size.
- `RecursiveCharacterTextSplitter`: Recommended for most use cases as it attempts to preserve larger text units by trying an ordered list of separators, from coarser boundaries such as paragraph breaks to finer boundaries such as line breaks, or spaces, down to individual characters.
- `CharacterTextSplitter`: Splits text using one defined separator or character sequence, such as a paragraph break, line break, space, or custom separator.
- **Token-based splitting:** The `TokenTextSplitter` ignores any potential text structure and simply creates chunks of a specified token length.
- **Sentence-based splitting:** Detects sentence boundaries first and then fills the chunks with whole sentences, so a chunk never ends in the middle of a sentence. Pick this for prose. A single sentence that is already longer than the target chunk size becomes an oversized chunk of its own, because the sentence is never cut.
- `NLTKTextSplitter`: Uses NLTK's Punkt sentence tokenizer, which is trained per language and therefore handles language-specific abbreviations such as `Dr.` in English or `z.B.` in German. Select the language of the input.
- `SpacyTextSplitter`: Uses spaCy. The rule-based sentencizer is fast and needs no model, but it applies English tokenization rules. The `en_core_web_sm` model is more accurate on English text, and is roughly five times slower and needs about ten times more memory.

Both sentence splitters keep the whitespace between the sentences, so a chunk
starts with the line break that follows the previous sentence. The chunk overlap
is applied in whole sentences: an overlapping sentence is only carried over if it
fits into the next chunk together with the following sentence, otherwise the
overlap is silently smaller than requested.

The NLTK splitter drops the text after the last detected sentence, usually the
trailing line break, so its chunks do not always add up to the complete input; a
warning is written to the tool log when that happens. The spaCy splitter keeps
everything.

<!-- TODO add sentences describing spaCy/NLTK Text Splitting -->
The NLTK splitter needs the `punkt_tab` resource for the selected language. It is
not part of the `nltk` package, so a Galaxy instance has to provide it locally,
for example below a directory listed in the `NLTK_DATA` environment variable.
Without it, any run of the NLTK splitter fails. The spaCy splitter has no such
requirement.

**Used Library for Token Counting**

Expand Down
102 changes: 87 additions & 15 deletions tools/langchain_text_splitters/split_text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,10 @@ def build_splitter(args, input_text, length_function, tiktoken_options):
}

if args.splitter_type == "nltk":
# separator="" together with strip_whitespace=False keeps the chunk text
# identical to the matching slice of the input, so the reported start
# indices stay usable. Note that the span based tokenizer drops whatever
# follows the last sentence, see the warning in main().
return NLTKTextSplitter(
**length_options,
language=args.sentence_language,
Expand All @@ -292,13 +296,23 @@ def build_splitter(args, input_text, length_function, tiktoken_options):
)

if args.splitter_type == "spacy":
return SpacyTextSplitter(
splitter = SpacyTextSplitter(
**length_options,
pipeline=args.spacy_pipeline,
max_length=args.spacy_max_length,
separator="",
strip_whitespace=False,
)
# langchain only forwards max_length to the pipelines it loads with
# spacy.load(). The sentencizer is built from English() instead and
# silently keeps the spaCy default of one million characters, so it is
# applied here for both. Guarded because _tokenizer is private API.
tokenizer = getattr(splitter, "_tokenizer", None)

if hasattr(tokenizer, "max_length"):
tokenizer.max_length = args.spacy_max_length

return splitter

character_options = {
**length_options,
Expand Down Expand Up @@ -413,6 +427,17 @@ def main():
"There is nothing to split."
)

# Checked before the tokenizer is built so that a job that cannot run does
# not first pay for loading the tiktoken encoding.
if args.splitter_type == "spacy" and len(input_text) > args.spacy_max_length:
sys.exit(
f"The input dataset holds {len(input_text)} characters, which is "
"more than the configured spaCy maximum input length of "
f"{args.spacy_max_length}. Raise 'Maximum input length' or split "
"the dataset into smaller parts first. Mind the memory cost per "
"input character stated in the help of that setting."
)

length_mode = "token" if args.splitter_type == "token" else args.length_mode

tiktoken_options = get_tiktoken_options(args)
Expand Down Expand Up @@ -442,37 +467,57 @@ def main():
f"Original error: {error}"
)
raise

try:
documents = splitter.create_documents([input_text])
except LookupError as error:
if args.splitter_type == "nltk":
# nltk is searching for the punkt_tab data by default here:
# -/usr/share/nltk_data
# -/usr/local/share/nltk_data
# -/usr/lib/nltk_data
# -/usr/local/lib/nltk_data
# alternative conda package:
# https://anaconda.org/channels/conda-forge/packages/nltk_data/overview
# but it is from 2022.05.27 so quite outdated
# here is the feedstock https://github.com/conda-forge/nltk_data-feedstock
# NLTKTextSplitter loads the punkt_tab data in its constructor, so this
# has to be caught around build_splitter() and not around
# create_documents().
# nltk is searching for the punkt_tab data by default here:
# -/usr/share/nltk_data
# -/usr/local/share/nltk_data
# -/usr/lib/nltk_data
# -/usr/local/lib/nltk_data
# alternative conda package:
# https://anaconda.org/channels/conda-forge/packages/nltk_data/overview
# but it is from 2022.05.27 and ships 'punkt' instead of 'punkt_tab'
# here is the feedstock https://github.com/conda-forge/nltk_data-feedstock
# KeyError and IndexError also derive from LookupError, so they are
# excluded to keep an unrelated failure from being reported as missing
# Punkt data.
if args.splitter_type == "nltk" and not isinstance(
error, (KeyError, IndexError)
):
sys.exit(
"The NLTK Punkt data required for the selected language is not "
"installed. The Galaxy tool environment must provide the "
"'punkt_tab' NLTK resource.\n"
"'punkt_tab' NLTK resource, for example below a directory "
"listed in the NLTK_DATA environment variable. Use the spaCy "
"sentence splitter if the data cannot be installed.\n"
f"Original error: {error}"
)
raise

try:
documents = splitter.create_documents([input_text])
except ValueError as error:
fail_on_disallowed_special_token(error)
raise

chunks = []
start_index_warnings = []
empty_chunks = 0

for document in documents:
raw_chunk_text = document.page_content
start_index = document.metadata.get("start_index")

# A chunk that holds nothing but whitespace carries no content for a
# downstream step to work on. The sentence splitters produce one for the
# trailing line break of the input, because they keep the whitespace
# between the sentences and never strip a chunk.
if not raw_chunk_text.strip():
empty_chunks += 1
continue

chunk_number = len(chunks) + 1

if start_index is not None and start_index < 0:
Expand Down Expand Up @@ -512,6 +557,33 @@ def main():
flush=True,
)

# The NLTK splitter builds the chunks from the sentence spans reported by
# punkt, which end at the last sentence. Punkt trims trailing whitespace, so
# whatever follows is dropped and the chunks no longer add up to the input.
# The first span always starts at offset 0, so nothing is lost in front.
# The spaCy splitter keeps everything.
if empty_chunks:
print(
f"WARNING: {empty_chunks} chunk(s) held nothing but whitespace and "
"were left out of the outputs.",
flush=True,
)

if args.splitter_type == "nltk" and chunks:
last_text = chunks[-1]["text"]
last_start = input_text.rfind(last_text)
dropped = (
len(input_text) - (last_start + len(last_text)) if last_start >= 0 else 0
)

if dropped > 0:
print(
f"WARNING: The NLTK sentence splitter dropped the {dropped} "
"character(s) after the last sentence. Use the spaCy sentence "
"splitter to keep the complete input.",
flush=True,
)

metadata = {
"input_characters": len(input_text),
"input_length": length_function(input_text),
Expand Down
Loading