From 71524aa5f512f7b97ae695b139396b3baf4ebdb9 Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Mon, 3 Aug 2026 16:58:04 +0200 Subject: [PATCH 01/17] Add langchain_text_splitters --- tools/langchain_text_splitters/.shed.yml | 16 + .../dev_utils/README.md | 48 + .../dev_utils/get_encodings.py | 27 + .../langchain_text_splitters.xml | 839 ++++++++++++++++++ tools/langchain_text_splitters/macros.xml | 112 +++ .../macros_for_testing.xml | 10 + tools/langchain_text_splitters/split_text.py | 409 +++++++++ .../test-data/.gitignore | 1 + .../test-data/custom_separator.txt | 1 + .../test-data/langchain_docs_sample.txt | 33 + .../test-data/overlap_words.txt | 1 + .../test-data/recursive_default.txt | 5 + .../test-data/simple_separator.txt | 1 + .../test-data/special_token.txt | 1 + .../test-data/token_words.txt | 1 + .../test-data/tsv_escape.txt | 2 + .../test-data/tsv_escape_expected.tsv | 1 + .../test-data/unicode_separator.txt | 1 + 18 files changed, 1509 insertions(+) create mode 100644 tools/langchain_text_splitters/.shed.yml create mode 100644 tools/langchain_text_splitters/dev_utils/README.md create mode 100644 tools/langchain_text_splitters/dev_utils/get_encodings.py create mode 100644 tools/langchain_text_splitters/langchain_text_splitters.xml create mode 100644 tools/langchain_text_splitters/macros.xml create mode 100644 tools/langchain_text_splitters/macros_for_testing.xml create mode 100644 tools/langchain_text_splitters/split_text.py create mode 100644 tools/langchain_text_splitters/test-data/.gitignore create mode 100644 tools/langchain_text_splitters/test-data/custom_separator.txt create mode 100644 tools/langchain_text_splitters/test-data/langchain_docs_sample.txt create mode 100644 tools/langchain_text_splitters/test-data/overlap_words.txt create mode 100644 tools/langchain_text_splitters/test-data/recursive_default.txt create mode 100644 tools/langchain_text_splitters/test-data/simple_separator.txt create mode 100644 tools/langchain_text_splitters/test-data/special_token.txt create mode 100644 tools/langchain_text_splitters/test-data/token_words.txt create mode 100644 tools/langchain_text_splitters/test-data/tsv_escape.txt create mode 100644 tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv create mode 100644 tools/langchain_text_splitters/test-data/unicode_separator.txt diff --git a/tools/langchain_text_splitters/.shed.yml b/tools/langchain_text_splitters/.shed.yml new file mode 100644 index 0000000000..9a514eaf8f --- /dev/null +++ b/tools/langchain_text_splitters/.shed.yml @@ -0,0 +1,16 @@ +name: langchain_text_splitters +owner: bgruening +description: Split text into chunks using LangChain text splitters. +long_description: | + Split plain-text-like datasets into chunks for LLM and RAG workflows using + langchain-text-splitters. The tool currently supports recursive character, character, + and tiktoken-based token splitting, and reports per-chunk lengths and start + offsets as both readable text and structured JSON. +type: unrestricted +categories: + - Natural Language Processing +remote_repository_url: https://github.com/bgruening/galaxytools/tree/master/tools/langchain_text_splitters +homepage_url: https://github.com/langchain-ai/langchain/tree/master/libs/text-splitters +auto_tool_repositories: + name_template: "{{ tool_id }}" + description_template: "Wrapper for: {{ tool_name }}" \ No newline at end of file diff --git a/tools/langchain_text_splitters/dev_utils/README.md b/tools/langchain_text_splitters/dev_utils/README.md new file mode 100644 index 0000000000..17fc20f139 --- /dev/null +++ b/tools/langchain_text_splitters/dev_utils/README.md @@ -0,0 +1,48 @@ +# LangChain Text Splitters Galaxy Tool Dev Utilities + +## Populate tiktoken cache directory + +Download the OpenAI encoding files using the script `get_encodings.py` in this folder. +This will populate the cache directory `/tmp/tiktoken-cache`. + +## Testing tiktoken without internet access in Galaxy + +! Prerequisite: The OpenAI encoding files must be downloaded and the cache directory populated as described above. + +To test whether a Galaxy tool can use a locally cached tiktoken encoding without internet access, +save the following configuration as job_conf_offline.xml: + +```xml + + + + + + + + + /tmp/tiktoken-cache + + http://127.0.0.1:9 + http://127.0.0.1:9 + + + +``` +Start Planemo using the job configuration above: +`planemo serve --job_config_file dev_utils/job_conf_offline.xml` + +You can test as follows that the tool works without internet access by setting +**Text splitting boundaries** to: + +**(A) Between tokens**. + +**(B) Character**, then set **Target chunk size should be determined by the count of** to **Tokens**. + +To double check that the tool actually fails when the cache is empty, remove or rename the cache directory `/tmp/tiktoken-cache` and run it as described above. It should run into a +`ConnectionRefusedError` when trying to download the encoding files from the OpenAI servers. \ No newline at end of file diff --git a/tools/langchain_text_splitters/dev_utils/get_encodings.py b/tools/langchain_text_splitters/dev_utils/get_encodings.py new file mode 100644 index 0000000000..18db45f852 --- /dev/null +++ b/tools/langchain_text_splitters/dev_utils/get_encodings.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import sys +import os +from pathlib import Path +from tiktoken import get_encoding + +TIKTOKEN_CACHE_DIR = "/tmp/tiktoken-cache" +os.environ["TIKTOKEN_CACHE_DIR"] = TIKTOKEN_CACHE_DIR + +TIKTOKEN_ENCODINGS = [ + "o200k_harmony", + "o200k_base", + "cl100k_base", + "p50k_edit", + "p50k_base", + "r50k_base", + "gpt2", +] + +Path(TIKTOKEN_CACHE_DIR).mkdir(parents=True, exist_ok=True) + +print(f"Populating tiktoken cache directory {TIKTOKEN_CACHE_DIR}") + +for encoding_name in TIKTOKEN_ENCODINGS: + print(f"Getting {encoding_name}...") + get_encoding(encoding_name) diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml new file mode 100644 index 0000000000..1e16a9467a --- /dev/null +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -0,0 +1,839 @@ + + Split text into chunks for LLM and RAG workflows + + macros.xml + macros_for_testing.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + + + + + + +**What it does** + +This tool splits a text dataset into chunks with `langchain-text-splitters`. +It is useful before retrieval-augmented generation, embedding, summarization, +and other LLM workflows where long text needs to be bounded by a chunk size. + +**Leveraged Methods of `langchain-text-splitters`** + +- **Character-based splitting:** Splits text at character-based separators. The target chunk size can be measured in characters or tokens. + - `RecursiveCharacterTextSplitter`: Recommended for most use cases as it attempts to preserve larger text units by trying an ordered list of separators, from coarser boundaries such as paragraph breaks to finer boundaries such as line breaks, or spaces, down to individual characters. + - `CharacterTextSplitter`: Splits text using one defined separator or character sequence, such as a paragraph break, line break, space, or custom separator. +- **Token-based splitting:** The `TokenTextSplitter` ignores any potential text structure and simply creates chunks of a specified token length. + +**Used Library for Token Counting** + +OpenAI's fast Byte-Pair Encoding (BPE) tokenizer `tiktoken` library is used +to determine the number of tokens in a chunk. + +**Outputs** + +- Text and JSON file with input/chunk metadata, where each chunk comes along with its size and start index in the input text. +- TSV file with the following columns: chunk number, content, size in characters or tokens +- Galaxy collection of text files, with one raw chunk per file + +**License** +Langchain-Text-Splitters is licensed under the MIT License. + + + + @software{Chase_LangChain_2022, + author = {Chase, Harrison}, + month = oct, + title = {{LangChain}}, + url = {https://github.com/langchain-ai/langchain}, + year = {2022} + } + + + diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml new file mode 100644 index 0000000000..4a5d9666cb --- /dev/null +++ b/tools/langchain_text_splitters/macros.xml @@ -0,0 +1,112 @@ + + 1.1.2 + 0 + 25.1 + + + langchain-text-splitters + tiktoken + + + + + + + + + + + /tmp/tiktoken-cache + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + diff --git a/tools/langchain_text_splitters/macros_for_testing.xml b/tools/langchain_text_splitters/macros_for_testing.xml new file mode 100644 index 0000000000..48ec21e315 --- /dev/null +++ b/tools/langchain_text_splitters/macros_for_testing.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py new file mode 100644 index 0000000000..2feedef3ba --- /dev/null +++ b/tools/langchain_text_splitters/split_text.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 + +import argparse +import csv +import json +import re +import sys +from pathlib import Path + +from langchain_text_splitters import ( + CharacterTextSplitter, + RecursiveCharacterTextSplitter, + TokenTextSplitter, +) + +SEPARATOR_MAP = { + "paragraph_break": "\n\n", + "line_break": "\n", + "tab": "\t", + "space": " ", + "ascii_full_stop": ".", + "ascii_comma": ",", + "ascii_semicolon": ";", + "zero_width_space": "\u200b", + "fullwidth_comma": "\uff0c", + "ideographic_comma": "\u3001", + "fullwidth_full_stop": "\uff0e", + "ideographic_full_stop": "\u3002", +} + +KEEP_SEPARATOR_VALUES = { + "start": "start", + "end": "end", + "false": False, +} + +CUSTOM_SEPARATOR_ESCAPE_PATTERN = re.compile( + r"\\(?:[\\nrt]|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})" +) + +def parse_args(): + parser = argparse.ArgumentParser( + description="Split text with langchain-text-splitters." + ) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output-text", type=Path, required=True) + parser.add_argument("--output-json", type=Path, required=True) + parser.add_argument("--output-tsv", type=Path, required=True) + parser.add_argument("--chunks-dir", type=Path, required=True) + parser.add_argument( + "--splitter-type", + choices=("recursive_character", "character", "token"), + default="recursive_character", + ) + parser.add_argument("--chunk-size", type=int, required=True) + parser.add_argument("--chunk-overlap", type=int, default=0) + parser.add_argument( + "--length-mode", + choices=("characters", "token"), + default="characters", + ) + parser.add_argument("--encoding-name", default="gpt2") + parser.add_argument("--model-name", default="") + parser.add_argument( + "--allowed-special", + choices=("none", "all"), + default="none", + ) + parser.add_argument( + "--keep-separator", + choices=("start", "end", "false"), + default="start", + ) + parser.add_argument( + "--separator-name", + choices=SEPARATOR_MAP, + ) + parser.add_argument("--separator") + parser.add_argument("--separator-specs", nargs="+", default=None) + parser.add_argument("--strip-whitespace", action="store_true") + return parser.parse_args() + + +def get_tiktoken_options(args): + allow_all_special = args.allowed_special == "all" + + return { + "encoding_name": args.encoding_name, + "model_name": args.model_name or None, + "allowed_special": "all" if allow_all_special else set(), + "disallowed_special": () if allow_all_special else "all", + } + + +def build_tiktoken_counter( + encoding_name, + model_name=None, + allowed_special=None, + disallowed_special="all", +): + import tiktoken + + if allowed_special is None: + allowed_special = set() + + # Note: below needs either the TIKTOKEN_CACHE_DIR environment variable set + # respectively the directory populated with `python3 get_encodings.py` in the folder dev_utils. + # or internet access to download the encodings from the OpenAI servers. + if model_name: + encoding = tiktoken.encoding_for_model(model_name) + else: + encoding = tiktoken.get_encoding(encoding_name) + + def count_tokens(text): + return len( + encoding.encode( + text, + allowed_special=allowed_special, + disallowed_special=disallowed_special, + ) + ) + + return count_tokens + +def decode_custom_separator(value): + """Decode supported escape notation without altering literal Unicode.""" + simple_escapes = { + r"\n": "\n", + r"\r": "\r", + r"\t": "\t", + r"\\": "\\", + } + + def replace_escape(match): + escape = match.group(0) + + if escape in simple_escapes: + return simple_escapes[escape] + + return chr(int(escape[2:], 16)) + + return CUSTOM_SEPARATOR_ESCAPE_PATTERN.sub(replace_escape, value) + + +def resolve_separator( + separator_name, + custom_separator, +): + if separator_name is not None: + return SEPARATOR_MAP[separator_name] + + if custom_separator is not None: + return decode_custom_separator(custom_separator) + + raise ValueError("No separator was provided.") + + +def resolve_separator_specs(specs): + separators = [] + + for spec in specs: + kind, value = spec.split(":", 1) + + if kind == "predefined": + separators.append(SEPARATOR_MAP[value]) + else: + separators.append(decode_custom_separator(value)) + + return separators + + +def build_splitter(args, input_text, length_function, tiktoken_options): + common_options = { + "chunk_size": args.chunk_size, + "chunk_overlap": args.chunk_overlap, + "add_start_index": True, + } + + if args.splitter_type == "token": + return TokenTextSplitter( + **common_options, + **tiktoken_options, + ) + + character_options = { + **common_options, + "strip_whitespace": args.strip_whitespace, + "length_function": length_function, + "keep_separator": KEEP_SEPARATOR_VALUES[args.keep_separator], + } + + if args.splitter_type == "character": + separator = resolve_separator( + separator_name=args.separator_name, + custom_separator=args.separator, + ) + + if separator not in input_text: + sys.exit( + "The selected separator " + f"{separator!r} was not found in the input text. " + "Select a separator that occurs in the input or use the " + "recursive character splitter." + ) + + return CharacterTextSplitter( + **character_options, + separator=separator, + ) + + if args.separator_specs is not None: + separators = ( + resolve_separator_specs(args.separator_specs) + if args.separator_specs is not None + else None + ) + # Append an empty string as the final fallback separator to ensure that the text can always be split, + # even if none of the tried separators before were able to split the text without exceeding the chunk size. + character_options["separators"] = [*separators, ""] + + return RecursiveCharacterTextSplitter(**character_options) + + +def write_text_output( + output_path, + metadata, + chunks, + count_key, + length_label, +): + lines = [ + f"Splitter: {metadata['splitter_type']}", + f"Length function: {metadata['length_function']}", + f"Input characters: {metadata['input_characters']}", + f"Input length: {metadata['input_length']} {length_label}", + f"Chunk size: {metadata['chunk_size']}", + f"Chunk overlap: {metadata['chunk_overlap']}", + f"Number of chunks: {metadata['number_of_chunks']}", + "", + ] + + for chunk in chunks: + lines.extend( + [ + ( + f"--- Chunk {chunk['index']}: " + f"{chunk[count_key]} {length_label}, " + f"start={chunk['start_index']} ---" + ), + chunk["text"], + "", + ] + ) + output_path.write_text("\n".join(lines)) + + +def write_chunk_files(chunks_dir, chunks): + chunks_dir.mkdir(parents=True, exist_ok=True) + + for chunk in chunks: + chunk_path = chunks_dir / f"chunk_{chunk['index']:04d}.txt" + chunk_path.write_text( + chunk["text"], + ) + + +def write_tsv_output(output_path, chunks, count_key): + with output_path.open("w") as handle: + writer = csv.writer( + handle, + delimiter="\t", + lineterminator="\n", + ) + + for chunk in chunks: + table_text = chunk["text"].replace("\t", r"\t").replace("\n", r"\n") + writer.writerow( + [ + chunk["index"], + table_text, + chunk[count_key], + ] + ) + + +def main(): + args = parse_args() + + if args.chunk_overlap >= args.chunk_size: + sys.exit("Chunk overlap must be smaller than chunk size.") + + input_text = args.input.read_text() + + length_mode = "token" if args.splitter_type == "token" else args.length_mode + + tiktoken_options = get_tiktoken_options(args) + + if length_mode == "token": + length_function = build_tiktoken_counter(**tiktoken_options) + count_key = "token_count" + length_label = "tokens" + tokenizer_label = args.model_name or args.encoding_name + length_function_label = f"tiktoken:{tokenizer_label}" + else: + length_function = len + count_key = "character_count" + length_label = "characters" + length_function_label = "characters" + + splitter = build_splitter( + args, + input_text, + length_function, + tiktoken_options, + ) + documents = splitter.create_documents([input_text]) + + chunks = [] + start_index_warnings = [] + + for document in documents: + raw_chunk_text = document.page_content + start_index = document.metadata.get("start_index") + chunk_number = len(chunks) + 1 + + if start_index is not None and start_index < 0: + # TODO: report upstream to langchain-text-splitters as a bug, since this should not happen. + # Potential cause: langchain text splitters might calculate the start index based on the length of the chunk in characters, even though the chunk size is measured in tokens. + # reproduce with Character splitter recursive + # input: test-data/langchain_docs_sample.txt + # Text splitting separators and their order of use: default + # Place separator at start of the following chunk start → ["First paragraph.", "\n\nSecond paragraph."] + # Strip whitespace around chunks: true + # Target chunk size should be determined by the number of tokens + # Target chunk size: 100 + # Chunk overlap: 20 + start_index_warnings.append((chunk_number, start_index)) + start_index = ( + "Potential upstream langchain-text-splitters bug: " + f"received invalid start index {start_index!r}" + ) + + chunks.append( + { + "index": chunk_number, + count_key: length_function(raw_chunk_text), + "start_index": start_index, + "text": raw_chunk_text, + } + ) + + if start_index_warnings: + affected_chunks = ", ".join( + f"{chunk_number} (start_index: {start_index!r})" + for chunk_number, start_index in start_index_warnings + ) + print( + "WARNING: Potential upstream langchain-text-splitters bug: " + f"invalid start index returned for chunk(s): {affected_chunks}", + flush=True, + ) + + metadata = { + "input_characters": len(input_text), + "input_length": length_function(input_text), + "length_unit": length_label, + "length_function": length_function_label, + "splitter_type": args.splitter_type, + "chunk_size": args.chunk_size, + "chunk_overlap": args.chunk_overlap, + "number_of_chunks": len(chunks), + } + + if args.splitter_type != "token": + metadata.update( + { + "keep_separator": args.keep_separator, + "strip_whitespace": args.strip_whitespace, + } + ) + + args.output_json.write_text( + json.dumps( + {**metadata, "chunks": chunks}, + indent=2, + ) + + "\n", + ) + + write_text_output( + args.output_text, + metadata, + chunks, + count_key, + length_label, + ) + write_tsv_output( + args.output_tsv, + chunks, + count_key, + ) + write_chunk_files( + args.chunks_dir, + chunks, + ) + + +if __name__ == "__main__": + main() diff --git a/tools/langchain_text_splitters/test-data/.gitignore b/tools/langchain_text_splitters/test-data/.gitignore new file mode 100644 index 0000000000..5303bbab27 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/.gitignore @@ -0,0 +1 @@ +daniela_llm-hub_input.txt \ No newline at end of file diff --git a/tools/langchain_text_splitters/test-data/custom_separator.txt b/tools/langchain_text_splitters/test-data/custom_separator.txt new file mode 100644 index 0000000000..8bb2b54918 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/custom_separator.txt @@ -0,0 +1 @@ +alpha###beta###gamma diff --git a/tools/langchain_text_splitters/test-data/langchain_docs_sample.txt b/tools/langchain_text_splitters/test-data/langchain_docs_sample.txt new file mode 100644 index 0000000000..1afdfaf8e9 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/langchain_docs_sample.txt @@ -0,0 +1,33 @@ +Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. + +Members of Congress and the Cabinet. + +Justices of the Supreme Court. + +My fellow Americans. + +Last year COVID-19 kept us apart. + +This year we are finally together again. + +Tonight, we meet as Democrats Republicans and Independents. + +But most importantly as Americans. + +With a duty to one another to the American people to the Constitution. + +And with an unwavering resolve that freedom will always triumph over tyranny. + +Six days ago, Russia’s Vladimir Putin sought to shake the foundations of the free world thinking he could make it bend to his menacing ways. + +But he badly miscalculated. + +He thought he could roll into Ukraine and the world would roll over. + +Instead he met a wall of strength he never imagined. + +He met the Ukrainian people. + +From President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world. + +Groups of citizens blocking tanks with their bodies. \ No newline at end of file diff --git a/tools/langchain_text_splitters/test-data/overlap_words.txt b/tools/langchain_text_splitters/test-data/overlap_words.txt new file mode 100644 index 0000000000..cd4781dcb7 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/overlap_words.txt @@ -0,0 +1 @@ +one two three four five diff --git a/tools/langchain_text_splitters/test-data/recursive_default.txt b/tools/langchain_text_splitters/test-data/recursive_default.txt new file mode 100644 index 0000000000..7f57fb2e60 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/recursive_default.txt @@ -0,0 +1,5 @@ +Alpha beta. + +Gamma delta. + +Epsilon zeta. diff --git a/tools/langchain_text_splitters/test-data/simple_separator.txt b/tools/langchain_text_splitters/test-data/simple_separator.txt new file mode 100644 index 0000000000..fd35b2591f --- /dev/null +++ b/tools/langchain_text_splitters/test-data/simple_separator.txt @@ -0,0 +1 @@ +alpha.beta.gamma diff --git a/tools/langchain_text_splitters/test-data/special_token.txt b/tools/langchain_text_splitters/test-data/special_token.txt new file mode 100644 index 0000000000..74de15e945 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/special_token.txt @@ -0,0 +1 @@ +Before <|endoftext|> after diff --git a/tools/langchain_text_splitters/test-data/token_words.txt b/tools/langchain_text_splitters/test-data/token_words.txt new file mode 100644 index 0000000000..e1a7d0645a --- /dev/null +++ b/tools/langchain_text_splitters/test-data/token_words.txt @@ -0,0 +1 @@ +one two three four five six seven eight diff --git a/tools/langchain_text_splitters/test-data/tsv_escape.txt b/tools/langchain_text_splitters/test-data/tsv_escape.txt new file mode 100644 index 0000000000..d633373a9e --- /dev/null +++ b/tools/langchain_text_splitters/test-data/tsv_escape.txt @@ -0,0 +1,2 @@ +alpha beta +omega diff --git a/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv b/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv new file mode 100644 index 0000000000..1fe0d23d97 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv @@ -0,0 +1 @@ +1 alpha\tbeta\nomega\n 17 diff --git a/tools/langchain_text_splitters/test-data/unicode_separator.txt b/tools/langchain_text_splitters/test-data/unicode_separator.txt new file mode 100644 index 0000000000..2e100bac28 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/unicode_separator.txt @@ -0,0 +1 @@ +alpha。beta。gamma From a50e82144541387aad387203ba24687934d7a2b1 Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Mon, 3 Aug 2026 22:02:31 +0200 Subject: [PATCH 02/17] Address first langchain_text_splitters code reviews - Remove the folder dev_utils and its content - Remove .gitignore inside the test-data folder - Merge macros_for_testing.xml into macros.xml - Remove macro for setting the TIKTOKEN_CACHE_DIR env var - Add "AI4Social+" to the creators - Addresss flake8 warnings --- .../dev_utils/README.md | 48 ------------------- .../dev_utils/get_encodings.py | 27 ----------- .../langchain_text_splitters.xml | 2 - tools/langchain_text_splitters/macros.xml | 16 ++++--- .../macros_for_testing.xml | 10 ---- tools/langchain_text_splitters/split_text.py | 8 ++-- .../test-data/.gitignore | 1 - 7 files changed, 15 insertions(+), 97 deletions(-) delete mode 100644 tools/langchain_text_splitters/dev_utils/README.md delete mode 100644 tools/langchain_text_splitters/dev_utils/get_encodings.py delete mode 100644 tools/langchain_text_splitters/macros_for_testing.xml delete mode 100644 tools/langchain_text_splitters/test-data/.gitignore diff --git a/tools/langchain_text_splitters/dev_utils/README.md b/tools/langchain_text_splitters/dev_utils/README.md deleted file mode 100644 index 17fc20f139..0000000000 --- a/tools/langchain_text_splitters/dev_utils/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# LangChain Text Splitters Galaxy Tool Dev Utilities - -## Populate tiktoken cache directory - -Download the OpenAI encoding files using the script `get_encodings.py` in this folder. -This will populate the cache directory `/tmp/tiktoken-cache`. - -## Testing tiktoken without internet access in Galaxy - -! Prerequisite: The OpenAI encoding files must be downloaded and the cache directory populated as described above. - -To test whether a Galaxy tool can use a locally cached tiktoken encoding without internet access, -save the following configuration as job_conf_offline.xml: - -```xml - - - - - - - - - /tmp/tiktoken-cache - - http://127.0.0.1:9 - http://127.0.0.1:9 - - - -``` -Start Planemo using the job configuration above: -`planemo serve --job_config_file dev_utils/job_conf_offline.xml` - -You can test as follows that the tool works without internet access by setting -**Text splitting boundaries** to: - -**(A) Between tokens**. - -**(B) Character**, then set **Target chunk size should be determined by the count of** to **Tokens**. - -To double check that the tool actually fails when the cache is empty, remove or rename the cache directory `/tmp/tiktoken-cache` and run it as described above. It should run into a -`ConnectionRefusedError` when trying to download the encoding files from the OpenAI servers. \ No newline at end of file diff --git a/tools/langchain_text_splitters/dev_utils/get_encodings.py b/tools/langchain_text_splitters/dev_utils/get_encodings.py deleted file mode 100644 index 18db45f852..0000000000 --- a/tools/langchain_text_splitters/dev_utils/get_encodings.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from pathlib import Path -from tiktoken import get_encoding - -TIKTOKEN_CACHE_DIR = "/tmp/tiktoken-cache" -os.environ["TIKTOKEN_CACHE_DIR"] = TIKTOKEN_CACHE_DIR - -TIKTOKEN_ENCODINGS = [ - "o200k_harmony", - "o200k_base", - "cl100k_base", - "p50k_edit", - "p50k_base", - "r50k_base", - "gpt2", -] - -Path(TIKTOKEN_CACHE_DIR).mkdir(parents=True, exist_ok=True) - -print(f"Populating tiktoken cache directory {TIKTOKEN_CACHE_DIR}") - -for encoding_name in TIKTOKEN_ENCODINGS: - print(f"Getting {encoding_name}...") - get_encoding(encoding_name) diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index 1e16a9467a..7090aa03a5 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -2,7 +2,6 @@ Split text into chunks for LLM and RAG workflows macros.xml - macros_for_testing.xml @@ -76,7 +75,6 @@ --allowed-special '$splitter.tiktoken_options.allowed_special' #end if ]]> - diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml index 4a5d9666cb..e4b92ae3f1 100644 --- a/tools/langchain_text_splitters/macros.xml +++ b/tools/langchain_text_splitters/macros.xml @@ -11,14 +11,9 @@ + - - - - /tmp/tiktoken-cache - - @@ -109,4 +104,13 @@ + + + + + + + + + diff --git a/tools/langchain_text_splitters/macros_for_testing.xml b/tools/langchain_text_splitters/macros_for_testing.xml deleted file mode 100644 index 48ec21e315..0000000000 --- a/tools/langchain_text_splitters/macros_for_testing.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index 2feedef3ba..55b20924da 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -38,7 +38,9 @@ r"\\(?:[\\nrt]|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})" ) + def parse_args(): + parser = argparse.ArgumentParser( description="Split text with langchain-text-splitters." ) @@ -102,7 +104,7 @@ def build_tiktoken_counter( if allowed_special is None: allowed_special = set() - + # Note: below needs either the TIKTOKEN_CACHE_DIR environment variable set # respectively the directory populated with `python3 get_encodings.py` in the folder dev_utils. # or internet access to download the encodings from the OpenAI servers. @@ -122,8 +124,8 @@ def count_tokens(text): return count_tokens + def decode_custom_separator(value): - """Decode supported escape notation without altering literal Unicode.""" simple_escapes = { r"\n": "\n", r"\r": "\r", @@ -201,7 +203,7 @@ def build_splitter(args, input_text, length_function, tiktoken_options): f"{separator!r} was not found in the input text. " "Select a separator that occurs in the input or use the " "recursive character splitter." - ) + ) return CharacterTextSplitter( **character_options, diff --git a/tools/langchain_text_splitters/test-data/.gitignore b/tools/langchain_text_splitters/test-data/.gitignore deleted file mode 100644 index 5303bbab27..0000000000 --- a/tools/langchain_text_splitters/test-data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -daniela_llm-hub_input.txt \ No newline at end of file From d9c79776ce7fcce6cadbe01cf7d86f9ba03403c2 Mon Sep 17 00:00:00 2001 From: Arash <2973722+arash77@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:39:08 +0200 Subject: [PATCH 03/17] langchain_text_splitters: fix output correctness issues found in review Input handling: - Decode the input as UTF-8 instead of Path.read_text(), which silently rewrote CRLF/CR line endings and shifted the reported start indices. - Report invalid UTF-8 input with a clear message instead of a traceback. - Reject input that is empty or whitespace only instead of writing empty outputs. TSV output: - Escape backslash, tab, carriage return and newline so that the chunk text stays on one row and the escaping can be reversed. The previous escaping could not be undone, and csv.writer additionally quoted any chunk containing a double quote. Errors and metadata: - Report a disallowed special token with a clear message on the character splitter as well, not only on the token splitter. - Report an invalid start index as null instead of replacing the number with a string, so the JSON field keeps a single type. The warning on stdout is unchanged. - Use a single "length" key per chunk together with the existing "length_unit" instead of character_count/token_count. - Report strip_whitespace as false in token mode, where the splitter never applies it. - Write all outputs as UTF-8 instead of the system locale encoding. Command line and separators: - Pass a custom separator as --separator=VALUE so that a separator starting with a dash is not read as an option. - Reject unsupported and out-of-range custom separator escapes with a clear message instead of crashing or passing them through silently. - Make --encoding-name and --model-name mutually exclusive. Tests and docs: - Add tests for whitespace-only input, a disallowed special token on the character splitter, a dash-prefixed custom separator, and a predefined separator inside the repeat. - Expect null start indices in the two overlapping token tests, which hit the same upstream bug as the dedicated repro test. - Document the tiktoken encoding cache, the TSV escaping and the JSON fields in the tool help. - Deduplicate the two tiktoken command line blocks into a macro. --- .../langchain_text_splitters.xml | 200 ++++++++++++++++-- tools/langchain_text_splitters/macros.xml | 9 + tools/langchain_text_splitters/split_text.py | 187 +++++++++++----- .../test-data/dash_separator.txt | 1 + .../test-data/tsv_escape.txt | 1 + .../test-data/tsv_escape_expected.tsv | 2 +- .../test-data/whitespace_only.txt | 3 + 7 files changed, 331 insertions(+), 72 deletions(-) create mode 100644 tools/langchain_text_splitters/test-data/dash_separator.txt create mode 100644 tools/langchain_text_splitters/test-data/whitespace_only.txt diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index 7090aa03a5..bdccccf6bb 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -37,12 +37,8 @@ --length-mode '$splitter.length.length_mode' #if str($splitter.length.length_mode) == "token" - #if str($splitter.length.tiktoken_options.tokenizer.tokenizer_selection_method) == "model" - --model-name '$splitter.length.tiktoken_options.tokenizer.model_name' - #else - --encoding-name '$splitter.length.tiktoken_options.tokenizer.encoding_name' - #end if - --allowed-special '$splitter.length.tiktoken_options.allowed_special' + #set $tiktoken_options = $splitter.length.tiktoken_options + @TIKTOKEN_ARGS@ #end if #if $splitter.character_splitter.type == "recursive" @@ -58,7 +54,8 @@ #end if #else #if $splitter.character_splitter.separator_definition.sep_opt == "custom" - --separator '$splitter.character_splitter.separator_definition.value' + ## Passed as a single token so that a separator starting with a dash is not read as an option. + --separator='$splitter.character_splitter.separator_definition.value' #else --separator-name '$splitter.character_splitter.separator_definition.sep_opt.value' #end if @@ -67,12 +64,8 @@ #else --splitter-type token --length-mode token - #if str($splitter.tiktoken_options.tokenizer.tokenizer_selection_method) == "model" - --model-name '$splitter.tiktoken_options.tokenizer.model_name' - #else - --encoding-name '$splitter.tiktoken_options.tokenizer.encoding_name' - #end if - --allowed-special '$splitter.tiktoken_options.allowed_special' + #set $tiktoken_options = $splitter.tiktoken_options + @TIKTOKEN_ARGS@ #end if ]]> @@ -537,9 +530,13 @@ - + + + + + @@ -579,12 +576,18 @@ + + - + + + + + @@ -660,11 +663,16 @@ - + + + + + + - + @@ -787,13 +795,146 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + **What it does** @@ -811,13 +952,28 @@ and other LLM workflows where long text needs to be bounded by a chunk size. **Used Library for Token Counting** -OpenAI's fast Byte-Pair Encoding (BPE) tokenizer `tiktoken` library is used -to determine the number of tokens in a chunk. +OpenAI's fast Byte-Pair Encoding (BPE) tokenizer `tiktoken` library is used +to determine the number of tokens in a chunk. + +`tiktoken` does not ship the encoding files. On first use it downloads them from +the OpenAI servers, so a Galaxy instance without outgoing internet access on its +compute nodes has to provide them locally by pointing the `TIKTOKEN_CACHE_DIR` +environment variable at a directory that already contains the encoding files. +Without either, any run that counts tokens fails. + +**Inputs** + +The input dataset has to be UTF-8 encoded text. Line endings are preserved as +they are, so a dataset with Windows (`\r\n`) line endings keeps them in the +chunks. **Outputs** -- Text and JSON file with input/chunk metadata, where each chunk comes along with its size and start index in the input text. -- TSV file with the following columns: chunk number, content, size in characters or tokens +- Text and JSON file with input/chunk metadata, where each chunk comes along with its size and start index in the input text. + The chunk size is reported in the `length` field, and `length_unit` states whether it counts characters or tokens. + A `start_index` of `null` means that the splitter returned an invalid position for that chunk; a warning is written to the tool log in that case. +- TSV file with the following columns: chunk number, content, size in characters or tokens. + So that every chunk stays on a single row, backslash, tab, carriage return and newline in the content are escaped as `\\`, `\t`, `\r` and `\n`. Reverse the escaping to recover the exact chunk content, or use the collection below, which holds the raw text. - Galaxy collection of text files, with one raw chunk per file **License** diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml index e4b92ae3f1..447717a13e 100644 --- a/tools/langchain_text_splitters/macros.xml +++ b/tools/langchain_text_splitters/macros.xml @@ -2,6 +2,15 @@ 1.1.2 0 25.1 + + langchain-text-splitters diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index 55b20924da..6f88444ac4 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse -import csv import json import re import sys @@ -38,6 +37,21 @@ r"\\(?:[\\nrt]|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})" ) +MAX_UNICODE_CODE_POINT = 0x10FFFF +SURROGATE_RANGE = range(0xD800, 0xE000) + +# The chunk text is stored in a single TSV cell, so every character that would +# otherwise start a new row or a new column has to be escaped. The backslash is +# escaped first so that the transformation stays reversible. +TSV_ESCAPES = ( + ("\\", r"\\"), + ("\t", r"\t"), + ("\r", r"\r"), + ("\n", r"\n"), +) + +LENGTH_KEY = "length" + def parse_args(): @@ -61,8 +75,9 @@ def parse_args(): choices=("characters", "token"), default="characters", ) - parser.add_argument("--encoding-name", default="gpt2") - parser.add_argument("--model-name", default="") + tokenizer_group = parser.add_mutually_exclusive_group() + tokenizer_group.add_argument("--encoding-name", default="gpt2") + tokenizer_group.add_argument("--model-name", default="") parser.add_argument( "--allowed-special", choices=("none", "all"), @@ -83,6 +98,21 @@ def parse_args(): return parser.parse_args() +def read_input_text(input_path): + # The bytes are decoded explicitly instead of using Path.read_text(), which + # would apply universal newline translation and silently rewrite \r\n and \r + # line endings as \n. That would change both the chunk content and the + # reported start indices with respect to the input dataset. + try: + return input_path.read_bytes().decode("utf-8") + except UnicodeDecodeError as error: + sys.exit( + "The input dataset is not valid UTF-8 text " + f"(invalid byte at offset {error.start}). " + "Convert the dataset to UTF-8 before splitting it." + ) + + def get_tiktoken_options(args): allow_all_special = args.allowed_special == "all" @@ -94,6 +124,20 @@ def get_tiktoken_options(args): } +def fail_on_disallowed_special_token(error): + # Raised by tiktoken when the input contains a special token string such as + # <|endoftext|> while the user asked for those strings to be rejected. + if "disallowed special token" not in str(error): + raise error + + sys.exit( + "The input contains a special token string that is rejected by the " + "selected tokenizer. Set the special token handling to allow all " + "special token strings, or remove the special token from the input.\n" + f"Original error: {error}" + ) + + def build_tiktoken_counter( encoding_name, model_name=None, @@ -105,22 +149,27 @@ def build_tiktoken_counter( if allowed_special is None: allowed_special = set() - # Note: below needs either the TIKTOKEN_CACHE_DIR environment variable set - # respectively the directory populated with `python3 get_encodings.py` in the folder dev_utils. - # or internet access to download the encodings from the OpenAI servers. + # Note: below needs either internet access to download the encodings from + # the OpenAI servers, or the TIKTOKEN_CACHE_DIR environment variable + # pointing to a directory that already holds the encoding files. The + # tiktoken conda package does not ship them. if model_name: encoding = tiktoken.encoding_for_model(model_name) else: encoding = tiktoken.get_encoding(encoding_name) def count_tokens(text): - return len( - encoding.encode( + try: + tokens = encoding.encode( text, allowed_special=allowed_special, disallowed_special=disallowed_special, ) - ) + except ValueError as error: + fail_on_disallowed_special_token(error) + raise + + return len(tokens) return count_tokens @@ -133,15 +182,45 @@ def decode_custom_separator(value): r"\\": "\\", } - def replace_escape(match): + decoded = [] + position = 0 + + while position < len(value): + character = value[position] + + if character != "\\": + decoded.append(character) + position += 1 + continue + + match = CUSTOM_SEPARATOR_ESCAPE_PATTERN.match(value, position) + + if match is None: + sys.exit( + "The custom separator contains an unsupported escape sequence " + f"at position {position}: {value[position:position + 10]!r}. " + r"Supported escapes are \\, \n, \r, \t, " + r"\uXXXX (4 hex digits) and \UXXXXXXXX (8 hex digits)." + ) + escape = match.group(0) if escape in simple_escapes: - return simple_escapes[escape] + decoded.append(simple_escapes[escape]) + else: + code_point = int(escape[2:], 16) + + if code_point > MAX_UNICODE_CODE_POINT or code_point in SURROGATE_RANGE: + sys.exit( + f"The custom separator escape {escape} is not a valid " + "Unicode character." + ) + + decoded.append(chr(code_point)) - return chr(int(escape[2:], 16)) + position = match.end() - return CUSTOM_SEPARATOR_ESCAPE_PATTERN.sub(replace_escape, value) + return "".join(decoded) def resolve_separator( @@ -211,11 +290,7 @@ def build_splitter(args, input_text, length_function, tiktoken_options): ) if args.separator_specs is not None: - separators = ( - resolve_separator_specs(args.separator_specs) - if args.separator_specs is not None - else None - ) + separators = resolve_separator_specs(args.separator_specs) # Append an empty string as the final fallback separator to ensure that the text can always be split, # even if none of the tried separators before were able to split the text without exceeding the chunk size. character_options["separators"] = [*separators, ""] @@ -227,7 +302,6 @@ def write_text_output( output_path, metadata, chunks, - count_key, length_label, ): lines = [ @@ -246,14 +320,14 @@ def write_text_output( [ ( f"--- Chunk {chunk['index']}: " - f"{chunk[count_key]} {length_label}, " + f"{chunk[LENGTH_KEY]} {length_label}, " f"start={chunk['start_index']} ---" ), chunk["text"], "", ] ) - output_path.write_text("\n".join(lines)) + output_path.write_text("\n".join(lines), encoding="utf-8") def write_chunk_files(chunks_dir, chunks): @@ -263,26 +337,30 @@ def write_chunk_files(chunks_dir, chunks): chunk_path = chunks_dir / f"chunk_{chunk['index']:04d}.txt" chunk_path.write_text( chunk["text"], + encoding="utf-8", ) -def write_tsv_output(output_path, chunks, count_key): - with output_path.open("w") as handle: - writer = csv.writer( - handle, - delimiter="\t", - lineterminator="\n", - ) +def escape_tsv_text(text): + for raw, escaped in TSV_ESCAPES: + text = text.replace(raw, escaped) + + return text + +def write_tsv_output(output_path, chunks): + # The rows are written without the csv module on purpose. Its writer would + # additionally apply CSV quoting to any chunk containing a double quote, + # which the documented escaping above cannot undo. escape_tsv_text() already + # removes every character that could break the column or row structure. + with output_path.open("w", encoding="utf-8", newline="") as handle: for chunk in chunks: - table_text = chunk["text"].replace("\t", r"\t").replace("\n", r"\n") - writer.writerow( - [ - chunk["index"], - table_text, - chunk[count_key], - ] - ) + fields = [ + str(chunk["index"]), + escape_tsv_text(chunk["text"]), + str(chunk[LENGTH_KEY]), + ] + handle.write("\t".join(fields) + "\n") def main(): @@ -291,7 +369,13 @@ def main(): if args.chunk_overlap >= args.chunk_size: sys.exit("Chunk overlap must be smaller than chunk size.") - input_text = args.input.read_text() + input_text = read_input_text(args.input) + + if not input_text.strip(): + sys.exit( + "The input dataset is empty or contains only whitespace. " + "There is nothing to split." + ) length_mode = "token" if args.splitter_type == "token" else args.length_mode @@ -299,13 +383,11 @@ def main(): if length_mode == "token": length_function = build_tiktoken_counter(**tiktoken_options) - count_key = "token_count" length_label = "tokens" tokenizer_label = args.model_name or args.encoding_name length_function_label = f"tiktoken:{tokenizer_label}" else: length_function = len - count_key = "character_count" length_label = "characters" length_function_label = "characters" @@ -315,7 +397,12 @@ def main(): length_function, tiktoken_options, ) - documents = splitter.create_documents([input_text]) + + try: + documents = splitter.create_documents([input_text]) + except ValueError as error: + fail_on_disallowed_special_token(error) + raise chunks = [] start_index_warnings = [] @@ -336,16 +423,15 @@ def main(): # Target chunk size should be determined by the number of tokens # Target chunk size: 100 # Chunk overlap: 20 + # The invalid index is reported as null so that the JSON keeps a + # single type for the field. The warning below carries the detail. start_index_warnings.append((chunk_number, start_index)) - start_index = ( - "Potential upstream langchain-text-splitters bug: " - f"received invalid start index {start_index!r}" - ) + start_index = None chunks.append( { "index": chunk_number, - count_key: length_function(raw_chunk_text), + LENGTH_KEY: length_function(raw_chunk_text), "start_index": start_index, "text": raw_chunk_text, } @@ -358,7 +444,8 @@ def main(): ) print( "WARNING: Potential upstream langchain-text-splitters bug: " - f"invalid start index returned for chunk(s): {affected_chunks}", + f"invalid start index returned for chunk(s): {affected_chunks}. " + "The start index of these chunks is reported as null.", flush=True, ) @@ -373,7 +460,11 @@ def main(): "number_of_chunks": len(chunks), } - if args.splitter_type != "token": + if args.splitter_type == "token": + # The token splitter has no separators and never strips whitespace, so + # the value is reported as false regardless of what was requested. + metadata["strip_whitespace"] = False + else: metadata.update( { "keep_separator": args.keep_separator, @@ -393,13 +484,11 @@ def main(): args.output_text, metadata, chunks, - count_key, length_label, ) write_tsv_output( args.output_tsv, chunks, - count_key, ) write_chunk_files( args.chunks_dir, diff --git a/tools/langchain_text_splitters/test-data/dash_separator.txt b/tools/langchain_text_splitters/test-data/dash_separator.txt new file mode 100644 index 0000000000..4c4b2bfb35 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/dash_separator.txt @@ -0,0 +1 @@ +alpha---beta---gamma diff --git a/tools/langchain_text_splitters/test-data/tsv_escape.txt b/tools/langchain_text_splitters/test-data/tsv_escape.txt index d633373a9e..375c76aac5 100644 --- a/tools/langchain_text_splitters/test-data/tsv_escape.txt +++ b/tools/langchain_text_splitters/test-data/tsv_escape.txt @@ -1,2 +1,3 @@ alpha beta +C:\path omega diff --git a/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv b/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv index 1fe0d23d97..902660bb43 100644 --- a/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv +++ b/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv @@ -1 +1 @@ -1 alpha\tbeta\nomega\n 17 +1 alpha\tbeta\nC:\\path\nomega\n 25 diff --git a/tools/langchain_text_splitters/test-data/whitespace_only.txt b/tools/langchain_text_splitters/test-data/whitespace_only.txt new file mode 100644 index 0000000000..216aec0f98 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/whitespace_only.txt @@ -0,0 +1,3 @@ + + + From e1dd4a7d0be56eab21f59ad6d328063f74adda88 Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Wed, 5 Aug 2026 19:45:44 +0200 Subject: [PATCH 04/17] New TextSplitter POC - Add NLTKTextSplitter and SpacyTextSplitter - Add Arash as creator *Note this is so far still a proof of concept. There are no spaCy or NLTK tests yet. So not merge ready. --- .../langchain_text_splitters.xml | 69 +++++++++++- tools/langchain_text_splitters/macros.xml | 6 ++ tools/langchain_text_splitters/split_text.py | 100 ++++++++++++++++-- 3 files changed, 164 insertions(+), 11 deletions(-) diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index bdccccf6bb..d57517474e 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -10,7 +10,7 @@ Character + @@ -133,6 +151,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -950,6 +1015,8 @@ and other LLM workflows where long text needs to be bounded by a chunk size. - `CharacterTextSplitter`: Splits text using one defined separator or character sequence, such as a paragraph break, line break, space, or custom separator. - **Token-based splitting:** The `TokenTextSplitter` ignores any potential text structure and simply creates chunks of a specified token length. + + **Used Library for Token Counting** OpenAI's fast Byte-Pair Encoding (BPE) tokenizer `tiktoken` library is used diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml index 447717a13e..f7eba8d2d0 100644 --- a/tools/langchain_text_splitters/macros.xml +++ b/tools/langchain_text_splitters/macros.xml @@ -15,11 +15,17 @@ langchain-text-splitters tiktoken + nltk + spacy + spacy-model-en_core_web_sm + + python-gil + diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index 6f88444ac4..93ee4fbb9d 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -8,7 +8,9 @@ from langchain_text_splitters import ( CharacterTextSplitter, + NLTKTextSplitter, RecursiveCharacterTextSplitter, + SpacyTextSplitter, TokenTextSplitter, ) @@ -65,9 +67,21 @@ def parse_args(): parser.add_argument("--chunks-dir", type=Path, required=True) parser.add_argument( "--splitter-type", - choices=("recursive_character", "character", "token"), + choices=("recursive_character", "character", "nltk", "spacy", "token"), default="recursive_character", ) + parser.add_argument("--sentence-language", default="english") + parser.add_argument( + "--spacy-pipeline", + choices=("sentencizer", "en_core_web_sm"), + default="sentencizer", + ) + parser.add_argument( + "--spacy-max-length", + type=int, + default=1_000_000, + ) + parser.add_argument("--chunk-size", type=int, required=True) parser.add_argument("--chunk-overlap", type=int, default=0) parser.add_argument( @@ -263,10 +277,32 @@ def build_splitter(args, input_text, length_function, tiktoken_options): **tiktoken_options, ) - character_options = { + length_options = { **common_options, - "strip_whitespace": args.strip_whitespace, "length_function": length_function, + } + + if args.splitter_type == "nltk": + return NLTKTextSplitter( + **length_options, + language=args.sentence_language, + separator="", + use_span_tokenize=True, + strip_whitespace=False, + ) + + if args.splitter_type == "spacy": + return SpacyTextSplitter( + **length_options, + pipeline=args.spacy_pipeline, + max_length=args.spacy_max_length, + separator="", + strip_whitespace=False, + ) + + character_options = { + **length_options, + "strip_whitespace": args.strip_whitespace, "keep_separator": KEEP_SEPARATOR_VALUES[args.keep_separator], } @@ -391,15 +427,42 @@ def main(): length_label = "characters" length_function_label = "characters" - splitter = build_splitter( - args, - input_text, - length_function, - tiktoken_options, - ) + try: + splitter = build_splitter( + args, + input_text, + length_function, + tiktoken_options, + ) + except OSError as error: + if args.splitter_type == "spacy": + sys.exit( + f"The selected spaCy pipeline {args.spacy_pipeline!r} " + "is not installed in the tool environment.\n" + f"Original error: {error}" + ) + raise try: documents = splitter.create_documents([input_text]) + except LookupError as error: + if args.splitter_type == "nltk": + # nltk is searching for the punkt_tab data by default here: + # -/usr/share/nltk_data + # -/usr/local/share/nltk_data + # -/usr/lib/nltk_data + # -/usr/local/lib/nltk_data + # alternative conda package: + # https://anaconda.org/channels/conda-forge/packages/nltk_data/overview + # but it is from 2022.05.27 so quite outdated + # here is the feedstock https://github.com/conda-forge/nltk_data-feedstock + sys.exit( + "The NLTK Punkt data required for the selected language is not " + "installed. The Galaxy tool environment must provide the " + "'punkt_tab' NLTK resource.\n" + f"Original error: {error}" + ) + raise except ValueError as error: fail_on_disallowed_special_token(error) raise @@ -464,7 +527,8 @@ def main(): # The token splitter has no separators and never strips whitespace, so # the value is reported as false regardless of what was requested. metadata["strip_whitespace"] = False - else: + + elif args.splitter_type in ("character", "recursive_character"): metadata.update( { "keep_separator": args.keep_separator, @@ -472,6 +536,22 @@ def main(): } ) + elif args.splitter_type == "nltk": + metadata.update( + { + "sentence_language": args.sentence_language, + "strip_whitespace": False, + } + ) + + elif args.splitter_type == "spacy": + metadata.update( + { + "spacy_pipeline": args.spacy_pipeline, + "strip_whitespace": False, + } + ) + args.output_json.write_text( json.dumps( {**metadata, "chunks": chunks}, From d80e5c7b0c0d4a0c0100407c1ffe45a2a53890f8 Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Thu, 6 Aug 2026 11:57:51 +0200 Subject: [PATCH 05/17] Add test cases for NLTK and spaCy splitters - Tests 11, 12 and 14 are still failing --- .../langchain_text_splitters.xml | 269 ++++++++++++++++++ tools/langchain_text_splitters/macros.xml | 2 +- .../test-data/sentence_nltk_english.txt | 3 + .../test-data/sentence_nltk_german.txt | 3 + .../test-data/sentence_overlap.txt | 4 + .../test-data/sentence_spacy.txt | 3 + .../test-data/sentence_tokens.txt | 3 + 7 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 tools/langchain_text_splitters/test-data/sentence_nltk_english.txt create mode 100644 tools/langchain_text_splitters/test-data/sentence_nltk_german.txt create mode 100644 tools/langchain_text_splitters/test-data/sentence_overlap.txt create mode 100644 tools/langchain_text_splitters/test-data/sentence_spacy.txt create mode 100644 tools/langchain_text_splitters/test-data/sentence_tokens.txt diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index d57517474e..afc2665cff 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -621,6 +621,275 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml index f7eba8d2d0..d476b871b4 100644 --- a/tools/langchain_text_splitters/macros.xml +++ b/tools/langchain_text_splitters/macros.xml @@ -19,7 +19,7 @@ spacy spacy-model-en_core_web_sm - python-gil + python-gil diff --git a/tools/langchain_text_splitters/test-data/sentence_nltk_english.txt b/tools/langchain_text_splitters/test-data/sentence_nltk_english.txt new file mode 100644 index 0000000000..ea4f6368e4 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/sentence_nltk_english.txt @@ -0,0 +1,3 @@ +Dr. Smith arrived early. +The meeting started promptly. +Everyone took notes. \ No newline at end of file diff --git a/tools/langchain_text_splitters/test-data/sentence_nltk_german.txt b/tools/langchain_text_splitters/test-data/sentence_nltk_german.txt new file mode 100644 index 0000000000..5c5b619082 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/sentence_nltk_german.txt @@ -0,0 +1,3 @@ +Das ist z.B. wichtig. +Der Test beginnt jetzt. +Alles funktioniert gut. \ No newline at end of file diff --git a/tools/langchain_text_splitters/test-data/sentence_overlap.txt b/tools/langchain_text_splitters/test-data/sentence_overlap.txt new file mode 100644 index 0000000000..ac65da5036 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/sentence_overlap.txt @@ -0,0 +1,4 @@ +Alpha one. +Beta two. +Gamma three. +Delta four. \ No newline at end of file diff --git a/tools/langchain_text_splitters/test-data/sentence_spacy.txt b/tools/langchain_text_splitters/test-data/sentence_spacy.txt new file mode 100644 index 0000000000..8e1bcfa9d1 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/sentence_spacy.txt @@ -0,0 +1,3 @@ +First sentence is short. +Second sentence is also short. +Third sentence ends here. \ No newline at end of file diff --git a/tools/langchain_text_splitters/test-data/sentence_tokens.txt b/tools/langchain_text_splitters/test-data/sentence_tokens.txt new file mode 100644 index 0000000000..9bfa262fff --- /dev/null +++ b/tools/langchain_text_splitters/test-data/sentence_tokens.txt @@ -0,0 +1,3 @@ +one two. +three four. +five six. \ No newline at end of file From 9b7191fa5eab92450a6182f53572f65832804394 Mon Sep 17 00:00:00 2001 From: Arash <2973722+arash77@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:39:07 +0200 Subject: [PATCH 06/17] langchain_text_splitters: fix the failing sentence splitter tests Tests 11, 12 and 14 could not be fixed by adjusting the assertions, each had a different cause. Test 12 asserted "\nASecond sentence ..." with a stray A, and its third element missed the trailing newline. planemo stops at the first failing element, so only one of the two showed up. Test 11 is unreachable at chunk size 23. The sentences are 10, 10, 13 and 12 characters long and langchain drops the carried over sentence until the incoming one fits, so the third chunk only keeps the overlap at a chunk size of 25 or more. Retuned to 25 / 13. Test 14 ended in a fourth chunk holding a single newline. Galaxy's upload appends a trailing newline to the input, and the sentencizer reports it as a sentence of its own. Since the sentence splitters keep the whitespace between the sentences and never strip a chunk, that newline survived into the outputs. Chunks that hold nothing but whitespace are now left out and reported in a warning, so the test is back to three chunks. Fixes found while testing the new sentence splitters: - The LookupError handler for the missing NLTK Punkt data was unreachable. langchain loads the tokenizer in the constructor, so the error is raised in build_splitter() and not in create_documents(). KeyError and IndexError are excluded so that an unrelated lookup failure is not reported as missing Punkt data. - --spacy-max-length was ignored for the sentencizer, which langchain builds from English() without forwarding max_length. Any input above one million characters failed with a raw spaCy traceback. - The maximum input length was unbounded. en_core_web_sm keeps the dependency parser and needs roughly 3.5 kB per input character, so a large value could exhaust the memory of a shared compute node. - The input length is now checked before the tiktoken encoding is loaded, so a job that cannot run does not pay for it first. - The NLTK splitter drops the text after the last detected sentence, so its chunks do not add up to the input. This is now reported in a warning and documented. The spaCy splitter keeps everything. - The help section still held a TODO instead of the sentence splitter documentation, and .shed.yml did not mention sentence splitting. Note that punkt_tab is still not covered by a requirement. The conda-forge nltk_data package ships punkt and not punkt_tab, so the NLTK splitter keeps failing where the resource is not provided by the instance. The error message now says so instead of showing a traceback. --- tools/langchain_text_splitters/.shed.yml | 5 +- .../langchain_text_splitters.xml | 56 +++++++--- tools/langchain_text_splitters/split_text.py | 102 +++++++++++++++--- 3 files changed, 134 insertions(+), 29 deletions(-) diff --git a/tools/langchain_text_splitters/.shed.yml b/tools/langchain_text_splitters/.shed.yml index 9a514eaf8f..af0209a17c 100644 --- a/tools/langchain_text_splitters/.shed.yml +++ b/tools/langchain_text_splitters/.shed.yml @@ -4,8 +4,9 @@ description: Split text into chunks using LangChain text splitters. long_description: | Split plain-text-like datasets into chunks for LLM and RAG workflows using langchain-text-splitters. The tool currently supports recursive character, character, - and tiktoken-based token splitting, and reports per-chunk lengths and start - offsets as both readable text and structured JSON. + sentence-based splitting with NLTK or spaCy, and tiktoken-based token splitting, + and reports per-chunk lengths and start offsets as both readable text and + structured JSON. type: unrestricted categories: - Natural Language Processing diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index afc2665cff..75524fe48b 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -181,7 +181,10 @@ English model (en_core_web_sm) - + + @@ -710,7 +713,9 @@ - + @@ -721,14 +726,14 @@ - - + + - - + + @@ -787,12 +792,13 @@ - + - + + @@ -867,25 +873,33 @@ + + + + - + + - + - + @@ -1283,8 +1297,26 @@ and other LLM workflows where long text needs to be bounded by a chunk size. - `RecursiveCharacterTextSplitter`: Recommended for most use cases as it attempts to preserve larger text units by trying an ordered list of separators, from coarser boundaries such as paragraph breaks to finer boundaries such as line breaks, or spaces, down to individual characters. - `CharacterTextSplitter`: Splits text using one defined separator or character sequence, such as a paragraph break, line break, space, or custom separator. - **Token-based splitting:** The `TokenTextSplitter` ignores any potential text structure and simply creates chunks of a specified token length. +- **Sentence-based splitting:** Detects sentence boundaries first and then fills the chunks with whole sentences, so a chunk never ends in the middle of a sentence. Pick this for prose. A single sentence that is already longer than the target chunk size becomes an oversized chunk of its own, because the sentence is never cut. + - `NLTKTextSplitter`: Uses NLTK's Punkt sentence tokenizer, which is trained per language and therefore handles language-specific abbreviations such as `Dr.` in English or `z.B.` in German. Select the language of the input. + - `SpacyTextSplitter`: Uses spaCy. The rule-based sentencizer is fast and needs no model, but it applies English tokenization rules. The `en_core_web_sm` model is more accurate on English text, and is roughly five times slower and needs about ten times more memory. + +Both sentence splitters keep the whitespace between the sentences, so a chunk +starts with the line break that follows the previous sentence. The chunk overlap +is applied in whole sentences: an overlapping sentence is only carried over if it +fits into the next chunk together with the following sentence, otherwise the +overlap is silently smaller than requested. + +The NLTK splitter drops the text after the last detected sentence, usually the +trailing line break, so its chunks do not always add up to the complete input; a +warning is written to the tool log when that happens. The spaCy splitter keeps +everything. - +The NLTK splitter needs the `punkt_tab` resource for the selected language. It is +not part of the `nltk` package, so a Galaxy instance has to provide it locally, +for example below a directory listed in the `NLTK_DATA` environment variable. +Without it, any run of the NLTK splitter fails. The spaCy splitter has no such +requirement. **Used Library for Token Counting** diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index 93ee4fbb9d..6ed414f54e 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -283,6 +283,10 @@ def build_splitter(args, input_text, length_function, tiktoken_options): } if args.splitter_type == "nltk": + # separator="" together with strip_whitespace=False keeps the chunk text + # identical to the matching slice of the input, so the reported start + # indices stay usable. Note that the span based tokenizer drops whatever + # follows the last sentence, see the warning in main(). return NLTKTextSplitter( **length_options, language=args.sentence_language, @@ -292,13 +296,23 @@ def build_splitter(args, input_text, length_function, tiktoken_options): ) if args.splitter_type == "spacy": - return SpacyTextSplitter( + splitter = SpacyTextSplitter( **length_options, pipeline=args.spacy_pipeline, max_length=args.spacy_max_length, separator="", strip_whitespace=False, ) + # langchain only forwards max_length to the pipelines it loads with + # spacy.load(). The sentencizer is built from English() instead and + # silently keeps the spaCy default of one million characters, so it is + # applied here for both. Guarded because _tokenizer is private API. + tokenizer = getattr(splitter, "_tokenizer", None) + + if hasattr(tokenizer, "max_length"): + tokenizer.max_length = args.spacy_max_length + + return splitter character_options = { **length_options, @@ -413,6 +427,17 @@ def main(): "There is nothing to split." ) + # Checked before the tokenizer is built so that a job that cannot run does + # not first pay for loading the tiktoken encoding. + if args.splitter_type == "spacy" and len(input_text) > args.spacy_max_length: + sys.exit( + f"The input dataset holds {len(input_text)} characters, which is " + "more than the configured spaCy maximum input length of " + f"{args.spacy_max_length}. Raise 'Maximum input length' or split " + "the dataset into smaller parts first. Mind the memory cost per " + "input character stated in the help of that setting." + ) + length_mode = "token" if args.splitter_type == "token" else args.length_mode tiktoken_options = get_tiktoken_options(args) @@ -442,37 +467,57 @@ def main(): f"Original error: {error}" ) raise - - try: - documents = splitter.create_documents([input_text]) except LookupError as error: - if args.splitter_type == "nltk": - # nltk is searching for the punkt_tab data by default here: - # -/usr/share/nltk_data - # -/usr/local/share/nltk_data - # -/usr/lib/nltk_data - # -/usr/local/lib/nltk_data - # alternative conda package: - # https://anaconda.org/channels/conda-forge/packages/nltk_data/overview - # but it is from 2022.05.27 so quite outdated - # here is the feedstock https://github.com/conda-forge/nltk_data-feedstock + # NLTKTextSplitter loads the punkt_tab data in its constructor, so this + # has to be caught around build_splitter() and not around + # create_documents(). + # nltk is searching for the punkt_tab data by default here: + # -/usr/share/nltk_data + # -/usr/local/share/nltk_data + # -/usr/lib/nltk_data + # -/usr/local/lib/nltk_data + # alternative conda package: + # https://anaconda.org/channels/conda-forge/packages/nltk_data/overview + # but it is from 2022.05.27 and ships 'punkt' instead of 'punkt_tab' + # here is the feedstock https://github.com/conda-forge/nltk_data-feedstock + # KeyError and IndexError also derive from LookupError, so they are + # excluded to keep an unrelated failure from being reported as missing + # Punkt data. + if args.splitter_type == "nltk" and not isinstance( + error, (KeyError, IndexError) + ): sys.exit( "The NLTK Punkt data required for the selected language is not " "installed. The Galaxy tool environment must provide the " - "'punkt_tab' NLTK resource.\n" + "'punkt_tab' NLTK resource, for example below a directory " + "listed in the NLTK_DATA environment variable. Use the spaCy " + "sentence splitter if the data cannot be installed.\n" f"Original error: {error}" ) raise + + try: + documents = splitter.create_documents([input_text]) except ValueError as error: fail_on_disallowed_special_token(error) raise chunks = [] start_index_warnings = [] + empty_chunks = 0 for document in documents: raw_chunk_text = document.page_content start_index = document.metadata.get("start_index") + + # A chunk that holds nothing but whitespace carries no content for a + # downstream step to work on. The sentence splitters produce one for the + # trailing line break of the input, because they keep the whitespace + # between the sentences and never strip a chunk. + if not raw_chunk_text.strip(): + empty_chunks += 1 + continue + chunk_number = len(chunks) + 1 if start_index is not None and start_index < 0: @@ -512,6 +557,33 @@ def main(): flush=True, ) + # The NLTK splitter builds the chunks from the sentence spans reported by + # punkt, which end at the last sentence. Punkt trims trailing whitespace, so + # whatever follows is dropped and the chunks no longer add up to the input. + # The first span always starts at offset 0, so nothing is lost in front. + # The spaCy splitter keeps everything. + if empty_chunks: + print( + f"WARNING: {empty_chunks} chunk(s) held nothing but whitespace and " + "were left out of the outputs.", + flush=True, + ) + + if args.splitter_type == "nltk" and chunks: + last_text = chunks[-1]["text"] + last_start = input_text.rfind(last_text) + dropped = ( + len(input_text) - (last_start + len(last_text)) if last_start >= 0 else 0 + ) + + if dropped > 0: + print( + f"WARNING: The NLTK sentence splitter dropped the {dropped} " + "character(s) after the last sentence. Use the spaCy sentence " + "splitter to keep the complete input.", + flush=True, + ) + metadata = { "input_characters": len(input_text), "input_length": length_function(input_text), From 6b3338cad5df69bbe9539a8b290aa56ccebaa4ee Mon Sep 17 00:00:00 2001 From: Arash <2973722+arash77@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:23:27 +0200 Subject: [PATCH 07/17] langchain_text_splitters: separate the chunk diagnostics and harden the separator input Chunk diagnostics: - Detect a start index that is invalid without being negative. Once the splitter returns one negative index, the positions it derives from it stay positive but point at an earlier part of the input, which was reported as if it were correct. The position is now checked for order and for content. - Report text that the splitter altered separately from a wrong position. Splitting between tokens can cut a character that is encoded in several bytes, which replaces it with the Unicode replacement character. Those chunks used to be reported as an invalid start index, pointing the user at the wrong cause. Custom separator: - Widen the sanitizer so that characters such as | ; ~ % & $ @ < > " [ ] { } reach the script instead of being silently replaced by an X, which split the text at the wrong places. - Reject a separator that still contains a character Galaxy cannot pass on unchanged, with a message pointing at the escape syntax. Other: - Write the JSON output as UTF-8 as well, instead of the system locale encoding. - Let special_token_error() return the error to raise, so the special token handling has a single contract instead of an unreachable re-raise at both call sites. Tests: - Add a test for a separator that the sanitizer used to mangle. --- .../langchain_text_splitters.xml | 39 ++++++ tools/langchain_text_splitters/macros.xml | 18 +++ tools/langchain_text_splitters/split_text.py | 114 ++++++++++++++---- .../test-data/pipe_separator.txt | 1 + 4 files changed, 149 insertions(+), 23 deletions(-) create mode 100644 tools/langchain_text_splitters/test-data/pipe_separator.txt diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index 75524fe48b..16e34c152f 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -1190,6 +1190,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml index d476b871b4..12988d4b82 100644 --- a/tools/langchain_text_splitters/macros.xml +++ b/tools/langchain_text_splitters/macros.xml @@ -60,13 +60,31 @@ + + + + + + + + + + + + + + + + ^[A-Za-z0-9 \-=_.()/+*^,:?!\\#|;~%&$@<>"\[\]{}]+\Z diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index 6ed414f54e..5683dc035a 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -54,6 +54,10 @@ LENGTH_KEY = "length" +# Reasons why a chunk cannot be traced back to the input, see diagnose_chunk(). +CHUNK_TEXT_ALTERED = "chunk_text_altered" +START_INDEX_INVALID = "start_index_invalid" + def parse_args(): @@ -138,13 +142,17 @@ def get_tiktoken_options(args): } -def fail_on_disallowed_special_token(error): - # Raised by tiktoken when the input contains a special token string such as - # <|endoftext|> while the user asked for those strings to be rejected. +def special_token_error(error): + """Return the error to raise for a ValueError coming from tiktoken. + + tiktoken raises when the input contains a special token string such as + <|endoftext|> while the user asked for those strings to be rejected. Any + other ValueError is handed back unchanged. + """ if "disallowed special token" not in str(error): - raise error + return error - sys.exit( + return SystemExit( "The input contains a special token string that is rejected by the " "selected tokenizer. Set the special token handling to allow all " "special token strings, or remove the special token from the input.\n" @@ -180,8 +188,7 @@ def count_tokens(text): disallowed_special=disallowed_special, ) except ValueError as error: - fail_on_disallowed_special_token(error) - raise + raise special_token_error(error) return len(tokens) @@ -348,6 +355,33 @@ def build_splitter(args, input_text, length_function, tiktoken_options): return RecursiveCharacterTextSplitter(**character_options) +def diagnose_chunk( + start_index, + chunk_text, + input_text, + previous_start_index, +): + """Return None when the chunk and its reported position are sound. + + Otherwise return the reason, so that the two very different causes can be + reported separately: text the splitter altered, and a position the splitter + got wrong. + """ + if ( + start_index >= 0 + and (previous_start_index is None or start_index > previous_start_index) + and input_text.startswith(chunk_text, start_index) + ): + return None + + # Only reached when something is already wrong, so scanning the whole input + # once more is acceptable here. + if chunk_text not in input_text: + return CHUNK_TEXT_ALTERED + + return START_INDEX_INVALID + + def write_text_output( output_path, metadata, @@ -499,11 +533,12 @@ def main(): try: documents = splitter.create_documents([input_text]) except ValueError as error: - fail_on_disallowed_special_token(error) - raise + raise special_token_error(error) chunks = [] start_index_warnings = [] + altered_text_warnings = [] + previous_start_index = None empty_chunks = 0 for document in documents: @@ -520,20 +555,41 @@ def main(): chunk_number = len(chunks) + 1 - if start_index is not None and start_index < 0: - # TODO: report upstream to langchain-text-splitters as a bug, since this should not happen. - # Potential cause: langchain text splitters might calculate the start index based on the length of the chunk in characters, even though the chunk size is measured in tokens. - # reproduce with Character splitter recursive - # input: test-data/langchain_docs_sample.txt - # Text splitting separators and their order of use: default - # Place separator at start of the following chunk start → ["First paragraph.", "\n\nSecond paragraph."] - # Strip whitespace around chunks: true - # Target chunk size should be determined by the number of tokens - # Target chunk size: 100 - # Chunk overlap: 20 - # The invalid index is reported as null so that the JSON keeps a - # single type for the field. The warning below carries the detail. - start_index_warnings.append((chunk_number, start_index)) + # TODO: report the invalid start index upstream to langchain-text-splitters, since this + # should not happen. Note that the invalid indices are not always negative: once one chunk + # gets a negative index, the positions derived from it stay positive but point at the wrong + # place, which is why diagnose_chunk() also checks the order and the content. + # Potential cause: langchain text splitters might calculate the start index based on the length of the chunk in characters, even though the chunk size is measured in tokens. + # reproduce with Character splitter recursive + # input: test-data/langchain_docs_sample.txt + # Text splitting separators and their order of use: default + # Place separator at start of the following chunk start → ["First paragraph.", "\n\nSecond paragraph."] + # Strip whitespace around chunks: true + # Target chunk size should be determined by the number of tokens + # Target chunk size: 100 + # Chunk overlap: 20 + problem = ( + None + if start_index is None + else diagnose_chunk( + start_index, + raw_chunk_text, + input_text, + previous_start_index, + ) + ) + + if problem is None: + if start_index is not None: + previous_start_index = start_index + else: + # The position is reported as null so that the JSON field keeps a + # single type. The warnings below carry the detail. + if problem == CHUNK_TEXT_ALTERED: + altered_text_warnings.append(chunk_number) + else: + start_index_warnings.append((chunk_number, start_index)) + start_index = None chunks.append( @@ -545,6 +601,17 @@ def main(): } ) + if altered_text_warnings: + print( + "WARNING: The text of the following chunk(s) does not occur in the " + f"input: {', '.join(str(number) for number in altered_text_warnings)}. " + "Splitting between tokens cuts the text at token boundaries, which " + "can fall inside a character that is encoded in several bytes and " + "replaces it with the Unicode replacement character. Use one of the " + "character based splitters for text that is not plain ASCII.", + flush=True, + ) + if start_index_warnings: affected_chunks = ", ".join( f"{chunk_number} (start_index: {start_index!r})" @@ -630,6 +697,7 @@ def main(): indent=2, ) + "\n", + encoding="utf-8", ) write_text_output( diff --git a/tools/langchain_text_splitters/test-data/pipe_separator.txt b/tools/langchain_text_splitters/test-data/pipe_separator.txt new file mode 100644 index 0000000000..df9c19ae89 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/pipe_separator.txt @@ -0,0 +1 @@ +alpha|beta|gamma From 47dba876d01ef7344d8348fecba3e6aea0550bb2 Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Thu, 6 Aug 2026 18:22:52 +0200 Subject: [PATCH 08/17] langchain_text_splitters: add nltk_data as requirement --- tools/langchain_text_splitters/macros.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml index 12988d4b82..0b5f0b3f72 100644 --- a/tools/langchain_text_splitters/macros.xml +++ b/tools/langchain_text_splitters/macros.xml @@ -16,6 +16,7 @@ langchain-text-splitters tiktoken nltk + nltk_data spacy spacy-model-en_core_web_sm From 25164333c5dc9540d3ed638013e1754d2f11ef7e Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Fri, 7 Aug 2026 20:59:11 +0200 Subject: [PATCH 09/17] langchain_text_splitters: - Add strip_whitespace option for SpacyTextSpliiter; refer to it in the tool help and change the params of one test case to strip_whitespace=True - Wrap strip_whitespace in macro - Add explanation why strip_whitespace is fixed to false for the NLTK text splitter - Populate the NLTK language options with all the in punkt_tab available 19 languages - Set the chunk_overlap default to "0" - Add a TODO related to galaxy adding by default a "\n" to each upload when there is no terminal new line and made one text fail on purpose so we remember to address this before the merge --- .../langchain_text_splitters.xml | 66 +++++++++++++------ tools/langchain_text_splitters/macros.xml | 5 +- tools/langchain_text_splitters/split_text.py | 18 ++++- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index 16e34c152f..86f989855a 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -70,6 +70,7 @@ #else --spacy-pipeline '$splitter.sentence_splitter.pipeline' --spacy-max-length '$splitter.sentence_splitter.max_length' + $splitter.strip_whitespace #end if #if str($splitter.length.length_mode) == "token" @@ -136,7 +137,7 @@ - + @@ -153,38 +154,51 @@ - - - + + + + + + - - + + + + + + + - + + + + + + Rule-based sentencizer (fast, no model required) + + English model (en_core_web_sm) + + @@ -625,6 +639,11 @@ + @@ -670,6 +689,9 @@ + + + @@ -766,6 +788,7 @@ + @@ -812,6 +835,7 @@ + @@ -832,12 +856,12 @@ - + - + @@ -851,6 +875,7 @@ + @@ -1340,9 +1365,10 @@ and other LLM workflows where long text needs to be bounded by a chunk size. - `NLTKTextSplitter`: Uses NLTK's Punkt sentence tokenizer, which is trained per language and therefore handles language-specific abbreviations such as `Dr.` in English or `z.B.` in German. Select the language of the input. - `SpacyTextSplitter`: Uses spaCy. The rule-based sentencizer is fast and needs no model, but it applies English tokenization rules. The `en_core_web_sm` model is more accurate on English text, and is roughly five times slower and needs about ten times more memory. -Both sentence splitters keep the whitespace between the sentences, so a chunk -starts with the line break that follows the previous sentence. The chunk overlap -is applied in whole sentences: an overlapping sentence is only carried over if it +Both sentence splitters preserve whitespace between sentences by default, +in that case a chunk starts with the line break that follows the previous sentence. +For spaCy, leading and trailing whitespace can optionally be removed from each returned chunk. +The chunk overlap is applied in whole sentences: an overlapping sentence is only carried over if it fits into the next chunk together with the following sentence, otherwise the overlap is silently smaller than requested. diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml index 0b5f0b3f72..4e817116c7 100644 --- a/tools/langchain_text_splitters/macros.xml +++ b/tools/langchain_text_splitters/macros.xml @@ -30,6 +30,9 @@ + + + @@ -136,7 +139,7 @@ - + diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index 5683dc035a..e094afadb0 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -122,7 +122,7 @@ def read_input_text(input_path): # line endings as \n. That would change both the chunk content and the # reported start indices with respect to the input dataset. try: - return input_path.read_bytes().decode("utf-8") + text = input_path.read_bytes().decode("utf-8") except UnicodeDecodeError as error: sys.exit( "The input dataset is not valid UTF-8 text " @@ -130,6 +130,18 @@ def read_input_text(input_path): "Convert the dataset to UTF-8 before splitting it." ) + # When a file without an empty last line gets uploaded to Galaxy, + # an \n will be append automatically see convert_newlines() in galaxy/lib/galaxy/datatypes/sniff.py + + # TODO: Thus, we need a way to address this in a robust way just + # return text.removesuffix("\n") leads to 6 six failing tests (6,12,14,15,16,17) + + # if we just keep the \n we face in the NLTK splitter case the warning below. + # WARNING: The NLTK sentence splitter dropped the 1 character(s) after the last sentence [...] + # To reproduce use sentence_nltk_english.txt, with target chunk size in characters: 30 + # Or run planemo test --test_index 9 + return text + def get_tiktoken_options(args): allow_all_special = args.allowed_special == "all" @@ -308,7 +320,7 @@ def build_splitter(args, input_text, length_function, tiktoken_options): pipeline=args.spacy_pipeline, max_length=args.spacy_max_length, separator="", - strip_whitespace=False, + strip_whitespace=args.strip_whitespace, ) # langchain only forwards max_length to the pipelines it loads with # spacy.load(). The sentencizer is built from English() instead and @@ -687,7 +699,7 @@ def main(): metadata.update( { "spacy_pipeline": args.spacy_pipeline, - "strip_whitespace": False, + "strip_whitespace": args.strip_whitespace, } ) From 1c5464e0dfe2d80653abf3f9155b170eb9720d2f Mon Sep 17 00:00:00 2001 From: Arash <2973722+arash77@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:16:52 +0200 Subject: [PATCH 10/17] langchain_text_splitters: fix whitespace handling in the sentence splitters Two bugs silently changed the user's text. spaCy with "strip whitespace" deleted the space between sentences. langchain strips every sentence before joining them and we join with an empty separator, so "mat. The" became "mat.The". The chunk then no longer occurred in the input, so its start index was reported as null. spaCy is now built with strip_whitespace=False and the stripping is done on the finished chunk, which is what the option promises and keeps the chunk a slice of the input. Whitespace-only chunks were dropped for every splitter. A run of blank lines is content for the token and character splitters, and dropping it renumbered the chunks that followed, so 64 characters of a chapter break disappeared. They are now only dropped when stripping was asked for and nothing is left. Also: - the spaCy input limit is set per pipeline; one limit of 10 million characters allowed about 44 GB with the English model - the "text does not occur in the input" warning no longer claims token splitting is the only cause - the NLTK dropped-text check uses the chunk's start index instead of rfind(), which found the last occurrence and so reported no loss when the dropped text repeated the final chunk - strip_whitespace is reported once from the argument instead of being hardcoded per splitter - the punkt_tab notes match the nltk_data requirement - test 9 pins the NLTK warning instead of failing on purpose - new test on space separated prose; every other sentence fixture separates with a line break, which is why this was never caught --- .../langchain_text_splitters.xml | 150 +++++++++++++----- tools/langchain_text_splitters/split_text.py | 137 +++++++++------- .../test-data/sentence_spacy_prose.txt | 1 + 3 files changed, 190 insertions(+), 98 deletions(-) create mode 100644 tools/langchain_text_splitters/test-data/sentence_spacy_prose.txt diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index 86f989855a..ab6c871098 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -68,8 +68,8 @@ #if str($splitter.sentence_splitter.type) == "nltk" --sentence-language '$splitter.sentence_splitter.language' #else - --spacy-pipeline '$splitter.sentence_splitter.pipeline' - --spacy-max-length '$splitter.sentence_splitter.max_length' + --spacy-pipeline '$splitter.sentence_splitter.pipeline_settings.pipeline' + --spacy-max-length '$splitter.sentence_splitter.pipeline_settings.max_length' $splitter.strip_whitespace #end if @@ -136,7 +136,8 @@ - + @@ -186,18 +187,29 @@ - - - - - - + + + + + + + + + + + + + @@ -638,12 +650,16 @@ - - + @@ -690,7 +706,7 @@ - + @@ -786,8 +802,10 @@ - - + + + + @@ -833,8 +851,10 @@ - - + + + + @@ -866,6 +886,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -873,8 +942,10 @@ - - + + + + @@ -899,17 +970,18 @@ - + upload appends as a fourth sentence. Whitespace-only + chunks are kept when stripping is off, so that the + chunks still add up to the whole input. --> + - + - + @@ -927,6 +999,11 @@ + + + + + @@ -1377,11 +1454,8 @@ trailing line break, so its chunks do not always add up to the complete input; a warning is written to the tool log when that happens. The spaCy splitter keeps everything. -The NLTK splitter needs the `punkt_tab` resource for the selected language. It is -not part of the `nltk` package, so a Galaxy instance has to provide it locally, -for example below a directory listed in the `NLTK_DATA` environment variable. -Without it, any run of the NLTK splitter fails. The spaCy splitter has no such -requirement. +The NLTK splitter needs the `punkt_tab` resource for the selected language. It +comes with the `nltk_data` requirement, so there is nothing to set up. **Used Library for Token Counting** diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index e094afadb0..27a76c264b 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -130,16 +130,17 @@ def read_input_text(input_path): "Convert the dataset to UTF-8 before splitting it." ) - # When a file without an empty last line gets uploaded to Galaxy, - # an \n will be append automatically see convert_newlines() in galaxy/lib/galaxy/datatypes/sniff.py - - # TODO: Thus, we need a way to address this in a robust way just - # return text.removesuffix("\n") leads to 6 six failing tests (6,12,14,15,16,17) - - # if we just keep the \n we face in the NLTK splitter case the warning below. - # WARNING: The NLTK sentence splitter dropped the 1 character(s) after the last sentence [...] - # To reproduce use sentence_nltk_english.txt, with target chunk size in characters: 30 - # Or run planemo test --test_index 9 + # Galaxy appends a newline to an uploaded file that has none, see + # convert_newlines() in galaxy/lib/galaxy/datatypes/sniff.py. It is + # deliberately kept here. An uploaded "abc" and an uploaded "abc\n" both + # arrive as "abc\n", so the two cases cannot be told apart, and removing the + # newline would cut a real character off every input that legitimately ends + # in one, which is the common case. The chunks would then no longer add up + # to the input, and the character based splitters would become lossy for no + # gain. It would not even settle the NLTK case, because punkt discards all + # trailing whitespace, so an input ending in a blank line still loses more + # than the one appended character. What NLTK drops is reported instead, see + # the warning at the end of main(). return text @@ -315,12 +316,16 @@ def build_splitter(args, input_text, length_function, tiktoken_options): ) if args.splitter_type == "spacy": + # strip_whitespace is handled on the finished chunks in main() instead of + # here. langchain strips every sentence before joining them, and + # separator="" joins with nothing, so letting it strip would also delete + # the space *between* two sentences and produce "mat.The". splitter = SpacyTextSplitter( **length_options, pipeline=args.spacy_pipeline, max_length=args.spacy_max_length, separator="", - strip_whitespace=args.strip_whitespace, + strip_whitespace=False, ) # langchain only forwards max_length to the pipelines it loads with # spacy.load(). The sentencizer is built from English() instead and @@ -517,15 +522,14 @@ def main(): # NLTKTextSplitter loads the punkt_tab data in its constructor, so this # has to be caught around build_splitter() and not around # create_documents(). - # nltk is searching for the punkt_tab data by default here: + # The data comes from the nltk_data requirement, so this is reached only + # on an environment that does not provide it, or provides it outside the + # directories nltk searches: # -/usr/share/nltk_data # -/usr/local/share/nltk_data # -/usr/lib/nltk_data # -/usr/local/lib/nltk_data - # alternative conda package: - # https://anaconda.org/channels/conda-forge/packages/nltk_data/overview - # but it is from 2022.05.27 and ships 'punkt' instead of 'punkt_tab' - # here is the feedstock https://github.com/conda-forge/nltk_data-feedstock + # -any directory listed in NLTK_DATA # KeyError and IndexError also derive from LookupError, so they are # excluded to keep an unrelated failure from being reported as missing # Punkt data. @@ -557,13 +561,30 @@ def main(): raw_chunk_text = document.page_content start_index = document.metadata.get("start_index") - # A chunk that holds nothing but whitespace carries no content for a - # downstream step to work on. The sentence splitters produce one for the - # trailing line break of the input, because they keep the whitespace - # between the sentences and never strip a chunk. - if not raw_chunk_text.strip(): - empty_chunks += 1 - continue + if args.strip_whitespace: + # Only the outer whitespace of the finished chunk is removed, which + # is what the option promises. The chunk therefore stays a verbatim + # slice of the input and its start index stays valid, so it only has + # to move by whatever came off the front. + # + # The splitter has already packed the chunks at this point, counting + # the whitespace that is removed here, so a stripped chunk can come + # out shorter than the requested size. That is the conservative + # direction, and measuring the stripped text instead would mean + # stripping before the packing, which is exactly what produces + # "mat.The". + leading = len(raw_chunk_text) - len(raw_chunk_text.lstrip()) + raw_chunk_text = raw_chunk_text.strip() + + if start_index is not None and start_index >= 0: + start_index += leading + + # Nothing is left for a downstream step to work on. Reachable only + # when the user asked for stripping, so no content is lost: without + # it every chunk is kept, whitespace included. + if not raw_chunk_text: + empty_chunks += 1 + continue chunk_number = len(chunks) + 1 @@ -617,10 +638,12 @@ def main(): print( "WARNING: The text of the following chunk(s) does not occur in the " f"input: {', '.join(str(number) for number in altered_text_warnings)}. " - "Splitting between tokens cuts the text at token boundaries, which " - "can fall inside a character that is encoded in several bytes and " - "replaces it with the Unicode replacement character. Use one of the " - "character based splitters for text that is not plain ASCII.", + "The splitter returned text it had modified, so no position in the " + "input describes it and the start index is reported as null. The " + "known cause is splitting between tokens: a cut can fall inside a " + "character that is encoded in several bytes, which then becomes the " + "Unicode replacement character. Use one of the character based " + "splitters for text that is not plain ASCII.", flush=True, ) @@ -636,21 +659,30 @@ def main(): flush=True, ) - # The NLTK splitter builds the chunks from the sentence spans reported by - # punkt, which end at the last sentence. Punkt trims trailing whitespace, so - # whatever follows is dropped and the chunks no longer add up to the input. - # The first span always starts at offset 0, so nothing is lost in front. - # The spaCy splitter keeps everything. if empty_chunks: print( f"WARNING: {empty_chunks} chunk(s) held nothing but whitespace and " - "were left out of the outputs.", + "were left out of the outputs, because stripping the whitespace " + "around the chunks left them empty.", flush=True, ) - if args.splitter_type == "nltk" and chunks: - last_text = chunks[-1]["text"] - last_start = input_text.rfind(last_text) + # The NLTK splitter builds the chunks from the sentence spans reported by + # punkt, which end at the last sentence. Punkt trims trailing whitespace, so + # whatever follows is dropped and the chunks no longer add up to the input. + # The first span always starts at offset 0, so nothing is lost in front. + # The spaCy splitter keeps everything. + if args.splitter_type == "nltk" and chunks and not args.strip_whitespace: + last_chunk = chunks[-1] + last_text = last_chunk["text"] + # The validated start index is preferred over searching for the text: + # rfind() returns the *last* occurrence, so it would report no loss + # whenever the dropped text happens to repeat the final chunk. + last_start = last_chunk["start_index"] + + if last_start is None: + last_start = input_text.rfind(last_text) + dropped = ( len(input_text) - (last_start + len(last_text)) if last_start >= 0 else 0 ) @@ -672,36 +704,21 @@ def main(): "chunk_size": args.chunk_size, "chunk_overlap": args.chunk_overlap, "number_of_chunks": len(chunks), + # Reported from the argument rather than per splitter, because the + # stripping is done above for whichever splitter produced the chunks. + # A splitter that does not offer the option never receives it, so the + # value stays false for it without having to be hardcoded here. + "strip_whitespace": args.strip_whitespace, } - if args.splitter_type == "token": - # The token splitter has no separators and never strips whitespace, so - # the value is reported as false regardless of what was requested. - metadata["strip_whitespace"] = False - - elif args.splitter_type in ("character", "recursive_character"): - metadata.update( - { - "keep_separator": args.keep_separator, - "strip_whitespace": args.strip_whitespace, - } - ) + if args.splitter_type in ("character", "recursive_character"): + metadata["keep_separator"] = args.keep_separator elif args.splitter_type == "nltk": - metadata.update( - { - "sentence_language": args.sentence_language, - "strip_whitespace": False, - } - ) + metadata["sentence_language"] = args.sentence_language elif args.splitter_type == "spacy": - metadata.update( - { - "spacy_pipeline": args.spacy_pipeline, - "strip_whitespace": args.strip_whitespace, - } - ) + metadata["spacy_pipeline"] = args.spacy_pipeline args.output_json.write_text( json.dumps( diff --git a/tools/langchain_text_splitters/test-data/sentence_spacy_prose.txt b/tools/langchain_text_splitters/test-data/sentence_spacy_prose.txt new file mode 100644 index 0000000000..4dd187abe6 --- /dev/null +++ b/tools/langchain_text_splitters/test-data/sentence_spacy_prose.txt @@ -0,0 +1 @@ +The cat sat on the mat. The dog barked loudly. Birds flew away. From 0c7213d9e6a521b0de53831fad4700a1686a7dca Mon Sep 17 00:00:00 2001 From: Ivo Leist Date: Thu, 27 Aug 2026 14:45:39 +0200 Subject: [PATCH 11/17] - allow strip_whitespace for the NLTK splitter - add disclaimer that strip_whitespace is not carried by langchain but by the main function in our python wrapper --- .../langchain_text_splitters.xml | 69 ++++++++++++++++--- tools/langchain_text_splitters/split_text.py | 3 +- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index ab6c871098..0d5f76a752 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -70,8 +70,8 @@ #else --spacy-pipeline '$splitter.sentence_splitter.pipeline_settings.pipeline' --spacy-max-length '$splitter.sentence_splitter.pipeline_settings.max_length' - $splitter.strip_whitespace #end if + $splitter.strip_whitespace #if str($splitter.length.length_mode) == "token" #set $tiktoken_options = $splitter.length.tiktoken_options @@ -155,7 +155,7 @@ - + @@ -210,9 +210,9 @@ - + @@ -709,6 +709,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -806,8 +849,8 @@ - + @@ -855,8 +898,8 @@ - + @@ -901,8 +944,8 @@ - + @@ -946,8 +989,8 @@ - + @@ -1442,9 +1485,15 @@ and other LLM workflows where long text needs to be bounded by a chunk size. - `NLTKTextSplitter`: Uses NLTK's Punkt sentence tokenizer, which is trained per language and therefore handles language-specific abbreviations such as `Dr.` in English or `z.B.` in German. Select the language of the input. - `SpacyTextSplitter`: Uses spaCy. The rule-based sentencizer is fast and needs no model, but it applies English tokenization rules. The `en_core_web_sm` model is more accurate on English text, and is roughly five times slower and needs about ten times more memory. -Both sentence splitters preserve whitespace between sentences by default, -in that case a chunk starts with the line break that follows the previous sentence. -For spaCy, leading and trailing whitespace can optionally be removed from each returned chunk. +Both sentence splitters preserve whitespace between sentences by default; in +that case a chunk starts with the line break that follows the previous sentence. +Leading and trailing whitespace can optionally be removed from each returned +chunk. However, for this we do not use the strip_whitespace option of the underlying splitters, +since in spaCy it would also delete the whitespace between sentences and in NLTK it could +cause the start index of a chunk to be reported incorrectly. Instead our python wrapper applies +the whitespace stripping only after LangChain has assembled a finished chunk. +Thus, preventing the two issues stated above. + The chunk overlap is applied in whole sentences: an overlapping sentence is only carried over if it fits into the next chunk together with the following sentence, otherwise the overlap is silently smaller than requested. diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py index 27a76c264b..0efec681a6 100644 --- a/tools/langchain_text_splitters/split_text.py +++ b/tools/langchain_text_splitters/split_text.py @@ -303,7 +303,8 @@ def build_splitter(args, input_text, length_function, tiktoken_options): } if args.splitter_type == "nltk": - # separator="" together with strip_whitespace=False keeps the chunk text + # strip_whitespace is handled on the finished chunks in main() instead of + # here. separator="" together with strip_whitespace=False keeps the chunk text # identical to the matching slice of the input, so the reported start # indices stay usable. Note that the span based tokenizer drops whatever # follows the last sentence, see the warning in main(). From 4331ee422f439efb4e7c57b69a6359348c53a90e Mon Sep 17 00:00:00 2001 From: Arash <2973722+arash77@users.noreply.github.com> Date: Tue, 1 Sep 2026 10:31:40 +0200 Subject: [PATCH 12/17] langchain_text_splitters: stop before Galaxy's dataset limit and fix two misleading warnings - fail early when the settings would produce more chunks than the instance's max_discovered_files, instead of splitting everything and then letting Galaxy fail the job on the file count - name the cause that fits the run in the altered-text warning instead of always blaming multi-byte tokens - stop reporting a null start index for overlapping chunks that legitimately begin at the same offset - set strip_whitespace once for every splitter instead of per branch - correct the help on whitespace at a chunk boundary, and let three tests assert the warning instead of pinning the upstream -1 --- .../langchain_text_splitters.xml | 36 +++++--- tools/langchain_text_splitters/split_text.py | 91 ++++++++++++++----- 2 files changed, 94 insertions(+), 33 deletions(-) diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml index 0d5f76a752..21d09a43d5 100644 --- a/tools/langchain_text_splitters/langchain_text_splitters.xml +++ b/tools/langchain_text_splitters/langchain_text_splitters.xml @@ -26,6 +26,9 @@ --chunks-dir chunks --chunk-size '$chunk_size' --chunk-overlap '$chunk_overlap' + ## The instance's own ceiling on datasets discovered per job, so the tool + ## stops early instead of being failed by Galaxy after all the work. + --max-chunk-files '$__app__.config.max_discovered_files' #if $splitter.method == "character" #if $splitter.character_splitter.type == "recursive" --splitter-type recursive_character @@ -44,7 +47,7 @@ #if $splitter.character_splitter.type == "recursive" #if $splitter.character_splitter.separator_settings.mode == "custom" --separator-specs - #for $separator in $splitter.character_splitter.separator_settings.separators + #for $separator in $splitter.character_splitter.separator_settings.separators #if $separator.separator_definition.sep_opt == "custom" 'custom:$separator.separator_definition.value' #else @@ -88,7 +91,7 @@ -