diff --git a/tools/langchain_text_splitters/.shed.yml b/tools/langchain_text_splitters/.shed.yml
new file mode 100644
index 0000000000..b4e17aba94
--- /dev/null
+++ b/tools/langchain_text_splitters/.shed.yml
@@ -0,0 +1,17 @@
+name: langchain_text_splitters
+owner: bgruening
+description: Split text into chunks using LangChain text splitters.
+long_description: |
+ Split plain-text-like datasets into chunks for LLM and RAG workflows using
+ langchain-text-splitters. The tool currently supports recursive character, character,
+ sentence-based splitting with NLTK or spaCy, and tiktoken-based token splitting,
+ and reports per-chunk lengths and start offsets as both readable text and
+ structured JSON.
+type: unrestricted
+categories:
+ - Natural Language Processing
+remote_repository_url: https://github.com/bgruening/galaxytools/tree/master/tools/langchain_text_splitters
+homepage_url: https://github.com/langchain-ai/langchain/tree/master/libs/text-splitters
+auto_tool_repositories:
+ name_template: "{{ tool_id }}"
+ description_template: "Wrapper for: {{ tool_name }}"
diff --git a/tools/langchain_text_splitters/langchain_text_splitters.xml b/tools/langchain_text_splitters/langchain_text_splitters.xml
new file mode 100644
index 0000000000..df777772c3
--- /dev/null
+++ b/tools/langchain_text_splitters/langchain_text_splitters.xml
@@ -0,0 +1,1629 @@
+
+ Split text into chunks for LLM and RAG workflows
+
+ macros.xml
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+**What it does**
+
+This tool splits a text dataset into chunks with `langchain-text-splitters`.
+It is useful before retrieval-augmented generation, embedding, summarization,
+and other LLM workflows where long text needs to be bounded by a chunk size.
+
+**Leveraged Methods of `langchain-text-splitters`**
+
+- **Character-based splitting:** Splits text at character-based separators. The target chunk size can be measured in characters or tokens.
+ - `RecursiveCharacterTextSplitter`: Recommended for most use cases as it attempts to preserve larger text units by trying an ordered list of separators, from coarser boundaries such as paragraph breaks to finer boundaries such as line breaks, or spaces, down to individual characters.
+ - `CharacterTextSplitter`: Splits text using one defined separator or character sequence, such as a paragraph break, line break, space, or custom separator.
+- **Token-based splitting:** The `TokenTextSplitter` ignores any potential text structure and simply creates chunks of a specified token length.
+- **Sentence-based splitting:** Detects sentence boundaries first and then fills the chunks with whole sentences, so a chunk never ends in the middle of a sentence. Pick this for prose. A single sentence that is already longer than the target chunk size becomes an oversized chunk of its own, because the sentence is never cut.
+ - `NLTKTextSplitter`: Uses NLTK's Punkt sentence tokenizer, which is trained per language and therefore handles language-specific abbreviations such as `Dr.` in English or `z.B.` in German. Select the language of the input.
+ - `SpacyTextSplitter`: Uses spaCy. The rule-based sentencizer is fast and needs no model, but it applies English tokenization rules. The `en_core_web_sm` model is more accurate on English text, and is roughly five times slower and needs about ten times more memory.
+
+For all splitters except `TokenTextSplitter`, leading and trailing whitespace can be removed while preserving whitespace
+between sentences within a chunk. We do not use the `strip_whitespace` option provided by [LangChain's
+text splitters](https://reference.langchain.com/python/langchain-text-splitters/base/TextSplitter),
+because it can produce incorrect chunk start indices relative to the input text.
+Additionally, some splitters, such as `SpacyTextSplitter`, may remove whitespace between
+sentences within the same chunk. Instead, our Python wrapper strips whitespace only after each chunk has been assembled.
+This preserves in between sentences whitespace and allows chunk positions to be determined more reliably.
+If a chunk still cannot be located in the input text, a warning is written to the Tool Standard Output.
+
+The chunk overlap is applied in whole sentences: an overlapping sentence is only carried over if it
+fits into the next chunk together with the following sentence, otherwise the
+overlap is silently smaller than requested.
+
+The NLTK splitter drops the text after the last detected sentence, usually the
+trailing line break, so its chunks do not always add up to the complete input.
+When that happens, a warning is written to the Tool Standard Output.
+The spaCy splitter in contrast keeps everything.
+
+**Used Library for Token Counting**
+
+OpenAI's fast Byte-Pair Encoding (BPE) tokenizer `tiktoken` library is used
+to determine the number of tokens in a chunk.
+
+`tiktoken` does not ship the encoding files. On first use it downloads them from
+the OpenAI servers, so a Galaxy instance without outgoing internet access on its
+compute nodes has to provide them locally by pointing the `TIKTOKEN_CACHE_DIR`
+environment variable at a directory that already contains the encoding files.
+Without either, any run that counts tokens fails.
+
+**Inputs**
+
+The input dataset has to be UTF-8 encoded text. Line endings are preserved as
+they are, so a dataset with Windows (`\r\n`) line endings keeps them in the
+chunks.
+
+**Outputs**
+
+- Text and JSON file with input/chunk metadata, where each chunk comes along with its size and start index in the input text.
+ The chunk size is reported in the `length` field, and `length_unit` states whether it counts characters or tokens.
+ A `start_index` of `null` means that the splitter returned an invalid position for that chunk; a warning is written to the tool log in that case.
+- TSV file with the following columns: chunk number, content, size in characters or tokens.
+ So that every chunk stays on a single row, backslash, tab, carriage return and newline in the content are escaped as `\\`, `\t`, `\r` and `\n`. Reverse the escaping to recover the exact chunk content, or use the collection below, which holds the raw text.
+- Galaxy collection of text files, with one raw chunk per file
+
+**License**
+Langchain-Text-Splitters is licensed under the MIT License.
+
+
+
+ @software{Chase_LangChain_2022,
+ author = {Chase, Harrison},
+ month = oct,
+ title = {{LangChain}},
+ url = {https://github.com/langchain-ai/langchain},
+ year = {2022}
+ }
+
+
+
diff --git a/tools/langchain_text_splitters/macros.xml b/tools/langchain_text_splitters/macros.xml
new file mode 100644
index 0000000000..4e817116c7
--- /dev/null
+++ b/tools/langchain_text_splitters/macros.xml
@@ -0,0 +1,153 @@
+
+ 1.1.2
+ 0
+ 25.1
+
+
+
+
+ langchain-text-splitters
+ tiktoken
+ nltk
+ nltk_data
+ spacy
+ spacy-model-en_core_web_sm
+
+ python-gil
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ^[A-Za-z0-9 \-=_.()/+*^,:?!\\#|;~%&$@<>"\[\]{}]+\Z
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tools/langchain_text_splitters/split_text.py b/tools/langchain_text_splitters/split_text.py
new file mode 100644
index 0000000000..0678cb3d55
--- /dev/null
+++ b/tools/langchain_text_splitters/split_text.py
@@ -0,0 +1,799 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+from langchain_text_splitters import (
+ CharacterTextSplitter,
+ NLTKTextSplitter,
+ RecursiveCharacterTextSplitter,
+ SpacyTextSplitter,
+ TokenTextSplitter,
+)
+
+SEPARATOR_MAP = {
+ "paragraph_break": "\n\n",
+ "line_break": "\n",
+ "tab": "\t",
+ "space": " ",
+ "ascii_full_stop": ".",
+ "ascii_comma": ",",
+ "ascii_semicolon": ";",
+ "zero_width_space": "\u200b",
+ "fullwidth_comma": "\uff0c",
+ "ideographic_comma": "\u3001",
+ "fullwidth_full_stop": "\uff0e",
+ "ideographic_full_stop": "\u3002",
+}
+
+KEEP_SEPARATOR_VALUES = {
+ "start": "start",
+ "end": "end",
+ "false": False,
+}
+
+CUSTOM_SEPARATOR_ESCAPE_PATTERN = re.compile(
+ r"\\(?:[\\nrt]|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})"
+)
+
+MAX_UNICODE_CODE_POINT = 0x10FFFF
+SURROGATE_RANGE = range(0xD800, 0xE000)
+
+# The chunk text is stored in a single TSV cell, so every character that would
+# otherwise start a new row or a new column has to be escaped. The backslash is
+# escaped first so that the transformation stays reversible.
+TSV_ESCAPES = (
+ ("\\", r"\\"),
+ ("\t", r"\t"),
+ ("\r", r"\r"),
+ ("\n", r"\n"),
+)
+
+LENGTH_KEY = "length"
+
+# Reasons why a chunk cannot be traced back to the input, see diagnose_chunk().
+CHUNK_TEXT_ALTERED = "chunk_text_altered"
+START_INDEX_INVALID = "start_index_invalid"
+
+
+def parse_args():
+
+ parser = argparse.ArgumentParser(
+ description="Split text with langchain-text-splitters."
+ )
+ parser.add_argument("--input", type=Path, required=True)
+ parser.add_argument("--output-text", type=Path, required=True)
+ parser.add_argument("--output-json", type=Path, required=True)
+ parser.add_argument("--output-tsv", type=Path, required=True)
+ parser.add_argument("--chunks-dir", type=Path, required=True)
+ parser.add_argument(
+ "--splitter-type",
+ choices=("recursive_character", "character", "nltk", "spacy", "token"),
+ default="recursive_character",
+ )
+ parser.add_argument("--sentence-language", default="english")
+ parser.add_argument(
+ "--spacy-pipeline",
+ choices=("sentencizer", "en_core_web_sm"),
+ default="sentencizer",
+ )
+ parser.add_argument(
+ "--spacy-max-length",
+ type=int,
+ default=1_000_000,
+ )
+
+ parser.add_argument("--chunk-size", type=int, required=True)
+ parser.add_argument("--chunk-overlap", type=int, default=0)
+ parser.add_argument(
+ "--length-mode",
+ choices=("characters", "token"),
+ default="characters",
+ )
+ tokenizer_group = parser.add_mutually_exclusive_group()
+ tokenizer_group.add_argument("--encoding-name", default="gpt2")
+ tokenizer_group.add_argument("--model-name", default="")
+ parser.add_argument(
+ "--allowed-special",
+ choices=("none", "all"),
+ default="none",
+ )
+ parser.add_argument(
+ "--keep-separator",
+ choices=("start", "end", "false"),
+ default="start",
+ )
+ parser.add_argument(
+ "--separator-name",
+ choices=SEPARATOR_MAP,
+ )
+ parser.add_argument("--separator")
+ parser.add_argument("--separator-specs", nargs="+", default=None)
+ parser.add_argument("--strip-whitespace", action="store_true")
+ parser.add_argument("--max-chunk-files", required=True)
+ return parser.parse_args()
+
+
+def read_input_text(input_path):
+ # The bytes are decoded explicitly instead of using Path.read_text(), which
+ # would apply universal newline translation and silently rewrite \r\n and \r
+ # line endings as \n. That would change both the chunk content and the
+ # reported start indices with respect to the input dataset.
+ try:
+ text = input_path.read_bytes().decode("utf-8")
+ except UnicodeDecodeError as error:
+ sys.exit(
+ "The input dataset is not valid UTF-8 text "
+ f"(invalid byte at offset {error.start}). "
+ "Convert the dataset to UTF-8 before splitting it."
+ )
+
+ # Galaxy appends a newline to an uploaded file that has none, see
+ # convert_newlines() in galaxy/lib/galaxy/datatypes/sniff.py. It is
+ # deliberately kept here. An uploaded "abc" and an uploaded "abc\n" both
+ # arrive as "abc\n", so the two cases cannot be told apart, and removing the
+ # newline would cut a real character off every input that legitimately ends
+ # in one, which is the common case. The chunks would then no longer add up
+ # to the input, and the character based splitters would become lossy for no
+ # gain. It would not even settle the NLTK case, because punkt discards all
+ # trailing whitespace, so an input ending in a blank line still loses more
+ # than the one appended character. What NLTK drops is reported instead, see
+ # the warning at the end of main().
+ return text
+
+
+def get_tiktoken_options(args):
+ allow_all_special = args.allowed_special == "all"
+
+ return {
+ "encoding_name": args.encoding_name,
+ "model_name": args.model_name or None,
+ "allowed_special": "all" if allow_all_special else set(),
+ "disallowed_special": () if allow_all_special else "all",
+ }
+
+
+def special_token_error(error):
+ """Return the error to raise for a ValueError coming from tiktoken.
+
+ tiktoken raises when the input contains a special token string such as
+ <|endoftext|> while the user asked for those strings to be rejected. Any
+ other ValueError is handed back unchanged.
+ """
+ if "disallowed special token" not in str(error):
+ return error
+
+ return SystemExit(
+ "The input contains a special token string that is rejected by the "
+ "selected tokenizer. Set the special token handling to allow all "
+ "special token strings, or remove the special token from the input.\n"
+ f"Original error: {error}"
+ )
+
+
+def build_tiktoken_counter(
+ encoding_name,
+ model_name=None,
+ allowed_special=None,
+ disallowed_special="all",
+):
+ import tiktoken
+
+ if allowed_special is None:
+ allowed_special = set()
+
+ # Note: below needs either internet access to download the encodings from
+ # the OpenAI servers, or the TIKTOKEN_CACHE_DIR environment variable
+ # pointing to a directory that already holds the encoding files. The
+ # tiktoken conda package does not ship them.
+ if model_name:
+ encoding = tiktoken.encoding_for_model(model_name)
+ else:
+ encoding = tiktoken.get_encoding(encoding_name)
+
+ def count_tokens(text):
+ try:
+ tokens = encoding.encode(
+ text,
+ allowed_special=allowed_special,
+ disallowed_special=disallowed_special,
+ )
+ except ValueError as error:
+ raise special_token_error(error)
+
+ return len(tokens)
+
+ return count_tokens
+
+
+def decode_custom_separator(value):
+ simple_escapes = {
+ r"\n": "\n",
+ r"\r": "\r",
+ r"\t": "\t",
+ r"\\": "\\",
+ }
+
+ decoded = []
+ position = 0
+
+ while position < len(value):
+ character = value[position]
+
+ if character != "\\":
+ decoded.append(character)
+ position += 1
+ continue
+
+ match = CUSTOM_SEPARATOR_ESCAPE_PATTERN.match(value, position)
+
+ if match is None:
+ sys.exit(
+ "The custom separator contains an unsupported escape sequence "
+ f"at position {position}: {value[position:position + 10]!r}. "
+ r"Supported escapes are \\, \n, \r, \t, "
+ r"\uXXXX (4 hex digits) and \UXXXXXXXX (8 hex digits)."
+ )
+
+ escape = match.group(0)
+
+ if escape in simple_escapes:
+ decoded.append(simple_escapes[escape])
+ else:
+ code_point = int(escape[2:], 16)
+
+ if code_point > MAX_UNICODE_CODE_POINT or code_point in SURROGATE_RANGE:
+ sys.exit(
+ f"The custom separator escape {escape} is not a valid "
+ "Unicode character."
+ )
+
+ decoded.append(chr(code_point))
+
+ position = match.end()
+
+ return "".join(decoded)
+
+
+def resolve_separator(
+ separator_name,
+ custom_separator,
+):
+ if separator_name is not None:
+ return SEPARATOR_MAP[separator_name]
+
+ if custom_separator is not None:
+ return decode_custom_separator(custom_separator)
+
+ raise ValueError("No separator was provided.")
+
+
+def resolve_separator_specs(specs):
+ separators = []
+
+ for spec in specs:
+ kind, value = spec.split(":", 1)
+
+ if kind == "predefined":
+ separators.append(SEPARATOR_MAP[value])
+ else:
+ separators.append(decode_custom_separator(value))
+
+ return separators
+
+
+def build_splitter(args, input_text, length_function, tiktoken_options):
+ common_options = {
+ "chunk_size": args.chunk_size,
+ "chunk_overlap": args.chunk_overlap,
+ "add_start_index": True,
+ # Never let langchain strip; main() does it once on the finished chunks.
+ # langchain would strip each sentence before joining them, welding
+ # "mat." to "The", and would misreport the start index. Defaults to True.
+ "strip_whitespace": False,
+ }
+
+ if args.splitter_type == "token":
+ return TokenTextSplitter(
+ **common_options,
+ **tiktoken_options,
+ )
+
+ length_options = {
+ **common_options,
+ "length_function": length_function,
+ }
+
+ if args.splitter_type == "nltk":
+ # separator="" keeps the chunk text identical to the matching slice of
+ # the input, so the reported start indices stay usable. The span based
+ # tokenizer drops whatever follows the last sentence, see main().
+ return NLTKTextSplitter(
+ **length_options,
+ language=args.sentence_language,
+ separator="",
+ use_span_tokenize=True,
+ )
+
+ if args.splitter_type == "spacy":
+ splitter = SpacyTextSplitter(
+ **length_options,
+ pipeline=args.spacy_pipeline,
+ max_length=args.spacy_max_length,
+ separator="",
+ )
+ # langchain only forwards max_length to the pipelines it loads with
+ # spacy.load(). The sentencizer is built from English() instead and
+ # silently keeps the spaCy default of one million characters, so it is
+ # applied here for both. Guarded because _tokenizer is private API.
+ tokenizer = getattr(splitter, "_tokenizer", None)
+
+ if hasattr(tokenizer, "max_length"):
+ tokenizer.max_length = args.spacy_max_length
+
+ return splitter
+
+ character_options = {
+ **length_options,
+ "keep_separator": KEEP_SEPARATOR_VALUES[args.keep_separator],
+ }
+
+ if args.splitter_type == "character":
+ separator = resolve_separator(
+ separator_name=args.separator_name,
+ custom_separator=args.separator,
+ )
+
+ if separator not in input_text:
+ sys.exit(
+ "The selected separator "
+ f"{separator!r} was not found in the input text. "
+ "Select a separator that occurs in the input or use the "
+ "recursive character splitter."
+ )
+
+ return CharacterTextSplitter(
+ **character_options,
+ separator=separator,
+ )
+
+ if args.separator_specs is not None:
+ separators = resolve_separator_specs(args.separator_specs)
+ # Append an empty string as the final fallback separator to ensure that the text can always be split,
+ # even if none of the tried separators before were able to split the text without exceeding the chunk size.
+ character_options["separators"] = [*separators, ""]
+
+ return RecursiveCharacterTextSplitter(**character_options)
+
+
+def altered_text_cause(args):
+ """Name the cause that fits this run instead of guessing a single one.
+
+ The two causes produce the same symptom but have opposite remedies, and
+ naming the wrong one sends the user in a circle.
+ """
+ if args.splitter_type == "token" or args.length_mode == "token":
+ return (
+ "A cut between two tokens can fall inside a character that is "
+ "encoded in several bytes, which then becomes the Unicode "
+ "replacement character. Use one of the character based splitters "
+ "for text that is not plain ASCII."
+ )
+
+ if args.keep_separator == "false":
+ return (
+ "Discarding the separator drops the empty pieces between two "
+ "adjacent separators, so a run of separators is rebuilt as a "
+ "single one and the characters in between are lost. Choose a "
+ "setting that keeps the separator, or split on a separator that "
+ "does not occur several times in a row."
+ )
+
+ return (
+ "The splitter did not return the input unchanged; the chunks can "
+ "therefore no longer be traced back to a position in it."
+ )
+
+
+def diagnose_chunk(
+ start_index,
+ chunk_text,
+ input_text,
+ previous_start_index,
+):
+ """Return None when the chunk and its reported position are sound.
+
+ Otherwise return the reason, so that the two very different causes can be
+ reported separately: text the splitter altered, and a position the splitter
+ got wrong.
+ """
+ if (
+ start_index >= 0
+ # Two chunks may legitimately begin at the same offset when the overlap
+ # repeats a whole split, so only a *backwards* jump is a real defect.
+ and (previous_start_index is None or start_index >= previous_start_index)
+ and input_text.startswith(chunk_text, start_index)
+ ):
+ return None
+
+ # Only reached when something is already wrong, so scanning the whole input
+ # once more is acceptable here.
+ if chunk_text not in input_text:
+ return CHUNK_TEXT_ALTERED
+
+ return START_INDEX_INVALID
+
+
+def write_text_output(
+ output_path,
+ metadata,
+ chunks,
+ length_label,
+):
+ lines = [
+ f"Splitter: {metadata['splitter_type']}",
+ f"Length function: {metadata['length_function']}",
+ f"Input characters: {metadata['input_characters']}",
+ f"Input length: {metadata['input_length']} {length_label}",
+ f"Chunk size: {metadata['chunk_size']}",
+ f"Chunk overlap: {metadata['chunk_overlap']}",
+ f"Number of chunks: {metadata['number_of_chunks']}",
+ "",
+ ]
+
+ for chunk in chunks:
+ lines.extend(
+ [
+ (
+ f"--- Chunk {chunk['index']}: "
+ f"{chunk[LENGTH_KEY]} {length_label}, "
+ f"start={chunk['start_index']} ---"
+ ),
+ chunk["text"],
+ "",
+ ]
+ )
+ output_path.write_text("\n".join(lines), encoding="utf-8")
+
+
+def write_chunk_files(chunks_dir, chunks):
+ chunks_dir.mkdir(parents=True, exist_ok=True)
+ # Galaxy sorts the discovered elements lexically by file name, so the pad
+ # has to be wide enough for the largest index or chunk_10000 would sort
+ # between chunk_1000 and chunk_1001.
+ width = max(4, len(str(len(chunks))))
+
+ for chunk in chunks:
+ chunk_path = chunks_dir / f"chunk_{chunk['index']:0{width}d}.txt"
+ chunk_path.write_text(
+ chunk["text"],
+ encoding="utf-8",
+ )
+
+
+def escape_tsv_text(text):
+ for raw, escaped in TSV_ESCAPES:
+ text = text.replace(raw, escaped)
+
+ return text
+
+
+def write_tsv_output(output_path, chunks):
+ # The rows are written without the csv module on purpose. Its writer would
+ # additionally apply CSV quoting to any chunk containing a double quote,
+ # which the documented escaping above cannot undo. escape_tsv_text() already
+ # removes every character that could break the column or row structure.
+ with output_path.open("w", encoding="utf-8", newline="") as handle:
+ for chunk in chunks:
+ fields = [
+ str(chunk["index"]),
+ escape_tsv_text(chunk["text"]),
+ str(chunk[LENGTH_KEY]),
+ ]
+ handle.write("\t".join(fields) + "\n")
+
+
+def main():
+ args = parse_args()
+
+ if args.chunk_overlap >= args.chunk_size:
+ sys.exit("Chunk overlap must be smaller than chunk size.")
+
+ input_text = read_input_text(args.input)
+
+ if not input_text.strip():
+ sys.exit(
+ "The input dataset is empty or contains only whitespace. "
+ "There is nothing to split."
+ )
+
+ # Checked before the tokenizer is built so that a job that cannot run does
+ # not first pay for loading the tiktoken encoding.
+ if args.splitter_type == "spacy" and len(input_text) > args.spacy_max_length:
+ sys.exit(
+ f"The input dataset holds {len(input_text)} characters, which is "
+ "more than the configured spaCy maximum input length of "
+ f"{args.spacy_max_length}. Raise 'Maximum input length' or split "
+ "the dataset into smaller parts first. Mind the memory cost per "
+ "input character stated in the help of that setting."
+ )
+
+ length_mode = "token" if args.splitter_type == "token" else args.length_mode
+
+ tiktoken_options = get_tiktoken_options(args)
+
+ if length_mode == "token":
+ length_function = build_tiktoken_counter(**tiktoken_options)
+ length_label = "tokens"
+ tokenizer_label = args.model_name or args.encoding_name
+ length_function_label = f"tiktoken:{tokenizer_label}"
+ else:
+ length_function = len
+ length_label = "characters"
+ length_function_label = "characters"
+
+ try:
+ splitter = build_splitter(
+ args,
+ input_text,
+ length_function,
+ tiktoken_options,
+ )
+ except OSError as error:
+ if args.splitter_type == "spacy":
+ sys.exit(
+ f"The selected spaCy pipeline {args.spacy_pipeline!r} "
+ "is not installed in the tool environment.\n"
+ f"Original error: {error}"
+ )
+ raise
+ except LookupError as error:
+ # NLTKTextSplitter loads the punkt_tab data in its constructor, so this
+ # has to be caught around build_splitter() and not around
+ # create_documents().
+ # The data comes from the nltk_data requirement, so this is reached only
+ # on an environment that does not provide it, or provides it outside the
+ # directories nltk searches:
+ # -/usr/share/nltk_data
+ # -/usr/local/share/nltk_data
+ # -/usr/lib/nltk_data
+ # -/usr/local/lib/nltk_data
+ # -any directory listed in NLTK_DATA
+ # KeyError and IndexError also derive from LookupError, so they are
+ # excluded to keep an unrelated failure from being reported as missing
+ # Punkt data.
+ if args.splitter_type == "nltk" and not isinstance(
+ error, (KeyError, IndexError)
+ ):
+ sys.exit(
+ "The NLTK Punkt data required for the selected language is not "
+ "installed. The Galaxy tool environment must provide the "
+ "'punkt_tab' NLTK resource, for example below a directory "
+ "listed in the NLTK_DATA environment variable. Use the spaCy "
+ "sentence splitter if the data cannot be installed.\n"
+ f"Original error: {error}"
+ )
+ raise
+
+ try:
+ documents = splitter.create_documents([input_text])
+ except ValueError as error:
+ raise special_token_error(error)
+
+ chunks = []
+ start_index_warnings = []
+ altered_text_warnings = []
+ previous_start_index = None
+ empty_chunks = 0
+
+ for document in documents:
+ raw_chunk_text = document.page_content
+ start_index = document.metadata.get("start_index")
+
+ if args.strip_whitespace:
+ # Only the outer whitespace of the finished chunk is removed, which
+ # is what the option promises. The chunk therefore stays a verbatim
+ # slice of the input and its start index stays valid, so it only has
+ # to move by whatever came off the front.
+ #
+ # The splitter has already packed the chunks at this point, counting
+ # the whitespace that is removed here, so a stripped chunk can come
+ # out shorter than the requested size. That is the conservative
+ # direction, and measuring the stripped text instead would mean
+ # stripping before the packing, which is exactly what produces
+ # "mat.The".
+ leading = len(raw_chunk_text) - len(raw_chunk_text.lstrip())
+ raw_chunk_text = raw_chunk_text.strip()
+
+ if start_index is not None and start_index >= 0:
+ start_index += leading
+
+ # Nothing is left for a downstream step to work on. Reachable only
+ # when the user asked for stripping, so no content is lost: without
+ # it every chunk is kept, whitespace included.
+ if not raw_chunk_text:
+ empty_chunks += 1
+ continue
+
+ chunk_number = len(chunks) + 1
+
+ # TODO: report the invalid start index upstream to langchain-text-splitters, since this
+ # should not happen. Note that the invalid indices are not always negative: once one chunk
+ # gets a negative index, the positions derived from it stay positive but point at the wrong
+ # place, which is why diagnose_chunk() also checks the order and the content.
+ # Potential cause: langchain text splitters might calculate the start index based on the length of the chunk in characters, even though the chunk size is measured in tokens.
+ # reproduce with Character splitter recursive
+ # input: test-data/langchain_docs_sample.txt
+ # Text splitting separators and their order of use: default
+ # Place separator at start of the following chunk start → ["First paragraph.", "\n\nSecond paragraph."]
+ # Strip whitespace around chunks: true
+ # Target chunk size should be determined by the number of tokens
+ # Target chunk size: 100
+ # Chunk overlap: 20
+ problem = (
+ None
+ if start_index is None
+ else diagnose_chunk(
+ start_index,
+ raw_chunk_text,
+ input_text,
+ previous_start_index,
+ )
+ )
+
+ if problem is None:
+ if start_index is not None:
+ previous_start_index = start_index
+ else:
+ # The position is reported as null so that the JSON field keeps a
+ # single type. The warnings below carry the detail.
+ if problem == CHUNK_TEXT_ALTERED:
+ altered_text_warnings.append(chunk_number)
+ else:
+ start_index_warnings.append((chunk_number, start_index))
+
+ start_index = None
+
+ chunks.append(
+ {
+ "index": chunk_number,
+ LENGTH_KEY: length_function(raw_chunk_text),
+ "start_index": start_index,
+ "text": raw_chunk_text,
+ }
+ )
+
+ if altered_text_warnings:
+ print(
+ "WARNING: The text of the following chunk(s) does not occur in the "
+ f"input: {', '.join(str(number) for number in altered_text_warnings)}. "
+ "The splitter returned text it had modified, so no position in the "
+ "input describes it and the start index is reported as null. "
+ + altered_text_cause(args),
+ flush=True,
+ )
+
+ if start_index_warnings:
+ affected_chunks = ", ".join(
+ f"{chunk_number} (start_index: {start_index!r})"
+ for chunk_number, start_index in start_index_warnings
+ )
+ print(
+ "WARNING: The reported position of the following chunk(s) does not "
+ f"describe where their text sits in the input: {affected_chunks}. "
+ "The start index of these chunks is reported as null.",
+ flush=True,
+ )
+
+ # Stop here, not after writing: Galaxy fails the job on the file count anyway.
+ # "None" means the admin set no limit, so isdigit() instead of int().
+ max_chunk_files = (
+ int(args.max_chunk_files) if args.max_chunk_files.isdigit() else 0
+ )
+
+ if max_chunk_files and len(chunks) > max_chunk_files:
+ sys.exit(
+ f"The selected settings produce {len(chunks)} chunks. The tool "
+ "writes one dataset per chunk, and this Galaxy instance refuses a "
+ f"job that produces more than {max_chunk_files} datasets, so the "
+ "job would be failed after the split had already run. Raise the "
+ f"target chunk size: the input holds {len(input_text)} characters, "
+ f"so a chunk size above {len(input_text) // max_chunk_files + 1} "
+ "keeps the count under the limit."
+ )
+
+ if empty_chunks:
+ print(
+ f"WARNING: {empty_chunks} chunk(s) held nothing but whitespace and "
+ "were left out of the outputs, because stripping the whitespace "
+ "around the chunks left them empty.",
+ flush=True,
+ )
+
+ # The NLTK splitter builds the chunks from the sentence spans reported by
+ # punkt, which end at the last sentence. Punkt trims trailing whitespace, so
+ # whatever follows is dropped and the chunks no longer add up to the input.
+ # The first span always starts at offset 0, so nothing is lost in front.
+ # The spaCy splitter keeps everything.
+ # Only reported when the chunks were not stripped. Punkt leaves nothing but
+ # whitespace after the last span, so with stripping on this could only ever
+ # report whitespace the user asked to discard, inflated by whatever strip()
+ # itself took off the end of the last chunk.
+ if args.splitter_type == "nltk" and chunks and not args.strip_whitespace:
+ last_chunk = chunks[-1]
+ last_text = last_chunk["text"]
+ # The validated start index is preferred over searching for the text:
+ # rfind() returns the *last* occurrence, so it would report no loss
+ # whenever the dropped text happens to repeat the final chunk.
+ last_start = last_chunk["start_index"]
+
+ if last_start is None:
+ last_start = input_text.rfind(last_text)
+
+ dropped = (
+ len(input_text) - (last_start + len(last_text)) if last_start >= 0 else 0
+ )
+
+ if dropped > 0:
+ print(
+ f"WARNING: The NLTK sentence splitter dropped the {dropped} "
+ "character(s) after the last sentence. Use the spaCy sentence "
+ "splitter to keep the complete input.",
+ flush=True,
+ )
+
+ metadata = {
+ "input_characters": len(input_text),
+ "input_length": length_function(input_text),
+ "length_unit": length_label,
+ "length_function": length_function_label,
+ "splitter_type": args.splitter_type,
+ "chunk_size": args.chunk_size,
+ "chunk_overlap": args.chunk_overlap,
+ "number_of_chunks": len(chunks),
+ # Reported from the argument rather than per splitter, because the
+ # stripping is done above for whichever splitter produced the chunks.
+ # A splitter that does not offer the option never receives it, so the
+ # value stays false for it without having to be hardcoded here.
+ "strip_whitespace": args.strip_whitespace,
+ }
+
+ if args.splitter_type in ("character", "recursive_character"):
+ metadata["keep_separator"] = args.keep_separator
+
+ elif args.splitter_type == "nltk":
+ metadata["sentence_language"] = args.sentence_language
+
+ elif args.splitter_type == "spacy":
+ metadata["spacy_pipeline"] = args.spacy_pipeline
+
+ args.output_json.write_text(
+ json.dumps(
+ {**metadata, "chunks": chunks},
+ indent=2,
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+
+ write_text_output(
+ args.output_text,
+ metadata,
+ chunks,
+ length_label,
+ )
+ write_tsv_output(
+ args.output_tsv,
+ chunks,
+ )
+ write_chunk_files(
+ args.chunks_dir,
+ chunks,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/langchain_text_splitters/test-data/custom_separator.txt b/tools/langchain_text_splitters/test-data/custom_separator.txt
new file mode 100644
index 0000000000..8bb2b54918
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/custom_separator.txt
@@ -0,0 +1 @@
+alpha###beta###gamma
diff --git a/tools/langchain_text_splitters/test-data/dash_separator.txt b/tools/langchain_text_splitters/test-data/dash_separator.txt
new file mode 100644
index 0000000000..4c4b2bfb35
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/dash_separator.txt
@@ -0,0 +1 @@
+alpha---beta---gamma
diff --git a/tools/langchain_text_splitters/test-data/langchain_docs_sample.txt b/tools/langchain_text_splitters/test-data/langchain_docs_sample.txt
new file mode 100644
index 0000000000..1afdfaf8e9
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/langchain_docs_sample.txt
@@ -0,0 +1,33 @@
+Madam Speaker, Madam Vice President, our First Lady and Second Gentleman.
+
+Members of Congress and the Cabinet.
+
+Justices of the Supreme Court.
+
+My fellow Americans.
+
+Last year COVID-19 kept us apart.
+
+This year we are finally together again.
+
+Tonight, we meet as Democrats Republicans and Independents.
+
+But most importantly as Americans.
+
+With a duty to one another to the American people to the Constitution.
+
+And with an unwavering resolve that freedom will always triumph over tyranny.
+
+Six days ago, Russia’s Vladimir Putin sought to shake the foundations of the free world thinking he could make it bend to his menacing ways.
+
+But he badly miscalculated.
+
+He thought he could roll into Ukraine and the world would roll over.
+
+Instead he met a wall of strength he never imagined.
+
+He met the Ukrainian people.
+
+From President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world.
+
+Groups of citizens blocking tanks with their bodies.
\ No newline at end of file
diff --git a/tools/langchain_text_splitters/test-data/multibyte_persian.txt b/tools/langchain_text_splitters/test-data/multibyte_persian.txt
new file mode 100644
index 0000000000..a6f5a1a283
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/multibyte_persian.txt
@@ -0,0 +1 @@
+سلام دنیا. این یک متن آزمایشی است.
diff --git a/tools/langchain_text_splitters/test-data/overlap_words.txt b/tools/langchain_text_splitters/test-data/overlap_words.txt
new file mode 100644
index 0000000000..cd4781dcb7
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/overlap_words.txt
@@ -0,0 +1 @@
+one two three four five
diff --git a/tools/langchain_text_splitters/test-data/pipe_separator.txt b/tools/langchain_text_splitters/test-data/pipe_separator.txt
new file mode 100644
index 0000000000..df9c19ae89
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/pipe_separator.txt
@@ -0,0 +1 @@
+alpha|beta|gamma
diff --git a/tools/langchain_text_splitters/test-data/recursive_default.txt b/tools/langchain_text_splitters/test-data/recursive_default.txt
new file mode 100644
index 0000000000..7f57fb2e60
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/recursive_default.txt
@@ -0,0 +1,5 @@
+Alpha beta.
+
+Gamma delta.
+
+Epsilon zeta.
diff --git a/tools/langchain_text_splitters/test-data/repeated_separator.txt b/tools/langchain_text_splitters/test-data/repeated_separator.txt
new file mode 100644
index 0000000000..2ae978d5fe
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/repeated_separator.txt
@@ -0,0 +1 @@
+alpha,,,beta,,,gamma,,,delta
diff --git a/tools/langchain_text_splitters/test-data/sentence_nltk_english.txt b/tools/langchain_text_splitters/test-data/sentence_nltk_english.txt
new file mode 100644
index 0000000000..ea4f6368e4
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/sentence_nltk_english.txt
@@ -0,0 +1,3 @@
+Dr. Smith arrived early.
+The meeting started promptly.
+Everyone took notes.
\ No newline at end of file
diff --git a/tools/langchain_text_splitters/test-data/sentence_nltk_german.txt b/tools/langchain_text_splitters/test-data/sentence_nltk_german.txt
new file mode 100644
index 0000000000..5c5b619082
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/sentence_nltk_german.txt
@@ -0,0 +1,3 @@
+Das ist z.B. wichtig.
+Der Test beginnt jetzt.
+Alles funktioniert gut.
\ No newline at end of file
diff --git a/tools/langchain_text_splitters/test-data/sentence_overlap.txt b/tools/langchain_text_splitters/test-data/sentence_overlap.txt
new file mode 100644
index 0000000000..ac65da5036
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/sentence_overlap.txt
@@ -0,0 +1,4 @@
+Alpha one.
+Beta two.
+Gamma three.
+Delta four.
\ No newline at end of file
diff --git a/tools/langchain_text_splitters/test-data/sentence_spacy.txt b/tools/langchain_text_splitters/test-data/sentence_spacy.txt
new file mode 100644
index 0000000000..8e1bcfa9d1
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/sentence_spacy.txt
@@ -0,0 +1,3 @@
+First sentence is short.
+Second sentence is also short.
+Third sentence ends here.
\ No newline at end of file
diff --git a/tools/langchain_text_splitters/test-data/sentence_spacy_prose.txt b/tools/langchain_text_splitters/test-data/sentence_spacy_prose.txt
new file mode 100644
index 0000000000..4dd187abe6
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/sentence_spacy_prose.txt
@@ -0,0 +1 @@
+The cat sat on the mat. The dog barked loudly. Birds flew away.
diff --git a/tools/langchain_text_splitters/test-data/sentence_tokens.txt b/tools/langchain_text_splitters/test-data/sentence_tokens.txt
new file mode 100644
index 0000000000..9bfa262fff
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/sentence_tokens.txt
@@ -0,0 +1,3 @@
+one two.
+three four.
+five six.
\ No newline at end of file
diff --git a/tools/langchain_text_splitters/test-data/simple_separator.txt b/tools/langchain_text_splitters/test-data/simple_separator.txt
new file mode 100644
index 0000000000..fd35b2591f
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/simple_separator.txt
@@ -0,0 +1 @@
+alpha.beta.gamma
diff --git a/tools/langchain_text_splitters/test-data/special_token.txt b/tools/langchain_text_splitters/test-data/special_token.txt
new file mode 100644
index 0000000000..74de15e945
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/special_token.txt
@@ -0,0 +1 @@
+Before <|endoftext|> after
diff --git a/tools/langchain_text_splitters/test-data/token_words.txt b/tools/langchain_text_splitters/test-data/token_words.txt
new file mode 100644
index 0000000000..e1a7d0645a
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/token_words.txt
@@ -0,0 +1 @@
+one two three four five six seven eight
diff --git a/tools/langchain_text_splitters/test-data/tsv_escape.txt b/tools/langchain_text_splitters/test-data/tsv_escape.txt
new file mode 100644
index 0000000000..375c76aac5
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/tsv_escape.txt
@@ -0,0 +1,3 @@
+alpha beta
+C:\path
+omega
diff --git a/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv b/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv
new file mode 100644
index 0000000000..902660bb43
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/tsv_escape_expected.tsv
@@ -0,0 +1 @@
+1 alpha\tbeta\nC:\\path\nomega\n 25
diff --git a/tools/langchain_text_splitters/test-data/unicode_separator.txt b/tools/langchain_text_splitters/test-data/unicode_separator.txt
new file mode 100644
index 0000000000..2e100bac28
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/unicode_separator.txt
@@ -0,0 +1 @@
+alpha。beta。gamma
diff --git a/tools/langchain_text_splitters/test-data/whitespace_only.txt b/tools/langchain_text_splitters/test-data/whitespace_only.txt
new file mode 100644
index 0000000000..216aec0f98
--- /dev/null
+++ b/tools/langchain_text_splitters/test-data/whitespace_only.txt
@@ -0,0 +1,3 @@
+
+
+