diff --git a/apps/worker/app/services/document_parser/support/filename_limits.py b/apps/worker/app/services/document_parser/support/filename_limits.py new file mode 100644 index 00000000..21597c22 --- /dev/null +++ b/apps/worker/app/services/document_parser/support/filename_limits.py @@ -0,0 +1,28 @@ +"""Filename safety helpers for parser working files.""" + +import os +from hashlib import sha256 + + +MAX_INTERNAL_FILENAME_BYTES = 240 +INTERNAL_FILENAME_HASH_LENGTH = 12 + + +def truncate_internal_filename(filename: str) -> str: + """Keep parser filenames below Linux NAME_MAX while retaining identity.""" + if len(os.fsencode(filename)) <= MAX_INTERNAL_FILENAME_BYTES: + return filename + + name_root, name_ext = os.path.splitext(filename) + filename_hash = sha256(filename.encode("utf-8")).hexdigest()[ + :INTERNAL_FILENAME_HASH_LENGTH + ] + suffix = f"-{filename_hash}{name_ext}" + available_root_bytes = MAX_INTERNAL_FILENAME_BYTES - len(os.fsencode(suffix)) + if available_root_bytes <= 0: + return f"document-{filename_hash}.bin" + + truncated_root = name_root.encode("utf-8")[:available_root_bytes].decode( + "utf-8", errors="ignore" + ) + return f"{truncated_root}{suffix}" diff --git a/apps/worker/app/services/document_parser/support/internal_parse_name.py b/apps/worker/app/services/document_parser/support/internal_parse_name.py index 568dc13c..7a59ed8c 100644 --- a/apps/worker/app/services/document_parser/support/internal_parse_name.py +++ b/apps/worker/app/services/document_parser/support/internal_parse_name.py @@ -5,6 +5,9 @@ from dataclasses import dataclass from app.services.common.file_utils import path_handle +from app.services.document_parser.support.filename_limits import ( + truncate_internal_filename, +) @dataclass(frozen=True) @@ -39,7 +42,7 @@ def normalize_internal_parse_name( effective_root = name_root or "document" internal_name = f"{effective_root}{effective_ext}" - return ( + normalized_name = ( internal_name.replace("(", "-") .replace(")", "-") .replace("[", "-") @@ -53,6 +56,7 @@ def normalize_internal_parse_name( .replace(chr(0x2015), "-") .replace(chr(0x2212), "-") ) + return truncate_internal_filename(normalized_name) def prepare_internal_parse_input( diff --git a/apps/worker/tests/unit/test_internal_parse_name.py b/apps/worker/tests/unit/test_internal_parse_name.py new file mode 100644 index 00000000..c43dcabb --- /dev/null +++ b/apps/worker/tests/unit/test_internal_parse_name.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from app.services.document_parser.support.internal_parse_name import ( + prepare_internal_parse_input, +) + + +def test_prepare_internal_parse_input_handles_long_encoded_filename( + tmp_path, +) -> None: + temporary_file_path = tmp_path / "temporary.pdf" + temporary_file_path.write_bytes(b"pdf") + encoded_filename = ( + "%E9%99%84%E4%BB%B65.%E5%8D%97%E4%BA%AC%E4%BF%A1%E6%81%AF%E5%B7%A5%E7%A8%8B" + * 20 + + ".pdf" + ) + + prepared_input = prepare_internal_parse_input( + str(temporary_file_path), + encoded_filename, + fallback_ext=".pdf", + prefer_fallback_ext=True, + ) + prepared_file_path = Path(prepared_input.file_path) + file_exists = prepared_file_path.exists() + file_contents = prepared_file_path.read_bytes() + + assert len(os.fsencode(prepared_input.internal_filename)) <= 240 + assert file_exists + assert file_contents == b"pdf"