Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""Filename safety helpers for parser working files."""

import os
from hashlib import sha256


MAX_INTERNAL_FILENAME_BYTES = 240
INTERNAL_FILENAME_HASH_LENGTH = 12


def truncate_internal_filename(filename: str) -> str:
"""Keep parser filenames below Linux NAME_MAX while retaining identity."""
if len(os.fsencode(filename)) <= MAX_INTERNAL_FILENAME_BYTES:
return filename

name_root, name_ext = os.path.splitext(filename)
filename_hash = sha256(filename.encode("utf-8")).hexdigest()[
:INTERNAL_FILENAME_HASH_LENGTH
]
suffix = f"-{filename_hash}{name_ext}"
available_root_bytes = MAX_INTERNAL_FILENAME_BYTES - len(os.fsencode(suffix))
if available_root_bytes <= 0:
return f"document-{filename_hash}.bin"

truncated_root = name_root.encode("utf-8")[:available_root_bytes].decode(
"utf-8", errors="ignore"
)
return f"{truncated_root}{suffix}"
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
from dataclasses import dataclass

from app.services.common.file_utils import path_handle
from app.services.document_parser.support.filename_limits import (
truncate_internal_filename,
)


@dataclass(frozen=True)
Expand Down Expand Up @@ -39,7 +42,7 @@ def normalize_internal_parse_name(
effective_root = name_root or "document"
internal_name = f"{effective_root}{effective_ext}"

return (
normalized_name = (
internal_name.replace("(", "-")
.replace(")", "-")
.replace("[", "-")
Expand All @@ -53,6 +56,7 @@ def normalize_internal_parse_name(
.replace(chr(0x2015), "-")
.replace(chr(0x2212), "-")
)
return truncate_internal_filename(normalized_name)


def prepare_internal_parse_input(
Expand Down
34 changes: 34 additions & 0 deletions apps/worker/tests/unit/test_internal_parse_name.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
from __future__ import annotations

import os
from pathlib import Path

from app.services.document_parser.support.internal_parse_name import (
prepare_internal_parse_input,
)


def test_prepare_internal_parse_input_handles_long_encoded_filename(
tmp_path,
) -> None:
temporary_file_path = tmp_path / "temporary.pdf"
temporary_file_path.write_bytes(b"pdf")
encoded_filename = (
"%E9%99%84%E4%BB%B65.%E5%8D%97%E4%BA%AC%E4%BF%A1%E6%81%AF%E5%B7%A5%E7%A8%8B"
* 20
+ ".pdf"
)

prepared_input = prepare_internal_parse_input(
str(temporary_file_path),
encoded_filename,
fallback_ext=".pdf",
prefer_fallback_ext=True,
)
prepared_file_path = Path(prepared_input.file_path)
file_exists = prepared_file_path.exists()
file_contents = prepared_file_path.read_bytes()

assert len(os.fsencode(prepared_input.internal_filename)) <= 240
assert file_exists
assert file_contents == b"pdf"
Loading