Merge pull request #414 from Ontos-AI/feat/wuchengke/office-compat-normalize

feat: normalize non-standard DOCX/XLSX before parse

Author: EricNGOntos

apps/worker/app/services/document_ingestion/office_compat_normalizer.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import NoReturn
+
+from app.services.document_parser.conversion.legacy_converter import (
+    normalize_docx_variant,
+    normalize_xlsx_variant,
+)
+from app.services.document_parser.orchestration.office_container_inspection import (
+    inspect_office_container,
+    read_override_content_type,
+)
+from loguru import logger
+
+from shared.core.exceptions.domain_exceptions import ValidationException
+
+OLE_CFBF_SIGNATURE = bytes.fromhex("D0CF11E0A1B11AE1")
+
+_STANDARD_MAIN_CONTENT_TYPES: dict[str, str] = {
+    ".docx": (
+        "application/vnd.openxmlformats-officedocument."
+        "wordprocessingml.document.main+xml"
+    ),
+    ".xlsx": (
+        "application/vnd.openxmlformats-officedocument."
+        "spreadsheetml.sheet.main+xml"
+    ),
+}
+_MAIN_PART_BY_EXTENSION: dict[str, str] = {
+    ".docx": "/word/document.xml",
+    ".xlsx": "/xl/workbook.xml",
+}
+_VARIANT_CONTENT_TYPE_MARKERS: tuple[str, ...] = (
+    "macroEnabled.main+xml",
+    "template.main+xml",
+)
+_NORMALIZED_OUTPUT_DIRNAME = "office_compat"
+
+
+@dataclass(frozen=True)
+class OfficeCompatNormalization:
+    file_path: str
+    conversion: dict[str, object] | None = None
+
+
+def normalize_office_source(file_path: str) -> OfficeCompatNormalization:
+    """Convert known DOCX/XLSX OOXML variants; reject encrypted OLE containers."""
+    extension = os.path.splitext(file_path)[1].lower()
+    if extension not in _STANDARD_MAIN_CONTENT_TYPES:
+        return OfficeCompatNormalization(file_path=file_path)
+
+    if _has_ole_cfbf_signature(file_path):
+        _raise_encrypted_office_file(extension)
+
+    inspection = inspect_office_container(file_path)
+    if inspection is None or inspection.content_types_xml is None:
+        return OfficeCompatNormalization(file_path=file_path)
+
+    content_type = read_override_content_type(
+        inspection.content_types_xml,
+        _MAIN_PART_BY_EXTENSION[extension],
+    )
+    if content_type is None:
+        return OfficeCompatNormalization(file_path=file_path)
+    if content_type == _STANDARD_MAIN_CONTENT_TYPES[extension]:
+        return OfficeCompatNormalization(file_path=file_path)
+    if not _is_known_ooxml_variant(content_type):
+        return OfficeCompatNormalization(file_path=file_path)
+
+    outdir = os.path.join(os.path.dirname(file_path), _NORMALIZED_OUTPUT_DIRNAME)
+    if extension == ".docx":
+        converted_path, _converted_name = normalize_docx_variant(file_path, outdir)
+    else:
+        converted_path, _converted_name = normalize_xlsx_variant(file_path, outdir)
+    logger.info(
+        "Office compatibility conversion applied: "
+        f"source_path={file_path}, normalized_path={converted_path}, "
+        f"content_type={content_type}"
+    )
+    return OfficeCompatNormalization(
+        file_path=converted_path,
+        conversion={"content_type": content_type},
+    )
+
+
+def _has_ole_cfbf_signature(file_path: str) -> bool:
+    with open(file_path, "rb") as handle:
+        header = handle.read(len(OLE_CFBF_SIGNATURE))
+    return header == OLE_CFBF_SIGNATURE
+
+
+def _is_known_ooxml_variant(content_type: str) -> bool:
+    return any(marker in content_type for marker in _VARIANT_CONTENT_TYPE_MARKERS)
+
+
+def _raise_encrypted_office_file(extension: str) -> NoReturn:
+    raise ValidationException(
+        user_message=(
+            f"Invalid file: the uploaded {extension} file is encrypted or "
+            "password-protected. Please unlock the file and upload again."
+        ),
+        violations=[
+            {
+                "field": "file",
+                "description": "Encrypted or password-protected Office file",
+            }
+        ],
+    )

apps/worker/app/services/document_ingestion/source_preparation.py

@@ -6,7 +6,13 @@
 from app.services.document_ingestion.file_size_policy import (
     build_file_size_limit_message,
 )
-from app.services.document_ingestion.processing_context import ParseJobContext
+from app.services.document_ingestion.office_compat_normalizer import (
+    normalize_office_source,
+)
+from app.services.document_ingestion.processing_context import (
+    ParseJobContext,
+    persist_job_metadata_updates,
+)
 from app.services.document_parser.support.internal_parse_name import (
     prepare_internal_parse_input,
 )
@@ -64,10 +70,18 @@ def prepare_source_file(
         f"local_path={prepared_parse_input.file_path}"
     )
 
+    normalized_source = normalize_office_source(prepared_parse_input.file_path)
+    if normalized_source.conversion is not None:
+        persist_job_metadata_updates(
+            job_id=job_id,
+            job_context=job_context,
+            metadata_updates={"office_compat": normalized_source.conversion},
+        )
+
     return PreparedSourceFile(
         source_file_name=source_file_name,
         internal_parse_name=prepared_parse_input.internal_filename,
-        local_file_path=prepared_parse_input.file_path,
+        local_file_path=normalized_source.file_path,
         file_extension=file_extension,
     )
 

apps/worker/app/services/document_parser/conversion/legacy_converter.py

@@ -51,7 +51,7 @@ def _convert_with_libreoffice(
     expected_output_ext: str,
     operation: str,
 ) -> tuple[str, str]:
-    """Convert a legacy Office document to an OOXML format via LibreOffice."""
+    """Convert an Office document to an OOXML format via LibreOffice."""
     soffice_path = resolve_libreoffice_binary()
 
     os.makedirs(outdir, exist_ok=True)
@@ -124,3 +124,25 @@ def xls_to_xlsx(xls_path: str, outdir: str = ".") -> tuple[str, str]:
         expected_output_ext="xlsx",
         operation="convert_xls_to_xlsx",
     )
+
+
+def normalize_docx_variant(source_path: str, outdir: str = ".") -> tuple[str, str]:
+    """Repair a non-standard DOCX container (macro/template content type) via LibreOffice."""
+    return _convert_with_libreoffice(
+        source_path=source_path,
+        outdir=outdir,
+        convert_to_arg="docx",
+        expected_output_ext="docx",
+        operation="normalize_docx_variant",
+    )
+
+
+def normalize_xlsx_variant(source_path: str, outdir: str = ".") -> tuple[str, str]:
+    """Repair a non-standard XLSX container (macro/template content type) via LibreOffice."""
+    return _convert_with_libreoffice(
+        source_path=source_path,
+        outdir=outdir,
+        convert_to_arg="xlsx",
+        expected_output_ext="xlsx",
+        operation="normalize_xlsx_variant",
+    )

apps/worker/app/services/document_parser/orchestration/office_container_inspection.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import zipfile
+from dataclasses import dataclass
+from xml.etree import ElementTree
+
+CONTENT_TYPES_MEMBER: str = "[Content_Types].xml"
+CONTENT_TYPES_NAMESPACE: str = (
+    "http://schemas.openxmlformats.org/package/2006/content-types"
+)
+
+
+@dataclass(frozen=True)
+class OfficeContainerInspection:
+    member_names: frozenset[str]
+    content_types_xml: bytes | None
+
+
+def inspect_office_container(file_path: str) -> OfficeContainerInspection | None:
+    """Return ZIP members and Content_Types.xml when the path is a readable ZIP."""
+    if not zipfile.is_zipfile(file_path):
+        return None
+    try:
+        with zipfile.ZipFile(file_path, "r") as archive:
+            member_names = frozenset(archive.namelist())
+            content_types_xml = (
+                archive.read(CONTENT_TYPES_MEMBER)
+                if CONTENT_TYPES_MEMBER in member_names
+                else None
+            )
+    except zipfile.BadZipFile:
+        return None
+    return OfficeContainerInspection(
+        member_names=member_names,
+        content_types_xml=content_types_xml,
+    )
+
+
+def read_override_content_type(
+    content_types_xml: bytes,
+    part_name: str,
+) -> str | None:
+    """Return the Override ContentType for ``part_name``, or None if absent."""
+    try:
+        root = ElementTree.fromstring(content_types_xml)
+    except ElementTree.ParseError:
+        return None
+
+    override_tag = f"{{{CONTENT_TYPES_NAMESPACE}}}Override"
+    for override in root.findall(override_tag):
+        if override.get("PartName") == part_name:
+            return override.get("ContentType")
+    return None
+
+
+__all__ = [
+    "CONTENT_TYPES_MEMBER",
+    "CONTENT_TYPES_NAMESPACE",
+    "OfficeContainerInspection",
+    "inspect_office_container",
+    "read_override_content_type",
+]

apps/worker/app/services/document_parser/orchestration/office_container_validator.py

@@ -1,9 +1,13 @@
 from __future__ import annotations
 
-import zipfile
 from dataclasses import dataclass
+from typing import NoReturn
 
 from app.services.document_parser.orchestration.format_router import DocumentFormat
+from app.services.document_parser.orchestration.office_container_inspection import (
+    CONTENT_TYPES_MEMBER,
+    inspect_office_container,
+)
 
 from shared.core.exceptions.domain_exceptions import ValidationException
 
@@ -29,7 +33,6 @@ def violation_description(self) -> str:
         )
 
 
-_CONTENT_TYPES_MEMBER: str = "[Content_Types].xml"
 _OFFICE_CONTAINER_REQUIREMENTS: dict[
     DocumentFormat,
     _OfficeContainerRequirement,
@@ -61,23 +64,18 @@ def validate_office_container(
     if requirement is None:
         return
 
-    if not zipfile.is_zipfile(file_path):
+    inspection = inspect_office_container(file_path)
+    if inspection is None:
         _raise_invalid_office_file(requirement)
 
-    try:
-        with zipfile.ZipFile(file_path, "r") as archive:
-            member_names = set(archive.namelist())
-    except zipfile.BadZipFile as exc:
-        raise _build_invalid_office_file_exception(requirement) from exc
-
     if (
-        _CONTENT_TYPES_MEMBER not in member_names
-        or requirement.required_member not in member_names
+        CONTENT_TYPES_MEMBER not in inspection.member_names
+        or requirement.required_member not in inspection.member_names
     ):
         _raise_invalid_office_file(requirement)
 
 
-def _raise_invalid_office_file(requirement: _OfficeContainerRequirement) -> None:
+def _raise_invalid_office_file(requirement: _OfficeContainerRequirement) -> NoReturn:
     raise _build_invalid_office_file_exception(requirement)