From 670da0220fdcd1e07a0107ab1ff20204b155f077 Mon Sep 17 00:00:00 2001 From: malx-labs Date: Wed, 29 Jul 2026 15:32:51 +0100 Subject: [PATCH 01/11] feat(pe): complete static PE directory coverage (v0.7.6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the remaining structural PE directories — relocations, certificate table, debug directory, and TLS — as deterministic, pefile-independent struct decoders, bringing IOCX's static PE engine to full directory completeness. Parsers (raw struct decode; never raise; tombstone into errors/truncations): - pe_relocations: IMAGE_BASE_RELOCATION blocks + typed entries (HIGHLOW, DIR64, ...), dual-bounded walk, non-advancing SizeOfBlock guarded. - pe_certificates: WIN_CERTIFICATE table read from raw file bytes (DATA_DIRECTORY[4].VirtualAddress is a FILE OFFSET, not an RVA); records overlaps_image fact; PKCS#7 blob left opaque. - pe_debug: IMAGE_DEBUG_DIRECTORY entries + CodeView PDB path extraction (RSDS/NB10) with canonical mixed-endian GUID; file-pointer-first reads. - pe_tls: IMAGE_TLS_DIRECTORY (PE32/PE32+); VA->RVA callback resolution via ImageBase; dual-bounded callback walk; zero-length raw data valid. Validators: - Add relocations + debug validators (content-level only; directory placement remains owned by rva_graph to avoid double-counting). - Migrate signature + tls validators to consume certificate_struct / tls_struct from InternalMetadata instead of pefile-derived metadata. All existing reason codes and checks preserved; adds the four new codes. Fixes a latent VA/RVA unit mismatch in TLS pointer->section mapping. - tls: surface previously dropped parser tombstones (tls_image_base_unavailable, tls_callbacks_va_below_image_base) as tls_callback_rva_invalid; narrow tls_zero_length_directory so a zero-length raw-data region with valid callbacks is no longer a false positive. Schema / plumbing: - internal_schema: add RelocationStruct, CertificateStruct, DebugStruct, TlsStruct and the four InternalMetadata keys. - reason_codes: add certificate_table_malformed, certificate_offset_inside_image, tls_directory_truncated, tls_callback_rva_invalid (+ relocation_* / debug_* families). All lowercase snake_case, non-overlapping, snapshot-stable. - engine: populate relocation_struct, certificate_struct, debug_struct, tls_struct via the build_*_structure decoders. - dispatcher: register relocations + debug (ascending directory-index order, after the placement backbone, before entropy). signature/tls keep their existing slots. --- iocx/engine.py | 8 + iocx/parsers/pe_certificates.py | 294 ++++++++++++++++++++ iocx/parsers/pe_debug.py | 329 +++++++++++++++++++++++ iocx/parsers/pe_relocations.py | 240 +++++++++++++++++ iocx/parsers/pe_tls.py | 267 ++++++++++++++++++ iocx/reason_codes.py | 28 ++ iocx/schemas/internal_schema.py | 116 ++++++++ iocx/validators/__init__.py | 10 + iocx/validators/_directory_invariants.py | 111 ++++++++ iocx/validators/debug.py | 150 +++++++++++ iocx/validators/relocations.py | 159 +++++++++++ iocx/validators/signature.py | 166 +++++++++--- iocx/validators/tls.py | 222 ++++++++++++--- 13 files changed, 2028 insertions(+), 72 deletions(-) create mode 100644 iocx/parsers/pe_certificates.py create mode 100644 iocx/parsers/pe_debug.py create mode 100644 iocx/parsers/pe_relocations.py create mode 100644 iocx/parsers/pe_tls.py create mode 100644 iocx/validators/_directory_invariants.py create mode 100644 iocx/validators/debug.py create mode 100644 iocx/validators/relocations.py diff --git a/iocx/engine.py b/iocx/engine.py index fd63364..6bfaeff 100644 --- a/iocx/engine.py +++ b/iocx/engine.py @@ -16,6 +16,10 @@ from .parsers.pe_optional_header import extract_optional_header_metadata from .parsers.pe_exports import build_export_structure from .parsers.pe_delay_imports import build_delay_import_structure +from .parsers.pe_relocations import build_relocation_structure +from .parsers.pe_debug import build_debug_structure +from .parsers.pe_certificates import build_certificate_structure +from .parsers.pe_tls import build_tls_structure from .detectors import all_detectors from .models import Detection, PluginContext from .plugins.loader import PluginLoader @@ -169,6 +173,10 @@ def _pipeline_pe(self, path: str) -> Dict[str, Any]: self._internal_metadata["export_struct"] = build_export_structure(pe) self._internal_metadata["delay_import_struct"] = build_delay_import_structure(pe) self._internal_metadata["data_directories_raw"] = analyse_data_directories_raw(pe) + self._internal_metadata["relocation_struct"] = build_relocation_structure(pe) + self._internal_metadata["debug_struct"] = build_debug_structure(pe) + self._internal_metadata["certificate_struct"] = build_certificate_structure(pe) + self._internal_metadata["tls_struct"] = build_tls_structure(pe) self._internal_metadata.update(extract_optional_header_metadata(pe)) internal: InternalMetadata = self._internal_metadata structural = run_structural_validators(internal, metadata, analysis_dict) diff --git a/iocx/parsers/pe_certificates.py b/iocx/parsers/pe_certificates.py new file mode 100644 index 0000000..4443043 --- /dev/null +++ b/iocx/parsers/pe_certificates.py @@ -0,0 +1,294 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Deterministic structural extraction of the PE certificate (Attribute +Certificate) table. + +Independent of pefile's DIRECTORY_ENTRY_SECURITY interpretation. +Pefile is used only to: + - Locate the security directory (file offset, size) + - Provide the raw file bytes via pe.__data__ + - Provide section raw-data extents for the "outside the image" fact + +IMPORTANT: For the security directory, DATA_DIRECTORY[4].VirtualAddress +is a *file offset*, NOT an RVA. The certificate table is appended to the +file and is not mapped into the image, so it must be read from the raw +file bytes, never resolved through pe.get_data. + +Each entry is a WIN_CERTIFICATE: + DWORD dwLength # total length incl. this 8-byte header + WORD wRevision # 0x0100 (1.0) or 0x0200 (2.0) + WORD wCertificateType # WIN_CERT_TYPE_* + BYTE bCertificate[] # opaque blob (not parsed here) + +Entries are 8-byte (QWORD) aligned. We decode structure only; the +embedded PKCS#7 blob is deliberately left opaque (static, no crypto). + +Output contract: + None - no certificate directory present (not an error) + dict per the documented contract (see CertificateStruct in + iocx.schemas.internal_schema). +""" + +from __future__ import annotations + +import struct +from typing import Any, Dict, List, Optional, Tuple + +# IMAGE_DIRECTORY_ENTRY_SECURITY = 4 +_SECURITY_DIRECTORY_INDEX = 4 +_WIN_CERT_HEADER_SIZE = 8 # dwLength + wRevision + wCertificateType +_CERT_ALIGNMENT = 8 # entries are QWORD-aligned + +# Hard cap on the number of certificate entries. +_MAX_CERTIFICATES = 1024 + +# WIN_CERT_REVISION_* +_REVISION_NAMES = { + 0x0100: "REVISION_1_0", + 0x0200: "REVISION_2_0", +} + +# WIN_CERT_TYPE_* +_CERT_TYPE_NAMES = { + 0x0001: "X509", + 0x0002: "PKCS_SIGNED_DATA", + 0x0003: "RESERVED_1", + 0x0004: "TS_STACK_SIGNED", +} + + +def build_certificate_structure(pe) -> Optional[Dict[str, Any]]: + """ + Locate and structurally decode the PE certificate table. + + Returns None if no security directory is present. Otherwise returns a + dict per the module docstring contract. Never raises; decode failures + produce tombstone entries in `errors` and `truncations`. + """ + placement = _locate_security_directory(pe) + if placement is None: + return None + + offset, size = placement + truncations: List[str] = [] + errors: List[str] = [] + + data = _raw_file_bytes(pe) + if data is None: + errors.append("raw_file_unavailable") + return { + "offset": offset, + "size": size, + "file_size": None, + "image_raw_end": None, + "overlaps_image": None, + "certificates": [], + "certificate_count": 0, + "truncations": truncations, + "errors": errors, + } + + file_size = len(data) + image_raw_end = _image_raw_end(pe) + # Structural fact only — the validator decides whether an overlap is a + # defect. A certificate table that begins before the end of any + # section's raw data overlaps mapped content on disk. + overlaps_image = ( + image_raw_end is not None and offset < image_raw_end + ) + + certificates = _read_certificates( + data, offset, size, file_size, truncations, errors, + ) + + return { + "offset": offset, + "size": size, + "file_size": file_size, + "image_raw_end": image_raw_end, + "overlaps_image": overlaps_image, + "certificates": certificates, + "certificate_count": len(certificates), + "truncations": truncations, + "errors": errors, + } + + +# ================================================================= +# Locator +# ================================================================= + +def _locate_security_directory(pe) -> Optional[Tuple[int, int]]: + """Return (file_offset, size) of the security directory, or None.""" + try: + data_dir = pe.OPTIONAL_HEADER.DATA_DIRECTORY[_SECURITY_DIRECTORY_INDEX] + offset = int(data_dir.VirtualAddress) # file offset, not RVA + size = int(data_dir.Size) + except (AttributeError, IndexError, ValueError, TypeError): + return None + + if offset == 0 or size == 0: + return None + + return (offset, size) + + +def _raw_file_bytes(pe) -> Optional[bytes]: + """Return the raw file bytes backing the PE, or None if unavailable.""" + raw = getattr(pe, "__data__", None) + if raw is None: + return None + try: + return bytes(raw) + except (TypeError, ValueError): + return None + + +def _image_raw_end(pe) -> Optional[int]: + """ + Largest (PointerToRawData + SizeOfRawData) across sections. + + This is the on-disk end of mapped content; the certificate table + should begin at or after it. Returns None if sections are absent. + """ + try: + sections = pe.sections + except AttributeError: + return None + if not sections: + return None + + end = 0 + for section in sections: + try: + ptr = int(section.PointerToRawData) + raw_size = int(section.SizeOfRawData) + except (AttributeError, ValueError, TypeError): + continue + if ptr and raw_size: + end = max(end, ptr + raw_size) + return end or None + + +# ================================================================= +# Certificate array +# ================================================================= + +def _read_certificates( + data: bytes, + base_offset: int, + declared_size: int, + file_size: int, + truncations: List[str], + errors: List[str], +) -> List[Dict[str, Any]]: + """ + Walk the WIN_CERTIFICATE array within the declared directory window. + + Each entry's dwLength includes the 8-byte header; entries advance on + an 8-byte alignment. A dwLength that fails to advance the cursor is + fatal for the walk and tagged as malformed. + """ + certificates: List[Dict[str, Any]] = [] + pos = base_offset + end = base_offset + declared_size + + if base_offset > file_size: + errors.append("certificate_offset_past_eof") + return certificates + + if end > file_size: + truncations.append("certificate_table_truncated") + end = file_size + + for index in range(_MAX_CERTIFICATES): + if pos >= end: + break + + if pos + _WIN_CERT_HEADER_SIZE > end: + truncations.append("certificate_header_truncated") + break + + try: + dw_length, revision, cert_type = struct.unpack_from( + " Dict[str, Any]: + """Decode a single WIN_CERTIFICATE header (blob left opaque).""" + cert: Dict[str, Any] = { + "index": index, + "offset": entry_offset, + "length": dw_length, + "revision": revision, + "revision_name": _REVISION_NAMES.get(revision), + "cert_type": cert_type, + "cert_type_name": _CERT_TYPE_NAMES.get(cert_type), + "data_length": 0, + "errors": [], + } + + if dw_length < _WIN_CERT_HEADER_SIZE: + cert["errors"].append("length_too_small") + return cert + + data_length = dw_length - _WIN_CERT_HEADER_SIZE + available = dir_end - (entry_offset + _WIN_CERT_HEADER_SIZE) + if data_length > available: + truncations.append("certificate_blob_truncated") + data_length = max(0, available) + + cert["data_length"] = data_length + + if revision not in _REVISION_NAMES: + cert["errors"].append("unknown_revision") + if cert_type not in _CERT_TYPE_NAMES: + cert["errors"].append("unknown_cert_type") + + return cert + + +# ================================================================= +# Helpers +# ================================================================= + +def _align_up(value: int, alignment: int) -> int: + """Round `value` up to the next multiple of `alignment`.""" + if alignment <= 0: + return value + remainder = value % alignment + if remainder == 0: + return value + return value + (alignment - remainder) diff --git a/iocx/parsers/pe_debug.py b/iocx/parsers/pe_debug.py new file mode 100644 index 0000000..c0825d7 --- /dev/null +++ b/iocx/parsers/pe_debug.py @@ -0,0 +1,329 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Deterministic structural extraction of the PE debug directory. + +Independent of pefile's DIRECTORY_ENTRY_DEBUG interpretation. +Pefile is used only to: + - Locate the debug directory (RVA, size) + - Resolve RVAs to file offsets via pe.get_data + - Provide raw file bytes via pe.__data__ (for PointerToRawData reads) + +Each entry is a 28-byte IMAGE_DEBUG_DIRECTORY: + DWORD Characteristics + DWORD TimeDateStamp + WORD MajorVersion + WORD MinorVersion + DWORD Type + DWORD SizeOfData + DWORD AddressOfRawData # RVA of the debug data + DWORD PointerToRawData # file offset of the debug data + +For CodeView entries (Type == 2) we additionally decode the PDB path from +the RSDS (PDB 7.0) or NB10 (PDB 2.0) record. All other blob types are left +opaque; we report structure only. + +Output contract: + None - no debug directory present (not an error) + dict per the documented contract (see DebugStruct in + iocx.schemas.internal_schema). +""" + +from __future__ import annotations + +import struct +from typing import Any, Dict, List, Optional, Tuple + +# IMAGE_DIRECTORY_ENTRY_DEBUG = 6 +_DEBUG_DIRECTORY_INDEX = 6 +_DEBUG_ENTRY_SIZE = 28 # IMAGE_DEBUG_DIRECTORY is 28 bytes + +# Hard cap on debug directory entries. +_MAX_DEBUG_ENTRIES = 256 + +# PDB path length cap to defend against unterminated reads. +_PDB_PATH_MAX_LEN = 512 +# Upper bound on CodeView blob we will read to locate the path. +_CODEVIEW_MAX_LEN = 4096 + +# IMAGE_DEBUG_TYPE_* +_DEBUG_TYPE_CODEVIEW = 2 +_DEBUG_TYPE_NAMES = { + 0: "UNKNOWN", + 1: "COFF", + 2: "CODEVIEW", + 3: "FPO", + 4: "MISC", + 5: "EXCEPTION", + 6: "FIXUP", + 7: "OMAP_TO_SRC", + 8: "OMAP_FROM_SRC", + 9: "BORLAND", + 10: "RESERVED10", + 11: "CLSID", + 12: "VC_FEATURE", + 13: "POGO", + 14: "ILTCG", + 15: "MPX", + 16: "REPRO", + 17: "SPGO", + 20: "EX_DLLCHARACTERISTICS", +} + +# CodeView signatures +_CV_SIG_RSDS = b"RSDS" # PDB 7.0 +_CV_SIG_NB10 = b"NB10" # PDB 2.0 + + +def build_debug_structure(pe) -> Optional[Dict[str, Any]]: + """ + Locate and structurally decode the PE debug directory. + + Returns None if no debug directory is present. Otherwise returns a + dict per the module docstring contract. Never raises; decode failures + produce tombstone entries in `errors` and `truncations`. + """ + placement = _locate_debug_directory(pe) + if placement is None: + return None + + rva, size = placement + truncations: List[str] = [] + errors: List[str] = [] + + entries = _read_entries(pe, rva, size, truncations, errors) + + return { + "rva": rva, + "size": size, + "entries": entries, + "entry_count": len(entries), + "truncations": truncations, + "errors": errors, + } + + +# ================================================================= +# Locator +# ================================================================= + +def _locate_debug_directory(pe) -> Optional[Tuple[int, int]]: + """Return (rva, size) of the debug directory, or None if absent.""" + try: + data_dir = pe.OPTIONAL_HEADER.DATA_DIRECTORY[_DEBUG_DIRECTORY_INDEX] + rva = int(data_dir.VirtualAddress) + size = int(data_dir.Size) + except (AttributeError, IndexError, ValueError, TypeError): + return None + + if rva == 0 or size == 0: + return None + + return (rva, size) + + +# ================================================================= +# Entry array +# ================================================================= + +def _read_entries( + pe, + base_rva: int, + declared_size: int, + truncations: List[str], + errors: List[str], +) -> List[Dict[str, Any]]: + """Walk the fixed-size IMAGE_DEBUG_DIRECTORY entries.""" + entries: List[Dict[str, Any]] = [] + declared_count = declared_size // _DEBUG_ENTRY_SIZE + + if declared_size % _DEBUG_ENTRY_SIZE != 0: + truncations.append("debug_directory_size_not_entry_aligned") + + if declared_count > _MAX_DEBUG_ENTRIES: + truncations.append("debug_directory_entry_count_exceeds_max") + declared_count = _MAX_DEBUG_ENTRIES + + pos = base_rva + for index in range(declared_count): + try: + raw = bytes(pe.get_data(pos, _DEBUG_ENTRY_SIZE)) + except Exception: + truncations.append("debug_entry_read_failed") + break + + if len(raw) < _DEBUG_ENTRY_SIZE: + truncations.append("debug_entry_truncated") + break + + entry = _decode_entry(pe, index, raw) + entries.append(entry) + pos += _DEBUG_ENTRY_SIZE + + return entries + + +def _decode_entry(pe, index: int, raw: bytes) -> Dict[str, Any]: + """Unpack one 28-byte IMAGE_DEBUG_DIRECTORY and enrich CodeView.""" + try: + (characteristics, timestamp, major, minor, dtype, + size_of_data, addr_raw, ptr_raw) = struct.unpack_from( + " None: + """ + Read the CodeView record and extract the PDB path deterministically. + + Prefers PointerToRawData (raw file offset); falls back to + AddressOfRawData (RVA) if the file pointer is absent. + """ + size_of_data = entry["size_of_data"] + read_len = min(max(size_of_data, 0) or _CODEVIEW_MAX_LEN, _CODEVIEW_MAX_LEN) + + blob = _read_codeview_blob(pe, entry, read_len) + if blob is None: + entry["errors"].append("codeview_read_failed") + return + + if len(blob) < 4: + entry["errors"].append("codeview_too_short") + return + + signature = blob[:4] + if signature == _CV_SIG_RSDS: + entry["cv_signature"] = "RSDS" + _decode_rsds(blob, entry) + elif signature == _CV_SIG_NB10: + entry["cv_signature"] = "NB10" + _decode_nb10(blob, entry) + else: + entry["errors"].append("codeview_signature_unknown") + + +def _read_codeview_blob( + pe, entry: Dict[str, Any], read_len: int, +) -> Optional[bytes]: + """Read the CodeView blob via file pointer first, then RVA.""" + ptr_raw = entry["pointer_to_raw_data"] + addr_raw = entry["address_of_raw_data"] + + if ptr_raw: + raw = getattr(pe, "__data__", None) + if raw is not None: + try: + data = bytes(raw) + return data[ptr_raw:ptr_raw + read_len] + except (TypeError, ValueError): + pass + + if addr_raw: + try: + return bytes(pe.get_data(addr_raw, read_len)) + except Exception: + return None + + return None + + +def _decode_rsds(blob: bytes, entry: Dict[str, Any]) -> None: + """ + RSDS record: 'RSDS' + GUID(16) + Age(DWORD) + ASCIIZ PDB path. + Header is 24 bytes; path follows. + """ + if len(blob) < 24: + entry["errors"].append("codeview_rsds_truncated") + return + + guid_bytes = blob[4:20] + (age,) = struct.unpack_from(" None: + """ + NB10 record: 'NB10' + Offset(DWORD) + Timestamp(DWORD) + Age(DWORD) + + ASCIIZ PDB path. Header is 16 bytes; path follows. + """ + if len(blob) < 16: + entry["errors"].append("codeview_nb10_truncated") + return + + (_offset, _ts, age) = struct.unpack_from(" Optional[str]: + """Extract a NUL-terminated ASCII PDB path starting at `start`.""" + region = blob[start:start + _PDB_PATH_MAX_LEN] + nul_pos = region.find(b"\x00") + if nul_pos == -1: + # No terminator within the cap — take what we have and flag it. + entry["errors"].append("pdb_path_unterminated") + candidate = region + else: + candidate = region[:nul_pos] + + if not candidate: + return None + + try: + return candidate.decode("ascii") + except UnicodeDecodeError: + entry["errors"].append("pdb_path_non_ascii") + return candidate.decode("ascii", errors="replace") + + +def _format_guid(guid_bytes: bytes) -> Optional[str]: + """ + Format a 16-byte CodeView GUID as the canonical mixed-endian string + used by symbol servers (Data1/2/3 little-endian, Data4 big-endian). + """ + if len(guid_bytes) < 16: + return None + d1, d2, d3 = struct.unpack_from(" Optional[Dict[str, Any]]: + """ + Locate and structurally decode the PE base-relocation table. + + Returns None if no relocation directory is present. Otherwise returns + a dict per the module docstring contract. Never raises; decode + failures produce tombstone entries in `errors` and `truncations`. + """ + placement = _locate_reloc_directory(pe) + if placement is None: + return None + + rva, size = placement + truncations: List[str] = [] + errors: List[str] = [] + + blocks = _read_blocks(pe, rva, size, truncations, errors) + + entry_count = sum(len(b["entries"]) for b in blocks) + + return { + "rva": rva, + "size": size, + "blocks": blocks, + "block_count": len(blocks), + "entry_count": entry_count, + "truncations": truncations, + "errors": errors, + } + + +# ================================================================= +# Locator +# ================================================================= + +def _locate_reloc_directory(pe) -> Optional[Tuple[int, int]]: + """Return (rva, size) of the relocation directory, or None if absent.""" + try: + data_dir = pe.OPTIONAL_HEADER.DATA_DIRECTORY[_RELOC_DIRECTORY_INDEX] + rva = int(data_dir.VirtualAddress) + size = int(data_dir.Size) + except (AttributeError, IndexError, ValueError, TypeError): + return None + + if rva == 0 or size == 0: + return None + + return (rva, size) + + +# ================================================================= +# Block array +# ================================================================= + +def _read_blocks( + pe, + base_rva: int, + declared_size: int, + truncations: List[str], + errors: List[str], +) -> List[Dict[str, Any]]: + """ + Walk the array of IMAGE_BASE_RELOCATION blocks. + + Blocks are packed contiguously; each block advertises its own total + SizeOfBlock (header + entries). We stop at the declared directory + size and enforce a max block count to defend against pathological + inputs. A SizeOfBlock that does not advance the cursor is fatal for + the walk (would otherwise loop forever) and is tagged as such. + """ + blocks: List[Dict[str, Any]] = [] + pos = base_rva + end = base_rva + declared_size + + for index in range(_MAX_BLOCKS): + if pos >= end: + break + + if pos + _BLOCK_HEADER_SIZE > end: + truncations.append("relocation_block_header_truncated") + break + + try: + header = bytes(pe.get_data(pos, _BLOCK_HEADER_SIZE)) + except Exception: + truncations.append("relocation_block_read_failed") + break + + if len(header) < _BLOCK_HEADER_SIZE: + truncations.append("relocation_block_header_truncated") + break + + try: + page_rva, size_of_block = struct.unpack_from(" Dict[str, Any]: + """Decode a single relocation block header and its WORD entries.""" + block: Dict[str, Any] = { + "index": index, + "block_rva": block_rva, + "page_rva": page_rva, + "size_of_block": size_of_block, + "entry_count": 0, + "entries": [], + "errors": [], + } + + if size_of_block < _BLOCK_HEADER_SIZE: + block["errors"].append("size_of_block_too_small") + return block + + entries_bytes = size_of_block - _BLOCK_HEADER_SIZE + if entries_bytes % _ENTRY_SIZE != 0: + # Odd trailing byte — not a whole number of WORD entries. + block["errors"].append("size_of_block_not_word_aligned") + + declared_entries = entries_bytes // _ENTRY_SIZE + if declared_entries > _MAX_ENTRIES_PER_BLOCK: + block["errors"].append("entry_count_exceeds_max") + declared_entries = _MAX_ENTRIES_PER_BLOCK + + # Clamp the readable region to the declared directory end so a block + # advertising a SizeOfBlock past the directory cannot over-read. + entries_start = block_rva + _BLOCK_HEADER_SIZE + readable = max(0, min(entries_bytes, dir_end - entries_start)) + if readable < entries_bytes: + truncations.append("relocation_entries_truncated") + + readable_entries = min(declared_entries, readable // _ENTRY_SIZE) + + try: + raw = bytes(pe.get_data(entries_start, readable_entries * _ENTRY_SIZE)) + except Exception: + truncations.append("relocation_entries_read_failed") + raw = b"" + + if len(raw) < readable_entries * _ENTRY_SIZE: + truncations.append("relocation_entries_truncated") + readable_entries = len(raw) // _ENTRY_SIZE + + for i in range(readable_entries): + (word,) = struct.unpack_from("> 12) & 0xF + offset = word & 0x0FFF + block["entries"].append({ + "type": reloc_type, + "type_name": _RELOC_TYPE_NAMES.get(reloc_type), + "offset": offset, + "rva": page_rva + offset, + }) + + block["entry_count"] = len(block["entries"]) + return block diff --git a/iocx/parsers/pe_tls.py b/iocx/parsers/pe_tls.py new file mode 100644 index 0000000..2b4efaf --- /dev/null +++ b/iocx/parsers/pe_tls.py @@ -0,0 +1,267 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Deterministic structural extraction of the PE TLS directory. + +Independent of pefile's DIRECTORY_ENTRY_TLS interpretation. +Pefile is used only to: + - Locate the TLS directory (RVA, size) + - Determine PE32 vs PE32+ via OPTIONAL_HEADER.Magic + - Read OPTIONAL_HEADER.ImageBase (to convert VA -> RVA) + - Resolve RVAs to file offsets via pe.get_data + +IMPORTANT: the address fields in IMAGE_TLS_DIRECTORY are *virtual +addresses* (VAs), not RVAs. To read the callback array we convert +AddressOfCallBacks to an RVA by subtracting ImageBase. + +IMAGE_TLS_DIRECTORY32 (24 bytes) / IMAGE_TLS_DIRECTORY64 (40 bytes): + StartAddressOfRawData # VA + EndAddressOfRawData # VA + AddressOfIndex # VA + AddressOfCallBacks # VA -> NULL-terminated array of VAs + DWORD SizeOfZeroFill + DWORD Characteristics +( is 4 bytes on PE32, 8 bytes on PE32+) + +Output contract: + None - no TLS directory present (not an error) + dict per the documented contract (see TlsStruct in + iocx.schemas.internal_schema). +""" + +from __future__ import annotations + +import struct +from typing import Any, Dict, List, Optional, Tuple + +# IMAGE_DIRECTORY_ENTRY_TLS = 9 +_TLS_DIRECTORY_INDEX = 9 + +# OPTIONAL_HEADER.Magic values +_MAGIC_PE32 = 0x10B +_MAGIC_PE32_PLUS = 0x20B + +# Hard cap on TLS callbacks to defend against looping / pathological +# arrays. Real binaries have a handful at most. +_MAX_CALLBACKS = 4096 + + +def build_tls_structure(pe) -> Optional[Dict[str, Any]]: + """ + Locate and structurally decode the PE TLS directory. + + Returns None if no TLS directory is present. Otherwise returns a dict + per the module docstring contract. Never raises; decode failures + produce tombstone entries in `errors` and `truncations`. + """ + placement = _locate_tls_directory(pe) + if placement is None: + return None + + rva, size = placement + is_64bit = _is_pe32_plus(pe) + ptr_size = 8 if is_64bit else 4 + dir_size = 40 if is_64bit else 24 + image_base = _image_base(pe) + + truncations: List[str] = [] + errors: List[str] = [] + + fields = _read_directory(pe, rva, dir_size, ptr_size, errors) + if fields is None: + return { + "rva": rva, + "size": size, + "is_64bit": is_64bit, + "image_base": image_base, + "start_address_of_raw_data": None, + "end_address_of_raw_data": None, + "address_of_index": None, + "address_of_callbacks": None, + "size_of_zero_fill": None, + "characteristics": None, + "raw_data_size": None, + "callbacks": [], + "callback_count": 0, + "truncations": truncations, + "errors": errors, + } + + (start_va, end_va, index_va, callbacks_va, + size_of_zero_fill, characteristics) = fields + + raw_data_size = _raw_data_size(start_va, end_va, errors) + + callbacks = _read_callbacks( + pe, callbacks_va, image_base, ptr_size, truncations, errors, + ) + + return { + "rva": rva, + "size": size, + "is_64bit": is_64bit, + "image_base": image_base, + "start_address_of_raw_data": start_va, + "end_address_of_raw_data": end_va, + "address_of_index": index_va, + "address_of_callbacks": callbacks_va, + "size_of_zero_fill": size_of_zero_fill, + "characteristics": characteristics, + "raw_data_size": raw_data_size, + "callbacks": callbacks, + "callback_count": len(callbacks), + "truncations": truncations, + "errors": errors, + } + + +# ================================================================= +# Locator +# ================================================================= + +def _locate_tls_directory(pe) -> Optional[Tuple[int, int]]: + """Return (rva, size) of the TLS directory, or None if absent.""" + try: + data_dir = pe.OPTIONAL_HEADER.DATA_DIRECTORY[_TLS_DIRECTORY_INDEX] + rva = int(data_dir.VirtualAddress) + size = int(data_dir.Size) + except (AttributeError, IndexError, ValueError, TypeError): + return None + + if rva == 0 or size == 0: + return None + + return (rva, size) + + +def _is_pe32_plus(pe) -> bool: + """Determine PE32+ (64-bit) by reading OPTIONAL_HEADER.Magic.""" + try: + magic = int(pe.OPTIONAL_HEADER.Magic) + return magic == _MAGIC_PE32_PLUS + except (AttributeError, ValueError, TypeError): + # Default to 32-bit if we can't tell. Conservative — narrower + # pointers, lower risk of over-reading. + return False + + +def _image_base(pe) -> Optional[int]: + """Read OPTIONAL_HEADER.ImageBase, or None if unavailable.""" + try: + return int(pe.OPTIONAL_HEADER.ImageBase) + except (AttributeError, ValueError, TypeError): + return None + + +# ================================================================= +# Directory struct +# ================================================================= + +def _read_directory( + pe, + rva: int, + dir_size: int, + ptr_size: int, + errors: List[str], +) -> Optional[Tuple[int, int, int, int, int, int]]: + """Read and unpack the fixed IMAGE_TLS_DIRECTORY struct.""" + try: + raw = bytes(pe.get_data(rva, dir_size)) + except Exception: + errors.append("tls_directory_read_failed") + return None + + if len(raw) < dir_size: + errors.append("tls_directory_truncated") + return None + + ptr_fmt = "Q" if ptr_size == 8 else "I" + fmt = "<" + (ptr_fmt * 4) + "II" + try: + (start_va, end_va, index_va, callbacks_va, + size_of_zero_fill, characteristics) = struct.unpack_from(fmt, raw, 0) + except struct.error: + errors.append("tls_directory_unpack_failed") + return None + + return (start_va, end_va, index_va, callbacks_va, + size_of_zero_fill, characteristics) + + +def _raw_data_size( + start_va: int, end_va: int, errors: List[str], +) -> Optional[int]: + """ + Compute EndAddressOfRawData - StartAddressOfRawData. + + A zero-length region (start == end) is valid and common. An end that + precedes start is structurally invalid. + """ + if end_va < start_va: + errors.append("tls_raw_data_end_before_start") + return None + return end_va - start_va + + +# ================================================================= +# Callback array +# ================================================================= + +def _read_callbacks( + pe, + callbacks_va: int, + image_base: Optional[int], + ptr_size: int, + truncations: List[str], + errors: List[str], +) -> List[int]: + """ + Read the NULL-terminated array of callback VAs. + + AddressOfCallBacks is a VA; convert to RVA by subtracting ImageBase. + A zero AddressOfCallBacks means "no callbacks" (not an error). The + read is capped to defend against looping / non-terminating arrays. + """ + if callbacks_va == 0: + return [] + + if image_base is None: + errors.append("tls_image_base_unavailable") + return [] + + callbacks_rva = callbacks_va - image_base + if callbacks_rva < 0: + errors.append("tls_callbacks_va_below_image_base") + return [] + + callbacks: List[int] = [] + pos = callbacks_rva + fmt = " int + analysis["sections"] -> List[{"rva": int, "virtual_size": int}] + +If `sections` is absent we fall back to a SizeOfImage bound check only; +if `size_of_image` is also absent we skip the check entirely rather than +guess (an upstream gap is not a directory defect). +""" + +from typing import Any, Dict, List, Optional, Tuple + +__all__ = [ + "region_within_image", + "rva_in_any_section", + "region_in_any_section", +] + + +def region_within_image( + rva: Optional[int], + size: Optional[int], + size_of_image: Optional[int], +) -> Optional[bool]: + """ + True if [rva, rva+size) lies within SizeOfImage. + + Returns None when the check cannot be performed (missing inputs) so + callers can distinguish "unknown" from "out of bounds". + """ + if rva is None or size_of_image is None: + return None + span = size or 0 + if rva < 0 or span < 0: + return False + return (rva + span) <= size_of_image + + +def _sections(analysis: Dict[str, Any]) -> List[Tuple[int, int]]: + """Extract [(rva, virtual_size), ...] from analysis, tolerating keys.""" + out: List[Tuple[int, int]] = [] + for sec in analysis.get("sections", []) or []: + rva = sec.get("rva", sec.get("virtual_address")) + vsize = sec.get("virtual_size", sec.get("size")) + if rva is None or vsize is None: + continue + try: + out.append((int(rva), int(vsize))) + except (ValueError, TypeError): + continue + return out + + +def rva_in_any_section( + rva: Optional[int], + analysis: Dict[str, Any], +) -> Optional[bool]: + """ + True if `rva` falls inside any section's virtual extent. + + Falls back to a SizeOfImage bound check when section data is absent. + Returns None when neither is available. + """ + if rva is None: + return None + + sections = _sections(analysis) + if sections: + for base, vsize in sections: + if base <= rva < base + max(vsize, 0): + return True + return False + + return region_within_image(rva, 0, analysis.get("size_of_image")) + + +def region_in_any_section( + rva: Optional[int], + size: Optional[int], + analysis: Dict[str, Any], +) -> Optional[bool]: + """ + True if the whole region [rva, rva+size) fits inside a single section. + + Falls back to a SizeOfImage bound check when section data is absent. + Returns None when neither is available. + """ + if rva is None: + return None + span = size or 0 + + sections = _sections(analysis) + if sections: + for base, vsize in sections: + end = base + max(vsize, 0) + if base <= rva and (rva + span) <= end: + return True + return False + + return region_within_image(rva, span, analysis.get("size_of_image")) diff --git a/iocx/validators/debug.py b/iocx/validators/debug.py new file mode 100644 index 0000000..a95874c --- /dev/null +++ b/iocx/validators/debug.py @@ -0,0 +1,150 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Validate the debug-directory structure produced by parser pe_debug. + +Absence of a debug directory is NOT a structural defect. We only emit +codes when the directory is present and structurally malformed. + +Placement ownership: directory->section placement for the debug directory +(dir 6) is owned by the rva_graph validator, which runs earlier in the +dispatcher. This validator therefore descends into entry contents only and +does NOT re-check directory placement, to avoid double-counting. (The +per-entry AddressOfRawData check below is a distinct, content-level fact: +it maps each entry's *data region*, which rva_graph does not inspect.) + +This validator covers: + - IMAGE_DEBUG_DIRECTORY entry integrity + - debug entry data-region RVA validity + - truncated / malformed entry handling + +Reason codes emitted: + DEBUG_DIRECTORY_INVALID_HEADER + DEBUG_TABLE_TRUNCATED + DEBUG_DIRECTORY_ENTRY_MALFORMED + DEBUG_ENTRY_RVA_INVALID +""" + +from typing import Any, Dict, List + +from iocx.reason_codes import ReasonCodes +from iocx.validators.schema import StructuralIssue +from iocx.schemas.internal_schema import InternalMetadata +from iocx.schemas.analysis import AnalysisDict +from .decorators import depends_on +from ._directory_invariants import region_in_any_section + +# Priority-resolved per-entry pathologies. First match wins. +_ENTRY_ERROR_PRIORITY = [ + "entry_unpack_failed", + "codeview_read_failed", + "codeview_too_short", + "codeview_rsds_truncated", + "codeview_nb10_truncated", + "codeview_signature_unknown", + "pdb_path_unterminated", + "pdb_path_non_ascii", +] + + +@depends_on("internal", "analysis") +def validate_debug(metadata: InternalMetadata, + analysis: AnalysisDict) -> List[StructuralIssue]: + issues: List[StructuralIssue] = [] + + debug = metadata.get("debug_struct") + if debug is None: + return issues # no debug directory — not a defect + + # ---- Top-level decode failures short-circuit ---- + if debug.get("errors"): + issues.append(StructuralIssue( + issue=ReasonCodes.DEBUG_DIRECTORY_INVALID_HEADER, + details={"reason": "top_level_decode", + "errors": list(debug["errors"])}, + )) + return issues + + # NOTE: directory placement is intentionally NOT checked here — it is + # owned by rva_graph. See module docstring. + _validate_truncations(debug, issues) + _validate_entries(debug, analysis, issues) + + return issues + + +# ================================================================= +# Truncations +# ================================================================= + +def _validate_truncations(debug: Dict[str, Any], + issues: List[StructuralIssue]) -> None: + """Map parser truncation tags to one issue per truncated region.""" + for tag in debug.get("truncations", []) or []: + issues.append(StructuralIssue( + issue=ReasonCodes.DEBUG_TABLE_TRUNCATED, + details={"region": tag}, + )) + + +# ================================================================= +# Entry-level validation +# ================================================================= + +def _validate_entries(debug: Dict[str, Any], + analysis: AnalysisDict, + issues: List[StructuralIssue]) -> None: + """Emit per-entry malformation and data-region RVA issues.""" + for entry in debug.get("entries", []) or []: + index = entry.get("index") + entry_errors = entry.get("errors", []) or [] + + reason = _first_matching(entry_errors, _ENTRY_ERROR_PRIORITY) + if reason != "unknown": + issues.append(StructuralIssue( + issue=ReasonCodes.DEBUG_DIRECTORY_ENTRY_MALFORMED, + details={"index": index, + "type": entry.get("type"), + "type_name": entry.get("type_name"), + "reason": reason}, + )) + + _validate_entry_rva(entry, analysis, issues) + + +def _validate_entry_rva(entry: Dict[str, Any], + analysis: AnalysisDict, + issues: List[StructuralIssue]) -> None: + """ + Flag a debug entry whose AddressOfRawData region does not map to any + section. Entries with no RVA (file-pointer-only) are not flagged here. + + This is a content-level check on the entry's data region, distinct from + the directory placement owned by rva_graph. + """ + addr_raw = entry.get("address_of_raw_data") + size_of_data = entry.get("size_of_data") or 0 + if not addr_raw: + return + + mapped = region_in_any_section(addr_raw, size_of_data, analysis) + if mapped is False: + issues.append(StructuralIssue( + issue=ReasonCodes.DEBUG_ENTRY_RVA_INVALID, + details={"index": entry.get("index"), + "address_of_raw_data": addr_raw, + "size_of_data": size_of_data}, + )) + + +# ================================================================= +# Helpers +# ================================================================= + +def _first_matching(errors: List[str], candidates: List[str]) -> str: + """Return the first tag from `candidates` present in `errors`.""" + for c in candidates: + if c in errors: + return c + return "unknown" diff --git a/iocx/validators/relocations.py b/iocx/validators/relocations.py new file mode 100644 index 0000000..9cdf2e7 --- /dev/null +++ b/iocx/validators/relocations.py @@ -0,0 +1,159 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Validate the base-relocation structure produced by parser pe_relocations. + +Absence of a relocation directory is NOT a structural defect — stripped +or fixed-base binaries legitimately omit it. We only emit codes when the +directory is present and structurally malformed. + +Placement ownership: directory->section placement for the relocation +directory (dir 5) is owned by the rva_graph validator, which runs earlier +in the dispatcher. This validator therefore descends into block contents +only and does NOT re-check placement, to avoid double-counting. + +This validator covers: + - IMAGE_BASE_RELOCATION block integrity + - relocation entry RVA validity + - truncated / malformed block handling + +Reason codes emitted: + RELOCATION_DIRECTORY_INVALID_HEADER + RELOCATION_TABLE_TRUNCATED + RELOCATION_BLOCK_MALFORMED + RELOCATION_ENTRY_RVA_INVALID +""" + +from typing import Any, Dict, List + +from iocx.reason_codes import ReasonCodes +from iocx.validators.schema import StructuralIssue +from iocx.schemas.internal_schema import InternalMetadata +from iocx.schemas.analysis import AnalysisDict +from .decorators import depends_on +from ._directory_invariants import rva_in_any_section + +# Priority-resolved block-level pathologies. First match wins for +# deterministic emission. +_BLOCK_ERROR_PRIORITY = [ + "size_of_block_too_small", + "size_of_block_not_word_aligned", + "entry_count_exceeds_max", +] + +# Cap on how many invalid-entry issues a single block may raise, so a +# pathological block cannot flood the issue stream. The count is always +# reported in details regardless of how many individual issues emit. +_MAX_ENTRY_ISSUES_PER_BLOCK = 8 + + +@depends_on("internal", "analysis") +def validate_relocations(metadata: InternalMetadata, + analysis: AnalysisDict) -> List[StructuralIssue]: + issues: List[StructuralIssue] = [] + + reloc = metadata.get("relocation_struct") + if reloc is None: + return issues # no relocation directory — not a defect + + # ---- Top-level decode failures short-circuit ---- + if reloc.get("errors"): + issues.append(StructuralIssue( + issue=ReasonCodes.RELOCATION_DIRECTORY_INVALID_HEADER, + details={"reason": "top_level_decode", + "errors": list(reloc["errors"])}, + )) + return issues + + # NOTE: directory placement is intentionally NOT checked here — it is + # owned by rva_graph. See module docstring. + _validate_truncations(reloc, issues) + _validate_blocks(reloc, analysis, issues) + + return issues + + +# ================================================================= +# Truncations +# ================================================================= + +def _validate_truncations(reloc: Dict[str, Any], + issues: List[StructuralIssue]) -> None: + """Map parser truncation tags to one issue per truncated region.""" + for tag in reloc.get("truncations", []) or []: + issues.append(StructuralIssue( + issue=ReasonCodes.RELOCATION_TABLE_TRUNCATED, + details={"region": tag}, + )) + + +# ================================================================= +# Block-level validation +# ================================================================= + +def _validate_blocks(reloc: Dict[str, Any], + analysis: AnalysisDict, + issues: List[StructuralIssue]) -> None: + """Emit per-block structural issues and per-entry RVA issues.""" + for block in reloc.get("blocks", []) or []: + index = block.get("index") + block_errors = block.get("errors", []) or [] + + reason = _first_matching(block_errors, _BLOCK_ERROR_PRIORITY) + if reason != "unknown": + issues.append(StructuralIssue( + issue=ReasonCodes.RELOCATION_BLOCK_MALFORMED, + details={"index": index, + "page_rva": block.get("page_rva"), + "size_of_block": block.get("size_of_block"), + "reason": reason}, + )) + + _validate_block_entries(block, analysis, issues) + + +def _validate_block_entries(block: Dict[str, Any], + analysis: AnalysisDict, + issues: List[StructuralIssue]) -> None: + """ + Flag relocation entries whose target RVA does not map to any section. + ABSOLUTE (type 0) entries are padding and are never flagged. + + Conservative check: on well-formed binaries every fixup target maps to + a section. Consider gating behind a strict-mode flag if real system + binaries produce noise. + """ + index = block.get("index") + invalid_rvas: List[int] = [] + + for entry in block.get("entries", []) or []: + if entry.get("type") == 0: # IMAGE_REL_BASED_ABSOLUTE — padding + continue + target_rva = entry.get("rva") + mapped = rva_in_any_section(target_rva, analysis) + if mapped is False: + invalid_rvas.append(target_rva) + + if not invalid_rvas: + return + + for target_rva in invalid_rvas[:_MAX_ENTRY_ISSUES_PER_BLOCK]: + issues.append(StructuralIssue( + issue=ReasonCodes.RELOCATION_ENTRY_RVA_INVALID, + details={"block_index": index, + "rva": target_rva, + "invalid_entry_count": len(invalid_rvas)}, + )) + + +# ================================================================= +# Helpers +# ================================================================= + +def _first_matching(errors: List[str], candidates: List[str]) -> str: + """Return the first tag from `candidates` present in `errors`.""" + for c in candidates: + if c in errors: + return c + return "unknown" diff --git a/iocx/validators/signature.py b/iocx/validators/signature.py index d8e0d82..95b370b 100644 --- a/iocx/validators/signature.py +++ b/iocx/validators/signature.py @@ -1,68 +1,171 @@ # Copyright (c) 2026 MalX Labs and contributors # SPDX-License-Identifier: MPL-2.0 +""" +Validate the WIN_CERTIFICATE (Authenticode) directory. + +v0.7.6 migration: this validator now sources per-certificate structural +truth from the deterministic ``certificate_struct`` produced by +parser pe_certificates (read from InternalMetadata), rather than from +pefile's ``metadata["signatures"]`` list. The flag/metadata symmetry check +still consults the public ``has_signature`` flag, and the overlay / section +overlap checks still use the ``analysis`` geometry — so ALL existing checks +and reason codes are preserved. + +Two v0.7.6 reason codes are added, in territory the existing checks did not +cover (no double-counting): + + CERTIFICATE_TABLE_MALFORMED - structural decode failure / truncation + reported by the parser. Distinct from the + field-value checks (SIGNATURE_INVALID_*), + which continue to own length/revision/type. + CERTIFICATE_OFFSET_INSIDE_IMAGE - table-level "offset must lie outside the + image" invariant, driven by the parser's + `overlaps_image` fact (offset < end of any + section's on-disk raw data). + +Preserved reason codes: + SIGNATURE_FLAG_SET_BUT_NO_METADATA + SIGNATURE_PRESENT_BUT_FLAG_NOT_SET + SIGNATURE_MULTIPLE_CERTIFICATES + SIGNATURE_INVALID_LENGTH + SIGNATURE_INVALID_REVISION + SIGNATURE_INVALID_TYPE + SIGNATURE_OUT_OF_FILE_BOUNDS + SIGNATURE_OVERLAPS_OTHER_DATA +""" + from typing import Dict, Any, List + from iocx.reason_codes import ReasonCodes from iocx.validators.schema import StructuralIssue from iocx.schemas.public_metadata import PublicMetadata +from iocx.schemas.internal_schema import InternalMetadata from iocx.schemas.analysis import AnalysisDict from .decorators import depends_on +# Parser per-certificate error tags that indicate a genuine structural decode +# failure (as opposed to a bad field VALUE, which the SIGNATURE_INVALID_* +# checks below already own). Kept deliberately narrow to avoid double-counting. +_STRUCTURAL_CERT_ERROR_TAGS = { + "length_too_small", # also covered by SIGNATURE_INVALID_LENGTH; see note +} + -@depends_on("metadata", "analysis") -def validate_signature(metadata: PublicMetadata, analysis: AnalysisDict) -> List[StructuralIssue]: +@depends_on("internal", "metadata", "analysis") +def validate_signature(internal: InternalMetadata, + metadata: PublicMetadata, + analysis: AnalysisDict) -> List[StructuralIssue]: issues: List[StructuralIssue] = [] + cert_struct = internal.get("certificate_struct") + + # --------------------------------------------------------- + # 0) Structural decode failure takes precedence (NEW: CERTIFICATE_TABLE_MALFORMED) + # The parser returns a struct (not None) when a security directory is + # declared. If it could not decode that directory at all, report it as + # malformed FIRST — otherwise the empty `certificates` list would trip + # the symmetry check below and mis-report a broken directory as + # "flag set but no metadata". Distinct from the field-value checks + # (SIGNATURE_INVALID_*), which own length/revision/type. + # --------------------------------------------------------- + if cert_struct is not None and cert_struct.get("errors"): + issues.append(StructuralIssue( + issue=ReasonCodes.CERTIFICATE_TABLE_MALFORMED, + details={"reason": "top_level_decode", + "errors": list(cert_struct["errors"])}, + )) + return issues + + # Deterministic presence: the parser returns None when there is no + # security directory. This replaces the pefile `signatures` list as the + # source of "were any certificates actually parsed". + certs: List[Dict[str, Any]] = ( + cert_struct.get("certificates", []) if cert_struct else [] + ) or [] + present = bool(certs) + has_sig = bool(metadata.get("has_signature")) - sigs = metadata.get("signatures") or [] # --------------------------------------------------------- - # 1) Flag/metadata symmetry + # 1) Flag/metadata symmetry (PRESERVED) # --------------------------------------------------------- - if has_sig and not sigs: + if has_sig and not present: issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_FLAG_SET_BUT_NO_METADATA, details={}, )) return issues - if not has_sig and sigs: + if not has_sig and present: issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_PRESENT_BUT_FLAG_NOT_SET, - details={"count": len(sigs)}, + details={"count": len(certs)}, )) - # Continue validating the certificates anyway + # Continue validating the certificates anyway (preserved behaviour) - if not sigs: + if cert_struct is None: return issues # --------------------------------------------------------- - # 2) Multiplicity + # 1a) Table truncation (NEW: CERTIFICATE_TABLE_MALFORMED) + # --------------------------------------------------------- + for tag in cert_struct.get("truncations", []) or []: + issues.append(StructuralIssue( + issue=ReasonCodes.CERTIFICATE_TABLE_MALFORMED, + details={"reason": "truncation", "region": tag}, + )) + + # --------------------------------------------------------- + # 1b) Table offset must lie OUTSIDE the mapped image + # (NEW: CERTIFICATE_OFFSET_INSIDE_IMAGE). Table-level invariant from + # the parser's overlaps_image fact. This is a different owner from the + # per-certificate section-overlap check in step 4 (which is byte-range, + # per-section); both may co-fire on a pathological sample. Kept + # separate to preserve the existing check while meeting the spec. + # --------------------------------------------------------- + if cert_struct.get("overlaps_image") is True: + issues.append(StructuralIssue( + issue=ReasonCodes.CERTIFICATE_OFFSET_INSIDE_IMAGE, + details={"offset": cert_struct.get("offset"), + "size": cert_struct.get("size"), + "image_raw_end": cert_struct.get("image_raw_end")}, + )) + + # --------------------------------------------------------- + # 2) Multiplicity (PRESERVED) # --------------------------------------------------------- - if len(sigs) > 1: + if len(certs) > 1: issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_MULTIPLE_CERTIFICATES, - details={"count": len(sigs)}, + details={"count": len(certs)}, )) # --------------------------------------------------------- - # 3) Certificate sanity checks + # 3) Per-certificate field sanity (PRESERVED) # --------------------------------------------------------- + # file_size: prefer the analysis value to preserve the original bounds + # behaviour; fall back to the parser's file_size if analysis omits it. file_size = analysis.get("file_size") + if not isinstance(file_size, int): + fs = cert_struct.get("file_size") + file_size = fs if isinstance(fs, int) else None sections = analysis.get("sections", []) or [] overlay_offset = analysis.get("overlay_offset") - for sig in sigs: - offset = sig.get("file_offset") - size = sig.get("length") - revision = sig.get("revision") - cert_type = sig.get("certificate_type") + for cert in certs: + offset = cert.get("offset") # absolute file offset + size = cert.get("length") # dwLength (incl. 8-byte header) + revision = cert.get("revision") + cert_type = cert.get("cert_type") - # Skip malformed metadata + # Skip malformed metadata (preserved guard) if not isinstance(offset, int) or not isinstance(size, int): continue - # Length sanity + # Length sanity (PRESERVED). Owns the length<8 fact; we deliberately + # do NOT also emit CERTIFICATE_TABLE_MALFORMED for the parser's + # "length_too_small" tag, to avoid double-counting. if size < 8: issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_INVALID_LENGTH, @@ -70,39 +173,41 @@ def validate_signature(metadata: PublicMetadata, analysis: AnalysisDict) -> List )) continue - # Revision sanity + # Revision sanity (PRESERVED) if revision not in (0x0100, 0x0200): issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_INVALID_REVISION, details={"revision": revision}, )) - # Type sanity + # Type sanity (PRESERVED) if cert_type not in (0x0001, 0x0002): issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_INVALID_TYPE, details={"certificate_type": cert_type}, )) - # --------------------------------------------------------- - # 4) Bounds checks - # --------------------------------------------------------- + # ----------------------------------------------------- + # 4) Bounds + overlap checks (PRESERVED) + # ----------------------------------------------------- if isinstance(file_size, int): if offset < 0 or offset + size > file_size: issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_OUT_OF_FILE_BOUNDS, - details={"offset": offset, "length": size, "file_size": file_size}, + details={"offset": offset, "length": size, + "file_size": file_size}, )) continue - # Overlay check + # Overlay check (PRESERVED) if isinstance(overlay_offset, int) and offset < overlay_offset < offset + size: issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_OVERLAPS_OTHER_DATA, - details={"offset": offset, "length": size, "overlay_offset": overlay_offset}, + details={"offset": offset, "length": size, + "overlay_offset": overlay_offset}, )) - # Section overlap check + # Section overlap check (PRESERVED) for sec in sections: raw = sec.get("raw_address") raw_size = sec.get("raw_size") @@ -110,7 +215,8 @@ def validate_signature(metadata: PublicMetadata, analysis: AnalysisDict) -> List if max(offset, raw) < min(offset + size, raw + raw_size): issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_OVERLAPS_OTHER_DATA, - details={"offset": offset, "length": size, "section": sec.get("name")}, + details={"offset": offset, "length": size, + "section": sec.get("name")}, )) break diff --git a/iocx/validators/tls.py b/iocx/validators/tls.py index 7b9c0e1..53e649e 100644 --- a/iocx/validators/tls.py +++ b/iocx/validators/tls.py @@ -1,13 +1,71 @@ # Copyright (c) 2026 MalX Labs and contributors # SPDX-License-Identifier: MPL-2.0 +""" +Validate the IMAGE_TLS_DIRECTORY. + +v0.7.6 migration: this validator now sources TLS structural truth from the +deterministic ``tls_struct`` produced by parser pe_tls (read from +InternalMetadata), rather than from the pefile-derived ``extended`` marker. +All existing directory/pointer checks and reason codes are preserved; the +multiplicity check still consults ``analysis["extended"]``. + +Two axes are validated: + + * The PRESERVED cascade operates on the raw-data range + (Start/EndAddressOfRawData) and the AddressOfCallBacks POINTER, exactly + as before, including every early return. Note: struct addresses are + VAs, so section mapping for the pointer is done in RVA space + (rva = va - ImageBase). Range comparisons stay in VA space. This fixes a + latent VA/RVA unit mismatch from the pefile path while keeping the same + codes and control flow. + + * The NEW target-array checks operate on the RESOLVED callback array + (the list of callback target VAs the parser walked), which the pefile + single-value model could not express. This is where the two new v0.7.6 + codes live, so they do not double-count the pointer-based checks: + + TLS_DIRECTORY_TRUNCATED - header decode failure or a truncated / + looping callback array (parser tombstones). + TLS_CALLBACK_RVA_INVALID - a resolved callback TARGET whose VA cannot + form a valid RVA (below ImageBase) or does + not map to any section. + +Preserved reason codes: + TLS_MULTIPLE_DIRECTORIES + TLS_ZERO_LENGTH_DIRECTORY + TLS_INVALID_RANGE + TLS_CALLBACKS_MISSING + TLS_CALLBACK_OUTSIDE_RANGE + TLS_CALLBACK_NOT_MAPPED_TO_SECTION (pointer -> section) + TLS_CALLBACK_IN_NON_EXECUTABLE_SECTION + TLS_CALLBACK_IN_HEADERS + TLS_CALLBACK_IN_OVERLAY +""" + from typing import Dict, Any, List, Optional + from iocx.reason_codes import ReasonCodes from iocx.validators.schema import StructuralIssue from iocx.schemas.public_metadata import PublicMetadata +from iocx.schemas.internal_schema import InternalMetadata from iocx.schemas.analysis import AnalysisDict from .decorators import depends_on +IMAGE_SCN_MEM_EXECUTE = 0x20000000 + +# Parser top-level error tags that mean the fixed TLS header could not be +# decoded — the directory is unusable and maps to TLS_DIRECTORY_TRUNCATED. +_HEADER_DECODE_ERROR_TAGS = { + "tls_directory_read_failed", + "tls_directory_truncated", + "tls_directory_unpack_failed", +} + +# Cap on how many invalid-callback-target issues we raise, so a looping / +# hostile array cannot flood the stream. Count is always in details. +_MAX_CALLBACK_TARGET_ISSUES = 16 + def _map_rva_to_section(sections, rva) -> Optional[Dict[str, Any]]: for sec in sections: @@ -19,45 +77,74 @@ def _map_rva_to_section(sections, rva) -> Optional[Dict[str, Any]]: return None -@depends_on("metadata", "analysis") -def validate_tls(metadata: PublicMetadata, analysis: AnalysisDict) -> List[StructuralIssue]: +@depends_on("internal", "metadata", "analysis") +def validate_tls(internal: InternalMetadata, + metadata: PublicMetadata, + analysis: AnalysisDict) -> List[StructuralIssue]: issues: List[StructuralIssue] = [] + # --------------------------------------------------------- + # 1) Multiple TLS directories (PRESERVED — from extended markers) + # --------------------------------------------------------- tls_entries = [ e for e in analysis.get("extended", []) if isinstance(e, dict) and e.get("value") == "tls_directory" ] - - # --------------------------------------------------------- - # 1) Multiple TLS directories - # --------------------------------------------------------- if len(tls_entries) > 1: issues.append(StructuralIssue( issue=ReasonCodes.TLS_MULTIPLE_DIRECTORIES, details={"count": len(tls_entries)}, )) - if not tls_entries: + tls = internal.get("tls_struct") + if tls is None: + return issues # no TLS directory — not a defect + + # --------------------------------------------------------- + # 2) Header decode failure (NEW: TLS_DIRECTORY_TRUNCATED) + # Unrecoverable — the fixed struct could not be read/unpacked. + # --------------------------------------------------------- + header_errs = [e for e in (tls.get("errors") or []) + if e in _HEADER_DECODE_ERROR_TAGS] + if header_errs: + issues.append(StructuralIssue( + issue=ReasonCodes.TLS_DIRECTORY_TRUNCATED, + details={"reason": "header_decode", "errors": header_errs}, + )) return issues - # Only validate the first directory structurally - entry = tls_entries[0] - meta = entry.get("metadata") or {} + # --------------------------------------------------------- + # 3) Callback-array truncation / loop (NEW: TLS_DIRECTORY_TRUNCATED) + # --------------------------------------------------------- + for tag in tls.get("truncations", []) or []: + issues.append(StructuralIssue( + issue=ReasonCodes.TLS_DIRECTORY_TRUNCATED, + details={"reason": "callback_array", "region": tag}, + )) - start = meta.get("start_address") - end = meta.get("end_address") - callbacks = meta.get("callbacks") + # --------------------------------------------------------- + # 4) Resolved callback TARGET validation (NEW: TLS_CALLBACK_RVA_INVALID) + # Independent of the raw-data-range cascade below, so it always runs + # even when the cascade returns early (e.g. zero-length raw data). + # --------------------------------------------------------- + _validate_callback_targets(tls, analysis, issues) - if not isinstance(start, int) or not isinstance(end, int) or not isinstance(callbacks, int): + # --------------------------------------------------------- + # 5) PRESERVED cascade on raw-data range + AddressOfCallBacks pointer + # --------------------------------------------------------- + start = tls.get("start_address_of_raw_data") # VA + end = tls.get("end_address_of_raw_data") # VA + ptr = tls.get("address_of_callbacks") # VA of the callback array + image_base = tls.get("image_base") + + if not isinstance(start, int) or not isinstance(end, int) or not isinstance(ptr, int): return issues sections = analysis.get("sections", []) or [] overlay_offset = analysis.get("overlay_offset") - size_of_headers = metadata.get("optional_header", {}).get("size_of_headers") + size_of_headers = (metadata.get("optional_header") or {}).get("size_of_headers") - # --------------------------------------------------------- - # 2) Range sanity - # --------------------------------------------------------- + # Range sanity (VA space — PRESERVED) if start == end: issues.append(StructuralIssue( issue=ReasonCodes.TLS_ZERO_LENGTH_DIRECTORY, @@ -72,66 +159,117 @@ def validate_tls(metadata: PublicMetadata, analysis: AnalysisDict) -> List[Struc )) return issues - # --------------------------------------------------------- - # 3) Missing callbacks - # --------------------------------------------------------- - if callbacks == 0: + # Missing callbacks (PRESERVED) + if ptr == 0: issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACKS_MISSING, details={"start_address": start, "end_address": end}, )) return issues - # --------------------------------------------------------- - # 4) Callback outside TLS range - # --------------------------------------------------------- - if not (start <= callbacks < end): + # Callback pointer outside TLS range (VA space — PRESERVED) + if not (start <= ptr < end): issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACK_OUTSIDE_RANGE, - details={"callbacks": callbacks, "start_address": start, "end_address": end}, + details={"callbacks": ptr, "start_address": start, + "end_address": end}, )) - # Do not attempt further mapping - avoid cascading anomalies return issues - # --------------------------------------------------------- - # 5) Callback mapping - # --------------------------------------------------------- - sec = _map_rva_to_section(sections, callbacks) + # Pointer -> section mapping. Struct addresses are VAs; convert to RVA + # before mapping (sections are RVA-space). If ImageBase is unavailable we + # cannot convert, so we skip the mapping-dependent checks rather than + # emit against the wrong unit. + if not isinstance(image_base, int): + return issues + ptr_rva = ptr - image_base + + sec = _map_rva_to_section(sections, ptr_rva) if sec is None: issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACK_NOT_MAPPED_TO_SECTION, - details={"callbacks": callbacks}, + details={"callbacks": ptr, "callbacks_rva": ptr_rva}, )) return issues name = sec.get("name") chars = sec.get("characteristics", 0) - executable = bool(chars & 0x20000000) + executable = bool(chars & IMAGE_SCN_MEM_EXECUTE) if not executable: issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACK_IN_NON_EXECUTABLE_SECTION, - details={"callbacks": callbacks, "section": name}, + details={"callbacks": ptr, "section": name}, )) - # --------------------------------------------------------- - # 6) Overlay / header checks - # --------------------------------------------------------- - if isinstance(size_of_headers, int) and callbacks < size_of_headers: + # Overlay / header checks (RVA space — PRESERVED) + if isinstance(size_of_headers, int) and ptr_rva < size_of_headers: issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACK_IN_HEADERS, - details={"callbacks": callbacks, "size_of_headers": size_of_headers}, + details={"callbacks": ptr, "callbacks_rva": ptr_rva, + "size_of_headers": size_of_headers}, )) if isinstance(overlay_offset, int): raw = sec.get("raw_address") va = sec.get("virtual_address") if isinstance(raw, int) and isinstance(va, int): - raw_offset = raw + (callbacks - va) + raw_offset = raw + (ptr_rva - va) if raw_offset >= overlay_offset: issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACK_IN_OVERLAY, - details={"callbacks": callbacks, "raw_offset": raw_offset}, + details={"callbacks": ptr, "raw_offset": raw_offset}, )) return issues + + +# ================================================================= +# NEW: resolved callback-target validation +# ================================================================= + +def _validate_callback_targets(tls: Dict[str, Any], + analysis: AnalysisDict, + issues: List[StructuralIssue]) -> None: + """ + Flag resolved callback TARGET VAs that cannot form a valid RVA (below + ImageBase) or do not map to any section. Distinct subject from the + pointer-based TLS_CALLBACK_NOT_MAPPED_TO_SECTION check above (which maps + the AddressOfCallBacks pointer, not the individual targets). + """ + callbacks = tls.get("callbacks") or [] + if not callbacks: + return + + image_base = tls.get("image_base") + sections = analysis.get("sections", []) or [] + + if not isinstance(image_base, int): + # Cannot convert VA -> RVA; the parser records this too. One issue. + issues.append(StructuralIssue( + issue=ReasonCodes.TLS_CALLBACK_RVA_INVALID, + details={"reason": "image_base_unavailable", + "callback_count": len(callbacks)}, + )) + return + + invalid: List[Dict[str, Any]] = [] + for va in callbacks: + if not isinstance(va, int): + continue + rva = va - image_base + if rva < 0: + invalid.append({"callback_va": va, "reason": "below_image_base"}) + continue + if _map_rva_to_section(sections, rva) is None: + invalid.append({"callback_va": va, "callback_rva": rva, + "reason": "not_mapped"}) + + if not invalid: + return + + for item in invalid[:_MAX_CALLBACK_TARGET_ISSUES]: + issues.append(StructuralIssue( + issue=ReasonCodes.TLS_CALLBACK_RVA_INVALID, + details={**item, "invalid_callback_count": len(invalid)}, + )) From 2f2c4b022075f3943ea11c066937ddbab0150764 Mon Sep 17 00:00:00 2001 From: malx-labs Date: Wed, 29 Jul 2026 15:50:22 +0100 Subject: [PATCH 02/11] fix(tls): surface dropped tombstones and narrow zero-length false positive Three surgical edits to the TLS validator, no changes elsewhere. - Add _CALLBACK_RESOLUTION_ERROR_TAGS for the two parser tombstones that signalled an unresolvable callback array (tls_image_base_unavailable, tls_callbacks_va_below_image_base). - Surface those tags as tls_callback_rva_invalid at the top of _validate_callback_targets, before the guard that was silently swallowing them (parser sets callbacks=[] in both cases). Emitted via sorted(set(...)) for deterministic, de-duplicated output. - Narrow tls_zero_length_directory: only flag a zero-length raw-data region when there are NO resolved callbacks. A zero-length template alongside a valid callback array is legitimate and no longer a false positive. The early return is retained so the degenerate range never reaches the pointer cascade. Both tombstone tags map to the already-registered TLS_CALLBACK_RVA_INVALID; no reason-code or parser changes required. Verified: both findings fixed, all preserved TLS codes unchanged, and deterministic double-run output. --- iocx/validators/tls.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/iocx/validators/tls.py b/iocx/validators/tls.py index 53e649e..a5084aa 100644 --- a/iocx/validators/tls.py +++ b/iocx/validators/tls.py @@ -62,6 +62,16 @@ "tls_directory_unpack_failed", } +# Parser tombstones recorded when the callback ARRAY could not be resolved +# to an RVA at all. In these cases the parser returns callbacks=[], so the +# per-target loop below has nothing to walk; without surfacing these tags +# the structural anomaly would be silently dropped. They map to +# TLS_CALLBACK_RVA_INVALID (the callback array is unresolvable). +_CALLBACK_RESOLUTION_ERROR_TAGS = { + "tls_image_base_unavailable", + "tls_callbacks_va_below_image_base", +} + # Cap on how many invalid-callback-target issues we raise, so a looping / # hostile array cannot flood the stream. Count is always in details. _MAX_CALLBACK_TARGET_ISSUES = 16 @@ -146,10 +156,17 @@ def validate_tls(internal: InternalMetadata, # Range sanity (VA space — PRESERVED) if start == end: - issues.append(StructuralIssue( - issue=ReasonCodes.TLS_ZERO_LENGTH_DIRECTORY, - details={"start_address": start, "end_address": end}, - )) + # A zero-length raw-data region is only anomalous when the directory + # carries NO resolved callbacks. A zero-length template alongside a + # valid callback array is legitimate and common, so we do not flag it + # (the callback targets were already validated in step 4). Either way + # the degenerate range makes the pointer cascade below meaningless, so + # we return here. + if not (tls.get("callbacks") or []): + issues.append(StructuralIssue( + issue=ReasonCodes.TLS_ZERO_LENGTH_DIRECTORY, + details={"start_address": start, "end_address": end}, + )) return issues if start > end: @@ -237,6 +254,17 @@ def _validate_callback_targets(tls: Dict[str, Any], pointer-based TLS_CALLBACK_NOT_MAPPED_TO_SECTION check above (which maps the AddressOfCallBacks pointer, not the individual targets). """ + + # Surface callback-array resolution tombstones the parser recorded but + # that would otherwise be dropped (callbacks=[] in these cases). One + # deterministic issue per distinct tag, in a stable order. + errors = tls.get("errors") or [] + for tag in sorted(set(errors) & _CALLBACK_RESOLUTION_ERROR_TAGS): + issues.append(StructuralIssue( + issue=ReasonCodes.TLS_CALLBACK_RVA_INVALID, + details={"reason": tag}, + )) + callbacks = tls.get("callbacks") or [] if not callbacks: return From b6464ca98e655d7737b0ae30f1143bd75f648e3e Mon Sep 17 00:00:00 2001 From: malx-labs Date: Wed, 29 Jul 2026 16:22:47 +0100 Subject: [PATCH 03/11] fix(rva_graph): exclude SECURITY directory from RVA-based checks IMAGE_DIRECTORY_ENTRY_SECURITY (index 4) is the one data directory whose VirtualAddress field is a FILE OFFSET, not an RVA; the attribute certificate table is appended to the file and never mapped into the image. The rva_graph validator interpreted it as an RVA, producing a spurious data_directory_out_of_range finding (and potential overlap findings) that double-counted the same corruption already owned, with correct semantics, by the certificate parser / signature validator (certificate_offset_past_eof -> certificate_table_malformed, validated against file_size). Exclude the SECURITY directory from all RVA-based checks: - add _is_security_directory() (matches by index 4 or name) - skip it in the per-directory loop (out-of-range, headers, mapping, overlay) - skip it on both sides of the overlap-detection loop Placement/validity of the security directory is now single-owned by signature / pe_certificates. Pre-existing latent bug surfaced by the v0.7.6 certificate decoder providing a correct second opinion. Updated one affected snapshot golden accordingly. --- iocx/validators/rva_graph.py | 34 +++++++++++++++++++ .../corrupted_data_directories.full.json | 15 ++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/iocx/validators/rva_graph.py b/iocx/validators/rva_graph.py index 8cc9d16..cd1d7d4 100644 --- a/iocx/validators/rva_graph.py +++ b/iocx/validators/rva_graph.py @@ -14,6 +14,26 @@ REQUIRED_NONZERO_DIRS: set[str] = set() +# IMAGE_DIRECTORY_ENTRY_SECURITY (index 4) is special: its VirtualAddress +# field is a FILE OFFSET, not an RVA (the attribute certificate table is +# appended to the file and never mapped into the image). Every check here +# interprets the field as an RVA, so applying them to the security directory +# is a category error and double-counts with the certificate parser / +# signature validator, which own that directory's placement truth (validated +# against file_size, e.g. certificate_offset_past_eof). Exclude it from ALL +# RVA-based checks — both the per-directory loop and the overlap detection. +_SECURITY_DIRECTORY_INDEX = 4 +_SECURITY_DIRECTORY_NAME = "IMAGE_DIRECTORY_ENTRY_SECURITY" + + +def _is_security_directory(d: Dict[str, Any]) -> bool: + """True for IMAGE_DIRECTORY_ENTRY_SECURITY, by index or name.""" + return ( + d.get("index") == _SECURITY_DIRECTORY_INDEX + or d.get("name") == _SECURITY_DIRECTORY_NAME + ) + + @depends_on("metadata", "analysis") def validate_rva_graph(metadata: PublicMetadata, analysis: AnalysisDict) -> List[StructuralIssue]: issues: List[StructuralIssue] = [] @@ -45,6 +65,11 @@ def validate_rva_graph(metadata: PublicMetadata, analysis: AnalysisDict) -> List # Directory validation # --------------------------------------------------------- for d in dirs: + # Skip for the security directory: its VirtualAddress is a file offset, not an RVA. + # Owned by signature / pe_certificates. See note above. + if _is_security_directory(d): + continue + rva = d.get("rva") size = d.get("size") name = d.get("name") or d.get("index") @@ -180,6 +205,11 @@ def validate_rva_graph(metadata: PublicMetadata, analysis: AnalysisDict) -> List # --------------------------------------------------------- for i in range(len(dirs)): a = dirs[i] + + # Exclude security: its offset is not an RVA, so an RVA-space overlap comparison is wrong + if _is_security_directory(a): + continue + rva_a = a.get("rva") size_a = a.get("size") if not isinstance(rva_a, int) or not isinstance(size_a, int): @@ -188,6 +218,10 @@ def validate_rva_graph(metadata: PublicMetadata, analysis: AnalysisDict) -> List for j in range(i + 1, len(dirs)): b = dirs[j] + + if _is_security_directory(b): + continue + rva_b = b.get("rva") size_b = b.get("size") if not isinstance(rva_b, int) or not isinstance(size_b, int): diff --git a/tests/contract/snapshots/layer3_adversarial/corrupted_data_directories.full.json b/tests/contract/snapshots/layer3_adversarial/corrupted_data_directories.full.json index 603309f..6c26f1a 100644 --- a/tests/contract/snapshots/layer3_adversarial/corrupted_data_directories.full.json +++ b/tests/contract/snapshots/layer3_adversarial/corrupted_data_directories.full.json @@ -182,11 +182,9 @@ "end": 0, "category": "pe_heuristic", "metadata": { - "reason": "data_directory_out_of_range", - "directory": "IMAGE_DIRECTORY_ENTRY_SECURITY", - "rva": 4294967280, - "size": 256, - "size_of_image": 12288 + "reason": "data_directory_overlap", + "directory_a": "IMAGE_DIRECTORY_ENTRY_RESOURCE", + "directory_b": "IMAGE_DIRECTORY_ENTRY_EXCEPTION" } }, { @@ -195,9 +193,10 @@ "end": 0, "category": "pe_heuristic", "metadata": { - "reason": "data_directory_overlap", - "directory_a": "IMAGE_DIRECTORY_ENTRY_RESOURCE", - "directory_b": "IMAGE_DIRECTORY_ENTRY_EXCEPTION" + "reason": "top_level_decode", + "errors": [ + "certificate_offset_past_eof" + ] } } ] From 9de10d3ff7b2d9d8a6a03bd8c6007b8cae2217a3 Mon Sep 17 00:00:00 2001 From: malx-labs Date: Wed, 29 Jul 2026 16:59:37 +0100 Subject: [PATCH 04/11] Doc changes: structural validation deterministic heuristics --- ...ral-validation-deterministic-heuristics.md | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/docs/specs/structural-validation-deterministic-heuristics.md b/docs/specs/structural-validation-deterministic-heuristics.md index 2fa68cd..adbf88b 100644 --- a/docs/specs/structural-validation-deterministic-heuristics.md +++ b/docs/specs/structural-validation-deterministic-heuristics.md @@ -31,7 +31,7 @@ This is **structural verification**. # **2. The Validator Suite** Each validator inspects a distinct subsystem of the PE format. -Together, they form a complete, deterministic structural model of the binary. +Together, they form a comprehensive, deterministic structural model across the covered subsystems. Some structural metadata extracted by parsers is **producer-facing**: it exists to enable validators and heuristics, not to be exposed via the public IOC schema. Examples include the export and delay-load structural details. Other structural metadata is **consumer-facing** and intended for public exposure: version-info string fields are an example of this, planned for promotion in a future release. @@ -121,7 +121,7 @@ This validator enforces: - Directories must not map into overlay data. - Zero‑length sections are invalid mapping targets. -This validator is the backbone of structural correctness for imports, exports, resources, relocations, TLS, and security directories. +This validator is the backbone of structural correctness for imports, exports, resources, relocations, and TLS directories. The security directory (index 4) is deliberately excluded from all RVA-based checks here; its VirtualAddress is a file offset, not an RVA, and its placement is owned by the signature validator (§2.7), so the two never double-count. --- @@ -164,6 +164,8 @@ This validator enforces: This ensures the Authenticode block is structurally valid before any trust decisions are made. +**v0.7.6 structural decoder.** The certificate subsystem is now backed by a pure `struct`-level decoder (pe_certificates) that walks the `WIN_CERTIFICATE` array independently of pefile's `DIRECTORY_ENTRY_SECURITY` interpretation. The decoder treats `DATA_DIRECTORY[4].VirtualAddress` as a *file offset*, not an RVA, and reads from the raw file bytes, since the certificate table is appended to the file and never mapped into the image. It extracts each entry's revision, type, and length, decodes on the 8-byte (QWORD) entry alignment, and records the structural fact of whether the table offset falls before the on-disk end of any section (`overlaps_image`). This decoder establishes raw structural truth via two new reason codes: `CERTIFICATE_OFFSET_INSIDE_IMAGE` (the table offset falls before the on-disk end of any section) and `CERTIFICATE_TABLE_MALFORMED` (top-level decode failure or a truncation tag surfaced with reason: "truncation"). The placement/overlap fact has a single owner to avoid double-counting with the RVA-graph backbone, and the signature validator continues to interpret the trust-facing symmetry above it. + --- # **2.8 TLS Validator** @@ -182,6 +184,8 @@ This validator enforces: TLS callbacks are a common malware trick; this validator ensures the structure is sound before heuristics interpret it. +**v0.7.6 structural decoder.** The TLS subsystem is now backed by a pure `struct`-level decoder (pe_tls) that reads `IMAGE_TLS_DIRECTORY` independently of pefile's `DIRECTORY_ENTRY_TLS` interpretation. The address fields are *virtual addresses*, not RVAs, so the decoder converts `AddressOfCallBacks` to an RVA by subtracting `ImageBase` before walking the NULL-terminated callback array; PE32 vs PE32+ pointer width is taken from `OPTIONAL_HEADER.Magic` once at parse-start. The callback walk is dual-bounded; NULL-terminator detection plus a hard limit (4096), so a looping or non-terminating array cannot destabilise the walk. A zero-length raw-data region (start == end) is decoded as valid by the parser; the validator flags `TLS_ZERO_LENGTH_DIRECTORY` only when the directory carries no resolved callbacks, eliminating the false positive on zero-length templates that still ship a valid callback array. This decoder adds two new reason codes - `TLS_DIRECTORY_TRUNCATED` (header decode failure, or a truncated/looping callback array) and `TLS_CALLBACK_RVA_INVALID` (a resolved callback target that cannot form a valid RVA or does not map to any section), feeding the executability and range interpretation performed above it. Directory placement remains owned by the RVA-graph backbone to avoid double-counting. + --- # **2.9 Load Config Directory Validator** @@ -291,6 +295,61 @@ The delay-load parser is implemented as a pure `struct`-level decoder over `pe.g --- +## 2.13 Relocations Validator + +### Validates the structural integrity of the PE base-relocation table extracted by pe_relocations. + +This validator performs: + +- Top-level decode failure detection and short-circuit for unrecoverable directory placement. +- Relocation directory placement within `SizeOfImage`. +- Truncation reporting across the block array and per-block entry regions. +- Per-block structural validation: `SizeOfBlock` below the 8-byte header minimum, `SizeOfBlock` not aligned to the WORD entry stride, and declared entry counts exceeding the per-block ceiling. +- Per-entry relocation-target validation: each non-`ABSOLUTE` entry's `page_rva + offset` must map to a real section. + +Absence of a relocation directory is not treated as a structural defect (stripped or fixed-base binaries legitimately omit it), and `IMAGE_REL_BASED_ABSOLUTE` (type 0) entries are padding and are never flagged. + +The relocation table is a chain of variable-length blocks whose walk depends entirely on a self-declared size field, which makes it a quiet divergence surface. Two properties make general-purpose relocation parsers prone to inconsistent output: each block advances the cursor by its own `SizeOfBlock` rather than by a count, so a block advertising a size that does not advance the cursor (zero, or below the header minimum) will loop a naive walker indefinitely or silently desynchronise the block stream; and each 16-bit entry packs a 4-bit type in the high nibble with a 12-bit page offset in the low bits, so parsers that mask the wrong width, or that resolve the offset against the wrong page base, emit relocation targets that disagree across tools while the raw bytes are identical. + +The relocation parser is implemented as a pure `struct`-level decoder over `pe.get_data`-acquired byte buffers: + +- The 8-byte `IMAGE_BASE_RELOCATION` header (`VirtualAddress`, `SizeOfBlock`) is unpacked via a single `struct.unpack_from` call. No reliance on pefile's `DIRECTORY_ENTRY_BASERELOC` interpretation. +- The block walk is dual-bounded: a hard block-count limit (65536) plus an explicit stop at the declared directory end, with a non-advancing `SizeOfBlock` treated as fatal for the walk rather than as a loop, tagged deterministically. +- Each entry is decoded by masking `(word >> 12) & 0xF` for the type and `word & 0x0FFF` for the offset; the target RVA is derived as `page_rva + offset` by fixed arithmetic, never by inference. +- The readable entry region is clamped to the declared directory end so a block advertising a size past the directory cannot over-read; the shortfall is reported as a truncation tag rather than a partial read. + +The validator then maps these structural states to a small, well-defined set of reason codes (`RELOCATION_DIRECTORY_INVALID_HEADER`, `RELOCATION_DIRECTORY_OUT_OF_BOUNDS`, `RELOCATION_TABLE_TRUNCATED`, `RELOCATION_BLOCK_MALFORMED`, `RELOCATION_ENTRY_RVA_INVALID`), which downstream heuristics and IOC consumers can rely on as a stable contract. Per-block malformations are priority-resolved so a block carrying several defects emits one deterministic sub-reason, and the count of invalid entry targets is always reported in the issue details even when the per-entry emission is capped. + +--- + +## 2.14 Debug Directory Validator + +### Validates the structural integrity of the PE debug directory extracted by pe_debug. + +This validator performs: + +- Top-level decode failure detection and short-circuit for unrecoverable directory placement. +- Debug directory placement within `SizeOfImage`. +- Truncation reporting across the fixed-size entry array, including non-entry-aligned directory sizes. +- Per-entry structural validation: entry unpack failure, CodeView blob read failure, and malformed or unrecognised CodeView records. +- Per-entry data-region validation: each entry's `AddressOfRawData` region must map to a real section. +- Deterministic PDB-path extraction from CodeView records (RSDS / NB10), including GUID and age. + +Absence of a debug directory is not treated as a structural defect. Entries whose debug data is reachable only via a raw file pointer (no `AddressOfRawData`) are not flagged for mapping, since they carry no RVA to validate against the section table. + +The debug directory is a fixed-stride array of 28-byte entries, but the CodeView entry type embeds a second, self-describing record whose layout is selected by a four-byte signature, and that inner record is a common divergence surface. Two properties make general-purpose debug parsers prone to inconsistent output: the debug data may be addressed by an RVA (`AddressOfRawData`) or by a raw file offset (`PointerToRawData`), and the two need not agree, so parsers that trust one field unconditionally read different bytes on binaries where the mapping is inconsistent; and the CodeView PDB path is a NUL-terminated string of unbounded declared length appended after a fixed header, so parsers that do not cap the scan, or that decode the GUID with the wrong field endianness, produce PDB paths and symbol-server keys that differ across tools while the raw record is identical. + +The debug parser is implemented as a pure `struct`-level decoder over both `pe.get_data`-acquired and raw-file byte buffers: + +- The 28-byte `IMAGE_DEBUG_DIRECTORY` structure is unpacked via a single `struct.unpack_from` call. No reliance on pefile's `DIRECTORY_ENTRY_DEBUG` interpretation. +- CodeView blobs are read via `PointerToRawData` (raw file offset) first, with a fallback to `AddressOfRawData` (RVA), so extraction is deterministic regardless of which addressing field the producer populated. +- The RSDS (PDB 7.0) and NB10 (PDB 2.0) records are decoded against their fixed header layouts; the GUID is formatted in the canonical mixed-endian symbol-server form (Data1/2/3 little-endian, Data4 big-endian) by fixed arithmetic, not library formatting. +- The PDB path scan is bounded (512 bytes); an absent terminator emits a deterministic tombstone tag rather than an unbounded read, and non-ASCII bytes are reported rather than silently normalised. + +The validator then maps these structural states to a small, well-defined set of reason codes (`DEBUG_DIRECTORY_INVALID_HEADER`, `DEBUG_DIRECTORY_OUT_OF_BOUNDS`, `DEBUG_TABLE_TRUNCATED`, `DEBUG_DIRECTORY_ENTRY_MALFORMED`, `DEBUG_ENTRY_RVA_INVALID`), which downstream heuristics and IOC consumers can rely on as a stable contract. Per-entry malformations are priority-resolved so an entry carrying several defects emits one deterministic sub-reason. The PDB path is a high-signal forensic surface; build paths routinely leak project names, usernames, and toolchain layout, so deterministic extraction is a prerequisite for treating it as a reliable triage signal. + +--- + # **3. Deterministic Heuristics Layer** ### *Heuristics interpret structural truth — they never override it.* From 765bb5021cbed0c2bd7e337084cbee11186d0d94 Mon Sep 17 00:00:00 2001 From: malx-labs Date: Thu, 30 Jul 2026 16:24:56 +0100 Subject: [PATCH 05/11] (tests): 100% coverage on relocations parser and validator --- iocx/parsers/pe_relocations.py | 2 +- tests/unit/parsers/test_pe_relocations.py | 314 +++++++++++++++++ tests/unit/parsers/test_pe_relocations_ext.py | 116 +++++++ .../validators/test_validator_relocations.py | 320 ++++++++++++++++++ 4 files changed, 751 insertions(+), 1 deletion(-) create mode 100644 tests/unit/parsers/test_pe_relocations.py create mode 100644 tests/unit/parsers/test_pe_relocations_ext.py create mode 100644 tests/unit/validators/test_validator_relocations.py diff --git a/iocx/parsers/pe_relocations.py b/iocx/parsers/pe_relocations.py index 92889f8..d1b6f53 100644 --- a/iocx/parsers/pe_relocations.py +++ b/iocx/parsers/pe_relocations.py @@ -149,7 +149,7 @@ def _read_blocks( try: page_rva, size_of_block = struct.unpack_from(" bytes: + """One WORD relocation entry: 4-bit type high nibble, 12-bit offset.""" + return struct.pack(" bytes: + """Build an IMAGE_BASE_RELOCATION block: 8-byte header + WORD entries.""" + body = b"".join(_pack_entry(t, off) for t, off in entries) + size_of_block = _BLOCK_HEADER_SIZE + len(body) + return struct.pack(" bytes: + """Build a block with an explicit (possibly malformed) SizeOfBlock.""" + return struct.pack(" bytes: + if rva < 0 or rva > len(self._image): + raise ValueError("unmapped RVA") + return bytes(self._image[rva:rva + size]) + + +def _pe_with_reloc(blocks: bytes, base_rva: int = 0x1000, + dir_size: Optional[int] = None, + image_size: int = 0x8000) -> _FakePE: + """Place `blocks` at base_rva in a zeroed image and point dir 5 at it.""" + if dir_size is None: + dir_size = len(blocks) + image = bytearray(image_size) + image[base_rva:base_rva + len(blocks)] = blocks + return _FakePE(bytes(image), _FakeDataDir(base_rva, dir_size)) + + +# ================================================================= +# _locate_reloc_directory +# ================================================================= + +class TestLocator: + def test_valid_returns_tuple(self): + pe = _FakePE(reloc_dir=_FakeDataDir(0x1000, 0x40)) + assert _locate_reloc_directory(pe) == (0x1000, 0x40) + + def test_zero_rva_returns_none(self): + pe = _FakePE(reloc_dir=_FakeDataDir(0, 0x40)) + assert _locate_reloc_directory(pe) is None + + def test_zero_size_returns_none(self): + pe = _FakePE(reloc_dir=_FakeDataDir(0x1000, 0)) + assert _locate_reloc_directory(pe) is None + + def test_empty_directory_returns_none(self): + pe = _FakePE(reloc_dir=_FakeDataDir(0, 0)) + assert _locate_reloc_directory(pe) is None + + def test_missing_optional_header_returns_none(self): + pe = _FakePE(has_optional_header=False) + assert _locate_reloc_directory(pe) is None + + def test_missing_entry_returns_none(self): + # OPTIONAL_HEADER present but dir 5 left as None + pe = _FakePE() + assert _locate_reloc_directory(pe) is None + + +# ================================================================= +# _decode_block +# ================================================================= + +class TestDecodeBlock: + def test_type_and_offset_decoding(self): + block_bytes = _build_block(0x2000, [(3, 0x10), (10, 0x20)]) + pe = _pe_with_reloc(block_bytes) + trunc: List[str] = [] + block = _decode_block(pe, 0, 0x1000, 0x2000, len(block_bytes), + 0x1000 + len(block_bytes), trunc) + assert block["entries"][0] == { + "type": 3, "type_name": "HIGHLOW", "offset": 0x10, "rva": 0x2010} + assert block["entries"][1] == { + "type": 10, "type_name": "DIR64", "offset": 0x20, "rva": 0x2020} + assert block["entry_count"] == 2 + assert block["errors"] == [] + + def test_unknown_type_name_is_none(self): + # type 11 has no label in _RELOC_TYPE_NAMES + block_bytes = _build_block(0x2000, [(11, 0x4)]) + pe = _pe_with_reloc(block_bytes) + block = _decode_block(pe, 0, 0x1000, 0x2000, len(block_bytes), + 0x1000 + len(block_bytes), []) + assert block["entries"][0]["type"] == 11 + assert block["entries"][0]["type_name"] is None + + def test_size_of_block_too_small(self): + block = _decode_block(_FakePE(), 0, 0x1000, 0x2000, 4, 0x2000, []) + assert block["errors"] == ["size_of_block_too_small"] + assert block["entries"] == [] + + def test_size_of_block_not_word_aligned(self): + # header(8) + 3 bytes -> not a whole number of WORD entries + body = _pack_entry(3, 0x10) + b"\x01" + raw = _build_block_raw(0x2000, _BLOCK_HEADER_SIZE + len(body), body) + pe = _pe_with_reloc(raw) + block = _decode_block(pe, 0, 0x1000, 0x2000, len(raw), + 0x1000 + len(raw), []) + assert "size_of_block_not_word_aligned" in block["errors"] + + def test_entry_count_exceeds_max_is_tagged_and_capped(self): + # SizeOfBlock large enough to declare > _MAX_ENTRIES_PER_BLOCK entries + big = _BLOCK_HEADER_SIZE + (_MAX_ENTRIES_PER_BLOCK + 100) * _ENTRY_SIZE + image = bytearray(0x1000 + big + 0x10) + pe = _FakePE(bytes(image), _FakeDataDir(0x1000, big)) + block = _decode_block(pe, 0, 0x1000, 0x2000, big, 0x1000 + big, []) + assert "entry_count_exceeds_max" in block["errors"] + assert block["entry_count"] <= _MAX_ENTRIES_PER_BLOCK + + +# ================================================================= +# _read_blocks +# ================================================================= + +class TestReadBlocks: + def test_two_blocks_walked(self): + blocks = _build_block(0x2000, [(3, 0x10), (0, 0)]) + blocks += _build_block(0x3000, [(3, 0x4)]) + pe = _pe_with_reloc(blocks) + trunc: List[str] = [] + errs: List[str] = [] + out = _read_blocks(pe, 0x1000, len(blocks), trunc, errs) + assert len(out) == 2 + assert [b["page_rva"] for b in out] == [0x2000, 0x3000] + assert trunc == [] and errs == [] + + def test_block_too_small_stops_walk_no_infinite_loop(self): + raw = _build_block_raw(0x2000, 4) # non-advancing SizeOfBlock + pe = _pe_with_reloc(raw, dir_size=0x100) + out = _read_blocks(pe, 0x1000, 0x100, [], []) + assert len(out) == 1 + assert out[0]["errors"] == ["size_of_block_too_small"] + + def test_header_truncated_when_window_too_small(self): + # declared directory window shorter than an 8-byte header + pe = _pe_with_reloc(b"\x00\x20\x00\x00", dir_size=4) + trunc: List[str] = [] + out = _read_blocks(pe, 0x1000, 4, trunc, []) + assert out == [] + assert "relocation_block_header_truncated" in trunc + + def test_entries_truncated_when_block_exceeds_window(self): + header = _build_block_raw(0x2000, _BLOCK_HEADER_SIZE + 0x40) + partial = header + _pack_entry(3, 0x4) # only one entry present + pe = _pe_with_reloc(partial, dir_size=10) # header + 1 entry + trunc: List[str] = [] + _read_blocks(pe, 0x1000, 10, trunc, []) + assert "relocation_entries_truncated" in trunc + + +# ================================================================= +# build_relocation_structure — full roundtrips +# ================================================================= + +class TestBuildRelocationStructure: + def test_absent_directory_returns_none(self): + assert build_relocation_structure(_FakePE()) is None + + def test_basic_roundtrip(self): + blocks = _build_block(0x2000, [(3, 0x10), (10, 0x20), (0, 0)]) + blocks += _build_block(0x3000, [(3, 0x4), (3, 0x8)]) + out = build_relocation_structure(_pe_with_reloc(blocks)) + assert out["block_count"] == 2 + assert out["entry_count"] == 5 + assert out["rva"] == 0x1000 + assert out["errors"] == [] and out["truncations"] == [] + + def test_dir64_page(self): + block = _build_block(0x5000, [(10, 0x0), (10, 0x8), (0, 0)]) + out = build_relocation_structure(_pe_with_reloc(block)) + types = {e["type_name"] for e in out["blocks"][0]["entries"]} + assert "DIR64" in types and "ABSOLUTE" in types + + def test_all_absolute_padding(self): + block = _build_block(0x2000, [(0, 0), (0, 0), (0, 0)]) + out = build_relocation_structure(_pe_with_reloc(block)) + assert all(e["type"] == 0 for e in out["blocks"][0]["entries"]) + assert out["errors"] == [] + + def test_never_raises_on_short_read(self): + # dir points beyond the backing image -> tombstone, no exception + pe = _FakePE(bytes(0x100), _FakeDataDir(0x2000, 0x40)) + out = build_relocation_structure(pe) + assert out is not None + assert out["truncations"] # read failure recorded + + +# ================================================================= +# Output contract +# ================================================================= + +class TestOutputContract: + def _out(self): + blocks = _build_block(0x2000, [(3, 0x10), (0, 0)]) + return build_relocation_structure(_pe_with_reloc(blocks)) + + def test_required_top_level_keys(self): + out = self._out() + for key in ("rva", "size", "blocks", "block_count", + "entry_count", "truncations", "errors"): + assert key in out + + def test_block_keys(self): + block = self._out()["blocks"][0] + for key in ("index", "block_rva", "page_rva", "size_of_block", + "entry_count", "entries", "errors"): + assert key in block + + def test_entry_keys(self): + entry = self._out()["blocks"][0]["entries"][0] + assert set(entry) == {"type", "type_name", "offset", "rva"} + + def test_json_serializable(self): + import json + json.dumps(self._out()) # must not raise + + +# ================================================================= +# Determinism +# ================================================================= + +class TestDeterminism: + def test_repeated_calls_identical(self): + import json + blocks = _build_block(0x2000, [(3, 0x10), (10, 0x20), (0, 0)]) + blocks += _build_block(0x3000, [(3, 0x4)]) + a = build_relocation_structure(_pe_with_reloc(blocks)) + b = build_relocation_structure(_pe_with_reloc(blocks)) + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) + + def test_adversarial_output_stable(self): + import json + raw = _build_block_raw(0x2000, 4) + a = build_relocation_structure(_pe_with_reloc(raw, dir_size=0x100)) + b = build_relocation_structure(_pe_with_reloc(raw, dir_size=0x100)) + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) diff --git a/tests/unit/parsers/test_pe_relocations_ext.py b/tests/unit/parsers/test_pe_relocations_ext.py new file mode 100644 index 0000000..9b6a893 --- /dev/null +++ b/tests/unit/parsers/test_pe_relocations_ext.py @@ -0,0 +1,116 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Coverage-gap tests for iocx.parsers.pe_relocations. + +Targets the branches missed by the main suite (88% -> 100%): + - short block-header read (relocation_block_header_truncated) + - block-header unpack failure (block_header_unpack_failed_at_*) + - _MAX_BLOCKS exhaustion (relocation_block_max_exceeded) + - entries read raises (relocation_entries_read_failed) + - entries short read (relocation_entries_truncated) + +Each of these needs a get_data() that behaves differently from a simple +flat-image stub (short reads, raising reads), plus two monkeypatched cases +for the defensive/loop-bound branches. +""" + +from __future__ import annotations + +import struct + +import pytest + +import iocx.parsers.pe_relocations as R +from iocx.parsers.pe_relocations import build_relocation_structure + +_BASE = 0x1000 # locator treats rva == 0 as "absent" + + +# ================================================================= +# Fake PE variants +# ================================================================= + +class _DataDir: + def __init__(self, va, size): + self.VirtualAddress = va + self.Size = size + + +class _OptHdr: + def __init__(self, va, size): + self.DATA_DIRECTORY = [None] * 16 + self.DATA_DIRECTORY[R._RELOC_DIRECTORY_INDEX] = _DataDir(va, size) + + +class _FlatPE: + """RVA == offset into image; get_data clamps, raises only if rva > len.""" + def __init__(self, block_bytes: bytes, dir_size: int): + self._img = bytearray(_BASE) + bytearray(block_bytes) + self.OPTIONAL_HEADER = _OptHdr(_BASE, dir_size) + + def get_data(self, rva, length): + if rva < 0 or rva > len(self._img): + raise ValueError("unmapped RVA") + return bytes(self._img[rva:rva + length]) + + +class _EntriesRaisePE: + """Header read at the block base succeeds; any later (entries) read raises.""" + def __init__(self, header: bytes, dir_size: int): + self._header = header + self.OPTIONAL_HEADER = _OptHdr(_BASE, dir_size) + + def get_data(self, rva, length): + if rva == _BASE: + return bytes(self._header[:length]) + raise ValueError("boom on entries read") + + +def _hdr(page_rva: int, size_of_block: int) -> bytes: + return struct.pack(" loop exhausts -> else + pe = _FlatPE(_hdr(0x1000, 8) * 4, dir_size=0x1000) + out = build_relocation_structure(pe) + assert "relocation_block_max_exceeded" in out["truncations"] + assert out["block_count"] == 3 + + +class TestEntriesReadRaises: + def test_entries_read_failure_tombstoned(self): + # header decodes; the entries read raises -> read_failed, empty entries + pe = _EntriesRaisePE(_hdr(0x2000, 0x10), dir_size=0x100) + out = build_relocation_structure(pe) + assert "relocation_entries_read_failed" in out["truncations"] + assert out["blocks"][0]["entry_count"] == 0 + + +class TestEntriesShortRead: + def test_short_entries_read_recomputes_count(self): + # header ok; dir_end covers the full block (no clamp), but the image is + # short so get_data returns fewer entry bytes than requested. + pe = _FlatPE(_hdr(0x2000, 0x10) + b"\xAA\xAA", dir_size=0x10) + out = build_relocation_structure(pe) + assert "relocation_entries_truncated" in out["truncations"] + # 2 bytes -> exactly one WORD entry decoded + assert out["blocks"][0]["entry_count"] == 1 diff --git a/tests/unit/validators/test_validator_relocations.py b/tests/unit/validators/test_validator_relocations.py new file mode 100644 index 0000000..3c010ae --- /dev/null +++ b/tests/unit/validators/test_validator_relocations.py @@ -0,0 +1,320 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Unit tests for iocx.validators.relocations.validate_relocations. + +Strategy: +- Input is the relocation_struct dict produced by parser pe_relocations, + carried under metadata["relocation_struct"]. +- Build dicts directly to isolate validator logic from parser behaviour. +- Tests assert on emitted REASONCODES and the details payload. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from iocx.reason_codes import ReasonCodes +from iocx.validators.relocations import ( + validate_relocations, + _MAX_ENTRY_ISSUES_PER_BLOCK, +) + + +# ================================================================= +# Input builders +# ================================================================= + +def _make_analysis( + size_of_image: Optional[int] = 0x100000, + sections: Optional[List[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + analysis: Dict[str, Any] = {"size_of_image": size_of_image} + if sections is not None: + analysis["sections"] = sections + return analysis + + +def _whole_image_sections() -> List[Dict[str, Any]]: + """One section spanning the fixtures' target RVAs -> targets map cleanly.""" + return [{"virtual_address": 0x1000, "virtual_size": 0xFF000}] + + +def _tiny_section() -> List[Dict[str, Any]]: + """A section too small to contain any real target -> targets miss.""" + return [{"virtual_address": 0x1000, "virtual_size": 0x8}] + + +def _make_entry( + reloc_type: int = 3, + type_name: Optional[str] = "HIGHLOW", + offset: int = 0x10, + rva: int = 0x2010, +) -> Dict[str, Any]: + return {"type": reloc_type, "type_name": type_name, + "offset": offset, "rva": rva} + + +def _make_block( + index: int = 0, + block_rva: int = 0x1000, + page_rva: int = 0x2000, + size_of_block: int = 0x10, + entries: Optional[List[Dict[str, Any]]] = None, + errors: Optional[List[str]] = None, +) -> Dict[str, Any]: + ents = entries or [] + return {"index": index, "block_rva": block_rva, "page_rva": page_rva, + "size_of_block": size_of_block, "entry_count": len(ents), + "entries": ents, "errors": errors or []} + + +def _make_reloc( + rva: int = 0x1000, + size: int = 0x40, + blocks: Optional[List[Dict[str, Any]]] = None, + truncations: Optional[List[str]] = None, + errors: Optional[List[str]] = None, +) -> Dict[str, Any]: + blks = blocks or [] + return {"rva": rva, "size": size, "blocks": blks, + "block_count": len(blks), + "entry_count": sum(len(b["entries"]) for b in blks), + "truncations": truncations or [], "errors": errors or []} + + +def _run(reloc: Optional[Dict[str, Any]], analysis: Dict[str, Any]): + return validate_relocations({"relocation_struct": reloc}, analysis) + + +def _codes(issues) -> List: + return [i["issue"] for i in issues] + + +def _details_for(issues, code) -> List[Dict[str, Any]]: + return [i["details"] for i in issues if i["issue"] == code] + + +# ================================================================= +# Absence +# ================================================================= + +class TestAbsence: + def test_none_struct_no_issues(self): + assert _run(None, _make_analysis()) == [] + + def test_missing_key_no_issues(self): + assert validate_relocations({}, _make_analysis()) == [] + + +# ================================================================= +# Top-level decode short-circuit +# ================================================================= + +class TestTopLevelDecodeFailure: + def test_errors_emit_invalid_header(self): + reloc = _make_reloc(errors=["block_header_unpack_failed_at_0"]) + issues = _run(reloc, _make_analysis()) + assert _codes(issues) == [ReasonCodes.RELOCATION_DIRECTORY_INVALID_HEADER] + details = _details_for(issues, ReasonCodes.RELOCATION_DIRECTORY_INVALID_HEADER)[0] + assert details["reason"] == "top_level_decode" + assert details["errors"] == ["block_header_unpack_failed_at_0"] + + def test_short_circuit_skips_blocks_and_truncations(self): + reloc = _make_reloc( + errors=["x"], + truncations=["relocation_entries_truncated"], + blocks=[_make_block(errors=["size_of_block_too_small"])], + ) + issues = _run(reloc, _make_analysis()) + # only the header issue; block/truncation checks never run + assert _codes(issues) == [ReasonCodes.RELOCATION_DIRECTORY_INVALID_HEADER] + + +# ================================================================= +# Truncations +# ================================================================= + +class TestTruncations: + def test_one_issue_per_tag(self): + reloc = _make_reloc(truncations=[ + "relocation_entries_truncated", + "relocation_block_header_truncated"]) + issues = _run(reloc, _make_analysis()) + assert _codes(issues) == [ + ReasonCodes.RELOCATION_TABLE_TRUNCATED, + ReasonCodes.RELOCATION_TABLE_TRUNCATED] + + def test_region_detail_preserved_in_order(self): + reloc = _make_reloc(truncations=["relocation_entries_truncated", + "relocation_block_read_failed"]) + issues = _run(reloc, _make_analysis()) + regions = [d["region"] for d in + _details_for(issues, ReasonCodes.RELOCATION_TABLE_TRUNCATED)] + assert regions == ["relocation_entries_truncated", + "relocation_block_read_failed"] + + +# ================================================================= +# Block validation +# ================================================================= + +class TestBlockValidation: + def test_size_too_small_flagged(self): + reloc = _make_reloc(blocks=[ + _make_block(errors=["size_of_block_too_small"])]) + issues = _run(reloc, _make_analysis()) + assert ReasonCodes.RELOCATION_BLOCK_MALFORMED in _codes(issues) + d = _details_for(issues, ReasonCodes.RELOCATION_BLOCK_MALFORMED)[0] + assert d["reason"] == "size_of_block_too_small" + assert d["index"] == 0 + + def test_priority_first_match_wins(self): + # both errors present -> the higher-priority one is reported, once + reloc = _make_reloc(blocks=[_make_block(errors=[ + "size_of_block_not_word_aligned", "size_of_block_too_small"])]) + issues = _run(reloc, _make_analysis()) + malformed = _details_for(issues, ReasonCodes.RELOCATION_BLOCK_MALFORMED) + assert len(malformed) == 1 + assert malformed[0]["reason"] == "size_of_block_too_small" + + def test_clean_block_no_issue(self): + reloc = _make_reloc(blocks=[_make_block( + entries=[_make_entry(rva=0x2010)])]) + issues = _run(reloc, _make_analysis(sections=_whole_image_sections())) + assert issues == [] + + +# ================================================================= +# Entry validation +# ================================================================= + +class TestEntryValidation: + def test_absolute_entries_never_flagged(self): + reloc = _make_reloc(blocks=[_make_block(entries=[ + {"type": 0, "type_name": "ABSOLUTE", "offset": 0, "rva": 0x2000}])]) + # even with a tiny section, ABSOLUTE padding is ignored + issues = _run(reloc, _make_analysis(sections=_tiny_section())) + assert issues == [] + + def test_unmapped_target_flagged(self): + reloc = _make_reloc(blocks=[_make_block(entries=[ + _make_entry(rva=0x9000)])]) + issues = _run(reloc, _make_analysis(sections=_tiny_section())) + assert _codes(issues) == [ReasonCodes.RELOCATION_ENTRY_RVA_INVALID] + d = _details_for(issues, ReasonCodes.RELOCATION_ENTRY_RVA_INVALID)[0] + assert d["block_index"] == 0 + assert d["rva"] == 0x9000 + assert d["invalid_entry_count"] == 1 + + def test_mapped_target_no_issue(self): + reloc = _make_reloc(blocks=[_make_block(entries=[ + _make_entry(rva=0x1004)])]) + issues = _run(reloc, _make_analysis(sections=_whole_image_sections())) + assert issues == [] + + def test_count_reported_and_capped(self): + # 12 invalid entries -> capped at _MAX_ENTRY_ISSUES_PER_BLOCK issues, + # but invalid_entry_count carries the true total. + entries = [_make_entry(offset=i, rva=0x9000 + i) for i in range(12)] + reloc = _make_reloc(blocks=[_make_block(entries=entries)]) + issues = _run(reloc, _make_analysis(sections=_tiny_section())) + assert len(issues) == _MAX_ENTRY_ISSUES_PER_BLOCK + assert all(d["invalid_entry_count"] == 12 + for d in _details_for(issues, + ReasonCodes.RELOCATION_ENTRY_RVA_INVALID)) + + def test_no_sections_falls_back_to_size_of_image(self): + # No sections -> region_within_image bound check against size_of_image + reloc = _make_reloc(blocks=[_make_block(entries=[ + _make_entry(rva=0x200000)])]) # beyond size_of_image + issues = _run(reloc, _make_analysis(size_of_image=0x100000)) + assert _codes(issues) == [ReasonCodes.RELOCATION_ENTRY_RVA_INVALID] + + +# ================================================================= +# Placement (owned by rva_graph — NOT re-checked here) +# ================================================================= + +class TestPlacementNotChecked: + def test_out_of_bounds_directory_emits_no_placement_issue(self): + # A directory whose rva+size exceeds size_of_image must NOT produce a + # placement finding from this validator (rva_graph owns that). + reloc = _make_reloc(rva=0x90000, size=0x2000, + blocks=[_make_block(entries=[_make_entry(rva=0x1004)])]) + issues = _run(reloc, _make_analysis( + size_of_image=0x10000, sections=_whole_image_sections())) + assert issues == [] + + +# ================================================================= +# Combined scenarios +# ================================================================= + +class TestCombinedAnomalies: + def test_truncation_plus_block_malformed(self): + reloc = _make_reloc( + truncations=["relocation_entries_truncated"], + blocks=[_make_block(errors=["size_of_block_too_small"])]) + issues = _run(reloc, _make_analysis()) + assert _codes(issues) == [ + ReasonCodes.RELOCATION_TABLE_TRUNCATED, + ReasonCodes.RELOCATION_BLOCK_MALFORMED] + + def test_malformed_block_plus_invalid_entries(self): + block = _make_block( + errors=["size_of_block_not_word_aligned"], + entries=[_make_entry(rva=0x9000), _make_entry(rva=0x9004)]) + issues = _run(_make_reloc(blocks=[block]), + _make_analysis(sections=_tiny_section())) + codes = _codes(issues) + assert codes[0] == ReasonCodes.RELOCATION_BLOCK_MALFORMED + assert codes.count(ReasonCodes.RELOCATION_ENTRY_RVA_INVALID) == 2 + + +# ================================================================= +# Output contract +# ================================================================= + +class TestOutputContract: + def test_dependency_contract(self): + assert getattr(validate_relocations, "_depends_on") == ("internal", "analysis") + + def test_issue_shape(self): + reloc = _make_reloc(blocks=[_make_block(entries=[ + _make_entry(rva=0x9000)])]) + issues = _run(reloc, _make_analysis(sections=_tiny_section())) + assert issues + for i in issues: + assert set(i) == {"issue", "details"} + assert isinstance(i["issue"], str) + assert isinstance(i["details"], dict) + + def test_json_serializable(self): + import json + reloc = _make_reloc( + truncations=["relocation_entries_truncated"], + blocks=[_make_block(errors=["size_of_block_too_small"])]) + issues = _run(reloc, _make_analysis()) + json.dumps([i for i in issues]) # must not raise + + +# ================================================================= +# Determinism +# ================================================================= + +class TestDeterminism: + def test_repeated_calls_identical(self): + import json + block = _make_block( + errors=["size_of_block_not_word_aligned"], + entries=[_make_entry(rva=0x9000), _make_entry(rva=0x9004)]) + reloc = _make_reloc( + truncations=["relocation_entries_truncated"], blocks=[block]) + analysis = _make_analysis(sections=_tiny_section()) + a = [i for i in _run(reloc, analysis)] + b = [i for i in _run(reloc, analysis)] + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) From f113ee1190ebc78522cc31889617e588015a054f Mon Sep 17 00:00:00 2001 From: malx-labs Date: Thu, 30 Jul 2026 16:26:42 +0100 Subject: [PATCH 06/11] (tests): Fix failing heuristic, signature and tls validator tests since the v0.7.6 migration --- tests/unit/analysis/test_heuristics.py | 35 +- .../validators/test_validator_signatures.py | 203 +++++++----- tests/unit/validators/test_validator_tls.py | 302 +++++++++--------- 3 files changed, 311 insertions(+), 229 deletions(-) diff --git a/tests/unit/analysis/test_heuristics.py b/tests/unit/analysis/test_heuristics.py index 3f957a9..14bfdcb 100644 --- a/tests/unit/analysis/test_heuristics.py +++ b/tests/unit/analysis/test_heuristics.py @@ -111,7 +111,24 @@ def test_tls_callback_outside_range(): ], ) - analysis["structural"] = run_structural_validators({}, metadata, analysis) + # v0.7.6: validate_tls reads internal["tls_struct"], not the extended marker. + # Synthesise the struct as parser pe_tls would emit it. image_base=0 keeps + # VA==RVA so the range/section numbers match the extended metadata. + internal = { + "tls_struct": { + "rva": 0x1000, "size": 24, "is_64bit": False, "image_base": 0, + "start_address_of_raw_data": 0x1000, + "end_address_of_raw_data": 0x2000, + "address_of_index": 0, + "address_of_callbacks": 0x3000, # pointer outside [start, end) + "size_of_zero_fill": 0, "characteristics": 0, + "raw_data_size": 0x1000, + "callbacks": [], "callback_count": 0, + "truncations": [], "errors": [], + } + } + + analysis["structural"] = run_structural_validators(internal, metadata, analysis) dets = analyse_pe_heuristics(metadata, analysis) d = _find(dets, "pe_structure_anomaly", "callback_outside_tls_range") @@ -290,7 +307,21 @@ def test_synthetic_triggers_all_heuristics(): ], ) - analysis["structural"] = run_structural_validators({}, metadata, analysis) + internal = { + "tls_struct": { + "rva": 0x1000, "size": 24, "is_64bit": False, "image_base": 0, + "start_address_of_raw_data": 0x1000, + "end_address_of_raw_data": 0x2000, + "address_of_index": 0, + "address_of_callbacks": 0x3000, # pointer outside [start, end) + "size_of_zero_fill": 0, "characteristics": 0, + "raw_data_size": 0x1000, + "callbacks": [], "callback_count": 0, + "truncations": [], "errors": [], + } + } + + analysis["structural"] = run_structural_validators(internal, metadata, analysis) dets = analyse_pe_heuristics(metadata, analysis) expected = { diff --git a/tests/unit/validators/test_validator_signatures.py b/tests/unit/validators/test_validator_signatures.py index 4a83154..e893663 100644 --- a/tests/unit/validators/test_validator_signatures.py +++ b/tests/unit/validators/test_validator_signatures.py @@ -1,7 +1,25 @@ # Copyright (c) 2026 MalX Labs and contributors # SPDX-License-Identifier: MPL-2.0 +""" +Unit tests for iocx.validators.signature.validate_signature. + +v0.7.6 migration: the validator now takes THREE positional arguments +(internal, metadata, analysis) via @depends_on("internal","metadata", +"analysis"), and sources per-certificate structural truth from +internal["certificate_struct"] (produced by parser pe_certificates), +NOT from the old pefile-derived metadata["signatures"] list. + +Per-certificate field names follow the parser's CertificateStruct: + offset, length, revision, cert_type (was: file_offset, length, + revision, certificate_type) + +Only metadata["has_signature"] and the analysis geometry +(file_size / overlay_offset / sections) are still read from those dicts. +""" + import pytest + from iocx.validators.signature import validate_signature from iocx.reason_codes import ReasonCodes from iocx.validators.schema import StructuralIssue @@ -11,30 +29,52 @@ def make_issue_list(result): return [i["issue"] for i in result] +def _cert_struct(certificates, *, errors=None, truncations=None, + overlaps_image=False, offset=0x800, size=0x200, + file_size=None, image_raw_end=None): + """Build an internal['certificate_struct'] as parser pe_certificates emits.""" + return { + "offset": offset, "size": size, "file_size": file_size, + "image_raw_end": image_raw_end, "overlaps_image": overlaps_image, + "certificates": certificates, "certificate_count": len(certificates), + "truncations": truncations or [], "errors": errors or [], + } + + +def _internal(cert_struct): + return {"certificate_struct": cert_struct} + + # --------------------------------------------------------- # 1) Flag/metadata symmetry # --------------------------------------------------------- def test_flag_set_but_no_metadata(): - metadata = {"has_signature": True, "signatures": []} + # has_signature True, but no certificates decoded + internal = _internal(_cert_struct([])) + metadata = {"has_signature": True} analysis = {} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert make_issue_list(issues) == [ ReasonCodes.SIGNATURE_FLAG_SET_BUT_NO_METADATA ] def test_signature_present_but_flag_not_set(): - metadata = {"has_signature": False, "signatures": [{"file_offset": 0, "length": 16}]} + internal = _internal(_cert_struct( + [{"offset": 0, "length": 16, "revision": 0x0200, "cert_type": 0x0002, + "errors": []}])) + metadata = {"has_signature": False} analysis = {} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_PRESENT_BUT_FLAG_NOT_SET in make_issue_list(issues) def test_no_sigs_and_flag_false_returns_clean(): - metadata = {"has_signature": False, "signatures": []} + internal = _internal(_cert_struct([])) + metadata = {"has_signature": False} analysis = {} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert issues == [] @@ -43,15 +83,13 @@ def test_no_sigs_and_flag_false_returns_clean(): # --------------------------------------------------------- def test_multiple_signatures_detected(): - metadata = { - "has_signature": True, - "signatures": [ - {"file_offset": 0, "length": 16}, - {"file_offset": 100, "length": 16}, - ], - } + internal = _internal(_cert_struct([ + {"offset": 0, "length": 16, "revision": 0x0200, "cert_type": 0x0002, "errors": []}, + {"offset": 100, "length": 16, "revision": 0x0200, "cert_type": 0x0002, "errors": []}, + ])) + metadata = {"has_signature": True} analysis = {} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_MULTIPLE_CERTIFICATES in make_issue_list(issues) @@ -60,32 +98,30 @@ def test_multiple_signatures_detected(): # --------------------------------------------------------- def test_invalid_length(): - metadata = { - "has_signature": True, - "signatures": [{"file_offset": 0, "length": 4}], - } + internal = _internal(_cert_struct([{"offset": 0, "length": 4, "errors": []}])) + metadata = {"has_signature": True} analysis = {} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_INVALID_LENGTH in make_issue_list(issues) def test_invalid_revision(): - metadata = { - "has_signature": True, - "signatures": [{"file_offset": 0, "length": 16, "revision": 0x9999}], - } + internal = _internal(_cert_struct( + [{"offset": 0, "length": 16, "revision": 0x9999, "cert_type": 0x0002, + "errors": []}])) + metadata = {"has_signature": True} analysis = {} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_INVALID_REVISION in make_issue_list(issues) def test_invalid_type(): - metadata = { - "has_signature": True, - "signatures": [{"file_offset": 0, "length": 16, "certificate_type": 0x9999}], - } + internal = _internal(_cert_struct( + [{"offset": 0, "length": 16, "revision": 0x0200, "cert_type": 0x9999, + "errors": []}])) + metadata = {"has_signature": True} analysis = {} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_INVALID_TYPE in make_issue_list(issues) @@ -94,36 +130,32 @@ def test_invalid_type(): # --------------------------------------------------------- def test_signature_out_of_bounds(): - metadata = { - "has_signature": True, - "signatures": [{"file_offset": 900, "length": 200}], - } + internal = _internal(_cert_struct( + [{"offset": 900, "length": 200, "revision": 0x0200, "cert_type": 0x0002, + "errors": []}])) + metadata = {"has_signature": True} analysis = {"file_size": 1000} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_OUT_OF_FILE_BOUNDS in make_issue_list(issues) def test_signature_overlaps_overlay(): - metadata = { - "has_signature": True, - "signatures": [{"file_offset": 100, "length": 200}], - } + internal = _internal(_cert_struct( + [{"offset": 100, "length": 200, "revision": 0x0200, "cert_type": 0x0002, + "errors": []}])) + metadata = {"has_signature": True} analysis = {"overlay_offset": 150} - issues = validate_signature(metadata, analysis) + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_OVERLAPS_OTHER_DATA in make_issue_list(issues) def test_signature_overlaps_section(): - metadata = { - "has_signature": True, - "signatures": [{"file_offset": 100, "length": 200}], - } - analysis = { - "sections": [ - {"name": ".text", "raw_address": 150, "raw_size": 50} - ] - } - issues = validate_signature(metadata, analysis) + internal = _internal(_cert_struct( + [{"offset": 100, "length": 200, "revision": 0x0200, "cert_type": 0x0002, + "errors": []}])) + metadata = {"has_signature": True} + analysis = {"sections": [{"name": ".text", "raw_address": 150, "raw_size": 50}]} + issues = validate_signature(internal, metadata, analysis) assert ReasonCodes.SIGNATURE_OVERLAPS_OTHER_DATA in make_issue_list(issues) @@ -132,42 +164,57 @@ def test_signature_overlaps_section(): # --------------------------------------------------------- def test_valid_signature_no_issues(): - metadata = { - "has_signature": True, - "signatures": [{ - "file_offset": 100, - "length": 64, - "revision": 0x0200, - "certificate_type": 0x0001, - }], - } - analysis = { - "file_size": 1000, - "overlay_offset": 2000, - "sections": [], - } - issues = validate_signature(metadata, analysis) + internal = _internal(_cert_struct( + [{"offset": 100, "length": 64, "revision": 0x0200, "cert_type": 0x0001, + "errors": []}])) + metadata = {"has_signature": True} + analysis = {"file_size": 1000, "overlay_offset": 2000, "sections": []} + issues = validate_signature(internal, metadata, analysis) assert issues == [] + # --------------------------------------------------------- # 6) Malformed case # --------------------------------------------------------- def test_malformed_signature_metadata_skips_entry(): - metadata = { - "has_signature": True, - "signatures": [ - {"file_offset": "not-an-int", "length": 16}, # triggers continue - ], - } + internal = _internal(_cert_struct( + [{"offset": "not-an-int", "length": 16, "errors": []}])) # triggers continue + metadata = {"has_signature": True} + analysis = {} + issues = validate_signature(internal, metadata, analysis) + # non-int offset -> entry skipped, no per-cert issue raised + assert ReasonCodes.SIGNATURE_INVALID_LENGTH not in make_issue_list(issues) + assert ReasonCodes.SIGNATURE_OUT_OF_FILE_BOUNDS not in make_issue_list(issues) - analysis = { - "file_size": 500, - "sections": [], - "overlay_offset": None, - } - issues = validate_signature(metadata, analysis) +# --------------------------------------------------------- +# 7) NEW v0.7.6 codes +# --------------------------------------------------------- + +def test_certificate_offset_inside_image(): + internal = _internal(_cert_struct( + [{"offset": 0x400, "length": 16, "revision": 0x0200, "cert_type": 0x0002, + "errors": []}], + overlaps_image=True, offset=0x400, image_raw_end=0x800)) + metadata = {"has_signature": True} + analysis = {"file_size": 0x1000} + issues = validate_signature(internal, metadata, analysis) + assert ReasonCodes.CERTIFICATE_OFFSET_INSIDE_IMAGE in make_issue_list(issues) - # The malformed entry should be skipped entirely — no issues from it. - assert issues == [] + +def test_certificate_table_malformed_top_level(): + internal = _internal(_cert_struct([], errors=["raw_file_unavailable"])) + metadata = {"has_signature": True} + analysis = {} + issues = validate_signature(internal, metadata, analysis) + # decode failure short-circuits and is reported as malformed (NOT as + # flag_set_but_no_metadata) + assert make_issue_list(issues) == [ReasonCodes.CERTIFICATE_TABLE_MALFORMED] + + +def test_absent_directory_returns_clean(): + metadata = {"has_signature": False} + analysis = {} + assert validate_signature({"certificate_struct": None}, metadata, analysis) == [] + assert validate_signature({}, metadata, analysis) == [] diff --git a/tests/unit/validators/test_validator_tls.py b/tests/unit/validators/test_validator_tls.py index e12e2c8..52c3c35 100644 --- a/tests/unit/validators/test_validator_tls.py +++ b/tests/unit/validators/test_validator_tls.py @@ -1,7 +1,30 @@ # Copyright (c) 2026 MalX Labs and contributors # SPDX-License-Identifier: MPL-2.0 +""" +Unit tests for iocx.validators.tls.validate_tls. + +v0.7.6 migration: the validator now takes THREE positional arguments +(internal, metadata, analysis) via @depends_on("internal","metadata", +"analysis"), and sources TLS structural truth from internal["tls_struct"] +(produced by parser pe_tls), NOT from the old pefile-derived +analysis["extended"] marker. + +Key shape changes vs the pre-migration tests: + * Address fields are VAs on the struct: + start_address_of_raw_data / end_address_of_raw_data + address_of_callbacks (the callback-array POINTER) + image_base (used to convert VA -> RVA for section mapping) + callbacks (LIST of resolved callback target VAs) + * The old single-int "callbacks" (a pointer) maps to address_of_callbacks. + * Section mapping is done in RVA space: rva = va - image_base. Tests below + use image_base = 0 so the VA and RVA spaces coincide, keeping the + original section/offset numbers intact. + * The multiplicity check still reads analysis["extended"]. +""" + import pytest + from iocx.validators.tls import validate_tls from iocx.reason_codes import ReasonCodes @@ -10,83 +33,90 @@ def make_issue_list(result): return [i["issue"] for i in result] +def _tls_struct(*, start, end, address_of_callbacks, callbacks=None, + image_base=0, errors=None, truncations=None): + """Build an internal['tls_struct'] as parser pe_tls emits.""" + return { + "rva": 0x1000, "size": 24, "is_64bit": False, "image_base": image_base, + "start_address_of_raw_data": start, + "end_address_of_raw_data": end, + "address_of_index": 0, + "address_of_callbacks": address_of_callbacks, + "size_of_zero_fill": 0, "characteristics": 0, + "raw_data_size": (None if (isinstance(start, int) and isinstance(end, int) + and end < start) + else (end - start if isinstance(start, int) + and isinstance(end, int) else None)), + "callbacks": callbacks or [], + "callback_count": len(callbacks or []), + "truncations": truncations or [], "errors": errors or [], + } + + +def _internal(tls_struct): + return {"tls_struct": tls_struct} + + # --------------------------------------------------------- # 1) No TLS entries # --------------------------------------------------------- def test_no_tls_entries_returns_clean(): + internal = {} # no tls_struct metadata = {} analysis = {"extended": []} - issues = validate_tls(metadata, analysis) + issues = validate_tls(internal, metadata, analysis) assert issues == [] # --------------------------------------------------------- -# 2) Multiple TLS directories +# 2) Multiple TLS directories (still from analysis["extended"]) # --------------------------------------------------------- def test_multiple_tls_directories(): + internal = {} # struct absent; multiplicity is an extended-marker check metadata = {} - analysis = { - "extended": [ - {"value": "tls_directory", "metadata": {}}, - {"value": "tls_directory", "metadata": {}}, - ] - } - issues = validate_tls(metadata, analysis) + analysis = {"extended": [ + {"value": "tls_directory", "metadata": {}}, + {"value": "tls_directory", "metadata": {}}, + ]} + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_MULTIPLE_DIRECTORIES in make_issue_list(issues) # --------------------------------------------------------- -# 3) Malformed TLS metadata (early return) +# 3) Malformed TLS metadata (early return on non-int fields) # --------------------------------------------------------- def test_malformed_tls_metadata_skips_validation(): + internal = _internal(_tls_struct( + start="bad", end=200, address_of_callbacks=150)) metadata = {} - analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": "bad", - "end_address": 200, - "callbacks": 150, - }} - ] - } - issues = validate_tls(metadata, analysis) + analysis = {"extended": []} + issues = validate_tls(internal, metadata, analysis) assert issues == [] # --------------------------------------------------------- -# 4) Invalid range (start >= end) +# 4) Invalid range (start > end) / zero-length # --------------------------------------------------------- def test_tls_invalid_range(): + internal = _internal(_tls_struct( + start=300, end=200, address_of_callbacks=250)) metadata = {} - analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 300, - "end_address": 200, - "callbacks": 250, - }} - ] - } - issues = validate_tls(metadata, analysis) + analysis = {"extended": []} + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_INVALID_RANGE in make_issue_list(issues) def test_tls_zero_length_directory(): + # start == end AND no resolved callbacks -> flagged (narrowed in v0.7.6) + internal = _internal(_tls_struct( + start=200, end=200, address_of_callbacks=200, callbacks=[])) metadata = {} - analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 200, - "end_address": 200, - "callbacks": 200, - }} - ] - } - issues = validate_tls(metadata, analysis) + analysis = {"extended": []} + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_ZERO_LENGTH_DIRECTORY in make_issue_list(issues) @@ -95,56 +125,37 @@ def test_tls_zero_length_directory(): # --------------------------------------------------------- def test_tls_callbacks_missing(): + internal = _internal(_tls_struct( + start=100, end=200, address_of_callbacks=0)) metadata = {} - analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 100, - "end_address": 200, - "callbacks": 0, - }} - ] - } - issues = validate_tls(metadata, analysis) + analysis = {"extended": []} + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_CALLBACKS_MISSING in make_issue_list(issues) # --------------------------------------------------------- -# 6) Callback outside TLS range +# 6) Callback pointer outside TLS range # --------------------------------------------------------- def test_tls_callback_outside_range(): + internal = _internal(_tls_struct( + start=100, end=200, address_of_callbacks=500)) metadata = {} - analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 100, - "end_address": 200, - "callbacks": 500, - }} - ] - } - issues = validate_tls(metadata, analysis) + analysis = {"extended": []} + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_CALLBACK_OUTSIDE_RANGE in make_issue_list(issues) # --------------------------------------------------------- -# 7) Callback not mapped to any section +# 7) Callback pointer not mapped to any section # --------------------------------------------------------- def test_tls_callback_not_mapped_to_section(): + internal = _internal(_tls_struct( + start=100, end=200, address_of_callbacks=150, image_base=0)) metadata = {} - analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 100, - "end_address": 200, - "callbacks": 150, - }} - ], - "sections": [], # no mapping possible - } - issues = validate_tls(metadata, analysis) + analysis = {"extended": [], "sections": []} # no mapping possible + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_CALLBACK_NOT_MAPPED_TO_SECTION in make_issue_list(issues) @@ -153,25 +164,15 @@ def test_tls_callback_not_mapped_to_section(): # --------------------------------------------------------- def test_tls_callback_in_non_executable_section(): + internal = _internal(_tls_struct( + start=100, end=200, address_of_callbacks=150, image_base=0)) metadata = {} analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 100, - "end_address": 200, - "callbacks": 150, - }} - ], - "sections": [ - { - "name": ".data", - "virtual_address": 100, - "virtual_size": 100, - "characteristics": 0x0, # NOT executable - } - ], + "extended": [], + "sections": [{"name": ".data", "virtual_address": 100, + "virtual_size": 100, "characteristics": 0x0}], } - issues = validate_tls(metadata, analysis) + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_CALLBACK_IN_NON_EXECUTABLE_SECTION in make_issue_list(issues) @@ -180,27 +181,15 @@ def test_tls_callback_in_non_executable_section(): # --------------------------------------------------------- def test_tls_callback_in_headers(): - metadata = { - "optional_header": {"size_of_headers": 300} - } + internal = _internal(_tls_struct( + start=100, end=400, address_of_callbacks=150, image_base=0)) + metadata = {"optional_header": {"size_of_headers": 300}} analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 100, - "end_address": 400, - "callbacks": 150, - }} - ], - "sections": [ - { - "name": ".text", - "virtual_address": 100, - "virtual_size": 300, - "characteristics": 0x20000000, # executable - } - ], + "extended": [], + "sections": [{"name": ".text", "virtual_address": 100, + "virtual_size": 300, "characteristics": 0x20000000}], } - issues = validate_tls(metadata, analysis) + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_CALLBACK_IN_HEADERS in make_issue_list(issues) @@ -209,28 +198,17 @@ def test_tls_callback_in_headers(): # --------------------------------------------------------- def test_tls_callback_in_overlay(): + internal = _internal(_tls_struct( + start=100, end=400, address_of_callbacks=150, image_base=0)) metadata = {} analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 100, - "end_address": 400, - "callbacks": 150, - }} - ], - "overlay_offset": 120, # overlay starts inside section - "sections": [ - { - "name": ".text", - "virtual_address": 100, - "virtual_size": 300, - "raw_address": 100, - "raw_size": 300, - "characteristics": 0x20000000, - } - ], + "extended": [], + "overlay_offset": 120, # overlay starts inside section + "sections": [{"name": ".text", "virtual_address": 100, + "virtual_size": 300, "raw_address": 100, + "raw_size": 300, "characteristics": 0x20000000}], } - issues = validate_tls(metadata, analysis) + issues = validate_tls(internal, metadata, analysis) assert ReasonCodes.TLS_CALLBACK_IN_OVERLAY in make_issue_list(issues) @@ -239,28 +217,54 @@ def test_tls_callback_in_overlay(): # --------------------------------------------------------- def test_tls_valid_no_issues(): - metadata = { - "optional_header": {"size_of_headers": 50} - } + internal = _internal(_tls_struct( + start=100, end=400, address_of_callbacks=150, image_base=0)) + metadata = {"optional_header": {"size_of_headers": 50}} analysis = { - "extended": [ - {"value": "tls_directory", "metadata": { - "start_address": 100, - "end_address": 400, - "callbacks": 150, - }} - ], - "sections": [ - { - "name": ".text", - "virtual_address": 100, - "virtual_size": 300, - "raw_address": 100, - "raw_size": 300, - "characteristics": 0x20000000, # executable - } - ], + "extended": [], + "sections": [{"name": ".text", "virtual_address": 100, + "virtual_size": 300, "raw_address": 100, + "raw_size": 300, "characteristics": 0x20000000}], "overlay_offset": 999999, } - issues = validate_tls(metadata, analysis) + issues = validate_tls(internal, metadata, analysis) assert issues == [] + + +# --------------------------------------------------------- +# 12) NEW v0.7.6 codes +# --------------------------------------------------------- + +def test_tls_directory_truncated_header(): + internal = _internal(_tls_struct( + start=None, end=None, address_of_callbacks=None, + errors=["tls_directory_truncated"])) + metadata = {} + analysis = {"extended": []} + issues = validate_tls(internal, metadata, analysis) + assert make_issue_list(issues) == [ReasonCodes.TLS_DIRECTORY_TRUNCATED] + + +def test_tls_callback_rva_invalid_target(): + # resolved callback target below image base -> rva_invalid + internal = _internal(_tls_struct( + start=0x2000, end=0x2100, address_of_callbacks=0x2050, + callbacks=[0x100], image_base=0x400000)) + metadata = {} + analysis = {"extended": [], "sections": []} + issues = validate_tls(internal, metadata, analysis) + assert ReasonCodes.TLS_CALLBACK_RVA_INVALID in make_issue_list(issues) + + +def test_tls_zero_length_with_callbacks_not_flagged(): + # zero-length raw data BUT valid callbacks -> no false positive + internal = _internal(_tls_struct( + start=0x1000, end=0x1000, address_of_callbacks=0x1500, + callbacks=[0x1200], image_base=0)) + metadata = {} + analysis = {"extended": [], "sections": [ + {"name": ".text", "virtual_address": 0x1000, "virtual_size": 0x1000, + "characteristics": 0x20000000}]} + issues = validate_tls(internal, metadata, analysis) + assert ReasonCodes.TLS_ZERO_LENGTH_DIRECTORY not in make_issue_list(issues) + assert ReasonCodes.TLS_CALLBACK_RVA_INVALID not in make_issue_list(issues) From e38f28bfee60dc032f0a5882532661fd827f85fe Mon Sep 17 00:00:00 2001 From: malx-labs Date: Thu, 30 Jul 2026 16:47:39 +0100 Subject: [PATCH 07/11] =?UTF-8?q?(tests):=20debug=20parser=20and=20validat?= =?UTF-8?q?or=20at=20100%=20coverage.=20test=5Fpe=5Fdebug.py:=201.=20TestL?= =?UTF-8?q?ocator=20=E2=80=94=20valid=20/=20zero-rva=20/=20zero-size=20/?= =?UTF-8?q?=20missing-optional-header=20/=20missing-entry.=202.=20TestRead?= =?UTF-8?q?Entries=20=E2=80=94=20the=20four=20array-level=20paths:=20size?= =?UTF-8?q?=5Fnot=5Fentry=5Faligned,=20entry=5Fcount=5Fexceeds=5Fmax,=20en?= =?UTF-8?q?try=5Fread=5Ffailed=20(via=20an=20always-raising=20PE),=20entry?= =?UTF-8?q?=5Ftruncated=20(short=20read).=203.=20TestDecodeEntry=20?= =?UTF-8?q?=E2=80=94=20field=20decode,=20unknown=20type-name=20=E2=86=92?= =?UTF-8?q?=20None,=20and=20the=20struct.error=20=E2=86=92=20entry=5Funpac?= =?UTF-8?q?k=5Ffailed=20tombstone.=204.=20CodeView:=20RSDS=20via=20file=20?= =?UTF-8?q?pointer=20and=20the=20RVA=20fallback;=20NB10;=20unknown=20signa?= =?UTF-8?q?ture;=20too-short;=20RSDS/NB10=20truncated;=20PDB=20path=20unte?= =?UTF-8?q?rminated,=20non-ASCII,=20and=20empty=E2=86=92None;=20size=5Fof?= =?UTF-8?q?=5Fdata=3D=3D0=20=E2=86=92=20max-len=20fallback.=205.=20TestRea?= =?UTF-8?q?dCodeViewBlob=20=E2=80=94=20the=20three=20fallback=20branches,?= =?UTF-8?q?=20including=20the=20one=20genuine=20gap=20I=20found=20and=20cl?= =?UTF-8?q?osed:=20=5F=5Fdata=5F=5F=20that=20raises=20on=20bytes()=20(line?= =?UTF-8?q?=20252's=20except=E2=80=A6:=20pass),=20proven=20to=20fall=20thr?= =?UTF-8?q?ough=20to=20the=20RVA=20read.=206.=20=5Fextract=5Fasciiz=5Fpath?= =?UTF-8?q?=20and=20=5Fformat=5Fguid=20exercised=20directly=20(canonical?= =?UTF-8?q?=20mixed-endian=20GUID;=20short=E2=86=92None).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_validator_debug.py: 1. Absence, top-level-decode short-circuit (proves entries/truncations are skipped), truncations (one issue per tag). 2. Entry malformation — priority-first-match, unknown-tag-ignored, clean-entry-clean. 3. Entry RVA validation — unmapped flagged, mapped clean, addr==0 skipped, missing size_of_data→0, no-sections→size_of_image fallback. 4. Combined — truncation+malformed, and malformed and RVA-invalid on the same entry (two issues, one entry). 5. Contract — _depends_on == (internal,analysis), issue shape, JSON-safety, determinism. --- tests/unit/parsers/test_pe_debug.py | 463 ++++++++++++++++++ tests/unit/validators/test_validator_debug.py | 296 +++++++++++ 2 files changed, 759 insertions(+) create mode 100644 tests/unit/parsers/test_pe_debug.py create mode 100644 tests/unit/validators/test_validator_debug.py diff --git a/tests/unit/parsers/test_pe_debug.py b/tests/unit/parsers/test_pe_debug.py new file mode 100644 index 0000000..eecb5dd --- /dev/null +++ b/tests/unit/parsers/test_pe_debug.py @@ -0,0 +1,463 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Unit tests for iocx.parsers.pe_debug. + +Strategy: +- Byte-level fixture builders construct 28-byte IMAGE_DEBUG_DIRECTORY + entries and CodeView (RSDS / NB10) records directly via struct.pack. +- Fake PE objects expose OPTIONAL_HEADER.DATA_DIRECTORY[6], get_data(), + and __data__, so the parser's RVA reads (get_data) and file-offset reads + (__data__) can be steered independently — including short reads and + raising reads that a flat-image stub cannot produce. +- Determinism tests assert byte-for-byte stable output across runs. +""" + +from __future__ import annotations + +import struct +from typing import Any, Dict, List, Optional + +import pytest + +from iocx.parsers.pe_debug import ( + build_debug_structure, + _decode_entry, + _extract_asciiz_path, + _format_guid, + _locate_debug_directory, + _read_codeview_blob, + _read_entries, + _DEBUG_DIRECTORY_INDEX, + _DEBUG_ENTRY_SIZE, + _MAX_DEBUG_ENTRIES, + _PDB_PATH_MAX_LEN, +) + +_GUID16 = bytes(range(16)) # 000102...0f + + +# ================================================================= +# Byte-level builders +# ================================================================= + +def _entry(dtype: int = 4, size_of_data: int = 0, addr_raw: int = 0, + ptr_raw: int = 0, characteristics: int = 0, timestamp: int = 0, + major: int = 0, minor: int = 0) -> bytes: + """Build one 28-byte IMAGE_DEBUG_DIRECTORY.""" + return struct.pack(" bytes: + body = b"RSDS" + guid + struct.pack(" bytes: + return b"NB10" + struct.pack("len. + """ + def __init__(self, image: bytes = b"", + dir_dd: Optional[_DataDir] = None, + file: Optional[bytes] = None, + has_optional_header: bool = True, + has_data: bool = True): + self._img = bytearray(image) + if has_data: + self.__data__ = bytes(file if file is not None else image) + if has_optional_header: + self.OPTIONAL_HEADER = _OptHdr(dir_dd) + + def get_data(self, rva: int, size: int) -> bytes: + if rva < 0 or rva > len(self._img): + raise ValueError("unmapped RVA") + return bytes(self._img[rva:rva + size]) + + +class _EntryReadRaisePE: + """get_data always raises — to drive debug_entry_read_failed.""" + def __init__(self, dir_dd: _DataDir): + self.OPTIONAL_HEADER = _OptHdr(dir_dd) + self.__data__ = b"" + + def get_data(self, rva, size): + raise ValueError("boom") + + +def _pe_one_entry(entry_bytes: bytes, base_rva: int = 0x1000, + blob: bytes = b"", blob_at: int = 0x2000, + image_size: int = 0x8000, + as_file_offset: bool = False) -> _FakePE: + """ + Place one directory entry at base_rva; optionally place a CodeView blob. + If as_file_offset, the blob is written only into the __data__ file buffer + at blob_at (exercising the PointerToRawData path); otherwise into the + image (RVA path). + """ + image = bytearray(image_size) + image[base_rva:base_rva + len(entry_bytes)] = entry_bytes + file_buf = None + if blob: + if as_file_offset: + file_buf = bytearray(image_size) + file_buf[blob_at:blob_at + len(blob)] = blob + else: + image[blob_at:blob_at + len(blob)] = blob + return _FakePE(bytes(image), _DataDir(base_rva, len(entry_bytes)), + file=bytes(file_buf) if file_buf is not None else None) + + +# ================================================================= +# _locate_debug_directory +# ================================================================= + +class TestLocator: + def test_valid(self): + pe = _FakePE(dir_dd=_DataDir(0x1000, 0x1C)) + assert _locate_debug_directory(pe) == (0x1000, 0x1C) + + def test_zero_rva_none(self): + assert _locate_debug_directory(_FakePE(dir_dd=_DataDir(0, 0x1C))) is None + + def test_zero_size_none(self): + assert _locate_debug_directory(_FakePE(dir_dd=_DataDir(0x1000, 0))) is None + + def test_missing_optional_header_none(self): + assert _locate_debug_directory(_FakePE(has_optional_header=False)) is None + + def test_missing_entry_none(self): + assert _locate_debug_directory(_FakePE()) is None + + +# ================================================================= +# _read_entries — array-level truncation paths +# ================================================================= + +class TestReadEntries: + def test_size_not_entry_aligned(self): + trunc: List[str] = [] + errs: List[str] = [] + # declared size not a multiple of 28, but < one entry so loop no-ops + _read_entries(_FakePE(bytes(0x100)), 0x0, 10, trunc, errs) + assert "debug_directory_size_not_entry_aligned" in trunc + + def test_entry_count_exceeds_max(self): + trunc: List[str] = [] + big = (_MAX_DEBUG_ENTRIES + 5) * _DEBUG_ENTRY_SIZE + image = bytearray((_MAX_DEBUG_ENTRIES + 5) * _DEBUG_ENTRY_SIZE + 0x10) + pe = _FakePE(bytes(image)) + _read_entries(pe, 0x0, big, trunc, []) + assert "debug_directory_entry_count_exceeds_max" in trunc + + def test_entry_read_failed(self): + trunc: List[str] = [] + pe = _EntryReadRaisePE(_DataDir(0x1000, _DEBUG_ENTRY_SIZE)) + _read_entries(pe, 0x1000, _DEBUG_ENTRY_SIZE, trunc, []) + assert "debug_entry_read_failed" in trunc + + def test_entry_truncated_short_read(self): + trunc: List[str] = [] + # image holds only 12 of the 28 entry bytes + image = bytearray(0x1000 + 12) + pe = _FakePE(bytes(image), _DataDir(0x1000, _DEBUG_ENTRY_SIZE)) + _read_entries(pe, 0x1000, _DEBUG_ENTRY_SIZE, trunc, []) + assert "debug_entry_truncated" in trunc + + +# ================================================================= +# _decode_entry — header + non-CodeView +# ================================================================= + +class TestDecodeEntry: + def test_fields_decoded(self): + raw = _entry(dtype=4, size_of_data=0x40, addr_raw=0x3000, + ptr_raw=0x900, timestamp=0x600D, major=1, minor=2) + e = _decode_entry(_FakePE(), 0, raw) + assert e["type"] == 4 and e["type_name"] == "MISC" + assert e["size_of_data"] == 0x40 + assert e["address_of_raw_data"] == 0x3000 + assert e["pointer_to_raw_data"] == 0x900 + assert e["timestamp"] == 0x600D + assert e["major_version"] == 1 and e["minor_version"] == 2 + assert e["errors"] == [] + # non-CodeView entries are not enriched + assert e["cv_signature"] is None and e["pdb_path"] is None + + def test_unknown_type_name_none(self): + e = _decode_entry(_FakePE(), 0, _entry(dtype=99)) + assert e["type"] == 99 and e["type_name"] is None + + def test_unpack_failure_tombstone(self): + # fewer than 28 bytes -> struct.error -> entry_unpack_failed + e = _decode_entry(_FakePE(), 5, b"\x00" * 10) + assert e == {"index": 5, "errors": ["entry_unpack_failed"]} + + +# ================================================================= +# CodeView enrichment — RSDS / NB10 / opaque +# ================================================================= + +class TestCodeViewRSDS: + def test_rsds_via_file_pointer(self): + blob = _rsds(pdb=r"C:\src\app.pdb", age=7) + raw = _entry(dtype=2, size_of_data=len(blob), ptr_raw=0x2000) + pe = _pe_one_entry(raw, blob=blob, blob_at=0x2000, as_file_offset=True) + out = build_debug_structure(pe) + e = out["entries"][0] + assert e["cv_signature"] == "RSDS" + assert e["pdb_path"] == r"C:\src\app.pdb" + assert e["age"] == 7 + assert e["guid"] == "03020100-0504-0706-0809-0A0B0C0D0E0F" + assert e["errors"] == [] + + def test_rsds_via_rva_fallback(self): + # ptr_raw = 0 forces the AddressOfRawData (RVA) branch + blob = _rsds(pdb="rva.pdb") + raw = _entry(dtype=2, size_of_data=len(blob), addr_raw=0x2000, ptr_raw=0) + pe = _pe_one_entry(raw, blob=blob, blob_at=0x2000, as_file_offset=False) + e = build_debug_structure(pe)["entries"][0] + assert e["cv_signature"] == "RSDS" and e["pdb_path"] == "rva.pdb" + + def test_rsds_truncated(self): + # RSDS signature but < 24 bytes total + blob = b"RSDS" + b"\x00" * 10 + raw = _entry(dtype=2, size_of_data=len(blob), ptr_raw=0x2000) + pe = _pe_one_entry(raw, blob=blob, as_file_offset=True) + e = build_debug_structure(pe)["entries"][0] + assert "codeview_rsds_truncated" in e["errors"] + + def test_rsds_unterminated_pdb_path(self): + blob = _rsds(pdb="A" * 40, terminator=False) + raw = _entry(dtype=2, size_of_data=len(blob), ptr_raw=0x2000) + pe = _pe_one_entry(raw, blob=blob, as_file_offset=True) + e = build_debug_structure(pe)["entries"][0] + assert "pdb_path_unterminated" in e["errors"] + assert e["pdb_path"] == "A" * 40 # took what it had + + def test_rsds_non_ascii_pdb_path(self): + body = b"RSDS" + _GUID16 + struct.pack(" blob None + raw = _entry(dtype=2, size_of_data=0x20, addr_raw=0, ptr_raw=0) + pe = _pe_one_entry(raw) # no blob placed + e = build_debug_structure(pe)["entries"][0] + assert "codeview_read_failed" in e["errors"] + + def test_codeview_rva_read_raises(self): + # ptr_raw=0 so RVA branch taken; addr_raw beyond image -> get_data raises + raw = _entry(dtype=2, size_of_data=0x20, addr_raw=0x99999, ptr_raw=0) + image = bytearray(0x2000) + image[0x1000:0x1000 + len(raw)] = raw + pe = _FakePE(bytes(image), _DataDir(0x1000, len(raw))) + e = build_debug_structure(pe)["entries"][0] + assert "codeview_read_failed" in e["errors"] + + def test_size_of_data_zero_uses_max_len(self): + # size_of_data == 0 -> read_len falls back to _CODEVIEW_MAX_LEN, + # and a valid RSDS blob is still decoded. + blob = _rsds(pdb="zero.pdb") + raw = _entry(dtype=2, size_of_data=0, ptr_raw=0x2000) + pe = _pe_one_entry(raw, blob=blob, as_file_offset=True) + e = build_debug_structure(pe)["entries"][0] + assert e["pdb_path"] == "zero.pdb" + + +# ================================================================= +# _read_codeview_blob — direct unit coverage of fallbacks +# ================================================================= + +class TestReadCodeViewBlob: + def test_no_data_attr_falls_through_to_rva(self): + # __data__ absent -> file-pointer branch skipped; RVA branch used + image = bytearray(0x3000) + image[0x2000:0x2004] = b"RSDS" + pe = _FakePE(bytes(image), has_data=False) + entry = {"pointer_to_raw_data": 0x900, "address_of_raw_data": 0x2000} + blob = _read_codeview_blob(pe, entry, 16) + assert blob[:4] == b"RSDS" + + def test_both_pointers_zero_returns_none(self): + pe = _FakePE(bytes(0x100)) + entry = {"pointer_to_raw_data": 0, "address_of_raw_data": 0} + assert _read_codeview_blob(pe, entry, 16) is None + + def test_data_bytes_conversion_raises_falls_through_to_rva(self): + # __data__ present but bytes(__data__) raises TypeError -> the + # except (TypeError, ValueError): pass branch runs, then the RVA + # fallback (address_of_raw_data) is used instead. + class _BadData: + def __bytes__(self): + raise TypeError("cannot convert") + + image = bytearray(0x3000) + image[0x2000:0x2004] = b"RSDS" + pe = _FakePE(bytes(image)) # normal image for get_data + pe.__data__ = _BadData() # but __data__ refuses bytes() + entry = {"pointer_to_raw_data": 0x900, "address_of_raw_data": 0x2000} + blob = _read_codeview_blob(pe, entry, 16) + assert blob[:4] == b"RSDS" # RVA fallback succeeded + + +# ================================================================= +# _extract_asciiz_path — direct edge cases +# ================================================================= + +class TestExtractAsciizPath: + def test_terminated(self): + e = {"errors": []} + assert _extract_asciiz_path(b"abc\x00xxxx", 0, e) == "abc" + assert e["errors"] == [] + + def test_unterminated_within_cap(self): + e = {"errors": []} + region = b"Z" * (_PDB_PATH_MAX_LEN + 10) # no NUL + out = _extract_asciiz_path(region, 0, e) + assert "pdb_path_unterminated" in e["errors"] + assert out == "Z" * _PDB_PATH_MAX_LEN # capped + + def test_empty_returns_none(self): + e = {"errors": []} + assert _extract_asciiz_path(b"\x00rest", 0, e) is None + + +# ================================================================= +# _format_guid +# ================================================================= + +class TestFormatGuid: + def test_canonical_mixed_endian(self): + assert _format_guid(_GUID16) == "03020100-0504-0706-0809-0A0B0C0D0E0F" + + def test_short_returns_none(self): + assert _format_guid(b"\x00" * 8) is None + + +# ================================================================= +# build_debug_structure — full roundtrips & contract +# ================================================================= + +class TestBuildDebugStructure: + def test_absent_returns_none(self): + assert build_debug_structure(_FakePE()) is None + + def test_multiple_entries(self): + cv = _rsds(pdb="multi.pdb") + e0 = _entry(dtype=2, size_of_data=len(cv), ptr_raw=0x2000) + e1 = _entry(dtype=13, size_of_data=0x40, addr_raw=0x3000) # POGO opaque + table = e0 + e1 + image = bytearray(0x8000) + image[0x1000:0x1000 + len(table)] = table + file_buf = bytearray(0x8000) + file_buf[0x2000:0x2000 + len(cv)] = cv + pe = _FakePE(bytes(image), _DataDir(0x1000, len(table)), + file=bytes(file_buf)) + out = build_debug_structure(pe) + assert out["entry_count"] == 2 + assert out["entries"][0]["type_name"] == "CODEVIEW" + assert out["entries"][1]["type_name"] == "POGO" + + def test_contract_keys(self): + raw = _entry(dtype=4) + out = build_debug_structure(_pe_one_entry(raw)) + for k in ("rva", "size", "entries", "entry_count", + "truncations", "errors"): + assert k in out + entry = out["entries"][0] + for k in ("index", "characteristics", "timestamp", "major_version", + "minor_version", "type", "type_name", "size_of_data", + "address_of_raw_data", "pointer_to_raw_data", "pdb_path", + "cv_signature", "guid", "age", "errors"): + assert k in entry + + def test_json_serializable(self): + import json + raw = _entry(dtype=2, size_of_data=len(_rsds()), ptr_raw=0x2000) + out = build_debug_structure(_pe_one_entry(raw, blob=_rsds(), + as_file_offset=True)) + json.dumps(out) # must not raise + + +# ================================================================= +# Determinism +# ================================================================= + +class TestDeterminism: + def test_repeated_calls_identical(self): + import json + raw = _entry(dtype=2, size_of_data=len(_rsds()), ptr_raw=0x2000) + a = build_debug_structure(_pe_one_entry(raw, blob=_rsds(), as_file_offset=True)) + b = build_debug_structure(_pe_one_entry(raw, blob=_rsds(), as_file_offset=True)) + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) diff --git a/tests/unit/validators/test_validator_debug.py b/tests/unit/validators/test_validator_debug.py new file mode 100644 index 0000000..f82adca --- /dev/null +++ b/tests/unit/validators/test_validator_debug.py @@ -0,0 +1,296 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Unit tests for iocx.validators.debug.validate_debug. + +Strategy: +- Input is the debug_struct dict produced by parser pe_debug, carried under + metadata["debug_struct"] (the dispatcher binds it via @depends_on with the + first positional arg named `metadata` receiving `internal`). +- Build dicts directly to isolate validator logic from parser behaviour. +- Tests assert on emitted REASONCODES and the details payload. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from iocx.reason_codes import ReasonCodes +from iocx.validators.debug import validate_debug + + +# ================================================================= +# Input builders +# ================================================================= + +def _make_analysis( + size_of_image: Optional[int] = 0x100000, + sections: Optional[List[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + analysis: Dict[str, Any] = {"size_of_image": size_of_image} + if sections is not None: + analysis["sections"] = sections + return analysis + + +def _whole_image_sections() -> List[Dict[str, Any]]: + return [{"virtual_address": 0x1000, "virtual_size": 0xFF000}] + + +def _tiny_section() -> List[Dict[str, Any]]: + return [{"virtual_address": 0x1000, "virtual_size": 0x8}] + + +def _make_entry( + index: int = 0, + dtype: int = 2, + type_name: Optional[str] = "CODEVIEW", + size_of_data: int = 0x20, + address_of_raw_data: int = 0x2000, + pointer_to_raw_data: int = 0x900, + errors: Optional[List[str]] = None, +) -> Dict[str, Any]: + return { + "index": index, "type": dtype, "type_name": type_name, + "size_of_data": size_of_data, + "address_of_raw_data": address_of_raw_data, + "pointer_to_raw_data": pointer_to_raw_data, + "errors": errors or [], + } + + +def _make_debug( + rva: int = 0x1000, + size: int = 0x1C, + entries: Optional[List[Dict[str, Any]]] = None, + truncations: Optional[List[str]] = None, + errors: Optional[List[str]] = None, +) -> Dict[str, Any]: + ents = entries or [] + return {"rva": rva, "size": size, "entries": ents, + "entry_count": len(ents), + "truncations": truncations or [], "errors": errors or []} + + +def _run(debug: Optional[Dict[str, Any]], analysis: Dict[str, Any]): + return validate_debug({"debug_struct": debug}, analysis) + + +def _codes(issues) -> List: + return [i["issue"] for i in issues] + + +def _details_for(issues, code) -> List[Dict[str, Any]]: + return [i["details"] for i in issues if i["issue"] == code] + + +# ================================================================= +# Absence +# ================================================================= + +class TestAbsence: + def test_none_struct_no_issues(self): + assert _run(None, _make_analysis()) == [] + + def test_missing_key_no_issues(self): + assert validate_debug({}, _make_analysis()) == [] + + +# ================================================================= +# Top-level decode short-circuit +# ================================================================= + +class TestTopLevelDecodeFailure: + def test_errors_emit_invalid_header(self): + debug = _make_debug(errors=["entry_unpack_failed"]) + issues = _run(debug, _make_analysis()) + assert _codes(issues) == [ReasonCodes.DEBUG_DIRECTORY_INVALID_HEADER] + d = _details_for(issues, ReasonCodes.DEBUG_DIRECTORY_INVALID_HEADER)[0] + assert d["reason"] == "top_level_decode" + assert d["errors"] == ["entry_unpack_failed"] + + def test_short_circuit_skips_entries_and_truncations(self): + debug = _make_debug( + errors=["x"], + truncations=["debug_entry_truncated"], + entries=[_make_entry(errors=["codeview_too_short"])]) + issues = _run(debug, _make_analysis()) + assert _codes(issues) == [ReasonCodes.DEBUG_DIRECTORY_INVALID_HEADER] + + +# ================================================================= +# Truncations +# ================================================================= + +class TestTruncations: + def test_one_issue_per_tag(self): + debug = _make_debug(truncations=[ + "debug_directory_size_not_entry_aligned", + "debug_entry_truncated"]) + issues = _run(debug, _make_analysis()) + assert _codes(issues) == [ + ReasonCodes.DEBUG_TABLE_TRUNCATED, + ReasonCodes.DEBUG_TABLE_TRUNCATED] + + def test_region_detail_preserved(self): + debug = _make_debug(truncations=["debug_entry_read_failed"]) + issues = _run(debug, _make_analysis()) + assert _details_for(issues, ReasonCodes.DEBUG_TABLE_TRUNCATED)[0] == { + "region": "debug_entry_read_failed"} + + +# ================================================================= +# Entry malformation (priority-resolved) +# ================================================================= + +class TestEntryMalformation: + def test_single_reason_flagged(self): + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0, pointer_to_raw_data=0, + errors=["codeview_signature_unknown"])]) + issues = _run(debug, _make_analysis()) + assert ReasonCodes.DEBUG_DIRECTORY_ENTRY_MALFORMED in _codes(issues) + d = _details_for(issues, ReasonCodes.DEBUG_DIRECTORY_ENTRY_MALFORMED)[0] + assert d["reason"] == "codeview_signature_unknown" + assert d["index"] == 0 and d["type_name"] == "CODEVIEW" + + def test_priority_first_match_wins(self): + # entry_unpack_failed outranks codeview_signature_unknown + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0, pointer_to_raw_data=0, + errors=["codeview_signature_unknown", "entry_unpack_failed"])]) + issues = _run(debug, _make_analysis()) + malformed = _details_for(issues, ReasonCodes.DEBUG_DIRECTORY_ENTRY_MALFORMED) + assert len(malformed) == 1 + assert malformed[0]["reason"] == "entry_unpack_failed" + + def test_unknown_error_not_flagged(self): + # an error tag not in the priority list -> no malformation issue + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0, pointer_to_raw_data=0, + errors=["some_unlisted_tag"])]) + issues = _run(debug, _make_analysis()) + assert ReasonCodes.DEBUG_DIRECTORY_ENTRY_MALFORMED not in _codes(issues) + + def test_clean_entry_no_issue(self): + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0x1004, size_of_data=0x10, errors=[])]) + issues = _run(debug, _make_analysis(sections=_whole_image_sections())) + assert issues == [] + + +# ================================================================= +# Entry data-region RVA validation +# ================================================================= + +class TestEntryRvaValidation: + def test_unmapped_region_flagged(self): + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0x9000, size_of_data=0x10, errors=[])]) + issues = _run(debug, _make_analysis(sections=_tiny_section())) + assert _codes(issues) == [ReasonCodes.DEBUG_ENTRY_RVA_INVALID] + d = _details_for(issues, ReasonCodes.DEBUG_ENTRY_RVA_INVALID)[0] + assert d["index"] == 0 + assert d["address_of_raw_data"] == 0x9000 + assert d["size_of_data"] == 0x10 + + def test_mapped_region_no_issue(self): + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0x1004, size_of_data=0x4, errors=[])]) + issues = _run(debug, _make_analysis(sections=_whole_image_sections())) + assert issues == [] + + def test_zero_addr_not_checked(self): + # AddressOfRawData == 0 (file-pointer-only entry) -> not flagged + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0, pointer_to_raw_data=0x900, errors=[])]) + issues = _run(debug, _make_analysis(sections=_tiny_section())) + assert ReasonCodes.DEBUG_ENTRY_RVA_INVALID not in _codes(issues) + + def test_missing_size_defaults_zero(self): + entry = _make_entry(address_of_raw_data=0x9000, errors=[]) + del entry["size_of_data"] # size_of_data absent -> treated as 0 + debug = _make_debug(entries=[entry]) + issues = _run(debug, _make_analysis(sections=_tiny_section())) + assert _codes(issues) == [ReasonCodes.DEBUG_ENTRY_RVA_INVALID] + assert _details_for(issues, ReasonCodes.DEBUG_ENTRY_RVA_INVALID)[0]["size_of_data"] == 0 + + def test_no_sections_falls_back_to_size_of_image(self): + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0x200000, size_of_data=0x10, errors=[])]) + issues = _run(debug, _make_analysis(size_of_image=0x100000)) + assert _codes(issues) == [ReasonCodes.DEBUG_ENTRY_RVA_INVALID] + + +# ================================================================= +# Combined scenarios +# ================================================================= + +class TestCombinedAnomalies: + def test_truncation_plus_entry_malformed(self): + debug = _make_debug( + truncations=["debug_entry_truncated"], + entries=[_make_entry(address_of_raw_data=0, pointer_to_raw_data=0, + errors=["codeview_rsds_truncated"])]) + issues = _run(debug, _make_analysis()) + assert _codes(issues) == [ + ReasonCodes.DEBUG_TABLE_TRUNCATED, + ReasonCodes.DEBUG_DIRECTORY_ENTRY_MALFORMED] + + def test_malformed_and_rva_invalid_same_entry(self): + # a CodeView entry that is both malformed AND points its data region + # at an unmapped RVA -> two issues for the one entry + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0x9000, size_of_data=0x10, + errors=["pdb_path_non_ascii"])]) + issues = _run(debug, _make_analysis(sections=_tiny_section())) + assert ReasonCodes.DEBUG_DIRECTORY_ENTRY_MALFORMED in _codes(issues) + assert ReasonCodes.DEBUG_ENTRY_RVA_INVALID in _codes(issues) + + +# ================================================================= +# Output contract +# ================================================================= + +class TestOutputContract: + def test_dependency_contract(self): + assert getattr(validate_debug, "_depends_on") == ("internal", "analysis") + + def test_issue_shape(self): + debug = _make_debug(entries=[_make_entry( + address_of_raw_data=0x9000, errors=["codeview_too_short"])]) + issues = _run(debug, _make_analysis(sections=_tiny_section())) + assert issues + for i in issues: + assert set(i) == {"issue", "details"} + assert isinstance(i["issue"], str) + assert isinstance(i["details"], dict) + + def test_json_serializable(self): + import json + debug = _make_debug( + truncations=["debug_entry_truncated"], + entries=[_make_entry(errors=["codeview_too_short"], + address_of_raw_data=0, pointer_to_raw_data=0)]) + issues = _run(debug, _make_analysis()) + json.dumps([i for i in issues]) + + +# ================================================================= +# Determinism +# ================================================================= + +class TestDeterminism: + def test_repeated_calls_identical(self): + import json + debug = _make_debug( + truncations=["debug_entry_truncated"], + entries=[_make_entry(address_of_raw_data=0x9000, size_of_data=0x10, + errors=["codeview_signature_unknown"])]) + analysis = _make_analysis(sections=_tiny_section()) + a = [i for i in _run(debug, analysis)] + b = [i for i in _run(debug, analysis)] + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) From 55e3c1469a9e293f148b946199cb5f7470644c28 Mon Sep 17 00:00:00 2001 From: malx-labs Date: Fri, 31 Jul 2026 11:10:28 +0100 Subject: [PATCH 08/11] test: achieve 100% coverage across the codebase (1620 tests) Complete the v0.7.6 test sweep, bringing statement coverage to 100% across the entire IOCX codebase with 1620 passing tests. Parsers (byte-level fixtures via struct.pack; fake PE stubs steering get_data / __data__ / sections, including short and raising reads): - pe_relocations: block-header truncation, unpack failure, MAX_BLOCKS exhaustion, entries read-failure and short-read paths. - pe_certificates: locator, __data__ absent/unconvertible, image-raw-end section extents (raising/zero-field sections), full WIN_CERTIFICATE walk (past-EOF, table/header/blob truncation, length-too-small, max-exceeded), revision/type maps, align_up. - pe_debug: entry-array truncations, RSDS/NB10 CodeView decode, PDB path unterminated/non-ASCII/empty, file-pointer->RVA fallback, GUID formatter. - pe_tls: PE32/PE32+ widths, VA->RVA callback resolution, directory and callback-array read/short/unpack paths, image-base-unavailable, looping array cap. Validators (dispatcher-faithful arg binding; assert on reason codes + details): - relocations, debug: block/entry integrity, truncations, per-entry RVA mapping, priority-resolved sub-reasons, placement-owned-by-rva_graph. - signature, tls: post-migration struct sourcing, the new v0.7.6 codes (certificate_table_malformed / certificate_offset_inside_image / tls_directory_truncated / tls_callback_rva_invalid), and the guarded short-circuit boundaries between them. - _directory_invariants: tri-state (True/False/None) contract, key-alias tolerance, int() coercion + except paths, section vs SizeOfImage fallback, and the zero-length-region boundary semantics. All tests are deterministic and JSON-safe; --- tests/unit/parsers/test_pe_certificates.py | 405 +++++++++++++++++ tests/unit/parsers/test_pe_tls.py | 414 ++++++++++++++++++ .../validators/test_directory_invariants.py | 256 +++++++++++ .../test_validator_signatures_ext.py | 107 +++++ .../unit/validators/test_validator_tls_ext.py | 209 +++++++++ 5 files changed, 1391 insertions(+) create mode 100644 tests/unit/parsers/test_pe_certificates.py create mode 100644 tests/unit/parsers/test_pe_tls.py create mode 100644 tests/unit/validators/test_directory_invariants.py create mode 100644 tests/unit/validators/test_validator_signatures_ext.py create mode 100644 tests/unit/validators/test_validator_tls_ext.py diff --git a/tests/unit/parsers/test_pe_certificates.py b/tests/unit/parsers/test_pe_certificates.py new file mode 100644 index 0000000..40c397e --- /dev/null +++ b/tests/unit/parsers/test_pe_certificates.py @@ -0,0 +1,405 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Unit tests for iocx.parsers.pe_certificates. + +Strategy: +- Byte-level fixture builders construct WIN_CERTIFICATE entries directly via + struct.pack (dwLength + wRevision + wCertificateType + opaque blob). +- Fake PE objects expose OPTIONAL_HEADER.DATA_DIRECTORY[4], __data__ (the + raw file bytes), and .sections, so the parser's file-offset reads and the + "outside the image" section-extent computation can be steered directly — + including __data__ that is absent or refuses bytes(), and sections that + raise on int(). +- Determinism tests assert byte-for-byte stable output across runs. + +Note: the security directory's VirtualAddress is a FILE OFFSET, not an RVA, +so all offsets here index into the __data__ buffer. +""" + +from __future__ import annotations + +import struct +from typing import Any, Dict, List, Optional + +import pytest + +from iocx.parsers.pe_certificates import ( + build_certificate_structure, + _align_up, + _decode_certificate, + _image_raw_end, + _locate_security_directory, + _raw_file_bytes, + _read_certificates, + _CERT_ALIGNMENT, + _MAX_CERTIFICATES, + _SECURITY_DIRECTORY_INDEX, + _WIN_CERT_HEADER_SIZE, +) + + +# ================================================================= +# Byte-level builders +# ================================================================= + +def _win_cert(revision: int = 0x0200, cert_type: int = 0x0002, + blob: bytes = b"", dw_length: Optional[int] = None, + align: bool = True) -> bytes: + """Build a WIN_CERTIFICATE. dw_length defaults to 8 + len(blob).""" + if dw_length is None: + dw_length = _WIN_CERT_HEADER_SIZE + len(blob) + entry = struct.pack(" exercises the per-section except path.""" + @property + def PointerToRawData(self): + raise ValueError("bad ptr") + + @property + def SizeOfRawData(self): + return 0x200 + + +class _FakePE: + def __init__(self, file_bytes: Optional[bytes] = b"", + dd: Optional[_DataDir] = None, + sections: Any = (), + has_optional_header: bool = True, + has_data: bool = True, + has_sections: bool = True): + if has_data: + self.__data__ = file_bytes + if has_optional_header: + self.OPTIONAL_HEADER = _OptHdr(dd) + if has_sections: + self.sections = sections + + +def _pe(file_bytes: bytes, sec_offset: int, sec_size: int, + sections=()) -> _FakePE: + """A PE with the security dir at (sec_offset, sec_size) over file_bytes.""" + return _FakePE(file_bytes, _DataDir(sec_offset, sec_size), sections=sections) + + +# ================================================================= +# _locate_security_directory +# ================================================================= + +class TestLocator: + def test_valid(self): + pe = _FakePE(dd=_DataDir(0x800, 0x40)) + assert _locate_security_directory(pe) == (0x800, 0x40) + + def test_zero_offset_none(self): + assert _locate_security_directory(_FakePE(dd=_DataDir(0, 0x40))) is None + + def test_zero_size_none(self): + assert _locate_security_directory(_FakePE(dd=_DataDir(0x800, 0))) is None + + def test_missing_optional_header_none(self): + assert _locate_security_directory( + _FakePE(has_optional_header=False)) is None + + def test_missing_entry_none(self): + assert _locate_security_directory(_FakePE()) is None + + +# ================================================================= +# _raw_file_bytes +# ================================================================= + +class TestRawFileBytes: + def test_bytes_returned(self): + assert _raw_file_bytes(_FakePE(b"\x01\x02")) == b"\x01\x02" + + def test_missing_data_none(self): + assert _raw_file_bytes(_FakePE(has_data=False)) is None + + def test_data_is_none_none(self): + assert _raw_file_bytes(_FakePE(file_bytes=None)) is None + + def test_bytes_conversion_raises_none(self): + class _Bad: + def __bytes__(self): + raise TypeError("nope") + pe = _FakePE.__new__(_FakePE) + pe.__data__ = _Bad() + assert _raw_file_bytes(pe) is None + + +# ================================================================= +# _image_raw_end +# ================================================================= + +class TestImageRawEnd: + def test_no_sections_attr_none(self): + pe = _FakePE(has_sections=False) + assert _image_raw_end(pe) is None + + def test_empty_sections_none(self): + assert _image_raw_end(_FakePE(sections=[])) is None + + def test_max_across_sections(self): + secs = [_Section(0x200, 0x400), _Section(0x600, 0x200)] # ends 0x600, 0x800 + assert _image_raw_end(_FakePE(sections=secs)) == 0x800 + + def test_raising_section_skipped(self): + secs = [_RaisingSection(), _Section(0x200, 0x400)] + assert _image_raw_end(_FakePE(sections=secs)) == 0x600 + + def test_zero_fields_skipped(self): + # ptr==0 or raw_size==0 -> not counted; all-zero -> end stays 0 -> None + assert _image_raw_end(_FakePE(sections=[_Section(0, 0x400)])) is None + assert _image_raw_end(_FakePE(sections=[_Section(0x200, 0)])) is None + + +# ================================================================= +# _align_up +# ================================================================= + +class TestAlignUp: + def test_already_aligned(self): + assert _align_up(16, 8) == 16 + + def test_rounds_up(self): + assert _align_up(9, 8) == 16 + assert _align_up(1, 8) == 8 + + def test_nonpositive_alignment_returns_value(self): + assert _align_up(13, 0) == 13 + assert _align_up(13, -4) == 13 + + +# ================================================================= +# _decode_certificate +# ================================================================= + +class TestDecodeCertificate: + def test_known_revision_and_type(self): + cert = _decode_certificate(0, 0x100, 0x0200, 0x0002, 0x800, 0x1000, []) + assert cert["revision_name"] == "REVISION_2_0" + assert cert["cert_type_name"] == "PKCS_SIGNED_DATA" + assert cert["data_length"] == 0x100 - 8 + assert cert["errors"] == [] + + def test_length_too_small(self): + cert = _decode_certificate(0, 4, 0x0200, 0x0002, 0x800, 0x1000, []) + assert cert["errors"] == ["length_too_small"] + assert cert["data_length"] == 0 + + def test_unknown_revision(self): + cert = _decode_certificate(0, 0x20, 0x0999, 0x0002, 0x800, 0x1000, []) + assert "unknown_revision" in cert["errors"] + + def test_unknown_cert_type(self): + cert = _decode_certificate(0, 0x20, 0x0200, 0x00FF, 0x800, 0x1000, []) + assert "unknown_cert_type" in cert["errors"] + + def test_blob_truncated_clamps_data_length(self): + trunc: List[str] = [] + # dw_length claims 0x100 payload, but only 0x10 available to dir_end + cert = _decode_certificate(0, 0x100, 0x0200, 0x0002, + entry_offset=0x800, dir_end=0x818, + truncations=trunc) + assert "certificate_blob_truncated" in trunc + assert cert["data_length"] == 0x818 - (0x800 + 8) # clamped + + +# ================================================================= +# _read_certificates — array walk +# ================================================================= + +class TestReadCertificates: + def test_single_certificate(self): + cert = _win_cert(blob=b"\xAA" * 16) + data = bytes(0x800) + cert + trunc: List[str] = [] + errs: List[str] = [] + out = _read_certificates(data, 0x800, len(cert), len(data), trunc, errs) + assert len(out) == 1 + assert out[0]["length"] == 8 + 16 + assert trunc == [] and errs == [] + + def test_multiple_8byte_aligned(self): + table = _win_cert(blob=b"\xAA" * 20) + _win_cert(blob=b"\xBB" * 12) + data = bytes(0x800) + table + out = _read_certificates(data, 0x800, len(table), len(data), [], []) + assert len(out) == 2 + + def test_offset_past_eof(self): + errs: List[str] = [] + out = _read_certificates(bytes(0x100), 0x800, 0x40, 0x100, [], errs) + assert out == [] + assert "certificate_offset_past_eof" in errs + + def test_table_truncated_clamps_end(self): + trunc: List[str] = [] + # declared end (0x800 + 0x400) runs past file_size 0x820 + cert = _win_cert(blob=b"\x00" * 8) + data = bytes(0x800) + cert + out = _read_certificates(data, 0x800, 0x400, len(data), trunc, []) + assert "certificate_table_truncated" in trunc + assert len(out) == 1 + + def test_header_truncated(self): + trunc: List[str] = [] + # window leaves < 8 bytes for a header (end 4 bytes past base) + data = bytes(0x800) + b"\x10\x00\x00\x00" # 4 stray bytes + out = _read_certificates(data, 0x800, 4, len(data), trunc, []) + assert out == [] + assert "certificate_header_truncated" in trunc + + def test_length_too_small_stops_walk(self): + # dwLength = 4 (< 8) -> decoder tags length_too_small; walk stops + bad = struct.pack(" loop exhausts -> else branch + one = struct.pack("= 8 bytes, so this + # handler is unreachable in practice. Force it via monkeypatch to + # exercise the certificate_header_unpack_failed_* tombstone. + import iocx.parsers.pe_certificates as M + orig = M.struct.unpack_from + + def boom(fmt, buf, off=0): + if fmt == " raw_file_unavailable + pe = _FakePE(dd=_DataDir(0x800, 0x40), has_data=False) + out = build_certificate_structure(pe) + assert out["errors"] == ["raw_file_unavailable"] + assert out["file_size"] is None + assert out["overlaps_image"] is None + assert out["certificates"] == [] + + def test_overlaps_image_true(self): + # cert offset 0x400 before section raw end 0x800 -> overlaps_image True + cert = _win_cert(blob=b"\x00" * 8) + data = bytearray(0x1000) + data[0x400:0x400 + len(cert)] = cert + pe = _pe(bytes(data), 0x400, len(cert), + sections=[_Section(0x200, 0x600)]) # raw end 0x800 + out = build_certificate_structure(pe) + assert out["overlaps_image"] is True + assert out["image_raw_end"] == 0x800 + + def test_overlaps_image_false(self): + cert = _win_cert(blob=bytes(range(32))) + data = bytes(0x800) + cert + pe = _pe(data, 0x800, len(cert), sections=[_Section(0x200, 0x600)]) + out = build_certificate_structure(pe) + assert out["overlaps_image"] is False + assert out["certificate_count"] == 1 + + def test_overlaps_image_none_when_no_sections(self): + # image_raw_end None (no sections) -> overlaps_image evaluates False, + # but we assert on the underlying image_raw_end being None. + cert = _win_cert(blob=b"\x00" * 8) + data = bytes(0x800) + cert + pe = _pe(data, 0x800, len(cert), sections=[]) + out = build_certificate_structure(pe) + assert out["image_raw_end"] is None + assert out["overlaps_image"] is False + + def test_contract_keys(self): + cert = _win_cert(blob=b"\x00" * 8) + data = bytes(0x800) + cert + out = build_certificate_structure( + _pe(data, 0x800, len(cert), sections=[_Section(0x200, 0x600)])) + for k in ("offset", "size", "file_size", "image_raw_end", + "overlaps_image", "certificates", "certificate_count", + "truncations", "errors"): + assert k in out + for k in ("index", "offset", "length", "revision", "revision_name", + "cert_type", "cert_type_name", "data_length", "errors"): + assert k in out["certificates"][0] + + def test_json_serializable(self): + import json + cert = _win_cert(blob=b"\x00" * 8) + data = bytes(0x800) + cert + out = build_certificate_structure( + _pe(data, 0x800, len(cert), sections=[_Section(0x200, 0x600)])) + json.dumps(out) # must not raise + + +# ================================================================= +# Determinism +# ================================================================= + +class TestDeterminism: + def _pe(self): + table = _win_cert(blob=b"\xAA" * 20) + _win_cert(0x0999, 0x00FF, b"\xBB" * 4) + data = bytes(0x800) + table + return _pe(data, 0x800, len(table), sections=[_Section(0x200, 0x600)]) + + def test_repeated_calls_identical(self): + import json + a = build_certificate_structure(self._pe()) + b = build_certificate_structure(self._pe()) + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) diff --git a/tests/unit/parsers/test_pe_tls.py b/tests/unit/parsers/test_pe_tls.py new file mode 100644 index 0000000..ed6c671 --- /dev/null +++ b/tests/unit/parsers/test_pe_tls.py @@ -0,0 +1,414 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Unit tests for iocx.parsers.pe_tls. + +Strategy: +- Byte-level builders construct IMAGE_TLS_DIRECTORY structs (24-byte PE32 / + 40-byte PE32+) and NULL-terminated callback VA arrays via struct.pack. +- Fake PE objects expose OPTIONAL_HEADER.Magic / ImageBase / + DATA_DIRECTORY[9] and get_data(), steering PE32-vs-PE32+ width, the + VA->RVA callback conversion, and read failures / short reads at chosen + RVAs (which a flat-image stub alone cannot produce). +- Determinism tests assert byte-for-byte stable output across runs. + +Note: TLS address fields are VAs. Callback resolution subtracts ImageBase, +so fixtures use image_base = 0x400000 and callback VAs = image_base + RVA. +""" + +from __future__ import annotations + +import struct +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +import iocx.parsers.pe_tls as M +from iocx.parsers.pe_tls import ( + build_tls_structure, + _image_base, + _is_pe32_plus, + _locate_tls_directory, + _raw_data_size, + _read_callbacks, + _read_directory, + _MAGIC_PE32, + _MAGIC_PE32_PLUS, + _TLS_DIRECTORY_INDEX, +) + +_IB = 0x400000 # ImageBase used throughout + + +# ================================================================= +# Byte-level builders +# ================================================================= + +def _tls_dir(ptr_size: int, start: int, end: int, index: int, + callbacks: int, zerofill: int = 0, chars: int = 0) -> bytes: + fmt = " bytes: + fmt = " bytes: + if rva in self._raise_at: + raise ValueError("boom") + if rva < 0 or rva > len(self._img): + raise ValueError("unmapped RVA") + return bytes(self._img[rva:rva + length]) + + +def _build_pe(ptr_size: int, *, tls_rva: int = 0x1000, cb_rva: int = 0x2000, + start=_IB + 0x3000, end=_IB + 0x3000, callback_vas=None, + image_size: int = 0x8000, magic=None, + image_base: int = _IB, raise_at=()) -> _FakePE: + """Assemble a PE with a TLS dir and (optional) callback array.""" + if magic is None: + magic = _MAGIC_PE32_PLUS if ptr_size == 8 else _MAGIC_PE32 + callbacks_va = (image_base + cb_rva) if callback_vas is not None else 0 + directory = _tls_dir(ptr_size, start, end, image_base + 0x4000, + callbacks_va) + image = bytearray(image_size) + image[tls_rva:tls_rva + len(directory)] = directory + if callback_vas is not None: + arr = _cb_array(ptr_size, callback_vas) + image[cb_rva:cb_rva + len(arr)] = arr + dir_size = 40 if ptr_size == 8 else 24 + return _FakePE(bytes(image), _DataDir(tls_rva, dir_size), magic=magic, + image_base=image_base, raise_at=raise_at) + + +# ================================================================= +# _locate_tls_directory +# ================================================================= + +class TestLocator: + def test_valid(self): + pe = _FakePE(dd=_DataDir(0x1000, 24)) + assert _locate_tls_directory(pe) == (0x1000, 24) + + def test_zero_rva_none(self): + assert _locate_tls_directory(_FakePE(dd=_DataDir(0, 24))) is None + + def test_zero_size_none(self): + assert _locate_tls_directory(_FakePE(dd=_DataDir(0x1000, 0))) is None + + def test_missing_optional_header_none(self): + assert _locate_tls_directory( + _FakePE(has_optional_header=False)) is None + + def test_missing_entry_none(self): + assert _locate_tls_directory(_FakePE()) is None + + +# ================================================================= +# _is_pe32_plus +# ================================================================= + +class TestIsPe32Plus: + def test_pe32_plus_true(self): + assert _is_pe32_plus(_FakePE(magic=_MAGIC_PE32_PLUS)) is True + + def test_pe32_false(self): + assert _is_pe32_plus(_FakePE(magic=_MAGIC_PE32)) is False + + def test_missing_magic_defaults_false(self): + assert _is_pe32_plus(_FakePE(has_magic=False)) is False + + def test_missing_optional_header_defaults_false(self): + assert _is_pe32_plus(_FakePE(has_optional_header=False)) is False + + +# ================================================================= +# _image_base +# ================================================================= + +class TestImageBase: + def test_valid(self): + assert _image_base(_FakePE(image_base=_IB)) == _IB + + def test_missing_none(self): + assert _image_base(_FakePE(has_image_base=False)) is None + + def test_missing_optional_header_none(self): + assert _image_base(_FakePE(has_optional_header=False)) is None + + +# ================================================================= +# _read_directory +# ================================================================= + +class TestReadDirectory: + def test_success_pe32(self): + directory = _tls_dir(4, _IB + 0x100, _IB + 0x200, _IB + 0x300, + _IB + 0x400, 0x10, 0x20) + pe = _FakePE(bytes(0x1000) + directory) + errs: List[str] = [] + out = _read_directory(pe, 0x1000, 24, 4, errs) + assert out == (_IB + 0x100, _IB + 0x200, _IB + 0x300, _IB + 0x400, + 0x10, 0x20) + assert errs == [] + + def test_read_raises(self): + pe = _FakePE(bytes(0x1000), raise_at={0x1000}) + errs: List[str] = [] + assert _read_directory(pe, 0x1000, 24, 4, errs) is None + assert errs == ["tls_directory_read_failed"] + + def test_short_read_truncated(self): + # image ends 10 bytes into the directory -> short read + pe = _FakePE(bytes(0x1000 + 10)) + errs: List[str] = [] + assert _read_directory(pe, 0x1000, 24, 4, errs) is None + assert errs == ["tls_directory_truncated"] + + def test_unpack_error_is_defensive(self, monkeypatch): + # After the length guard there are always dir_size bytes, so + # struct.unpack_from cannot fail with real input. Force it. + orig = M.struct.unpack_from + + def boom(fmt, buf, off=0): + if len(fmt) >= 7: # the directory formats " short read + trunc: List[str] = [] + image = bytearray(0x2000 + 2) # only 2 bytes at rva 0x2000 + pe = self._flat(bytes(image)) + out = _read_callbacks(pe, _IB + 0x2000, _IB, 4, trunc, []) + assert out == [] + assert "tls_callbacks_truncated" in trunc + + def test_unpack_error_is_defensive(self, monkeypatch): + orig = M.struct.unpack_from + + def boom(fmt, buf, off=0): + if fmt in (" fields None branch + pe = _build_pe(4, raise_at={0x1000}) + out = build_tls_structure(pe) + assert out["errors"] == ["tls_directory_read_failed"] + assert out["start_address_of_raw_data"] is None + assert out["callbacks"] == [] and out["callback_count"] == 0 + assert out["is_64bit"] is False + + def test_roundtrip_pe32_no_callbacks(self): + # AddressOfCallBacks == 0 -> empty callbacks, zero-length raw valid + pe = _build_pe(4, callback_vas=None, + start=_IB + 0x3000, end=_IB + 0x3000) + out = build_tls_structure(pe) + assert out["is_64bit"] is False + assert out["address_of_callbacks"] == 0 + assert out["callbacks"] == [] + assert out["raw_data_size"] == 0 + assert out["errors"] == [] and out["truncations"] == [] + + def test_roundtrip_pe32_with_callbacks(self): + pe = _build_pe(4, callback_vas=[_IB + 0x1111, _IB + 0x2222], + start=_IB + 0x3000, end=_IB + 0x3100) + out = build_tls_structure(pe) + assert out["callback_count"] == 2 + assert out["callbacks"] == [_IB + 0x1111, _IB + 0x2222] + assert out["raw_data_size"] == 0x100 + + def test_roundtrip_pe32_plus_with_callbacks(self): + pe = _build_pe(8, callback_vas=[_IB + 0xABCD]) + out = build_tls_structure(pe) + assert out["is_64bit"] is True + assert out["callbacks"] == [_IB + 0xABCD] + + def test_end_before_start_tombstone(self): + pe = _build_pe(4, callback_vas=None, + start=_IB + 0x4000, end=_IB + 0x3000) + out = build_tls_structure(pe) + assert "tls_raw_data_end_before_start" in out["errors"] + assert out["raw_data_size"] is None + + def test_contract_keys(self): + pe = _build_pe(4, callback_vas=[_IB + 0x10]) + out = build_tls_structure(pe) + for k in ("rva", "size", "is_64bit", "image_base", + "start_address_of_raw_data", "end_address_of_raw_data", + "address_of_index", "address_of_callbacks", + "size_of_zero_fill", "characteristics", "raw_data_size", + "callbacks", "callback_count", "truncations", "errors"): + assert k in out + + def test_json_serializable(self): + import json + out = build_tls_structure(_build_pe(4, callback_vas=[_IB + 0x10])) + json.dumps(out) # must not raise + + +# ================================================================= +# Determinism +# ================================================================= + +class TestDeterminism: + def test_repeated_calls_identical(self): + import json + a = build_tls_structure(_build_pe(8, callback_vas=[_IB + 0x1, _IB + 0x2])) + b = build_tls_structure(_build_pe(8, callback_vas=[_IB + 0x1, _IB + 0x2])) + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) diff --git a/tests/unit/validators/test_directory_invariants.py b/tests/unit/validators/test_directory_invariants.py new file mode 100644 index 0000000..e40369b --- /dev/null +++ b/tests/unit/validators/test_directory_invariants.py @@ -0,0 +1,256 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Unit tests for iocx.validators._directory_invariants. + +Pure-logic module (no PE object needed): the "analysis" input is a plain +dict, so every branch is driven directly. Covers: + - region_within_image: None-guards, negative rva/size, in/out of bounds + - _sections: key tolerance (rva/virtual_address, virtual_size/size), + None-field skips, int() coercion, and the (ValueError, TypeError) except + - rva_in_any_section: section hit/miss + SizeOfImage fallback + None + - region_in_any_section: whole-region fit/miss + fallback + None + +The tri-state contract (True / False / None) is asserted explicitly, since +callers rely on None meaning "unknown", not "out of bounds". +""" + +from __future__ import annotations + +import pytest + +from iocx.validators._directory_invariants import ( + region_within_image, + rva_in_any_section, + region_in_any_section, + _sections, +) + + +# ================================================================= +# region_within_image +# ================================================================= + +class TestRegionWithinImage: + def test_none_rva_returns_none(self): + assert region_within_image(None, 0x10, 0x1000) is None + + def test_none_size_of_image_returns_none(self): + assert region_within_image(0x100, 0x10, None) is None + + def test_negative_rva_false(self): + assert region_within_image(-1, 0x10, 0x1000) is False + + def test_negative_size_false(self): + assert region_within_image(0x100, -1, 0x1000) is False + + def test_within_bounds_true(self): + assert region_within_image(0x100, 0x10, 0x1000) is True + + def test_exact_end_boundary_true(self): + # rva + size == size_of_image is inclusive (<=) + assert region_within_image(0xFF0, 0x10, 0x1000) is True + + def test_one_past_end_false(self): + assert region_within_image(0xFF0, 0x11, 0x1000) is False + + def test_none_size_treated_as_zero(self): + # size None -> span 0; a bare rva within the image is True + assert region_within_image(0x100, None, 0x1000) is True + + def test_zero_size_at_size_of_image_boundary_true(self): + assert region_within_image(0x1000, 0, 0x1000) is True + + +# ================================================================= +# _sections +# ================================================================= + +class TestSections: + def test_empty_when_absent(self): + assert _sections({}) == [] + + def test_none_sections_value(self): + assert _sections({"sections": None}) == [] + + def test_rva_and_virtual_size_keys(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x200}]} + assert _sections(analysis) == [(0x1000, 0x200)] + + def test_virtual_address_and_size_aliases(self): + analysis = {"sections": [{"virtual_address": 0x2000, "size": 0x100}]} + assert _sections(analysis) == [(0x2000, 0x100)] + + def test_missing_rva_skipped(self): + analysis = {"sections": [{"virtual_size": 0x200}]} + assert _sections(analysis) == [] + + def test_missing_vsize_skipped(self): + analysis = {"sections": [{"rva": 0x1000}]} + assert _sections(analysis) == [] + + def test_int_coercible_strings_coerced(self): + analysis = {"sections": [{"rva": "4096", "virtual_size": "512"}]} + assert _sections(analysis) == [(0x1000, 0x200)] + + def test_non_coercible_value_excepted_and_skipped(self): + # int("nope") raises ValueError -> section skipped, others kept + analysis = {"sections": [ + {"rva": "nope", "virtual_size": 0x200}, + {"rva": 0x3000, "virtual_size": 0x10}, + ]} + assert _sections(analysis) == [(0x3000, 0x10)] + + def test_typeerror_value_excepted(self): + # int(object()) raises TypeError -> skipped + analysis = {"sections": [{"rva": object(), "virtual_size": 0x200}]} + assert _sections(analysis) == [] + + def test_multiple_sections_order_preserved(self): + analysis = {"sections": [ + {"rva": 0x1000, "virtual_size": 0x100}, + {"rva": 0x2000, "virtual_size": 0x200}, + ]} + assert _sections(analysis) == [(0x1000, 0x100), (0x2000, 0x200)] + + +# ================================================================= +# rva_in_any_section +# ================================================================= + +class TestRvaInAnySection: + def test_none_rva_returns_none(self): + assert rva_in_any_section(None, {"size_of_image": 0x1000}) is None + + def test_hit_inside_section(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x1000}]} + assert rva_in_any_section(0x1500, analysis) is True + + def test_at_section_base_true(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x1000}]} + assert rva_in_any_section(0x1000, analysis) is True + + def test_at_section_end_exclusive_false(self): + # base + vsize is exclusive + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x1000}]} + assert rva_in_any_section(0x2000, analysis) is False + + def test_miss_all_sections_false(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x10}]} + assert rva_in_any_section(0x9000, analysis) is False + + def test_negative_vsize_clamped(self): + # max(vsize, 0): a negative virtual_size yields an empty extent + analysis = {"sections": [{"rva": 0x1000, "virtual_size": -5}]} + assert rva_in_any_section(0x1000, analysis) is False + + def test_fallback_within_size_of_image_true(self): + # no sections -> SizeOfImage bound check + assert rva_in_any_section(0x500, {"size_of_image": 0x1000}) is True + + def test_fallback_out_of_size_of_image_false(self): + assert rva_in_any_section(0x2000, {"size_of_image": 0x1000}) is False + + def test_no_sections_and_no_size_of_image_none(self): + assert rva_in_any_section(0x500, {}) is None + + def test_empty_sections_list_uses_fallback(self): + # [] is falsy -> fallback path, not the section loop + assert rva_in_any_section(0x500, {"sections": [], + "size_of_image": 0x1000}) is True + + +# ================================================================= +# region_in_any_section +# ================================================================= + +class TestRegionInAnySection: + def test_none_rva_returns_none(self): + assert region_in_any_section(None, 0x10, + {"size_of_image": 0x1000}) is None + + def test_whole_region_fits_true(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x1000}]} + assert region_in_any_section(0x1000, 0x100, analysis) is True + + def test_region_spilling_past_section_false(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x100}]} + # region ends at 0x1200, section ends at 0x1100 + assert region_in_any_section(0x1000, 0x200, analysis) is False + + def test_region_at_exact_section_end_true(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x100}]} + # rva+size == base+vsize (inclusive <=) + assert region_in_any_section(0x1000, 0x100, analysis) is True + + def test_none_size_treated_as_zero(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x100}]} + assert region_in_any_section(0x1050, None, analysis) is True + + def test_region_before_section_false(self): + analysis = {"sections": [{"rva": 0x2000, "virtual_size": 0x100}]} + assert region_in_any_section(0x1000, 0x10, analysis) is False + + def test_negative_vsize_clamped(self): + # max(vsize, 0) makes the section a zero-length extent [0x1000, 0x1000). + # A NON-zero region cannot fit -> False ... + analysis = {"sections": [{"rva": 0x1000, "virtual_size": -5}]} + assert region_in_any_section(0x1000, 0x10, analysis) is False + + def test_zero_region_at_clamped_empty_section_true(self): + # ... but a ZERO-length region at that base does "fit": end == base, + # and 0x1000 <= 0x1000 and (0x1000 + 0) <= 0x1000. Documenting the + # boundary explicitly so the behaviour is intentional, not accidental. + analysis = {"sections": [{"rva": 0x1000, "virtual_size": -5}]} + assert region_in_any_section(0x1000, 0, analysis) is True + + def test_spans_two_sections_not_single_false(self): + # region straddles the gap; no single section contains it + analysis = {"sections": [ + {"rva": 0x1000, "virtual_size": 0x100}, + {"rva": 0x1100, "virtual_size": 0x100}, + ]} + assert region_in_any_section(0x1080, 0x100, analysis) is False + + def test_fallback_within_size_of_image_true(self): + assert region_in_any_section(0x100, 0x10, + {"size_of_image": 0x1000}) is True + + def test_fallback_out_of_size_of_image_false(self): + assert region_in_any_section(0xFF0, 0x100, + {"size_of_image": 0x1000}) is False + + def test_no_sections_and_no_size_of_image_none(self): + assert region_in_any_section(0x100, 0x10, {}) is None + + def test_empty_sections_list_uses_fallback(self): + assert region_in_any_section(0x100, 0x10, + {"sections": [], + "size_of_image": 0x1000}) is True + + +# ================================================================= +# Tri-state contract & purity +# ================================================================= + +class TestContract: + def test_tri_state_values(self): + # explicit True / False / None across the three public helpers + secs = {"sections": [{"rva": 0x1000, "virtual_size": 0x1000}]} + assert rva_in_any_section(0x1500, secs) is True + assert rva_in_any_section(0x9000, secs) is False + assert rva_in_any_section(None, secs) is None + assert region_in_any_section(0x1000, 0x10, secs) is True + assert region_in_any_section(0x9000, 0x10, secs) is False + assert region_in_any_section(None, 0x10, secs) is None + + def test_no_mutation_of_analysis(self): + analysis = {"sections": [{"rva": 0x1000, "virtual_size": 0x100}], + "size_of_image": 0x1000} + import copy + snapshot = copy.deepcopy(analysis) + rva_in_any_section(0x1050, analysis) + region_in_any_section(0x1050, 0x10, analysis) + region_within_image(0x100, 0x10, 0x1000) + assert analysis == snapshot # side-effect-free diff --git a/tests/unit/validators/test_validator_signatures_ext.py b/tests/unit/validators/test_validator_signatures_ext.py new file mode 100644 index 0000000..3c4661a --- /dev/null +++ b/tests/unit/validators/test_validator_signatures_ext.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Targeted tests for the table-truncation block in validate_signature (1a): + + for tag in cert_struct.get("truncations", []) or []: + issues.append(StructuralIssue( + issue=ReasonCodes.CERTIFICATE_TABLE_MALFORMED, + details={"reason": "truncation", "region": tag})) + +Reaching this block requires a certificate_struct that: + - has an EMPTY top-level `errors` list (a non-empty one short-circuits at + step 0 with reason "top_level_decode" and returns before 1a), and + - has one or more parser `truncations` tags. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from iocx.reason_codes import ReasonCodes +from iocx.validators.signature import validate_signature + + +def _cert_struct(certificates=None, *, truncations=None, errors=None, + overlaps_image=False, offset=0x800, size=0x200, + file_size=0x1000, image_raw_end=None) -> Dict[str, Any]: + certs = certificates or [] + return { + "offset": offset, "size": size, "file_size": file_size, + "image_raw_end": image_raw_end, "overlaps_image": overlaps_image, + "certificates": certs, "certificate_count": len(certs), + "truncations": truncations or [], "errors": errors or [], + } + + +def _run(cert_struct: Optional[Dict[str, Any]], + has_signature: bool = True, + analysis: Optional[Dict[str, Any]] = None): + return validate_signature( + {"certificate_struct": cert_struct}, + {"has_signature": has_signature}, + analysis if analysis is not None else {}, + ) + + +def _codes(issues) -> List: + return [i["issue"] for i in issues] + + +def _details_for(issues, code) -> List[Dict[str, Any]]: + return [i["details"] for i in issues if i["issue"] == code] + + +class TestTableTruncation: + def test_single_truncation_tag_emits_malformed(self): + # one valid cert so the symmetry check passes to reach block 1a + cert = {"offset": 0x800, "length": 0x40, "revision": 0x0200, + "cert_type": 0x0002, "errors": []} + cs = _cert_struct([cert], truncations=["certificate_blob_truncated"]) + issues = _run(cs, analysis={"file_size": 0x1000}) + assert ReasonCodes.CERTIFICATE_TABLE_MALFORMED in _codes(issues) + details = _details_for(issues, ReasonCodes.CERTIFICATE_TABLE_MALFORMED) + assert details == [{"reason": "truncation", + "region": "certificate_blob_truncated"}] + + def test_truncation_with_no_certificates(self): + # truncations present, empty errors, no certs. has_signature=False so + # the present/flag symmetry does not short-circuit before block 1a. + cs = _cert_struct([], truncations=["certificate_table_truncated"]) + issues = _run(cs, has_signature=False) + assert _codes(issues) == [ReasonCodes.CERTIFICATE_TABLE_MALFORMED] + assert _details_for(issues, ReasonCodes.CERTIFICATE_TABLE_MALFORMED) == [ + {"reason": "truncation", "region": "certificate_table_truncated"}] + + def test_multiple_truncation_tags_one_issue_each_in_order(self): + cs = _cert_struct( + [], truncations=["certificate_table_truncated", + "certificate_header_truncated", + "certificate_blob_truncated"]) + issues = _run(cs, has_signature=False) + malformed = _details_for(issues, ReasonCodes.CERTIFICATE_TABLE_MALFORMED) + assert [d["region"] for d in malformed] == [ + "certificate_table_truncated", + "certificate_header_truncated", + "certificate_blob_truncated"] + assert all(d["reason"] == "truncation" for d in malformed) + + def test_no_truncations_does_not_emit_malformed(self): + cs = _cert_struct([], truncations=[]) + issues = _run(cs, has_signature=False) + assert ReasonCodes.CERTIFICATE_TABLE_MALFORMED not in _codes(issues) + + def test_top_level_error_short_circuits_before_1a(self): + # A non-empty top-level `errors` must be reported as top_level_decode + # and MUST NOT also emit a truncation-region issue from block 1a. + cs = _cert_struct( + [], errors=["raw_file_unavailable"], + truncations=["certificate_blob_truncated"]) + issues = _run(cs) + assert _codes(issues) == [ReasonCodes.CERTIFICATE_TABLE_MALFORMED] + # single issue, and it is the decode variant (not the truncation one) + assert _details_for(issues, ReasonCodes.CERTIFICATE_TABLE_MALFORMED) == [ + {"reason": "top_level_decode", "errors": ["raw_file_unavailable"]}] diff --git a/tests/unit/validators/test_validator_tls_ext.py b/tests/unit/validators/test_validator_tls_ext.py new file mode 100644 index 0000000..51d7393 --- /dev/null +++ b/tests/unit/validators/test_validator_tls_ext.py @@ -0,0 +1,209 @@ +# Copyright (c) 2026 MalX Labs and contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Coverage-gap tests for iocx.validators.tls.validate_tls. + +Targets the branches the main suite missed: + - 130 callback-array truncation loop body (TLS_DIRECTORY_TRUNCATED) + - 201 cascade early-return when ImageBase is not an int + - 263 resolution-tombstone loop body (TLS_CALLBACK_RVA_INVALID) + - 277-282 callbacks present but ImageBase not int (image_base_unavailable) + - 287 `continue` when a callback VA is not an int + - 293 resolved callback target that does not map to any section + +Each fixture is shaped to reach exactly one target while keeping the rest of +the validator quiet, so the assertions stay unambiguous. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from iocx.reason_codes import ReasonCodes +from iocx.validators.tls import validate_tls + + +# ================================================================= +# Builders +# ================================================================= + +def _tls(**kw) -> Dict[str, Any]: + """A tls_struct with sensible, quiet defaults; override per test.""" + base: Dict[str, Any] = { + "rva": 0x1000, "size": 24, "is_64bit": False, "image_base": 0x400000, + "start_address_of_raw_data": None, + "end_address_of_raw_data": None, + "address_of_index": 0, + "address_of_callbacks": None, + "size_of_zero_fill": 0, "characteristics": 0, + "raw_data_size": None, + "callbacks": [], "callback_count": 0, + "truncations": [], "errors": [], + } + base.update(kw) + return base + + +def _run(tls: Optional[Dict[str, Any]], + analysis: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None): + return validate_tls( + {"tls_struct": tls}, + metadata or {}, + analysis if analysis is not None else {"extended": []}, + ) + + +def _codes(issues) -> List: + return [i["issue"] for i in issues] + + +def _details_for(issues, code) -> List[Dict[str, Any]]: + return [i["details"] for i in issues if i["issue"] == code] + + +# ================================================================= +# 130 — callback-array truncation loop body +# ================================================================= + +class TestCallbackArrayTruncation: + def test_truncation_tag_emits_directory_truncated(self): + # No header-decode error (that would short-circuit at step 2), a + # truncations tag present -> line 130 loop body runs. + tls = _tls(truncations=["tls_callbacks_truncated"]) + issues = _run(tls) + assert ReasonCodes.TLS_DIRECTORY_TRUNCATED in _codes(issues) + assert _details_for(issues, ReasonCodes.TLS_DIRECTORY_TRUNCATED) == [ + {"reason": "callback_array", "region": "tls_callbacks_truncated"}] + + def test_multiple_truncation_tags_in_order(self): + tls = _tls(truncations=["tls_callbacks_read_failed", + "tls_callbacks_max_exceeded"]) + issues = _run(tls) + regions = [d["region"] for d in + _details_for(issues, ReasonCodes.TLS_DIRECTORY_TRUNCATED)] + assert regions == ["tls_callbacks_read_failed", + "tls_callbacks_max_exceeded"] + + +# ================================================================= +# 201 — cascade early-return when ImageBase is not an int +# ================================================================= + +class TestCascadeImageBaseNotInt: + def test_in_range_pointer_but_no_image_base_returns(self): + # Valid range, non-zero in-range pointer, but image_base is None -> + # the pointer cannot be converted to an RVA, so line 201 returns + # without emitting a mapping issue. callbacks=[] so step 4 is a no-op. + tls = _tls(start_address_of_raw_data=0x1000, + end_address_of_raw_data=0x2000, + address_of_callbacks=0x1500, # in [start, end) + image_base=None, + callbacks=[]) + issues = _run(tls, analysis={"extended": [], + "sections": [{"virtual_address": 0, + "virtual_size": 0x9000}]}) + # No pointer-mapping / executability issues emitted; clean early exit. + assert ReasonCodes.TLS_CALLBACK_NOT_MAPPED_TO_SECTION not in _codes(issues) + assert ReasonCodes.TLS_CALLBACK_IN_NON_EXECUTABLE_SECTION not in _codes(issues) + assert issues == [] + + +# ================================================================= +# 263 — resolution-tombstone loop body +# ================================================================= + +class TestResolutionTombstones: + def test_va_below_image_base_surfaced(self): + # Parser recorded a resolution tombstone (not a header-decode tag), + # callbacks=[] -> line 263 loop emits one rva_invalid per tag. + tls = _tls(errors=["tls_callbacks_va_below_image_base"], callbacks=[]) + issues = _run(tls) + assert _codes(issues) == [ReasonCodes.TLS_CALLBACK_RVA_INVALID] + assert _details_for(issues, ReasonCodes.TLS_CALLBACK_RVA_INVALID) == [ + {"reason": "tls_callbacks_va_below_image_base"}] + + def test_both_resolution_tags_sorted_and_deduped(self): + tls = _tls(errors=["tls_image_base_unavailable", + "tls_callbacks_va_below_image_base", + "tls_image_base_unavailable"], # dup + callbacks=[]) + issues = _run(tls) + reasons = [d["reason"] for d in + _details_for(issues, ReasonCodes.TLS_CALLBACK_RVA_INVALID)] + # set() dedupes, sorted() orders deterministically + assert reasons == ["tls_callbacks_va_below_image_base", + "tls_image_base_unavailable"] + + def test_header_decode_tag_does_not_reach_263(self): + # A header-decode tag short-circuits at step 2; the resolution loop + # (263) must not run, even though callbacks=[] would otherwise reach it. + tls = _tls(errors=["tls_directory_truncated"], callbacks=[]) + issues = _run(tls) + assert _codes(issues) == [ReasonCodes.TLS_DIRECTORY_TRUNCATED] + + +# ================================================================= +# 277-282 — callbacks present but ImageBase not int +# ================================================================= + +class TestCallbacksPresentNoImageBase: + def test_image_base_unavailable_with_callbacks(self): + # callbacks non-empty AND image_base not int -> the 277-282 branch + # emits a single image_base_unavailable issue and returns. + tls = _tls(callbacks=[0x401000, 0x402000], image_base=None) + issues = _run(tls) + assert _codes(issues).count(ReasonCodes.TLS_CALLBACK_RVA_INVALID) == 1 + assert _details_for(issues, ReasonCodes.TLS_CALLBACK_RVA_INVALID) == [ + {"reason": "image_base_unavailable", "callback_count": 2}] + + +# ================================================================= +# 287 — non-int callback VA is skipped +# ================================================================= + +class TestNonIntCallbackSkipped: + def test_non_int_va_continues(self): + # One non-int VA (skipped at 287) and one valid, mapped VA -> no + # rva_invalid issue is produced (the non-int is not flagged). + image_base = 0x400000 + tls = _tls(image_base=image_base, + callbacks=["not-an-int", image_base + 0x1500]) + analysis = {"extended": [], + "sections": [{"virtual_address": 0x1000, + "virtual_size": 0x1000}]} # covers rva 0x1500 + issues = _run(tls, analysis=analysis) + assert ReasonCodes.TLS_CALLBACK_RVA_INVALID not in _codes(issues) + + +# ================================================================= +# 293 — resolved target does not map to any section +# ================================================================= + +class TestTargetNotMapped: + def test_unmapped_target_flagged(self): + image_base = 0x400000 + # rva = 0x8000, but the only section covers [0x1000, 0x1010) + tls = _tls(image_base=image_base, callbacks=[image_base + 0x8000]) + analysis = {"extended": [], + "sections": [{"virtual_address": 0x1000, + "virtual_size": 0x10}]} + issues = _run(tls, analysis=analysis) + d = _details_for(issues, ReasonCodes.TLS_CALLBACK_RVA_INVALID) + assert d == [{"callback_va": image_base + 0x8000, + "callback_rva": 0x8000, + "reason": "not_mapped", + "invalid_callback_count": 1}] + + def test_below_image_base_target_flagged(self): + # sibling branch: rva < 0 -> below_image_base (kept for contrast) + image_base = 0x400000 + tls = _tls(image_base=image_base, callbacks=[image_base - 0x10]) + issues = _run(tls, analysis={"extended": [], "sections": []}) + d = _details_for(issues, ReasonCodes.TLS_CALLBACK_RVA_INVALID) + assert d == [{"callback_va": image_base - 0x10, + "reason": "below_image_base", + "invalid_callback_count": 1}] From c5ab5f7370e6c7209e8df9cababc8c7da26c414c Mon Sep 17 00:00:00 2001 From: malx-labs Date: Mon, 10 Aug 2026 10:36:56 +0100 Subject: [PATCH 09/11] Pre-release documentation --- CHANGELOG.md | 48 +++++++++++++++++++++++++++++++++++++ README-pypi.md | 14 +++++------ README.md | 11 ++++++++- iocx/reason_codes.py | 21 +++++----------- iocx/validators/__init__.py | 8 +++---- pyproject.toml | 2 +- 6 files changed, 76 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee01e41..d9b0a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,51 @@ +# **v0.7.6 — Structural validator expansion: debug, relocations directories** +**Released: 2026‑08‑10** + +## Added +- **Expanded static PE directory coverage.** Deterministic, pefile-independent +struct decoders for the following directories: **Relocations**, +**Certificate Table**, **Debug Directory**, and **TLS**. +- **Relocation parsing** — `IMAGE_BASE_RELOCATION` blocks with typed +entries (HIGHLOW, DIR64, …), block-size and word-alignment validation, +and per-entry target-RVA checks. +- **Certificate table parsing** — `WIN_CERTIFICATE` entries (revision, +type, length) read from raw file bytes, with an "offset must lie outside +the image" invariant. The embedded PKCS#7 blob is left opaque. +- **Debug directory parsing** — `IMAGE_DEBUG_DIRECTORY` entries with +deterministic CodeView PDB-path extraction (RSDS/NB10) and canonical +mixed-endian GUID formatting. +- **TLS parsing** — `IMAGE_TLS_DIRECTORY` (PE32/PE32+) with VA→RVA callback +resolution via `ImageBase`, bounded callback walking, and zero-length +raw-data handling. +- Structured, JSON-safe metadata for all new directories +(`relocation_struct`, `certificate_struct`, `debug_struct`, `tls_struct`). +- New deterministic reason codes: `certificate_table_malformed`, +`certificate_offset_inside_image`, `tls_directory_truncated`, +`tls_callback_rva_invalid`, plus the `relocation_*` and `debug_*` +families. + +## Changed +- **Signature & TLS validators** now consume the new deterministic +`certificate_struct` / `tls_struct` from internal metadata instead of +pefile-derived data. All prior reason codes and checks are preserved. +- Structural validator dispatcher registers the new `relocations` and +`debug` validators; directory placement remains solely owned by the +RVA-graph validator to avoid double-counting. + +## Fixed +- Corrected a latent VA/RVA unit mismatch when mapping TLS callback +pointers to sections. +- TLS: surface parser tombstones that were previously dropped +(`tls_image_base_unavailable`, `tls_callbacks_va_below_image_base`). +- TLS: a zero-length raw-data region accompanied by a valid callback array +no longer raises a false-positive `tls_zero_length_directory`. + +## Notes +- Static-only and deterministic by design: no dynamic execution, +unpacking, emulation, ML, sandboxing, or network access. + +--- + # **v0.7.5 - Structural validator expansion** **Released: 2026‑07‑01** diff --git a/README-pypi.md b/README-pypi.md index dac56be..3669890 100644 --- a/README-pypi.md +++ b/README-pypi.md @@ -40,6 +40,13 @@ If you need predictable, automatable IOC extraction — IOCX is built for you. --- +## Version highlights (v0.7.6) + +- Added new PE structural validators for relocations and debug directories +- WIN_CERTIFICATE and tls validators now have pefile-independent struct parsers +- Never crashes on malformed input - byte-level parsing with structured error tombstones +- 1620 tests at 100% coverage - deterministic output, snapshot-stable + ## Version highlights (v0.7.5) - Added detection for malformed exports, delay-load tables, resources, VS_VERSIONINFO, and Optional Header fields via 24 structural reason codes @@ -47,13 +54,6 @@ If you need predictable, automatable IOC extraction — IOCX is built for you. - Never crashes on malformed input — byte-level parsing with structured error tombstones - 1370 tests at 100% coverage — deterministic output, snapshot-stable, cross-verified against `dumpbin` -## Version highlights (v0.7.4.1) - -- Removed the `python-magic` dependency, which caused import failures on Windows systems -- Added a pure‑Python file‑type detector for full cross‑platform portability -- No behavioural changes to IOC extraction -- The `--min-length` consistency fix is planned for **v0.7.6** - --- ## **Performance** diff --git a/README.md b/README.md index 82e73f6..a66016e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

- + @@ -202,6 +202,15 @@ Fast path — no PE parsing.

Show Version History
+### **v0.7.6 — Structural Validator Expansion: Debug and relocations directories** +- Two new PE structural validators - relocations and debug +- WIN_CERTIFICATE and tls validators now source structural truth from dedicated struct parsers, independent of pefile +- 12 new reason codes with priority-resolved sub-reason taxonomies +- Deterministic byte-level parsing - no reliance on pefile's lazy interpretation +- 1620 tests at 100% coverage + +--- + ### **v0.7.5 — Structural Validator Expansion** - Four new PE structural validators — exports, delay-load imports, VS_VERSIONINFO, and resource hierarchy - 24 new reason codes with priority-resolved sub-reason taxonomies diff --git a/iocx/reason_codes.py b/iocx/reason_codes.py index 581a552..dea8132 100644 --- a/iocx/reason_codes.py +++ b/iocx/reason_codes.py @@ -70,6 +70,9 @@ class ReasonCodes: # (future extension) TLS_CALLBACK_ARRAY_NOT_TERMINATED = "tls_callback_array_not_terminated" + TLS_DIRECTORY_TRUNCATED = "tls_directory_truncated" + TLS_CALLBACK_RVA_INVALID = "tls_callback_rva_invalid" + # --- Signature anomalies --- SIGNATURE_FLAG_SET_BUT_NO_METADATA = "signature_flag_set_but_no_metadata" SIGNATURE_PRESENT_BUT_FLAG_NOT_SET = "signature_present_but_flag_not_set" @@ -161,29 +164,17 @@ class ReasonCodes: # --- Delay-load entry anomalies --- DELAY_IMPORT_ENTRY_INVALID = "delay_import_entry_invalid" - # ---------------------------------------------------------------------- - # v0.7.6 — Certificate table (WIN_CERTIFICATE) - # ---------------------------------------------------------------------- + # —-- Certificate table (WIN_CERTIFICATE) --- CERTIFICATE_TABLE_MALFORMED = "certificate_table_malformed" CERTIFICATE_OFFSET_INSIDE_IMAGE = "certificate_offset_inside_image" - # ---------------------------------------------------------------------- - # v0.7.6 — TLS directory (IMAGE_TLS_DIRECTORY) - # ---------------------------------------------------------------------- - TLS_DIRECTORY_TRUNCATED = "tls_directory_truncated" - TLS_CALLBACK_RVA_INVALID = "tls_callback_rva_invalid" - - # ---------------------------------------------------------------------- - # v0.7.6 — Relocations (IMAGE_BASE_RELOCATION) - # ---------------------------------------------------------------------- + # --— Relocations (IMAGE_BASE_RELOCATION) --- RELOCATION_DIRECTORY_INVALID_HEADER = "relocation_directory_invalid_header" RELOCATION_TABLE_TRUNCATED = "relocation_table_truncated" RELOCATION_BLOCK_MALFORMED = "relocation_block_malformed" RELOCATION_ENTRY_RVA_INVALID = "relocation_entry_rva_invalid" - # ---------------------------------------------------------------------- - # v0.7.6 — Debug directory (IMAGE_DEBUG_DIRECTORY) - # ---------------------------------------------------------------------- + # --- Debug directory (IMAGE_DEBUG_DIRECTORY) --- DEBUG_DIRECTORY_INVALID_HEADER = "debug_directory_invalid_header" DEBUG_TABLE_TRUNCATED = "debug_table_truncated" DEBUG_DIRECTORY_ENTRY_MALFORMED = "debug_directory_entry_malformed" diff --git a/iocx/validators/__init__.py b/iocx/validators/__init__.py index 3abd404..f5a2e09 100644 --- a/iocx/validators/__init__.py +++ b/iocx/validators/__init__.py @@ -10,8 +10,8 @@ from .optional_header import validate_optional_header from .tls import validate_tls from .signature import validate_signature -from .relocations import validate_relocations # NEW in v0.7.6 -from .debug import validate_debug # NEW in v0.7.6 +from .relocations import validate_relocations +from .debug import validate_debug from .resources import validate_resources from .version_info import validate_version_info from .exports import validate_exports @@ -34,12 +34,12 @@ # Signature directory correctness "signature": validate_signature, # Base relocations (dir 5): block + entry structural correctness. - # NEW in v0.7.6. Placement owned by rva_graph; this descends into + # Placement owned by rva_graph; this descends into # block/entry contents only. Placed in ascending directory-index order # after the placement backbone and security cluster, before entropy. "relocations": validate_relocations, # Debug directory (dir 6): entry + CodeView PDB structural correctness. - # NEW in v0.7.6. Placement owned by rva_graph. + # Placement owned by rva_graph. "debug": validate_debug, # Resource directory correctness "resources": validate_resources, diff --git a/pyproject.toml b/pyproject.toml index 11e5217..cb6a652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "iocx" -version = "0.7.5" +version = "0.7.6" description = "A deterministic, high‑performance static‑analysis engine that extracts high‑signal IOCs from PE binaries, text, and logs — built for SOC automation and modern threat‑analysis pipelines." authors = [ { name = "MalX Labs" } From a1ca5f21dfa99444204e9629156ae04ab708ccc7 Mon Sep 17 00:00:00 2001 From: malx-labs Date: Mon, 10 Aug 2026 10:42:54 +0100 Subject: [PATCH 10/11] Fix formatting on internal schema --- iocx/schemas/internal_schema.py | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/iocx/schemas/internal_schema.py b/iocx/schemas/internal_schema.py index 9565cee..d7eabbe 100644 --- a/iocx/schemas/internal_schema.py +++ b/iocx/schemas/internal_schema.py @@ -194,7 +194,7 @@ class DelayImportStruct(TypedDict, total=False): # ------------------------- -# Relocation table (v0.7.6) +# Relocation table # ------------------------- class RelocationEntry(TypedDict): @@ -225,48 +225,48 @@ class RelocationStruct(TypedDict, total=False): # ------------------------- -# Certificate table (v0.7.6) +# Certificate table # ------------------------- class CertificateEntry(TypedDict): index: int - offset: int # file offset of this WIN_CERTIFICATE - length: int # dwLength (incl. 8-byte header) + offset: int # file offset of this WIN_CERTIFICATE + length: int # dwLength (incl. 8-byte header) revision: int revision_name: Optional[str] # WIN_CERT_REVISION_* name, None if unknown cert_type: int cert_type_name: Optional[str] # WIN_CERT_TYPE_* name, None if unknown - data_length: int # derived: bCertificate payload length + data_length: int # derived: bCertificate payload length errors: List[str] class CertificateStruct(TypedDict, total=False): - offset: int # file offset (NOT an RVA) per WIN_CERTIFICATE + offset: int # file offset (NOT an RVA) per WIN_CERTIFICATE size: int - file_size: Optional[int] # backing file length, None if unavailable - image_raw_end: Optional[int] # max(PointerToRawData + SizeOfRawData) - overlaps_image: Optional[bool] # offset < image_raw_end (defect if True) + file_size: Optional[int] # backing file length, None if unavailable + image_raw_end: Optional[int] # max(PointerToRawData + SizeOfRawData) + overlaps_image: Optional[bool] # offset < image_raw_end (defect if True) certificates: List[CertificateEntry] - certificate_count: int # derived: len(certificates) + certificate_count: int # derived: len(certificates) truncations: List[str] errors: List[str] # ------------------------- -# Debug directory (v0.7.6) +# Debug directory # ------------------------- class DebugEntry(TypedDict, total=False): index: int characteristics: int - timestamp: int # TimeDateStamp + timestamp: int # TimeDateStamp major_version: int minor_version: int type: int - type_name: Optional[str] # IMAGE_DEBUG_TYPE_* name, None if unknown + type_name: Optional[str] # IMAGE_DEBUG_TYPE_* name, None if unknown size_of_data: int - address_of_raw_data: int # RVA of the debug data - pointer_to_raw_data: int # file offset of the debug data + address_of_raw_data: int # RVA of the debug data + pointer_to_raw_data: int # file offset of the debug data pdb_path: Optional[str] # CodeView only; None otherwise cv_signature: Optional[str] # "RSDS" | "NB10" | None guid: Optional[str] # RSDS only; canonical mixed-endian @@ -284,23 +284,23 @@ class DebugStruct(TypedDict, total=False): # ------------------------- -# TLS directory (v0.7.6) +# TLS directory # ------------------------- class TlsStruct(TypedDict, total=False): rva: int size: int is_64bit: bool - image_base: Optional[int] # OPTIONAL_HEADER.ImageBase + image_base: Optional[int] # OPTIONAL_HEADER.ImageBase start_address_of_raw_data: Optional[int] # VA end_address_of_raw_data: Optional[int] # VA address_of_index: Optional[int] # VA address_of_callbacks: Optional[int] # VA of the callback array size_of_zero_fill: Optional[int] characteristics: Optional[int] - raw_data_size: Optional[int] # derived: end - start; None if end < start - callbacks: List[int] # resolved callback VAs (NULL-terminated) - callback_count: int # derived: len(callbacks) + raw_data_size: Optional[int] # derived: end - start; None if end < start + callbacks: List[int] # resolved callback VAs (NULL-terminated) + callback_count: int # derived: len(callbacks) truncations: List[str] errors: List[str] @@ -315,9 +315,9 @@ class InternalMetadata(TypedDict, total=False): data_directories_raw: List[DataDirectoryRaw] export_struct: Optional[ExportStruct] delay_import_struct: Optional[DelayImportStruct] - relocation_struct: Optional[RelocationStruct] # v0.7.6 - certificate_struct: Optional[CertificateStruct] # v0.7.6 - debug_struct: Optional[DebugStruct] # v0.7.6 - tls_struct: Optional[TlsStruct] # v0.7.6 + relocation_struct: Optional[RelocationStruct] + certificate_struct: Optional[CertificateStruct] + debug_struct: Optional[DebugStruct] + tls_struct: Optional[TlsStruct] optional_header_magic: int number_of_rva_and_sizes: int From 7267f33b8800717043b6732df88693583f8f4aaa Mon Sep 17 00:00:00 2001 From: malx-labs Date: Mon, 10 Aug 2026 10:55:21 +0100 Subject: [PATCH 11/11] Formatting on signature and tls validators --- iocx/validators/signature.py | 46 ++++++++++++++++++------------------ iocx/validators/tls.py | 34 +++++++++++++------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/iocx/validators/signature.py b/iocx/validators/signature.py index 95b370b..7739613 100644 --- a/iocx/validators/signature.py +++ b/iocx/validators/signature.py @@ -9,7 +9,7 @@ parser pe_certificates (read from InternalMetadata), rather than from pefile's ``metadata["signatures"]`` list. The flag/metadata symmetry check still consults the public ``has_signature`` flag, and the overlay / section -overlap checks still use the ``analysis`` geometry — so ALL existing checks +overlap checks still use the ``analysis`` geometry, so ALL existing checks and reason codes are preserved. Two v0.7.6 reason codes are added, in territory the existing checks did not @@ -61,13 +61,13 @@ def validate_signature(internal: InternalMetadata, cert_struct = internal.get("certificate_struct") # --------------------------------------------------------- - # 0) Structural decode failure takes precedence (NEW: CERTIFICATE_TABLE_MALFORMED) - # The parser returns a struct (not None) when a security directory is - # declared. If it could not decode that directory at all, report it as - # malformed FIRST — otherwise the empty `certificates` list would trip - # the symmetry check below and mis-report a broken directory as - # "flag set but no metadata". Distinct from the field-value checks - # (SIGNATURE_INVALID_*), which own length/revision/type. + # Structural decode failure takes precedence: + # The parser returns a struct (not None) when a security directory is + # declared. If it could not decode that directory at all, report it as + # malformed FIRST, otherwise the empty `certificates` list would trip + # the symmetry check below and mis-report a broken directory as + # "flag set but no metadata". Distinct from the field-value checks + # (SIGNATURE_INVALID_*), which own length/revision/type. # --------------------------------------------------------- if cert_struct is not None and cert_struct.get("errors"): issues.append(StructuralIssue( @@ -88,7 +88,7 @@ def validate_signature(internal: InternalMetadata, has_sig = bool(metadata.get("has_signature")) # --------------------------------------------------------- - # 1) Flag/metadata symmetry (PRESERVED) + # 1) Flag/metadata symmetry # --------------------------------------------------------- if has_sig and not present: issues.append(StructuralIssue( @@ -108,7 +108,7 @@ def validate_signature(internal: InternalMetadata, return issues # --------------------------------------------------------- - # 1a) Table truncation (NEW: CERTIFICATE_TABLE_MALFORMED) + # 1a) Table truncation # --------------------------------------------------------- for tag in cert_struct.get("truncations", []) or []: issues.append(StructuralIssue( @@ -118,11 +118,11 @@ def validate_signature(internal: InternalMetadata, # --------------------------------------------------------- # 1b) Table offset must lie OUTSIDE the mapped image - # (NEW: CERTIFICATE_OFFSET_INSIDE_IMAGE). Table-level invariant from - # the parser's overlaps_image fact. This is a different owner from the - # per-certificate section-overlap check in step 4 (which is byte-range, - # per-section); both may co-fire on a pathological sample. Kept - # separate to preserve the existing check while meeting the spec. + # CERTIFICATE_OFFSET_INSIDE_IMAGE: Table-level invariant from + # the parser's overlaps_image fact. This is a different owner from the + # per-certificate section-overlap check in step 4 (which is byte-range, + # per-section); both may co-fire on a pathological sample. Kept + # separate to preserve the existing check while meeting the spec. # --------------------------------------------------------- if cert_struct.get("overlaps_image") is True: issues.append(StructuralIssue( @@ -133,7 +133,7 @@ def validate_signature(internal: InternalMetadata, )) # --------------------------------------------------------- - # 2) Multiplicity (PRESERVED) + # 2) Multiplicity # --------------------------------------------------------- if len(certs) > 1: issues.append(StructuralIssue( @@ -142,7 +142,7 @@ def validate_signature(internal: InternalMetadata, )) # --------------------------------------------------------- - # 3) Per-certificate field sanity (PRESERVED) + # 3) Per-certificate field sanity # --------------------------------------------------------- # file_size: prefer the analysis value to preserve the original bounds # behaviour; fall back to the parser's file_size if analysis omits it. @@ -163,7 +163,7 @@ def validate_signature(internal: InternalMetadata, if not isinstance(offset, int) or not isinstance(size, int): continue - # Length sanity (PRESERVED). Owns the length<8 fact; we deliberately + # Length sanity: Owns the length<8 fact; we deliberately # do NOT also emit CERTIFICATE_TABLE_MALFORMED for the parser's # "length_too_small" tag, to avoid double-counting. if size < 8: @@ -173,14 +173,14 @@ def validate_signature(internal: InternalMetadata, )) continue - # Revision sanity (PRESERVED) + # Revision sanity if revision not in (0x0100, 0x0200): issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_INVALID_REVISION, details={"revision": revision}, )) - # Type sanity (PRESERVED) + # Type sanity if cert_type not in (0x0001, 0x0002): issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_INVALID_TYPE, @@ -188,7 +188,7 @@ def validate_signature(internal: InternalMetadata, )) # ----------------------------------------------------- - # 4) Bounds + overlap checks (PRESERVED) + # 4) Bounds + overlap checks # ----------------------------------------------------- if isinstance(file_size, int): if offset < 0 or offset + size > file_size: @@ -199,7 +199,7 @@ def validate_signature(internal: InternalMetadata, )) continue - # Overlay check (PRESERVED) + # Overlay check if isinstance(overlay_offset, int) and offset < overlay_offset < offset + size: issues.append(StructuralIssue( issue=ReasonCodes.SIGNATURE_OVERLAPS_OTHER_DATA, @@ -207,7 +207,7 @@ def validate_signature(internal: InternalMetadata, "overlay_offset": overlay_offset}, )) - # Section overlap check (PRESERVED) + # Section overlap check for sec in sections: raw = sec.get("raw_address") raw_size = sec.get("raw_size") diff --git a/iocx/validators/tls.py b/iocx/validators/tls.py index a5084aa..67f4828 100644 --- a/iocx/validators/tls.py +++ b/iocx/validators/tls.py @@ -12,7 +12,7 @@ Two axes are validated: - * The PRESERVED cascade operates on the raw-data range + * The preserved cascade operates on the raw-data range (Start/EndAddressOfRawData) and the AddressOfCallBacks POINTER, exactly as before, including every early return. Note: struct addresses are VAs, so section mapping for the pointer is done in RVA space @@ -20,7 +20,7 @@ latent VA/RVA unit mismatch from the pefile path while keeping the same codes and control flow. - * The NEW target-array checks operate on the RESOLVED callback array + * The new target-array checks operate on the resolved callback array (the list of callback target VAs the parser walked), which the pefile single-value model could not express. This is where the two new v0.7.6 codes live, so they do not double-count the pointer-based checks: @@ -62,7 +62,7 @@ "tls_directory_unpack_failed", } -# Parser tombstones recorded when the callback ARRAY could not be resolved +# Parser tombstones recorded when the callback array could not be resolved # to an RVA at all. In these cases the parser returns callbacks=[], so the # per-target loop below has nothing to walk; without surfacing these tags # the structural anomaly would be silently dropped. They map to @@ -94,7 +94,7 @@ def validate_tls(internal: InternalMetadata, issues: List[StructuralIssue] = [] # --------------------------------------------------------- - # 1) Multiple TLS directories (PRESERVED — from extended markers) + # 1) Multiple TLS directories # --------------------------------------------------------- tls_entries = [ e for e in analysis.get("extended", []) @@ -111,8 +111,8 @@ def validate_tls(internal: InternalMetadata, return issues # no TLS directory — not a defect # --------------------------------------------------------- - # 2) Header decode failure (NEW: TLS_DIRECTORY_TRUNCATED) - # Unrecoverable — the fixed struct could not be read/unpacked. + # 2) Header decode failure + # Unrecoverable - the fixed struct could not be read/unpacked. # --------------------------------------------------------- header_errs = [e for e in (tls.get("errors") or []) if e in _HEADER_DECODE_ERROR_TAGS] @@ -124,7 +124,7 @@ def validate_tls(internal: InternalMetadata, return issues # --------------------------------------------------------- - # 3) Callback-array truncation / loop (NEW: TLS_DIRECTORY_TRUNCATED) + # 3) Callback-array truncation / loop # --------------------------------------------------------- for tag in tls.get("truncations", []) or []: issues.append(StructuralIssue( @@ -133,14 +133,14 @@ def validate_tls(internal: InternalMetadata, )) # --------------------------------------------------------- - # 4) Resolved callback TARGET validation (NEW: TLS_CALLBACK_RVA_INVALID) - # Independent of the raw-data-range cascade below, so it always runs - # even when the cascade returns early (e.g. zero-length raw data). + # 4) Resolved callback target validation + # Independent of the raw-data-range cascade below, so it always runs + # even when the cascade returns early (e.g. zero-length raw data). # --------------------------------------------------------- _validate_callback_targets(tls, analysis, issues) # --------------------------------------------------------- - # 5) PRESERVED cascade on raw-data range + AddressOfCallBacks pointer + # 5) Preserved cascade on raw-data range + AddressOfCallBacks pointer # --------------------------------------------------------- start = tls.get("start_address_of_raw_data") # VA end = tls.get("end_address_of_raw_data") # VA @@ -154,7 +154,7 @@ def validate_tls(internal: InternalMetadata, overlay_offset = analysis.get("overlay_offset") size_of_headers = (metadata.get("optional_header") or {}).get("size_of_headers") - # Range sanity (VA space — PRESERVED) + # Range sanity (VA space) if start == end: # A zero-length raw-data region is only anomalous when the directory # carries NO resolved callbacks. A zero-length template alongside a @@ -176,7 +176,7 @@ def validate_tls(internal: InternalMetadata, )) return issues - # Missing callbacks (PRESERVED) + # Missing callbacks if ptr == 0: issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACKS_MISSING, @@ -184,7 +184,7 @@ def validate_tls(internal: InternalMetadata, )) return issues - # Callback pointer outside TLS range (VA space — PRESERVED) + # Callback pointer outside TLS range (VA space) if not (start <= ptr < end): issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACK_OUTSIDE_RANGE, @@ -219,7 +219,7 @@ def validate_tls(internal: InternalMetadata, details={"callbacks": ptr, "section": name}, )) - # Overlay / header checks (RVA space — PRESERVED) + # Overlay / header checks (RVA space) if isinstance(size_of_headers, int) and ptr_rva < size_of_headers: issues.append(StructuralIssue( issue=ReasonCodes.TLS_CALLBACK_IN_HEADERS, @@ -242,14 +242,14 @@ def validate_tls(internal: InternalMetadata, # ================================================================= -# NEW: resolved callback-target validation +# Resolved callback-target validation # ================================================================= def _validate_callback_targets(tls: Dict[str, Any], analysis: AnalysisDict, issues: List[StructuralIssue]) -> None: """ - Flag resolved callback TARGET VAs that cannot form a valid RVA (below + Flag resolved callback target VAs that cannot form a valid RVA (below ImageBase) or do not map to any section. Distinct subject from the pointer-based TLS_CALLBACK_NOT_MAPPED_TO_SECTION check above (which maps the AddressOfCallBacks pointer, not the individual targets).