diff --git a/.github/ISSUE_TEMPLATE/unsupported_gml_api.yml b/.github/ISSUE_TEMPLATE/unsupported_gml_api.yml index 0747d48..5f194e2 100644 --- a/.github/ISSUE_TEMPLATE/unsupported_gml_api.yml +++ b/.github/ISSUE_TEMPLATE/unsupported_gml_api.yml @@ -39,6 +39,6 @@ body: attributes: label: Versions description: GM2Godot commit or release, GameMaker version, Godot version, and target platform. - placeholder: "GM2Godot 0.7.64, GameMaker LTS 2026, Godot 4.7.1, Windows" + placeholder: "GM2Godot 0.7.65, GameMaker LTS 2026, Godot 4.7.1, Windows" validations: required: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 41352dc..f19782a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.7.65 - 2026-09-03 + +- Reused retained, verified Windows cleanup-parent bindings across present Included Files trees instead of recapturing and reverifying the same ancestor chain for every child. +- Made nested Windows cleanup perform an iterative full-tree preflight followed by bottom-up removal with live native handles bounded by directory depth, while preserving exact identity, no-follow, reparse/junction, mount, hard-link, digest, read-only, relocation/replacement, tombstone, and rollback defenses. +- Added deterministic flat, nested, and deep-tree operation-count and adversarial coverage for the optimized fallback while preserving GameMaker LTS 2026 conversion behavior and exact Godot 4.7.1 output. + ## 0.7.64 - 2026-09-03 - Documented `particles/` and `particlesystems/` in the complete generated-project tree and converter-managed output-root list. diff --git a/README.md b/README.md index a147d2b..0cd6713 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,8 @@ Version 0.7.63 makes the privileged dependency-submission validator compare nati Version 0.7.64 makes the canonical generated-project documentation enumerate every converter-managed output root in exact production order, including `particles/` and `particlesystems/`. An executable documentation-health check binds the marked list to `MANAGED_OUTPUT_DIRECTORIES`, so future ownership changes cannot silently drift from Wiki guidance. This documentation-only release leaves GameMaker LTS 2026 conversion behavior and exact Godot 4.7.1 output unchanged. +Version 0.7.65 reuses retained, verified Windows cleanup-parent bindings across present Included Files trees, removing repeated ancestor capture and re-verification for every child. Nested cleanup now uses iterative full-tree preflight and bottom-up removal with live native handles bounded by directory depth while retaining exact identity, no-follow, reparse/junction, mount, hard-link, digest, read-only, relocation/replacement, tombstone, and rollback defenses. Deterministic flat, nested, and deep-tree operation-count and adversarial tests cover the optimized fallback. GameMaker LTS 2026 conversion behavior and exact Godot 4.7.1 output remain unchanged. + ## What GM2Godot Is and Isn't **GM2Godot is:** @@ -106,7 +108,7 @@ The full compatibility roadmap lives in [`todo-list/`](todo-list/README.md). It ## Releases -Current source version: `0.7.64`. +Current source version: `0.7.65`. Downloadable releases include Windows (`.exe`), macOS (`.dmg` with `.app`), and Linux binaries. You can also run from source on Windows, macOS, and Linux. The packaged Linux artifact is validated on Ubuntu 24.04 x86_64. Its glibc 2.39 requirement is necessary but does not make other distributions a validated target; they must also supply compatible system, OpenGL/EGL, and X11 libraries. The reviewed Linux package manifest installs Ubuntu's `libegl1` and `libgl1` providers for QtGui together with the required XCB client libraries. The release job rejects unresolved-library warnings, extracts the final ZIP, and proves that its GUI reaches the event loop through the real `qxcb` platform under Xvfb before upload. diff --git a/docs/wiki/Compatibility-and-Limitations.md b/docs/wiki/Compatibility-and-Limitations.md index 3a4126e..3cac5fe 100644 --- a/docs/wiki/Compatibility-and-Limitations.md +++ b/docs/wiki/Compatibility-and-Limitations.md @@ -1,6 +1,6 @@ # Compatibility and Limitations -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 diff --git a/docs/wiki/Contributing-and-Testing.md b/docs/wiki/Contributing-and-Testing.md index 72a5000..8a5bc4e 100644 --- a/docs/wiki/Contributing-and-Testing.md +++ b/docs/wiki/Contributing-and-Testing.md @@ -1,6 +1,6 @@ # Contributing and Testing -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 diff --git a/docs/wiki/Diagnostics-and-Troubleshooting.md b/docs/wiki/Diagnostics-and-Troubleshooting.md index 88ad106..3a849ce 100644 --- a/docs/wiki/Diagnostics-and-Troubleshooting.md +++ b/docs/wiki/Diagnostics-and-Troubleshooting.md @@ -1,6 +1,6 @@ # Diagnostics and Troubleshooting -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 diff --git a/docs/wiki/Generated-Project-and-Runtime.md b/docs/wiki/Generated-Project-and-Runtime.md index e1d3afb..b40f336 100644 --- a/docs/wiki/Generated-Project-and-Runtime.md +++ b/docs/wiki/Generated-Project-and-Runtime.md @@ -1,6 +1,6 @@ # Generated Project and Runtime -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 diff --git a/docs/wiki/Home.md b/docs/wiki/Home.md index f744331..1e3613e 100644 --- a/docs/wiki/Home.md +++ b/docs/wiki/Home.md @@ -1,6 +1,6 @@ # GM2Godot Documentation -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 @@ -21,7 +21,7 @@ Maintainers should also read [Release and Wiki Maintenance](Maintainer-Release-a This documentation set describes: -- GM2Godot 0.7.64; +- GM2Godot 0.7.65; - GameMaker LTS 2026 source projects in the GMS2 runtime family; and - Godot 4.7.1 output and validation, pinned in CI as `4.7.1.stable.official.a13da4feb`. diff --git a/docs/wiki/Installation.md b/docs/wiki/Installation.md index acfb453..7ed8572 100644 --- a/docs/wiki/Installation.md +++ b/docs/wiki/Installation.md @@ -1,6 +1,6 @@ # Installation -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 @@ -46,7 +46,7 @@ On Windows, run `Get-FileHash -Algorithm SHA256 .\GM2Godot-windows.zip` in Power The packaged builds are produced as windowed applications. For the CLI commands in this Wiki, use a source installation. -After launch, confirm that the title bar or **Help → About GM2Godot** shows version `0.7.64`. Click the version in the bottom information bar to browse the ten newest release changelogs; **Show more** appends the next ten. +After launch, confirm that the title bar or **Help → About GM2Godot** shows version `0.7.65`. Click the version in the bottom information bar to browse the ten newest release changelogs; **Show more** appends the next ten. ## Run from source @@ -148,6 +148,6 @@ python main.py --version python main.py list-converters ``` -The first command should print `GM2Godot 0.7.64`; the second should list the conversion groups and the exact converter keys accepted by `--only`. The same CLI is also available through `python -m src.cli`. +The first command should print `GM2Godot 0.7.65`; the second should list the conversion groups and the exact converter keys accepted by `--only`. The same CLI is also available through `python -m src.cli`. Continue with [Quick Start Conversion](Quick-Start-Conversion). If launch or dependency setup fails, see [Diagnostics and Troubleshooting](Diagnostics-and-Troubleshooting). diff --git a/docs/wiki/Maintainer-Release-and-Wiki.md b/docs/wiki/Maintainer-Release-and-Wiki.md index 3327891..dcc4e9f 100644 --- a/docs/wiki/Maintainer-Release-and-Wiki.md +++ b/docs/wiki/Maintainer-Release-and-Wiki.md @@ -1,6 +1,6 @@ # Release and Wiki Maintenance -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 diff --git a/docs/wiki/Quick-Start-Conversion.md b/docs/wiki/Quick-Start-Conversion.md index d5aa5a7..1fa1761 100644 --- a/docs/wiki/Quick-Start-Conversion.md +++ b/docs/wiki/Quick-Start-Conversion.md @@ -1,6 +1,6 @@ # Quick Start Conversion -> **Applies to:** GM2Godot 0.7.64 · GameMaker LTS 2026 · Godot 4.7.1 +> **Applies to:** GM2Godot 0.7.65 · GameMaker LTS 2026 · Godot 4.7.1 > > **Last reviewed:** 2026-09-03 diff --git a/src/conversion/included_files.py b/src/conversion/included_files.py index 9b359f8..65d5075 100644 --- a/src/conversion/included_files.py +++ b/src/conversion/included_files.py @@ -11,6 +11,7 @@ import stat import sys import tempfile +from contextlib import ExitStack from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait from dataclasses import dataclass, field, replace from functools import lru_cache @@ -380,14 +381,45 @@ class _IncludedOutputSetCancelled(Exception): ) _WINDOWS_GENERIC_READ = 0x80000000 +_WINDOWS_FILE_TRAVERSE = 0x00000020 +_WINDOWS_FILE_READ_ATTRIBUTES = 0x00000080 _WINDOWS_FILE_SHARE_READ = 0x00000001 +_WINDOWS_FILE_SHARE_WRITE = 0x00000002 +_WINDOWS_FILE_SHARE_DELETE = 0x00000004 _WINDOWS_OPEN_EXISTING = 3 +_WINDOWS_FILE_ATTRIBUTE_DIRECTORY = 0x00000010 _WINDOWS_FILE_ATTRIBUTE_NORMAL = 0x00000080 +_WINDOWS_FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400 +_WINDOWS_FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 _WINDOWS_FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 _WINDOWS_FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 +_WINDOWS_FILE_TYPE_DISK = 1 +_WINDOWS_FILE_BASIC_INFO_CLASS = 0 +_WINDOWS_FILE_ID_INFO_CLASS = 18 _WINDOWS_MOVEFILE_WRITE_THROUGH = 0x00000008 +class _WindowsIncludedFileId128(ctypes.Structure): + _fields_ = (("Identifier", ctypes.c_uint8 * 16),) + + +class _WindowsIncludedFileIdInfo(ctypes.Structure): + _fields_ = ( + ("VolumeSerialNumber", ctypes.c_uint64), + ("FileId", _WindowsIncludedFileId128), + ) + + +class _WindowsIncludedFileBasicInfo(ctypes.Structure): + _fields_ = ( + ("CreationTime", ctypes.c_int64), + ("LastAccessTime", ctypes.c_int64), + ("LastWriteTime", ctypes.c_int64), + ("ChangeTime", ctypes.c_int64), + ("FileAttributes", ctypes.c_uint32), + ) + + def _included_descriptor_paths_supported() -> bool: return ( os.name != "nt" @@ -852,6 +884,48 @@ def _windows_included_file_read_api() -> Any: return kernel32 +@lru_cache(maxsize=1) +def _windows_included_cleanup_parent_api() -> Any: + """Return the Win32 calls used to pin one cleanup directory parent.""" + + if os.name != "nt": + raise OSError( + "Windows Included Files cleanup parent handles are unavailable" + ) + if ( + ctypes.sizeof(_WindowsIncludedFileId128) != 16 + or ctypes.sizeof(_WindowsIncludedFileIdInfo) != 24 + or _WindowsIncludedFileIdInfo.FileId.offset != 8 + or ctypes.sizeof(_WindowsIncludedFileBasicInfo) != 40 + or _WindowsIncludedFileBasicInfo.FileAttributes.offset != 32 + ): + raise OSError("Unsupported Windows Included Files cleanup ABI layout") + win_dll = cast(Callable[..., Any], getattr(ctypes, "WinDLL")) + kernel32 = win_dll("kernel32", use_last_error=True) + kernel32.CreateFileW.argtypes = ( + ctypes.c_wchar_p, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_void_p, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_void_p, + ) + kernel32.CreateFileW.restype = ctypes.c_void_p + kernel32.GetFileInformationByHandleEx.argtypes = ( + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_uint32, + ) + kernel32.GetFileInformationByHandleEx.restype = ctypes.c_int + kernel32.GetFileType.argtypes = (ctypes.c_void_p,) + kernel32.GetFileType.restype = ctypes.c_uint32 + kernel32.CloseHandle.argtypes = (ctypes.c_void_p,) + kernel32.CloseHandle.restype = ctypes.c_int + return kernel32 + + @lru_cache(maxsize=1) def _windows_included_transaction_api() -> Any: if os.name != "nt": @@ -892,6 +966,213 @@ def _windows_extended_included_path(path: str) -> str: return "\\\\?\\" + absolute_path +def _windows_included_cleanup_parent_identity( + kernel32: Any, + handle: int, + path: str, +) -> _PathIdentity: + identity_info = _WindowsIncludedFileIdInfo() + if not kernel32.GetFileInformationByHandleEx( + handle, + _WINDOWS_FILE_ID_INFO_CLASS, + ctypes.byref(identity_info), + ctypes.sizeof(identity_info), + ): + raise _windows_included_transaction_error( + "Could not identify Included Files cleanup parent handle", + path, + ) + return ( + int(identity_info.VolumeSerialNumber), + int.from_bytes(bytes(identity_info.FileId.Identifier), "little"), + ) + + +def _windows_included_cleanup_parent_attributes( + kernel32: Any, + handle: int, + path: str, +) -> int: + basic_info = _WindowsIncludedFileBasicInfo() + if not kernel32.GetFileInformationByHandleEx( + handle, + _WINDOWS_FILE_BASIC_INFO_CLASS, + ctypes.byref(basic_info), + ctypes.sizeof(basic_info), + ): + raise _windows_included_transaction_error( + "Could not inspect Included Files cleanup parent handle", + path, + ) + return int(basic_info.FileAttributes) + + +@dataclass +class _WindowsIncludedCleanupParentBinding: + """Keep one verified cleanup parent immovable for path-based operations. + + Windows has no Python ``dir_fd`` equivalent for the cleanup operations in + this module. The retained directory handle deliberately omits + ``FILE_SHARE_DELETE`` so its directory cannot be renamed or deleted while + the binding is live. Callers still revalidate the native file ID and path + before every group of path-based child operations. + """ + + path: str + identity: _PathIdentity + kernel32: Any + handle: int | None + + @classmethod + def open( + cls, + path: str, + expected_identity: _PathIdentity, + ) -> "_WindowsIncludedCleanupParentBinding": + if os.name != "nt": + raise OSError( + "Windows Included Files cleanup parent bindings are unavailable" + ) + absolute_path = os.path.abspath(path) + try: + path_stat = os.lstat(absolute_path) + except OSError as error: + raise OSError( + f"Included Files cleanup parent changed: {absolute_path}" + ) from error + if ( + _included_output_path_is_redirected(absolute_path, path_stat) + or not stat.S_ISDIR(path_stat.st_mode) + or (path_stat.st_dev, path_stat.st_ino) != expected_identity + ): + raise OSError( + f"Included Files cleanup parent changed: {absolute_path}" + ) + + kernel32 = _windows_included_cleanup_parent_api() + handle = kernel32.CreateFileW( + _windows_extended_included_path(absolute_path), + _WINDOWS_FILE_TRAVERSE | _WINDOWS_FILE_READ_ATTRIBUTES, + _WINDOWS_FILE_SHARE_READ | _WINDOWS_FILE_SHARE_WRITE, + None, + _WINDOWS_OPEN_EXISTING, + _WINDOWS_FILE_FLAG_BACKUP_SEMANTICS + | _WINDOWS_FILE_FLAG_OPEN_REPARSE_POINT, + None, + ) + invalid_handle = ctypes.c_void_p(-1).value + if handle is None or handle == invalid_handle: + raise _windows_included_transaction_error( + "Could not bind Included Files cleanup parent", + absolute_path, + ) + binding = cls( + path=absolute_path, + identity=expected_identity, + kernel32=kernel32, + handle=cast(int, handle), + ) + try: + binding.verify() + except BaseException as error: + try: + binding.close() + except BaseException as close_error: + error.add_note( + "Could not close rejected Included Files cleanup parent " + f"binding: {close_error}" + ) + raise + return binding + + def __enter__(self) -> "_WindowsIncludedCleanupParentBinding": + self.verify() + return self + + def __exit__( + self, + _exception_type: object, + active_error: BaseException | None, + _traceback: object, + ) -> bool | None: + try: + self.close() + except BaseException as close_error: + if active_error is None: + raise + active_error.add_note( + "Could not close Included Files cleanup parent binding: " + + str(close_error) + ) + return None + + def verify(self) -> None: + handle = self.handle + if handle is None: + raise OSError( + f"Included Files cleanup parent binding is closed: {self.path}" + ) + try: + path_stat = os.lstat(self.path) + except OSError as error: + raise OSError( + f"Included Files cleanup parent changed: {self.path}" + ) from error + attributes = _windows_included_cleanup_parent_attributes( + self.kernel32, + handle, + self.path, + ) + if ( + self.kernel32.GetFileType(handle) != _WINDOWS_FILE_TYPE_DISK + or _included_output_path_is_redirected(self.path, path_stat) + or not stat.S_ISDIR(path_stat.st_mode) + or (path_stat.st_dev, path_stat.st_ino) != self.identity + or _windows_included_cleanup_parent_identity( + self.kernel32, + handle, + self.path, + ) + != self.identity + or not attributes & _WINDOWS_FILE_ATTRIBUTE_DIRECTORY + or attributes & _WINDOWS_FILE_ATTRIBUTE_REPARSE_POINT + ): + raise OSError( + f"Included Files cleanup parent changed: {self.path}" + ) + + def close(self) -> None: + handle = self.handle + if handle is None: + return + self.handle = None + if not self.kernel32.CloseHandle(handle): + raise _windows_included_transaction_error( + "Could not close Included Files cleanup parent handle", + self.path, + ) + + +def _verify_windows_included_cleanup_parent_binding( + binding: _WindowsIncludedCleanupParentBinding, + parent_path: str, + expected_identity: _PathIdentity, +) -> None: + """Verify that a retained Windows handle still binds the requested parent.""" + + absolute_parent_path = os.path.abspath(parent_path) + if ( + os.name != "nt" + or os.path.normcase(binding.path) + != os.path.normcase(absolute_parent_path) + or binding.identity != expected_identity + ): + raise OSError( + f"Included Files cleanup parent binding mismatch: {absolute_parent_path}" + ) + binding.verify() + + def _open_included_file_validation_stream( path: str, *, @@ -3080,7 +3361,21 @@ def _quarantine_included_entry_fallback( def _unlink_exact_quarantined_entry_fallback( path: str, expected_identity: _PathIdentity, + *, + expected_parent_identity: _PathIdentity | None = None, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> None: + parent_path = os.path.dirname(os.path.abspath(path)) + if windows_parent_binding is not None: + if expected_parent_identity is None: + raise OSError( + "Included Files cleanup parent binding requires an exact identity" + ) + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) _before_included_cleanup_remove_fallback(path) current_stat = os.lstat(path) if ( @@ -3103,15 +3398,20 @@ def _unlink_exact_quarantined_entry_fallback( "Included Files cleanup file with multiple hard links; " f"recoverable quarantine retained at {path!r}" ) - parent_path = os.path.dirname(os.path.abspath(path)) - parent_identities = _capture_fallback_directory_ancestors(parent_path) - parent_identity = parent_identities[-1][1] + if windows_parent_binding is None: + parent_identity = _capture_fallback_directory_ancestors( + parent_path + )[-1][1] + else: + assert expected_parent_identity is not None + parent_identity = expected_parent_identity try: _chmod_exact_included_file( path, expected_identity, original_mode | stat.S_IWRITE, parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError as error: raise OSError( @@ -3132,19 +3432,31 @@ def _unlink_exact_quarantined_entry_fallback( f"Windows read-only attribute: {path!r}" ) _after_included_transaction_phase("cleanup-readonly-cleared") + if windows_parent_binding is not None: + assert expected_parent_identity is not None + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) try: os.unlink(path) except OSError as error: if windows_read_only: try: - parent_identity = _capture_fallback_directory_ancestors( - os.path.dirname(os.path.abspath(path)) - )[-1][1] + if windows_parent_binding is None: + parent_identity = _capture_fallback_directory_ancestors( + parent_path + )[-1][1] + else: + assert expected_parent_identity is not None + parent_identity = expected_parent_identity _chmod_exact_included_file( path, expected_identity, original_mode, parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError as restore_error: error.add_note( @@ -3163,13 +3475,38 @@ def _chmod_exact_included_directory_fallback( expected_identity: _PathIdentity, mode: int, expected_parent_identity: _PathIdentity, + *, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> None: parent_path = os.path.dirname(os.path.abspath(path)) - parent_identities = _capture_fallback_directory_ancestors(parent_path) - if parent_identities[-1][1] != expected_parent_identity: - raise OSError( - f"Included Files directory parent changed before chmod: {parent_path}" + parent_identities: tuple[tuple[str, _PathIdentity], ...] | None + if windows_parent_binding is None: + parent_identities = _capture_fallback_directory_ancestors(parent_path) + if parent_identities[-1][1] != expected_parent_identity: + raise OSError( + "Included Files directory parent changed before chmod: " + + parent_path + ) + else: + parent_identities = None + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, ) + + def verify_parent() -> None: + if windows_parent_binding is None: + if parent_identities is None: + raise AssertionError("Missing Included Files directory parent state") + _verify_fallback_directory_ancestors(parent_identities) + else: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + current_stat = os.lstat(path) if ( _included_output_path_is_redirected(path, current_stat) @@ -3187,7 +3524,7 @@ def _chmod_exact_included_directory_fallback( "Path-based Included Files directory chmod is only supported on " "Windows" ) - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() quarantined_path = _quarantine_included_entry_fallback( path, expected_identity, @@ -3233,6 +3570,8 @@ def _chmod_exact_included_directory_fallback( expected_identity, source_parent_identity=expected_parent_identity, destination_parent_identity=expected_parent_identity, + windows_source_parent_binding=windows_parent_binding, + windows_destination_parent_binding=windows_parent_binding, ) except BaseException as restore_error: error.add_note( @@ -3247,6 +3586,8 @@ def _chmod_exact_included_directory_fallback( expected_identity, source_parent_identity=expected_parent_identity, destination_parent_identity=expected_parent_identity, + windows_source_parent_binding=windows_parent_binding, + windows_destination_parent_binding=windows_parent_binding, ) final_stat = os.lstat(path) if ( @@ -3257,13 +3598,27 @@ def _chmod_exact_included_directory_fallback( != bool(mode & stat.S_IWRITE) ): raise OSError(f"Included Files directory changed after chmod: {path}") - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() def _rmdir_exact_quarantined_entry_fallback( path: str, expected_identity: _PathIdentity, + *, + expected_parent_identity: _PathIdentity | None = None, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> None: + parent_path = os.path.dirname(os.path.abspath(path)) + if windows_parent_binding is not None: + if expected_parent_identity is None: + raise OSError( + "Included Files cleanup parent binding requires an exact identity" + ) + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) _before_included_cleanup_remove_fallback(path) current_stat = os.lstat(path) if ( @@ -3280,15 +3635,20 @@ def _rmdir_exact_quarantined_entry_fallback( and not bool(current_stat.st_mode & stat.S_IWRITE) ) if windows_read_only: - parent_identity = _capture_fallback_directory_ancestors( - os.path.dirname(os.path.abspath(path)) - )[-1][1] + if windows_parent_binding is None: + parent_identity = _capture_fallback_directory_ancestors( + parent_path + )[-1][1] + else: + assert expected_parent_identity is not None + parent_identity = expected_parent_identity try: _chmod_exact_included_directory_fallback( path, expected_identity, original_mode | stat.S_IWRITE, parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError as error: raise OSError( @@ -3308,19 +3668,31 @@ def _rmdir_exact_quarantined_entry_fallback( "Included Files cleanup directory changed while clearing its " f"Windows read-only attribute: {path!r}" ) + if windows_parent_binding is not None: + assert expected_parent_identity is not None + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) try: os.rmdir(path) except OSError as error: if windows_read_only: try: - parent_identity = _capture_fallback_directory_ancestors( - os.path.dirname(os.path.abspath(path)) - )[-1][1] + if windows_parent_binding is None: + parent_identity = _capture_fallback_directory_ancestors( + parent_path + )[-1][1] + else: + assert expected_parent_identity is not None + parent_identity = expected_parent_identity _chmod_exact_included_directory_fallback( path, expected_identity, original_mode, parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError as restore_error: error.add_note( @@ -3564,6 +3936,8 @@ def _chmod_exact_included_file( expected_identity: _PathIdentity, mode: int, expected_parent_identity: _PathIdentity, + *, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> None: if _included_descriptor_paths_supported(): parent_fd, name = _open_pinned_included_parent(path) @@ -3592,11 +3966,32 @@ def _chmod_exact_included_file( finally: os.close(parent_fd) return - parent_identities = _capture_fallback_directory_ancestors( - os.path.dirname(os.path.abspath(path)) - ) - if parent_identities[-1][1] != expected_parent_identity: - raise OSError(f"Included Files file parent changed: {path}") + parent_path = os.path.dirname(os.path.abspath(path)) + parent_identities: tuple[tuple[str, _PathIdentity], ...] | None + if windows_parent_binding is None: + parent_identities = _capture_fallback_directory_ancestors(parent_path) + if parent_identities[-1][1] != expected_parent_identity: + raise OSError(f"Included Files file parent changed: {path}") + else: + parent_identities = None + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + + def verify_parent() -> None: + if windows_parent_binding is None: + if parent_identities is None: + raise AssertionError("Missing Included Files file parent state") + _verify_fallback_directory_ancestors(parent_identities) + else: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + try: path_stat = os.lstat(path) except FileNotFoundError as error: @@ -3607,7 +4002,7 @@ def _chmod_exact_included_file( or (path_stat.st_dev, path_stat.st_ino) != expected_identity ): raise OSError(f"Included Files file changed: {path}") - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() _before_included_fallback_chmod_open(path) file_descriptor = os.open( path, @@ -3621,7 +4016,7 @@ def _chmod_exact_included_file( or (opened_stat.st_dev, opened_stat.st_ino) != expected_identity ): raise OSError(f"Included Files file changed: {path}") - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() if os.chmod in os.supports_fd: os.chmod(file_descriptor, mode) elif os.name == "nt": @@ -3636,6 +4031,7 @@ def _chmod_exact_included_file( finally: os.close(file_descriptor) if path_chmod_required: + verify_parent() quarantined_path = _quarantine_included_entry_fallback( path, expected_identity, @@ -3671,6 +4067,8 @@ def _chmod_exact_included_file( expected_identity, source_parent_identity=expected_parent_identity, destination_parent_identity=expected_parent_identity, + windows_source_parent_binding=windows_parent_binding, + windows_destination_parent_binding=windows_parent_binding, ) except OSError as restore_error: error.add_note( @@ -3684,6 +4082,8 @@ def _chmod_exact_included_file( expected_identity, source_parent_identity=expected_parent_identity, destination_parent_identity=expected_parent_identity, + windows_source_parent_binding=windows_parent_binding, + windows_destination_parent_binding=windows_parent_binding, ) try: current_stat = os.lstat(path) @@ -3695,7 +4095,7 @@ def _chmod_exact_included_file( or (current_stat.st_dev, current_stat.st_ino) != expected_identity ): raise OSError(f"Included Files file changed: {path}") - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() def _unique_included_transaction_path( @@ -3742,6 +4142,12 @@ def _move_exact_included_entry( expect_directory: bool, source_parent_identity: _PathIdentity | None, destination_parent_identity: _PathIdentity | None, + windows_source_parent_binding: ( + _WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + _WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: source_parent_path = os.path.dirname(os.path.abspath(source)) destination_parent_path = os.path.dirname(os.path.abspath(destination)) @@ -3825,25 +4231,85 @@ def _move_exact_included_entry( os.close(source_parent_fd) return - source_parent_ancestors = _capture_fallback_directory_ancestors( - source_parent_path - ) - destination_parent_ancestors = _capture_fallback_directory_ancestors( - destination_parent_path + source_parent_ancestors: tuple[tuple[str, _PathIdentity], ...] | None + if windows_source_parent_binding is None: + source_parent_ancestors = _capture_fallback_directory_ancestors( + source_parent_path + ) + if ( + source_parent_identity is not None + and source_parent_ancestors[-1][1] != source_parent_identity + ): + raise OSError( + f"Included Files source parent changed: {source_parent_path}" + ) + else: + source_parent_ancestors = None + if source_parent_identity is None: + raise OSError( + "Included Files source parent binding requires an exact identity" + ) + _verify_windows_included_cleanup_parent_binding( + windows_source_parent_binding, + source_parent_path, + source_parent_identity, + ) + + destination_parent_ancestors: ( + tuple[tuple[str, _PathIdentity], ...] | None ) - if ( - source_parent_identity is not None - and source_parent_ancestors[-1][1] != source_parent_identity - ): - raise OSError(f"Included Files source parent changed: {source_parent_path}") - if ( - destination_parent_identity is not None - and destination_parent_ancestors[-1][1] - != destination_parent_identity - ): - raise OSError( - f"Included Files destination parent changed: {destination_parent_path}" + if windows_destination_parent_binding is None: + destination_parent_ancestors = _capture_fallback_directory_ancestors( + destination_parent_path + ) + if ( + destination_parent_identity is not None + and destination_parent_ancestors[-1][1] + != destination_parent_identity + ): + raise OSError( + "Included Files destination parent changed: " + + destination_parent_path + ) + else: + destination_parent_ancestors = None + if destination_parent_identity is None: + raise OSError( + "Included Files destination parent binding requires an exact identity" + ) + _verify_windows_included_cleanup_parent_binding( + windows_destination_parent_binding, + destination_parent_path, + destination_parent_identity, ) + + def verify_parents() -> None: + if windows_source_parent_binding is None: + if source_parent_ancestors is None: + raise AssertionError("Missing Included Files source parent state") + _verify_fallback_directory_ancestors(source_parent_ancestors) + else: + if source_parent_identity is None: + raise AssertionError("Missing Included Files source parent identity") + _verify_windows_included_cleanup_parent_binding( + windows_source_parent_binding, + source_parent_path, + source_parent_identity, + ) + if windows_destination_parent_binding is None: + if destination_parent_ancestors is None: + raise AssertionError("Missing Included Files destination parent state") + _verify_fallback_directory_ancestors(destination_parent_ancestors) + elif windows_destination_parent_binding is not windows_source_parent_binding: + if destination_parent_identity is None: + raise AssertionError( + "Missing Included Files destination parent identity" + ) + _verify_windows_included_cleanup_parent_binding( + windows_destination_parent_binding, + destination_parent_path, + destination_parent_identity, + ) source_stat = os.lstat(source) source_is_expected_kind = ( stat.S_ISDIR(source_stat.st_mode) @@ -3860,12 +4326,11 @@ def _move_exact_included_entry( raise OSError( f"Included Files transaction destination already exists: {destination}" ) - _verify_fallback_directory_ancestors(source_parent_ancestors) - _verify_fallback_directory_ancestors(destination_parent_ancestors) + verify_parents() _before_included_transaction_rename_fallback(source, destination) + verify_parents() _rename_included_transaction_entry(source, destination) - _verify_fallback_directory_ancestors(source_parent_ancestors) - _verify_fallback_directory_ancestors(destination_parent_ancestors) + verify_parents() destination_stat = os.lstat(destination) destination_is_expected_kind = ( stat.S_ISDIR(destination_stat.st_mode) @@ -3891,6 +4356,12 @@ def _move_exact_included_directory( *, source_parent_identity: _PathIdentity | None = None, destination_parent_identity: _PathIdentity | None = None, + windows_source_parent_binding: ( + _WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + _WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: _move_exact_included_entry( source, @@ -3899,6 +4370,8 @@ def _move_exact_included_directory( expect_directory=True, source_parent_identity=source_parent_identity, destination_parent_identity=destination_parent_identity, + windows_source_parent_binding=windows_source_parent_binding, + windows_destination_parent_binding=windows_destination_parent_binding, ) @@ -3909,6 +4382,12 @@ def _move_exact_included_file( *, source_parent_identity: _PathIdentity | None = None, destination_parent_identity: _PathIdentity | None = None, + windows_source_parent_binding: ( + _WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + _WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: _move_exact_included_entry( source, @@ -3917,6 +4396,8 @@ def _move_exact_included_file( expect_directory=False, source_parent_identity=source_parent_identity, destination_parent_identity=destination_parent_identity, + windows_source_parent_binding=windows_source_parent_binding, + windows_destination_parent_binding=windows_destination_parent_binding, ) @@ -6804,6 +7285,8 @@ def _included_cleanup_file_state( path: str, expected_identity: _PathIdentity, expected_parent_identity: _PathIdentity, + *, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> _IncludedCleanupFileState | None: if _included_descriptor_paths_supported(): parent_fd, name = _open_pinned_included_parent(path) @@ -6852,13 +7335,34 @@ def _included_cleanup_file_state( os.close(parent_fd) parent_path = os.path.dirname(os.path.abspath(path)) - parent_identities = _capture_fallback_directory_ancestors(parent_path) - if parent_identities[-1][1] != expected_parent_identity: - raise OSError(f"Included Files cleanup parent changed: {path}") + parent_identities: tuple[tuple[str, _PathIdentity], ...] | None + if windows_parent_binding is None: + parent_identities = _capture_fallback_directory_ancestors(parent_path) + if parent_identities[-1][1] != expected_parent_identity: + raise OSError(f"Included Files cleanup parent changed: {path}") + else: + parent_identities = None + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + + def verify_parent() -> None: + if windows_parent_binding is None: + if parent_identities is None: + raise AssertionError("Missing Included Files cleanup parent state") + _verify_fallback_directory_ancestors(parent_identities) + else: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) try: current_stat = os.lstat(path) except FileNotFoundError: - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() return None if ( _included_output_path_is_redirected(path, current_stat) @@ -6872,14 +7376,14 @@ def _included_cleanup_file_state( parent_path, expected_parent_identity, ) - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() content_sha256 = _digest_included_regular_file( path, current_stat, expected_device=expected_parent_identity[0], expected_mount_id=parent_mount_id, ) - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() final_stat = os.lstat(path) if ( _included_output_path_is_redirected(path, final_stat) @@ -6889,7 +7393,7 @@ def _included_cleanup_file_state( or final_stat.st_ctime_ns != expected_ctime_ns ): raise OSError(f"Included Files cleanup file changed: {path}") - _verify_fallback_directory_ancestors(parent_identities) + verify_parent() return ( stat.S_IMODE(final_stat.st_mode), content_sha256, @@ -6968,8 +7472,61 @@ def _included_cleanup_directory_state( path: str, expected_identity: _PathIdentity, expected_parent_identity: _PathIdentity, + *, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> bool | None: parent_path = os.path.dirname(os.path.abspath(path)) + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + try: + path_stat = os.lstat(path) + except FileNotFoundError: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + return None + if ( + _included_output_path_is_redirected(path, path_stat) + or not stat.S_ISDIR(path_stat.st_mode) + or (path_stat.st_dev, path_stat.st_ino) != expected_identity + ): + raise OSError(f"Included Files cleanup directory changed: {path}") + parent_mount_id = _included_directory_mount_id( + parent_path, + expected_parent_identity, + ) + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + _verify_included_mount_boundary_path( + path, + path_stat, + expected_parent_identity[0], + parent_mount_id, + expect_directory=True, + ) + final_stat = os.lstat(path) + if ( + _included_output_path_is_redirected(path, final_stat) + or not stat.S_ISDIR(final_stat.st_mode) + or (final_stat.st_dev, final_stat.st_ino) != expected_identity + ): + raise OSError(f"Included Files cleanup directory changed: {path}") + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) + return True + current_parent_identity = _included_directory_identity(parent_path) if current_parent_identity != expected_parent_identity: raise OSError(f"Included Files cleanup parent changed: {parent_path}") @@ -7010,9 +7567,16 @@ def _remove_included_cleanup_tombstone( parent_identity: _PathIdentity, *, expect_directory: bool, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> None: # The fallback removers must observe the original Windows READONLY state # themselves so they can restore that attribute after a sharing failure. + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + parent_identity, + ) if _included_descriptor_paths_supported(): parent_fd, name = _open_pinned_included_parent(path) try: @@ -7038,9 +7602,19 @@ def _remove_included_cleanup_tombstone( finally: os.close(parent_fd) elif expect_directory: - _rmdir_exact_quarantined_entry_fallback(path, expected_identity) + _rmdir_exact_quarantined_entry_fallback( + path, + expected_identity, + expected_parent_identity=parent_identity, + windows_parent_binding=windows_parent_binding, + ) else: - _unlink_exact_quarantined_entry_fallback(path, expected_identity) + _unlink_exact_quarantined_entry_fallback( + path, + expected_identity, + expected_parent_identity=parent_identity, + windows_parent_binding=windows_parent_binding, + ) _sync_included_directory(parent_path, parent_identity) @@ -7055,6 +7629,7 @@ def _cleanup_recorded_included_file( *, expected_fingerprint: _PathFingerprint | None = None, expected_mode: int | None = None, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> tuple[str, ...]: warnings: list[str] = [] parent_path = os.path.dirname(os.path.abspath(path)) @@ -7070,8 +7645,15 @@ def _cleanup_recorded_included_file( path, expected_identity, expected_parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError: + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) source_state = None if os.path.lexists(path): warnings.append( @@ -7082,8 +7664,15 @@ def _cleanup_recorded_included_file( tombstone_path, expected_identity, expected_parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError: + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) tombstone_state = None if os.path.lexists(tombstone_path): warnings.append( @@ -7110,12 +7699,19 @@ def _cleanup_recorded_included_file( + tombstone_path ) return tuple(warnings) + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) _remove_included_cleanup_tombstone( tombstone_path, expected_identity, parent_path, expected_parent_identity, expect_directory=False, + windows_parent_binding=windows_parent_binding, ) _after_included_transaction_phase( f"cleanup:{role}:{relative_path}:removed" @@ -7139,6 +7735,8 @@ def _cleanup_recorded_included_file( expected_identity, source_parent_identity=expected_parent_identity, destination_parent_identity=expected_parent_identity, + windows_source_parent_binding=windows_parent_binding, + windows_destination_parent_binding=windows_parent_binding, ) _sync_included_directory(parent_path, expected_parent_identity) _after_included_transaction_phase( @@ -7148,6 +7746,7 @@ def _cleanup_recorded_included_file( tombstone_path, expected_identity, expected_parent_identity, + windows_parent_binding=windows_parent_binding, ) if tombstone_state is None or not _included_cleanup_file_receipt_matches( tombstone_state, @@ -7160,12 +7759,19 @@ def _cleanup_recorded_included_file( "Included Files cleanup tombstone changed after publication: " + tombstone_path ) + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) _remove_included_cleanup_tombstone( tombstone_path, expected_identity, parent_path, expected_parent_identity, expect_directory=False, + windows_parent_binding=windows_parent_binding, ) _after_included_transaction_phase(f"cleanup:{role}:{relative_path}:removed") return tuple(warnings) @@ -7178,6 +7784,8 @@ def _cleanup_recorded_included_directory( transaction_id: str, role: str, relative_path: str, + *, + windows_parent_binding: _WindowsIncludedCleanupParentBinding | None = None, ) -> tuple[str, ...]: warnings: list[str] = [] parent_path = os.path.dirname(os.path.abspath(path)) @@ -7193,8 +7801,15 @@ def _cleanup_recorded_included_directory( path, expected_identity, expected_parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError: + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) source_state = None if os.path.lexists(path): warnings.append( @@ -7205,8 +7820,15 @@ def _cleanup_recorded_included_directory( tombstone_path, expected_identity, expected_parent_identity, + windows_parent_binding=windows_parent_binding, ) except OSError: + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) tombstone_state = None if os.path.lexists(tombstone_path): warnings.append( @@ -7221,18 +7843,31 @@ def _cleanup_recorded_included_directory( ) return tuple(warnings) if tombstone_state is not None: + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) if os.listdir(tombstone_path): warnings.append( "non-empty Included Files cleanup tombstone was preserved: " + tombstone_path ) return tuple(warnings) + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) _remove_included_cleanup_tombstone( tombstone_path, expected_identity, parent_path, expected_parent_identity, expect_directory=True, + windows_parent_binding=windows_parent_binding, ) _after_included_transaction_phase( f"cleanup:{role}:{relative_path}:removed" @@ -7240,39 +7875,75 @@ def _cleanup_recorded_included_directory( return tuple(warnings) if source_state is None: return tuple(warnings) + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) if os.listdir(path): warnings.append( f"non-empty Included Files cleanup directory was preserved: {path}" ) return tuple(warnings) - if ( - _included_directory_identity(path) != expected_identity - or _included_directory_identity(parent_path) != expected_parent_identity - ): - raise OSError(f"Included Files cleanup directory changed: {path}") + if windows_parent_binding is None: + if ( + _included_directory_identity(path) != expected_identity + or _included_directory_identity(parent_path) + != expected_parent_identity + ): + raise OSError(f"Included Files cleanup directory changed: {path}") + else: + current_stat = os.lstat(path) + if ( + _included_output_path_is_redirected(path, current_stat) + or not stat.S_ISDIR(current_stat.st_mode) + or (current_stat.st_dev, current_stat.st_ino) != expected_identity + ): + raise OSError(f"Included Files cleanup directory changed: {path}") + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) _move_exact_included_directory( path, tombstone_path, expected_identity, source_parent_identity=expected_parent_identity, destination_parent_identity=expected_parent_identity, + windows_source_parent_binding=windows_parent_binding, + windows_destination_parent_binding=windows_parent_binding, ) _sync_included_directory(parent_path, expected_parent_identity) _after_included_transaction_phase( f"cleanup:{role}:{relative_path}:quarantined" ) + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) if os.listdir(tombstone_path): warnings.append( "non-empty Included Files cleanup tombstone was preserved: " + tombstone_path ) return tuple(warnings) + if windows_parent_binding is not None: + _verify_windows_included_cleanup_parent_binding( + windows_parent_binding, + parent_path, + expected_parent_identity, + ) _remove_included_cleanup_tombstone( tombstone_path, expected_identity, parent_path, expected_parent_identity, expect_directory=True, + windows_parent_binding=windows_parent_binding, ) _after_included_transaction_phase(f"cleanup:{role}:{relative_path}:removed") return tuple(warnings) @@ -7332,84 +8003,371 @@ def _cleanup_recorded_included_tree( ".", ) - absent_directories: set[str] = set() - for entry in sorted( - (candidate for candidate in snapshot.entries if candidate.kind == "directory"), - key=lambda candidate: ( - candidate.relative_path.count("/"), - candidate.relative_path, - ), - ): - entry_path = recorded_entry_paths[entry.relative_path] - parent_relative = posixpath.dirname(entry.relative_path) - if parent_relative in absent_directories: - absent_directories.add(entry.relative_path) - continue - try: - entry_state = _included_cleanup_directory_state( - entry_path, - entry.fingerprint[:2], - directory_identities[parent_relative], - ) - except OSError: - return ( - "unknown or mounted Included Files cleanup directory was " - f"preserved: {entry_path}", + warnings: list[str] = [] + windows_bindings_enabled = ( + os.name == "nt" + and sys.platform == "win32" + and not _included_descriptor_paths_supported() + ) + if not windows_bindings_enabled or not snapshot.entries: + absent_directories: set[str] = set() + for entry in sorted( + ( + candidate + for candidate in snapshot.entries + if candidate.kind == "directory" + ), + key=lambda candidate: ( + candidate.relative_path.count("/"), + candidate.relative_path, + ), + ): + entry_path = recorded_entry_paths[entry.relative_path] + parent_relative = posixpath.dirname(entry.relative_path) + if parent_relative in absent_directories: + absent_directories.add(entry.relative_path) + continue + try: + entry_state = _included_cleanup_directory_state( + entry_path, + entry.fingerprint[:2], + directory_identities[parent_relative], + ) + except OSError: + return ( + "unknown or mounted Included Files cleanup directory was " + f"preserved: {entry_path}", + ) + if entry_state is None: + absent_directories.add(entry.relative_path) + + for entry in sorted( + ( + candidate + for candidate in snapshot.entries + if candidate.kind == "file" + ), + key=lambda candidate: candidate.relative_path, + reverse=True, + ): + parent_relative = posixpath.dirname(entry.relative_path) + if parent_relative in absent_directories: + # Bottom-up directory cleanup rechecks and preserves anything that + # reappears; probing every known-absent descendant is redundant. + continue + if entry.content_sha256 is None: + warnings.append( + "Included Files cleanup manifest omitted a file digest: " + + entry.relative_path + ) + continue + warnings.extend( + _cleanup_recorded_included_file( + recorded_entry_paths[entry.relative_path], + entry.fingerprint[:2], + entry.content_sha256, + directory_identities[parent_relative], + transaction_id, + role, + entry.relative_path, + expected_fingerprint=entry.fingerprint, + ) ) - if entry_state is None: - absent_directories.add(entry.relative_path) - warnings: list[str] = [] - for entry in sorted( - (candidate for candidate in snapshot.entries if candidate.kind == "file"), - key=lambda candidate: candidate.relative_path, - reverse=True, - ): - parent_relative = posixpath.dirname(entry.relative_path) - if parent_relative in absent_directories: - # Bottom-up directory cleanup rechecks and preserves anything that - # reappears; probing every known-absent descendant is redundant. - continue - parent_identity = directory_identities[parent_relative] - if entry.content_sha256 is None: - warnings.append( - "Included Files cleanup manifest omitted a file digest: " - + entry.relative_path + for entry in sorted( + ( + candidate + for candidate in snapshot.entries + if candidate.kind == "directory" + ), + key=lambda candidate: ( + candidate.relative_path.count("/"), + candidate.relative_path, + ), + reverse=True, + ): + parent_relative = posixpath.dirname(entry.relative_path) + warnings.extend( + _cleanup_recorded_included_directory( + recorded_entry_paths[entry.relative_path], + entry.fingerprint[:2], + directory_identities[parent_relative], + transaction_id, + role, + entry.relative_path, + ) ) - continue - warnings.extend( - _cleanup_recorded_included_file( - recorded_entry_paths[entry.relative_path], - entry.fingerprint[:2], - entry.content_sha256, - parent_identity, - transaction_id, - role, - entry.relative_path, - expected_fingerprint=entry.fingerprint, + else: + directory_entries_by_parent: dict[ + str, + list[_IncludedTreeEntry], + ] = {} + file_entries_by_parent: dict[str, list[_IncludedTreeEntry]] = {} + for entry in snapshot.entries: + parent_relative = posixpath.dirname(entry.relative_path) + entries_by_parent = ( + directory_entries_by_parent + if entry.kind == "directory" + else file_entries_by_parent + ) + entries_by_parent.setdefault(parent_relative, []).append(entry) + + def parent_path_for(relative_path: str) -> str: + return ( + path + if not relative_path + else recorded_entry_paths[relative_path] ) - ) - directories = sorted( - (candidate for candidate in snapshot.entries if candidate.kind == "directory"), - key=lambda candidate: ( - candidate.relative_path.count("/"), - candidate.relative_path, - ), - reverse=True, - ) - for entry in directories: - parent_relative = posixpath.dirname(entry.relative_path) - warnings.extend( - _cleanup_recorded_included_directory( - recorded_entry_paths[entry.relative_path], - entry.fingerprint[:2], - directory_identities[parent_relative], - transaction_id, - role, - entry.relative_path, + def verify_parent_binding( + binding: _WindowsIncludedCleanupParentBinding, + relative_path: str, + ) -> None: + _verify_windows_included_cleanup_parent_binding( + binding, + parent_path_for(relative_path), + directory_identities[relative_path], ) - ) + + with ExitStack() as root_binding_scope: + root_parent_binding = _WindowsIncludedCleanupParentBinding.open( + path, + root_identity, + ) + # ``open`` already verifies and closes on failure. Register its + # exception-aware ``__exit__`` without a second racy ``__enter__``. + root_binding_scope.push(root_parent_binding) + + def open_child_binding( + binding_scope: ExitStack, + active_bindings: dict[ + str, + _WindowsIncludedCleanupParentBinding, + ], + relative_path: str, + ) -> _WindowsIncludedCleanupParentBinding: + parent_relative = posixpath.dirname(relative_path) + parent_binding = active_bindings.get(parent_relative) + if parent_binding is None: + raise OSError( + "Missing Included Files cleanup ancestor binding: " + + parent_relative + ) + verify_parent_binding(parent_binding, parent_relative) + binding = _WindowsIncludedCleanupParentBinding.open( + parent_path_for(relative_path), + directory_identities[relative_path], + ) + # Register before the post-open parent check so both handles + # unwind if a parent changes during child acquisition. + binding_scope.push(binding) + verify_parent_binding(parent_binding, parent_relative) + active_bindings[relative_path] = binding + return binding + + absent_directories = set() + with ExitStack() as preflight_binding_scope: + active_preflight_bindings = {"": root_parent_binding} + preflight_actions: list[tuple[str, str]] = [("enter", "")] + while preflight_actions: + action, parent_relative = preflight_actions.pop() + if action == "leave": + binding = active_preflight_bindings.pop( + parent_relative + ) + binding.close() + continue + if action != "enter": + raise AssertionError( + "Unknown Included Files cleanup preflight action" + ) + parent_binding = active_preflight_bindings.get( + parent_relative + ) + if parent_binding is None: + parent_binding = open_child_binding( + preflight_binding_scope, + active_preflight_bindings, + parent_relative, + ) + present_child_parents: list[str] = [] + for entry in sorted( + directory_entries_by_parent.get(parent_relative, ()), + key=lambda candidate: candidate.relative_path, + ): + entry_path = recorded_entry_paths[entry.relative_path] + try: + entry_state = _included_cleanup_directory_state( + entry_path, + entry.fingerprint[:2], + directory_identities[parent_relative], + windows_parent_binding=parent_binding, + ) + except OSError: + verify_parent_binding( + parent_binding, + parent_relative, + ) + return ( + "unknown or mounted Included Files cleanup " + f"directory was preserved: {entry_path}", + ) + if entry_state is None: + absent_directories.add(entry.relative_path) + elif entry.relative_path in directory_entries_by_parent: + present_child_parents.append(entry.relative_path) + if parent_relative: + preflight_actions.append(("leave", parent_relative)) + preflight_actions.extend( + ("enter", child_relative) + for child_relative in reversed(present_child_parents) + ) + + for entry in sorted( + ( + candidate + for candidate in snapshot.entries + if candidate.kind == "directory" + ), + key=lambda candidate: ( + candidate.relative_path.count("/"), + candidate.relative_path, + ), + ): + if ( + posixpath.dirname(entry.relative_path) + in absent_directories + ): + absent_directories.add(entry.relative_path) + + with ExitStack() as cleanup_binding_scope: + active_cleanup_bindings = {"": root_parent_binding} + cleanup_actions: list[ + tuple[str, str, _IncludedTreeEntry | None] + ] = [("enter", "", None)] + while cleanup_actions: + action, relative_path, own_entry = cleanup_actions.pop() + if action == "remove": + if own_entry is None: + raise AssertionError( + "Missing Included Files cleanup directory entry" + ) + parent_relative = posixpath.dirname(relative_path) + parent_binding = active_cleanup_bindings.get( + parent_relative + ) + if parent_binding is None: + raise OSError( + "Missing Included Files cleanup parent binding: " + + parent_relative + ) + warnings.extend( + _cleanup_recorded_included_directory( + recorded_entry_paths[relative_path], + own_entry.fingerprint[:2], + directory_identities[parent_relative], + transaction_id, + role, + relative_path, + windows_parent_binding=parent_binding, + ) + ) + continue + if action == "leave": + if own_entry is None: + continue + binding = active_cleanup_bindings.pop(relative_path) + # Closing the child first permits its own removal while + # the complete parent chain remains retained. + binding.close() + parent_relative = posixpath.dirname(relative_path) + parent_binding = active_cleanup_bindings.get( + parent_relative + ) + if parent_binding is None: + raise OSError( + "Missing Included Files cleanup parent binding: " + + parent_relative + ) + warnings.extend( + _cleanup_recorded_included_directory( + recorded_entry_paths[relative_path], + own_entry.fingerprint[:2], + directory_identities[parent_relative], + transaction_id, + role, + relative_path, + windows_parent_binding=parent_binding, + ) + ) + continue + + if action != "enter" or ( + relative_path and own_entry is None + ): + raise AssertionError( + "Invalid Included Files cleanup traversal action" + ) + + parent_binding = active_cleanup_bindings.get(relative_path) + if parent_binding is None: + parent_binding = open_child_binding( + cleanup_binding_scope, + active_cleanup_bindings, + relative_path, + ) + for entry in sorted( + file_entries_by_parent.get(relative_path, ()), + key=lambda candidate: candidate.relative_path, + reverse=True, + ): + if entry.content_sha256 is None: + warnings.append( + "Included Files cleanup manifest omitted a file " + "digest: " + + entry.relative_path + ) + continue + warnings.extend( + _cleanup_recorded_included_file( + recorded_entry_paths[entry.relative_path], + entry.fingerprint[:2], + entry.content_sha256, + directory_identities[relative_path], + transaction_id, + role, + entry.relative_path, + expected_fingerprint=entry.fingerprint, + windows_parent_binding=parent_binding, + ) + ) + + cleanup_actions.append( + ("leave", relative_path, own_entry) + ) + child_entries = sorted( + directory_entries_by_parent.get(relative_path, ()), + key=lambda candidate: candidate.relative_path, + reverse=True, + ) + for entry in reversed(child_entries): + child_relative = entry.relative_path + has_recorded_children = ( + child_relative in directory_entries_by_parent + or child_relative in file_entries_by_parent + ) + child_action = ( + "enter" + if child_relative not in absent_directories + and has_recorded_children + else "remove" + ) + cleanup_actions.append( + (child_action, child_relative, entry) + ) + + # The binding deliberately denies deletion sharing, so release it before + # removing the now-empty root that it protected. warnings.extend( _cleanup_recorded_included_directory( path, diff --git a/src/version.py b/src/version.py index e738348..77fa6dc 100644 --- a/src/version.py +++ b/src/version.py @@ -1,4 +1,4 @@ -VERSION = "0.7.64" +VERSION = "0.7.65" def get_version(): diff --git a/tests/test_included_files.py b/tests/test_included_files.py index 5ad0c35..5a9bc60 100644 --- a/tests/test_included_files.py +++ b/tests/test_included_files.py @@ -14,6 +14,7 @@ import unittest from collections.abc import Collection, Iterable, Mapping from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import ExitStack from dataclasses import replace from types import SimpleNamespace from typing import BinaryIO, Callable @@ -82,6 +83,72 @@ def _included_files_transaction_debris(project_path: str) -> tuple[str, ...]: return tuple(sorted(debris)) +class _ModeledWindowsCleanupParentBinding: + """Model the path/identity contract of a retained native Windows handle.""" + + def __init__( + self, + path: str, + identity: tuple[int, int], + *, + close_error: BaseException | None = None, + ) -> None: + self.path = os.path.abspath(path) + self.identity = identity + self.closed = False + self.verify_count = 0 + self.close_count = 0 + self.close_error = close_error + + def __enter__(self) -> "_ModeledWindowsCleanupParentBinding": + self.verify() + return self + + def __exit__( + self, + _exception_type: object, + active_error: BaseException | None, + _traceback: object, + ) -> None: + try: + self.close() + except BaseException as close_error: + if active_error is None: + raise + active_error.add_note( + "Could not close modeled Included Files cleanup parent " + f"binding: {close_error}" + ) + + def verify(self) -> None: + self.verify_count += 1 + if self.closed: + raise OSError(f"Included Files cleanup parent binding is closed: {self.path}") + try: + path_stat = os.lstat(self.path) + except OSError as error: + raise OSError( + f"Included Files cleanup parent changed: {self.path}" + ) from error + if ( + included_files_module._included_output_path_is_redirected( + self.path, + path_stat, + ) + or not stat.S_ISDIR(path_stat.st_mode) + or (path_stat.st_dev, path_stat.st_ino) != self.identity + ): + raise OSError(f"Included Files cleanup parent changed: {self.path}") + + def close(self) -> None: + if self.closed: + return + self.closed = True + self.close_count += 1 + if self.close_error is not None: + raise self.close_error + + class TestIncludedFilesConverterBasic(unittest.TestCase): """Test IncludedFilesConverter copies datafiles to the Godot project.""" @@ -283,6 +350,12 @@ def publish_then_fail( *, source_parent_identity: tuple[int, int] | None = None, destination_parent_identity: tuple[int, int] | None = None, + windows_source_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: original_move( source, @@ -290,6 +363,10 @@ def publish_then_fail( expected_identity, source_parent_identity=source_parent_identity, destination_parent_identity=destination_parent_identity, + windows_source_parent_binding=windows_source_parent_binding, + windows_destination_parent_binding=( + windows_destination_parent_binding + ), ) if destination == registry_path: raise OSError("registry publication failed") @@ -371,6 +448,50 @@ def _open_modeled_windows_validation_stream( del deny_writes, no_follow return open(path, "rb") + def _modeled_windows_cleanup_context( + self, + binding_opener: Callable[ + [str, tuple[int, int]], + _ModeledWindowsCleanupParentBinding, + ], + ) -> ExitStack: + cleanup_context = ExitStack() + cleanup_context.enter_context( + patch.object( + included_files_module, + "_included_descriptor_paths_supported", + return_value=False, + ) + ) + cleanup_context.enter_context( + patch.object(included_files_module.os, "name", "nt") + ) + cleanup_context.enter_context( + patch.object(included_files_module.sys, "platform", "win32") + ) + cleanup_context.enter_context( + patch.object( + included_files_module._WindowsIncludedCleanupParentBinding, + "open", + side_effect=binding_opener, + ) + ) + cleanup_context.enter_context( + patch.object( + included_files_module, + "_rename_included_transaction_entry", + side_effect=os.rename, + ) + ) + cleanup_context.enter_context( + patch.object( + included_files_module, + "_open_included_file_validation_stream", + side_effect=self._open_modeled_windows_validation_stream, + ) + ) + return cleanup_context + def _converter(self, *, max_workers: int = 2) -> IncludedFilesConverter: return IncludedFilesConverter( self.gm_dir, @@ -1063,96 +1184,1300 @@ def modeled_mountpoint(path: str) -> bool: "ismount", side_effect=modeled_mountpoint, ): - warnings = included_files_module._cleanup_recorded_included_tree( + warnings = included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + (project_stat.st_dev, project_stat.st_ino), + "a" * 32, + "late-mount", + ) + + self.assertTrue(any("mounted" in warning for warning in warnings)) + with open(sentinel_path, "rb") as sentinel_file: + self.assertEqual(sentinel_file.read(), b"late mount sentinel") + self.assertTrue(os.path.isdir(root_path)) + + def test_cleanup_skips_file_probes_below_proven_absent_directory( + self, + ) -> None: + fallback_ancestor_counts: list[int] = [] + + for entry_count in (16, 256): + with self.subTest(entry_count=entry_count): + root_path = os.path.join( + self.godot_dir, + f"absent-stage-{entry_count}", + ) + nested_path = os.path.join(root_path, "included_files") + os.makedirs(nested_path) + for index in range(entry_count): + with open( + os.path.join(nested_path, f"entry-{index:04d}.txt"), + "wb", + ) as entry_file: + entry_file.write(b"x") + + project_stat = os.lstat(self.godot_dir) + project_identity = (project_stat.st_dev, project_stat.st_ino) + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + published_path = os.path.join( + self.godot_dir, + f"published-{entry_count}", + ) + os.rename(nested_path, published_path) + cleanup_file_state = ( + included_files_module._included_cleanup_file_state + ) + capture_ancestors = ( + included_files_module._capture_fallback_directory_ancestors + ) + + with ( + patch.object( + included_files_module, + "_included_descriptor_paths_supported", + return_value=False, + ), + patch.object(included_files_module.os, "name", "nt"), + patch.object( + included_files_module, + "_included_cleanup_file_state", + wraps=cleanup_file_state, + ) as file_state_mock, + patch.object( + included_files_module, + "_capture_fallback_directory_ancestors", + wraps=capture_ancestors, + ) as ancestor_mock, + ): + warnings = ( + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "a" * 32, + "stage", + ) + ) + + self.assertEqual(warnings, ()) + self.assertEqual(file_state_mock.call_count, 0) + fallback_ancestor_counts.append(ancestor_mock.call_count) + self.assertFalse(os.path.lexists(root_path)) + self.assertEqual(len(os.listdir(published_path)), entry_count) + + self.assertEqual( + fallback_ancestor_counts, + [fallback_ancestor_counts[0]] * len(fallback_ancestor_counts), + ) + self.assertLessEqual(fallback_ancestor_counts[0], 64) + + def test_windows_nested_absent_subtree_skips_descendant_work(self) -> None: + root_path = os.path.join(self.godot_dir, "windows-absent-stage") + nested_path = os.path.join(root_path, "included_files") + deep_path = os.path.join(nested_path, "deep") + os.makedirs(deep_path) + expected_contents: dict[str, bytes] = {} + for index in range(16): + filename = f"entry-{index:04d}.txt" + content = f"preserved payload {index}\n".encode() + expected_contents[filename] = content + with open(os.path.join(deep_path, filename), "wb") as entry_file: + entry_file.write(content) + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + published_path = os.path.join( + self.godot_dir, + "windows-published-stage", + ) + os.rename(nested_path, published_path) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + return binding + + cleanup_file_state = included_files_module._included_cleanup_file_state + cleanup_directory_state = ( + included_files_module._included_cleanup_directory_state + ) + with ( + self._modeled_windows_cleanup_context(open_binding), + patch.object( + included_files_module, + "_included_cleanup_file_state", + wraps=cleanup_file_state, + ) as file_state_mock, + patch.object( + included_files_module, + "_included_cleanup_directory_state", + wraps=cleanup_directory_state, + ) as directory_state_mock, + ): + warnings = included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "b" * 32, + "windows-absent-stage", + ) + + self.assertEqual(warnings, ()) + self.assertEqual(file_state_mock.call_count, 0) + self.assertEqual(len(bindings), 1) + self.assertEqual(bindings[0].path, os.path.abspath(root_path)) + self.assertTrue(bindings[0].closed) + self.assertEqual(bindings[0].close_count, 1) + probed_directory_paths = { + os.path.abspath(call.args[0]) + for call in directory_state_mock.call_args_list + } + self.assertNotIn(os.path.abspath(deep_path), probed_directory_paths) + self.assertFalse(os.path.lexists(root_path)) + published_deep_path = os.path.join(published_path, "deep") + self.assertEqual( + sorted(os.listdir(published_deep_path)), + sorted(expected_contents), + ) + for filename, expected_content in expected_contents.items(): + with open( + os.path.join(published_deep_path, filename), + "rb", + ) as published_file: + self.assertEqual(published_file.read(), expected_content) + + def test_windows_flat_present_cleanup_reuses_one_parent_binding(self) -> None: + ancestor_capture_counts: list[int] = [] + binding_verify_counts: list[int] = [] + + for entry_count in (16, 256): + with self.subTest(entry_count=entry_count): + root_path = os.path.join( + self.godot_dir, + f"flat-present-{entry_count}", + ) + os.mkdir(root_path) + for index in range(entry_count): + with open( + os.path.join(root_path, f"entry-{index:04d}.txt"), + "wb", + ) as entry_file: + entry_file.write(f"payload {index}\n".encode()) + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + capture_ancestors = ( + included_files_module._capture_fallback_directory_ancestors + ) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + return binding + + with ( + self._modeled_windows_cleanup_context(open_binding), + patch.object( + included_files_module, + "_capture_fallback_directory_ancestors", + wraps=capture_ancestors, + ) as ancestor_capture, + ): + warnings = ( + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "c" * 32, + "flat-present", + ) + ) + + self.assertEqual(warnings, ()) + self.assertEqual(len(bindings), 1) + self.assertEqual(bindings[0].path, os.path.abspath(root_path)) + self.assertEqual(bindings[0].identity, snapshot.identity) + self.assertTrue(bindings[0].closed) + self.assertEqual(bindings[0].close_count, 1) + self.assertFalse(os.path.lexists(root_path)) + ancestor_capture_counts.append(ancestor_capture.call_count) + binding_verify_counts.append(bindings[0].verify_count) + + self.assertEqual( + ancestor_capture_counts, + [ancestor_capture_counts[0]] * len(ancestor_capture_counts), + ) + self.assertLessEqual(ancestor_capture_counts[0], 32) + self.assertGreater(binding_verify_counts[1], binding_verify_counts[0]) + self.assertLessEqual( + binding_verify_counts[1], + binding_verify_counts[0] * 16 + 32, + ) + + def test_windows_nested_same_parent_cleanup_reuses_one_binding(self) -> None: + ancestor_capture_counts: list[int] = [] + nested_verify_counts: list[int] = [] + + for entry_count in (16, 256): + with self.subTest(entry_count=entry_count): + root_path = os.path.join( + self.godot_dir, + f"nested-same-parent-{entry_count}", + ) + nested_path = os.path.join(root_path, "nested") + os.makedirs(nested_path) + for index in range(entry_count): + with open( + os.path.join(nested_path, f"entry-{index:04d}.txt"), + "wb", + ) as entry_file: + entry_file.write(f"nested payload {index}\n".encode()) + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + capture_ancestors = ( + included_files_module._capture_fallback_directory_ancestors + ) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + return binding + + with ( + self._modeled_windows_cleanup_context(open_binding), + patch.object( + included_files_module, + "_capture_fallback_directory_ancestors", + wraps=capture_ancestors, + ) as ancestor_capture, + ): + warnings = ( + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "1" * 32, + "nested-same-parent", + ) + ) + + self.assertEqual(warnings, ()) + self.assertEqual(len(bindings), 2) + self.assertEqual( + [binding.path for binding in bindings], + [os.path.abspath(root_path), os.path.abspath(nested_path)], + ) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue( + all(binding.close_count == 1 for binding in bindings) + ) + self.assertFalse(os.path.lexists(root_path)) + ancestor_capture_counts.append(ancestor_capture.call_count) + nested_verify_counts.append(bindings[1].verify_count) + + self.assertEqual( + ancestor_capture_counts, + [ancestor_capture_counts[0]] * len(ancestor_capture_counts), + ) + self.assertLessEqual(ancestor_capture_counts[0], 32) + self.assertGreater(nested_verify_counts[1], nested_verify_counts[0]) + self.assertLessEqual( + nested_verify_counts[1], + nested_verify_counts[0] * 16 + 32, + ) + + def test_windows_multilevel_cleanup_binding_operations_are_linear( + self, + ) -> None: + ancestor_capture_counts: list[int] = [] + binding_open_counts: list[int] = [] + binding_verify_counts: list[int] = [] + + for branch_count in (16, 256): + with self.subTest(branch_count=branch_count): + root_path = os.path.join( + self.godot_dir, + f"nested-branches-{branch_count}", + ) + os.mkdir(root_path) + branch_paths: list[tuple[str, str]] = [] + for index in range(branch_count): + branch_path = os.path.join( + root_path, + f"branch-{index:04d}", + ) + leaf_path = os.path.join(branch_path, "leaf") + os.makedirs(leaf_path) + with open( + os.path.join(leaf_path, "payload.bin"), + "wb", + ) as payload_file: + payload_file.write( + f"multilevel payload {index}\n".encode() + ) + branch_paths.append((branch_path, leaf_path)) + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + capture_ancestors = ( + included_files_module._capture_fallback_directory_ancestors + ) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + maximum_live_bindings = 0 + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + nonlocal maximum_live_bindings + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + maximum_live_bindings = max( + maximum_live_bindings, + sum(not candidate.closed for candidate in bindings), + ) + return binding + + with ( + self._modeled_windows_cleanup_context(open_binding), + patch.object( + included_files_module, + "_capture_fallback_directory_ancestors", + wraps=capture_ancestors, + ) as ancestor_capture, + ): + warnings = ( + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "2" * 32, + "nested-branches", + ) + ) + + self.assertEqual(warnings, ()) + self.assertEqual( + len(bindings), + 1 + branch_count * 3, + ) + self.assertEqual(maximum_live_bindings, 3) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue( + all(binding.close_count == 1 for binding in bindings) + ) + path_open_counts: dict[str, int] = {} + for binding in bindings: + path_open_counts[binding.path] = ( + path_open_counts.get(binding.path, 0) + 1 + ) + self.assertEqual(path_open_counts[os.path.abspath(root_path)], 1) + for branch_path, leaf_path in branch_paths: + self.assertEqual( + path_open_counts[os.path.abspath(branch_path)], + 2, + ) + self.assertEqual( + path_open_counts[os.path.abspath(leaf_path)], + 1, + ) + self.assertFalse(os.path.lexists(root_path)) + ancestor_capture_counts.append(ancestor_capture.call_count) + binding_open_counts.append(len(bindings)) + binding_verify_counts.append( + sum(binding.verify_count for binding in bindings) + ) + + self.assertEqual( + ancestor_capture_counts, + [ancestor_capture_counts[0]] * len(ancestor_capture_counts), + ) + self.assertLessEqual(ancestor_capture_counts[0], 32) + self.assertEqual(binding_open_counts, [49, 769]) + self.assertGreater(binding_verify_counts[1], binding_verify_counts[0]) + self.assertLessEqual( + binding_verify_counts[1], + binding_verify_counts[0] * 16 + 64, + ) + + def test_windows_cleanup_binding_operations_scale_with_depth(self) -> None: + total_entry_count = 40 + metrics: list[tuple[int, int, int, int, int, int]] = [] + + for depth in (1, 8, 32): + with self.subTest(depth=depth): + root_path = os.path.join( + self.godot_dir, + f"depth-sweep-{depth}", + ) + os.mkdir(root_path) + leaf_directory = root_path + for _index in range(depth): + leaf_directory = os.path.join(leaf_directory, "d") + os.mkdir(leaf_directory) + for index in range(total_entry_count - depth): + with open( + os.path.join(leaf_directory, f"entry-{index:04d}.txt"), + "wb", + ) as entry_file: + entry_file.write(f"depth payload {index}\n".encode()) + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + with patch.object( + included_files_module, + "_included_descriptor_paths_supported", + return_value=False, + ): + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + self.assertEqual(len(snapshot.entries), total_entry_count) + + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + maximum_live_bindings = 0 + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + nonlocal maximum_live_bindings + binding = _ModeledWindowsCleanupParentBinding(path, identity) + # The real native ``open`` verifies its new handle once + # before returning it. Keep that verification in these + # operation counts even though the handle itself is modeled. + binding.verify() + bindings.append(binding) + maximum_live_bindings = max( + maximum_live_bindings, + sum(not candidate.closed for candidate in bindings), + ) + return binding + + binding_opener = MagicMock(side_effect=open_binding) + verify_parent_binding = ( + included_files_module._verify_windows_included_cleanup_parent_binding + ) + capture_ancestors = ( + included_files_module._capture_fallback_directory_ancestors + ) + with ( + self._modeled_windows_cleanup_context(binding_opener), + patch.object( + included_files_module, + "_verify_windows_included_cleanup_parent_binding", + wraps=verify_parent_binding, + ) as parent_binding_verifier, + patch.object( + included_files_module, + "_capture_fallback_directory_ancestors", + wraps=capture_ancestors, + ) as ancestor_capture, + ): + warnings = ( + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "7" * 32, + "depth-sweep", + ) + ) + + expected_open_count = depth * 2 + # With the entry count fixed, exchanging one leaf file for one + # directory adds only five retained-parent checks. It must not + # cause every entry to walk the complete ancestor chain. + expected_helper_verify_count = ( + total_entry_count * 19 + depth * 5 - 2 + ) + expected_native_verify_count = ( + expected_helper_verify_count + expected_open_count + ) + self.assertEqual(warnings, ()) + self.assertEqual(binding_opener.call_count, expected_open_count) + self.assertEqual( + parent_binding_verifier.call_count, + expected_helper_verify_count, + ) + self.assertEqual( + sum(binding.verify_count for binding in bindings), + expected_native_verify_count, + ) + self.assertEqual(ancestor_capture.call_count, 14) + self.assertEqual(maximum_live_bindings, depth + 1) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue( + all(binding.close_count == 1 for binding in bindings) + ) + self.assertFalse(os.path.lexists(root_path)) + metrics.append( + ( + depth, + binding_opener.call_count, + parent_binding_verifier.call_count, + sum(binding.verify_count for binding in bindings), + ancestor_capture.call_count, + maximum_live_bindings, + ) + ) + + self.assertEqual( + metrics, + [ + (1, 2, 763, 765, 14, 2), + (8, 16, 798, 814, 14, 9), + (32, 64, 918, 982, 14, 33), + ], + ) + + def test_windows_present_cleanup_chain_exceeds_recursion_limit(self) -> None: + depth = 128 + modeled_recursion_limit = 80 + self.assertGreater(depth, modeled_recursion_limit) + + root_path = os.path.join(self.godot_dir, "iterative-depth") + os.mkdir(root_path) + leaf_directory = root_path + for _index in range(depth): + leaf_directory = os.path.join(leaf_directory, "d") + os.mkdir(leaf_directory) + payload_path = os.path.join(leaf_directory, "payload.txt") + with open(payload_path, "wb") as payload_file: + payload_file.write(b"deep iterative cleanup payload\n") + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + with patch.object( + included_files_module, + "_included_descriptor_paths_supported", + return_value=False, + ): + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + self.assertEqual(len(snapshot.entries), depth + 1) + + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + maximum_live_bindings = 0 + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + nonlocal maximum_live_bindings + binding = _ModeledWindowsCleanupParentBinding(path, identity) + binding.verify() + bindings.append(binding) + maximum_live_bindings = max( + maximum_live_bindings, + sum(not candidate.closed for candidate in bindings), + ) + return binding + + binding_opener = MagicMock(side_effect=open_binding) + with self._modeled_windows_cleanup_context(binding_opener): + previous_recursion_limit = sys.getrecursionlimit() + sys.setrecursionlimit(modeled_recursion_limit) + try: + warnings = included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "8" * 32, + "iterative-depth", + ) + finally: + sys.setrecursionlimit(previous_recursion_limit) + + self.assertEqual(warnings, ()) + self.assertEqual(binding_opener.call_count, depth * 2) + self.assertEqual(maximum_live_bindings, depth + 1) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue(all(binding.close_count == 1 for binding in bindings)) + self.assertFalse(os.path.lexists(payload_path)) + self.assertFalse(os.path.lexists(root_path)) + + def test_windows_nested_bindings_close_before_parent_removal(self) -> None: + root_path = os.path.join(self.godot_dir, "nested-binding-lifetime") + nested_path = os.path.join(root_path, "nested") + deep_path = os.path.join(nested_path, "deep") + os.makedirs(deep_path) + payload_path = os.path.join(deep_path, "payload.txt") + with open(payload_path, "wb") as payload_file: + payload_file.write(b"binding lifetime payload\n") + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + live_bindings_before_move: dict[str, frozenset[str]] = {} + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + return binding + + def record_live_bindings(source: str, _destination: str) -> None: + live_bindings_before_move[os.path.abspath(source)] = frozenset( + binding.path for binding in bindings if not binding.closed + ) + + with ( + self._modeled_windows_cleanup_context(open_binding), + patch.object( + included_files_module, + "_before_included_transaction_rename_fallback", + side_effect=record_live_bindings, + ), + ): + warnings = included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "3" * 32, + "nested-lifetime", + ) + + absolute_root = os.path.abspath(root_path) + absolute_nested = os.path.abspath(nested_path) + absolute_deep = os.path.abspath(deep_path) + self.assertEqual(warnings, ()) + self.assertEqual( + live_bindings_before_move[os.path.abspath(payload_path)], + frozenset({absolute_root, absolute_nested, absolute_deep}), + ) + self.assertEqual( + live_bindings_before_move[absolute_deep], + frozenset({absolute_root, absolute_nested}), + ) + self.assertEqual( + live_bindings_before_move[absolute_nested], + frozenset({absolute_root}), + ) + self.assertEqual( + live_bindings_before_move[absolute_root], + frozenset(), + ) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue(all(binding.close_count == 1 for binding in bindings)) + self.assertFalse(os.path.lexists(root_path)) + + def test_windows_child_binding_open_rechecks_nested_parent(self) -> None: + root_path = os.path.join(self.godot_dir, "nested-open-race") + outer_path = os.path.join(root_path, "outer") + parked_outer_path = os.path.join(root_path, "outer-parked") + inner_path = os.path.join(outer_path, "inner") + owned_path = os.path.join(inner_path, "owned.txt") + os.makedirs(inner_path) + with open(owned_path, "wb") as owned_file: + owned_file.write(b"recorded acquisition-race content\n") + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + parent_changed = False + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + nonlocal parent_changed + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + if ( + not parent_changed + and os.path.abspath(path) == os.path.abspath(inner_path) + ): + self.assertTrue( + any( + candidate.path == os.path.abspath(outer_path) + and not candidate.closed + for candidate in bindings + ) + ) + os.rename(outer_path, parked_outer_path) + os.makedirs(inner_path) + with open(owned_path, "wb") as replacement_file: + replacement_file.write(b"unknown acquisition replacement\n") + parent_changed = True + return binding + + with ( + self._modeled_windows_cleanup_context(open_binding), + self.assertRaisesRegex(OSError, "cleanup parent changed"), + ): + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "5" * 32, + "nested-open-race", + ) + + self.assertTrue(parent_changed) + self.assertEqual(len(bindings), 4) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue(all(binding.close_count == 1 for binding in bindings)) + self.assertEqual(bindings[-1].path, os.path.abspath(inner_path)) + with open( + os.path.join(parked_outer_path, "inner", "owned.txt"), + "rb", + ) as parked_file: + self.assertEqual( + parked_file.read(), + b"recorded acquisition-race content\n", + ) + with open(owned_path, "rb") as replacement_file: + self.assertEqual( + replacement_file.read(), + b"unknown acquisition replacement\n", + ) + + def test_windows_nested_cleanup_keeps_primary_close_errors_as_notes( + self, + ) -> None: + root_path = os.path.join(self.godot_dir, "nested-close-errors") + nested_path = os.path.join(root_path, "nested") + owned_path = os.path.join(nested_path, "owned.txt") + os.makedirs(nested_path) + with open(owned_path, "wb") as owned_file: + owned_file.write(b"nested close-error content\n") + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + primary_error = RuntimeError("injected nested cleanup interruption") + root_close_error = OSError("injected root binding close failure") + nested_close_error = OSError("injected nested binding close failure") + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + close_error = ( + root_close_error + if os.path.abspath(path) == os.path.abspath(root_path) + else nested_close_error + ) + binding = _ModeledWindowsCleanupParentBinding( + path, + identity, + close_error=close_error, + ) + bindings.append(binding) + return binding + + def interrupt_after_quarantine(phase: str) -> None: + if phase == ( + "cleanup:nested-close-errors:nested/owned.txt:quarantined" + ): + raise primary_error + + with ( + self._modeled_windows_cleanup_context(open_binding), + patch.object( + included_files_module, + "_after_included_transaction_phase", + side_effect=interrupt_after_quarantine, + ), + self.assertRaises(RuntimeError) as raised, + ): + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "6" * 32, + "nested-close-errors", + ) + + self.assertIs(raised.exception, primary_error) + notes = getattr(raised.exception, "__notes__", ()) + self.assertTrue( + any(str(nested_close_error) in note for note in notes), + notes, + ) + self.assertTrue( + any(str(root_close_error) in note for note in notes), + notes, + ) + self.assertEqual(len(bindings), 2) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue(all(binding.close_count == 1 for binding in bindings)) + self.assertFalse(os.path.lexists(owned_path)) + tombstone_path = included_files_module._included_cleanup_tombstone_path( + owned_path, + "6" * 32, + "nested-close-errors", + "nested/owned.txt", + expect_directory=False, + ) + self.assertTrue(os.path.isfile(tombstone_path)) + + def _assert_windows_nested_cleanup_parent_change_is_preserved( + self, + *, + install_replacement: bool, + ) -> None: + label = "replacement" if install_replacement else "relocation" + root_path = os.path.join(self.godot_dir, f"nested-parent-{label}") + nested_path = os.path.join(root_path, "nested") + parked_path = os.path.join(root_path, "nested-parked") + owned_path = os.path.join(nested_path, "owned.txt") + os.makedirs(nested_path) + with open(owned_path, "wb") as owned_file: + owned_file.write(b"recorded nested content\n") + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + parent_changed = False + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + return binding + + def change_parent_before_move(source: str, _destination: str) -> None: + nonlocal parent_changed + if parent_changed or os.path.abspath(source) != os.path.abspath( + owned_path + ): + return + os.rename(nested_path, parked_path) + if install_replacement: + os.mkdir(nested_path) + with open(owned_path, "wb") as replacement_file: + replacement_file.write(b"unknown nested replacement\n") + parent_changed = True + + with ( + self._modeled_windows_cleanup_context(open_binding), + patch.object( + included_files_module, + "_before_included_transaction_rename_fallback", + side_effect=change_parent_before_move, + ), + self.assertRaisesRegex(OSError, "cleanup parent changed"), + ): + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "4" * 32, + f"nested-parent-{label}", + ) + + self.assertTrue(parent_changed) + self.assertEqual(len(bindings), 2) + self.assertTrue(all(binding.closed for binding in bindings)) + self.assertTrue(all(binding.close_count == 1 for binding in bindings)) + with open( + os.path.join(parked_path, "owned.txt"), + "rb", + ) as parked_file: + self.assertEqual(parked_file.read(), b"recorded nested content\n") + if install_replacement: + with open(owned_path, "rb") as replacement_file: + self.assertEqual( + replacement_file.read(), + b"unknown nested replacement\n", + ) + else: + self.assertFalse(os.path.lexists(nested_path)) + + def test_windows_nested_cleanup_rejects_parent_relocation(self) -> None: + self._assert_windows_nested_cleanup_parent_change_is_preserved( + install_replacement=False, + ) + + def test_windows_nested_cleanup_rejects_parent_replacement(self) -> None: + self._assert_windows_nested_cleanup_parent_change_is_preserved( + install_replacement=True, + ) + + def test_windows_flat_readonly_cleanup_reuses_parent_binding(self) -> None: + ancestor_capture_counts: list[int] = [] + + for entry_count in (16, 256): + with self.subTest(entry_count=entry_count): + root_path = os.path.join( + self.godot_dir, + f"flat-readonly-{entry_count}", + ) + os.mkdir(root_path) + for index in range(entry_count): + entry_path = os.path.join( + root_path, + f"entry-{index:04d}.txt", + ) + with open(entry_path, "wb") as entry_file: + entry_file.write(f"readonly payload {index}\n".encode()) + os.chmod(entry_path, 0o400) + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + capture_ancestors = ( + included_files_module._capture_fallback_directory_ancestors + ) + supports_without_chmod = set(os.supports_fd) + supports_without_chmod.discard(os.chmod) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + return binding + + with ( + patch.object( + included_files_module, + "_included_descriptor_paths_supported", + return_value=False, + ), + patch.object(included_files_module.os, "name", "nt"), + patch.object(included_files_module.sys, "platform", "win32"), + patch.object( + included_files_module.os, + "supports_fd", + supports_without_chmod, + ), + patch.object( + included_files_module._WindowsIncludedCleanupParentBinding, + "open", + side_effect=open_binding, + ) as binding_open, + patch.object( + included_files_module, + "_rename_included_transaction_entry", + side_effect=os.rename, + ), + patch.object( + included_files_module, + "_open_included_file_validation_stream", + side_effect=self._open_modeled_windows_validation_stream, + ), + patch.object( + included_files_module, + "_capture_fallback_directory_ancestors", + wraps=capture_ancestors, + ) as ancestor_capture, + patch.object( + included_files_module, + "_before_included_fallback_chmod_open", + ) as chmod_open, + ): + warnings = ( + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "e" * 32, + "flat-readonly", + ) + ) + + self.assertEqual(warnings, ()) + binding_open.assert_called_once_with(root_path, snapshot.identity) + self.assertEqual(chmod_open.call_count, entry_count) + self.assertEqual(len(bindings), 1) + self.assertTrue(bindings[0].closed) + self.assertEqual(bindings[0].close_count, 1) + self.assertFalse(os.path.lexists(root_path)) + ancestor_capture_counts.append(ancestor_capture.call_count) + + self.assertEqual( + ancestor_capture_counts, + [ancestor_capture_counts[0]] * len(ancestor_capture_counts), + ) + self.assertLessEqual(ancestor_capture_counts[0], 32) + + def test_windows_flat_readonly_cleanup_keeps_primary_close_error(self) -> None: + root_path = os.path.join(self.godot_dir, "flat-readonly-close-error") + os.mkdir(root_path) + owned_path = os.path.join(root_path, "owned.txt") + content = b"readonly close-error payload\n" + with open(owned_path, "wb") as owned_file: + owned_file.write(content) + os.chmod(owned_path, 0o400) + + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + root_identity = snapshot.identity + if root_identity is None: + self.fail("captured read-only cleanup root unexpectedly disappeared") + kernel32 = MagicMock() + kernel32.GetFileType.return_value = ( + included_files_module._WINDOWS_FILE_TYPE_DISK + ) + kernel32.CloseHandle.return_value = 0 + binding = included_files_module._WindowsIncludedCleanupParentBinding( + path=os.path.abspath(root_path), + identity=root_identity, + kernel32=kernel32, + handle=1234, + ) + supports_without_chmod = set(os.supports_fd) + supports_without_chmod.discard(os.chmod) + primary_error = RuntimeError("injected read-only cleanup interruption") + close_error = OSError("injected cleanup parent close failure") + + def interrupt_after_readonly_clear(phase: str) -> None: + if phase == "cleanup-readonly-cleared": + raise primary_error + + with ( + patch.object( + included_files_module, + "_included_descriptor_paths_supported", + return_value=False, + ), + patch.object(included_files_module.os, "name", "nt"), + patch.object(included_files_module.sys, "platform", "win32"), + patch.object( + included_files_module.os, + "supports_fd", + supports_without_chmod, + ), + patch.object( + included_files_module._WindowsIncludedCleanupParentBinding, + "open", + return_value=binding, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_identity", + return_value=root_identity, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_attributes", + return_value=( + included_files_module._WINDOWS_FILE_ATTRIBUTE_DIRECTORY + ), + ), + patch.object( + included_files_module, + "_windows_included_transaction_error", + return_value=close_error, + ), + patch.object( + included_files_module, + "_rename_included_transaction_entry", + side_effect=os.rename, + ), + patch.object( + included_files_module, + "_open_included_file_validation_stream", + side_effect=self._open_modeled_windows_validation_stream, + ), + patch.object( + included_files_module, + "_after_included_transaction_phase", + side_effect=interrupt_after_readonly_clear, + ), + self.assertRaises(RuntimeError) as raised, + ): + included_files_module._cleanup_recorded_included_tree( + root_path, + snapshot, + project_identity, + "f" * 32, + "flat-readonly-close-error", + ) + + self.assertIs(raised.exception, primary_error) + self.assertTrue( + any( + "Could not close Included Files cleanup parent binding" + in note + and str(close_error) in note + for note in getattr(raised.exception, "__notes__", ()) + ) + ) + self.assertIsNone(binding.handle) + kernel32.CloseHandle.assert_called_once_with(1234) + self.assertFalse(os.path.lexists(owned_path)) + tombstone_path = included_files_module._included_cleanup_tombstone_path( + owned_path, + "f" * 32, + "flat-readonly-close-error", + "owned.txt", + expect_directory=False, + ) + self.assertTrue(os.lstat(tombstone_path).st_mode & stat.S_IWRITE) + + def _assert_windows_flat_cleanup_parent_change_is_preserved( + self, + *, + install_replacement: bool, + ) -> None: + label = "replacement" if install_replacement else "relocation" + root_path = os.path.join(self.godot_dir, f"flat-parent-{label}") + parked_path = root_path + "-parked" + owned_path = os.path.join(root_path, "owned.txt") + os.mkdir(root_path) + with open(owned_path, "wb") as owned_file: + owned_file.write(b"recorded owned content\n") + project_stat = os.lstat(self.godot_dir) + project_identity = project_stat.st_dev, project_stat.st_ino + snapshot = included_files_module._capture_included_tree( + root_path, + expected_parent_identity=project_identity, + ) + bindings: list[_ModeledWindowsCleanupParentBinding] = [] + parent_changed = False + + def open_binding( + path: str, + identity: tuple[int, int], + ) -> _ModeledWindowsCleanupParentBinding: + binding = _ModeledWindowsCleanupParentBinding(path, identity) + bindings.append(binding) + return binding + + def change_parent_before_move(_source: str, _destination: str) -> None: + nonlocal parent_changed + if parent_changed: + return + os.rename(root_path, parked_path) + if install_replacement: + os.mkdir(root_path) + with open(owned_path, "wb") as replacement_file: + replacement_file.write(b"unknown replacement content\n") + parent_changed = True + + with ( + patch.object( + included_files_module, + "_included_descriptor_paths_supported", + return_value=False, + ), + patch.object(included_files_module.os, "name", "nt"), + patch.object(included_files_module.sys, "platform", "win32"), + patch.object( + included_files_module._WindowsIncludedCleanupParentBinding, + "open", + side_effect=open_binding, + ), + patch.object( + included_files_module, + "_rename_included_transaction_entry", + side_effect=os.rename, + ), + patch.object( + included_files_module, + "_open_included_file_validation_stream", + side_effect=self._open_modeled_windows_validation_stream, + ), + patch.object( + included_files_module, + "_before_included_transaction_rename_fallback", + side_effect=change_parent_before_move, + ), + self.assertRaisesRegex(OSError, "cleanup parent changed"), + ): + included_files_module._cleanup_recorded_included_tree( root_path, snapshot, - (project_stat.st_dev, project_stat.st_ino), - "a" * 32, - "late-mount", + project_identity, + "d" * 32, + f"flat-parent-{label}", ) - self.assertTrue(any("mounted" in warning for warning in warnings)) - with open(sentinel_path, "rb") as sentinel_file: - self.assertEqual(sentinel_file.read(), b"late mount sentinel") - self.assertTrue(os.path.isdir(root_path)) - - def test_cleanup_skips_file_probes_below_proven_absent_directory( - self, - ) -> None: - fallback_ancestor_counts: list[int] = [] - - for entry_count in (16, 256): - with self.subTest(entry_count=entry_count): - root_path = os.path.join( - self.godot_dir, - f"absent-stage-{entry_count}", - ) - nested_path = os.path.join(root_path, "included_files") - os.makedirs(nested_path) - for index in range(entry_count): - with open( - os.path.join(nested_path, f"entry-{index:04d}.txt"), - "wb", - ) as entry_file: - entry_file.write(b"x") - - project_stat = os.lstat(self.godot_dir) - project_identity = (project_stat.st_dev, project_stat.st_ino) - snapshot = included_files_module._capture_included_tree( - root_path, - expected_parent_identity=project_identity, - ) - published_path = os.path.join( - self.godot_dir, - f"published-{entry_count}", - ) - os.rename(nested_path, published_path) - cleanup_file_state = ( - included_files_module._included_cleanup_file_state - ) - capture_ancestors = ( - included_files_module._capture_fallback_directory_ancestors + self.assertTrue(parent_changed) + self.assertEqual(len(bindings), 1) + self.assertTrue(bindings[0].closed) + self.assertEqual(bindings[0].close_count, 1) + with open( + os.path.join(parked_path, "owned.txt"), + "rb", + ) as parked_file: + self.assertEqual(parked_file.read(), b"recorded owned content\n") + if install_replacement: + with open(owned_path, "rb") as replacement_file: + self.assertEqual( + replacement_file.read(), + b"unknown replacement content\n", ) + else: + self.assertFalse(os.path.lexists(root_path)) - with ( - patch.object( - included_files_module, - "_included_descriptor_paths_supported", - return_value=False, - ), - patch.object(included_files_module.os, "name", "nt"), - patch.object( - included_files_module, - "_included_cleanup_file_state", - wraps=cleanup_file_state, - ) as file_state_mock, - patch.object( - included_files_module, - "_capture_fallback_directory_ancestors", - wraps=capture_ancestors, - ) as ancestor_mock, - ): - warnings = ( - included_files_module._cleanup_recorded_included_tree( - root_path, - snapshot, - project_identity, - "a" * 32, - "stage", - ) - ) - - self.assertEqual(warnings, ()) - self.assertEqual(file_state_mock.call_count, 0) - fallback_ancestor_counts.append(ancestor_mock.call_count) - self.assertFalse(os.path.lexists(root_path)) - self.assertEqual(len(os.listdir(published_path)), entry_count) + def test_windows_flat_cleanup_rejects_parent_relocation(self) -> None: + self._assert_windows_flat_cleanup_parent_change_is_preserved( + install_replacement=False, + ) - self.assertEqual( - fallback_ancestor_counts, - [fallback_ancestor_counts[0]] * len(fallback_ancestor_counts), + def test_windows_flat_cleanup_rejects_parent_replacement(self) -> None: + self._assert_windows_flat_cleanup_parent_change_is_preserved( + install_replacement=True, ) - self.assertLessEqual(fallback_ancestor_counts[0], 64) def test_cleanup_preserves_directory_that_reappears_after_absence_proof( self, @@ -1182,12 +2507,17 @@ def create_replacement_after_absence( path: str, expected_identity: tuple[int, int], expected_parent_identity: tuple[int, int], + *, + windows_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, ) -> bool | None: nonlocal replacement_created state = cleanup_directory_state( path, expected_identity, expected_parent_identity, + windows_parent_binding=windows_parent_binding, ) if ( not replacement_created @@ -6087,6 +7417,328 @@ def test_windows_validation_stream_denies_writes_and_reparse_following( fdopen.assert_called_once_with(5678, "rb") kernel32.CloseHandle.assert_not_called() + def test_modeled_windows_cleanup_parent_binding_omits_delete_sharing( + self, + ) -> None: + parent_path = os.path.join(self.godot_dir, "cleanup-parent-binding") + os.mkdir(parent_path) + parent_stat = os.lstat(parent_path) + parent_identity = parent_stat.st_dev, parent_stat.st_ino + kernel32 = MagicMock() + kernel32.CreateFileW.return_value = 1234 + kernel32.GetFileType.return_value = ( + included_files_module._WINDOWS_FILE_TYPE_DISK + ) + kernel32.CloseHandle.return_value = 1 + + with ( + patch.object(included_files_module.os, "name", "nt"), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_api", + return_value=kernel32, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_identity", + return_value=parent_identity, + ) as identify, + patch.object( + included_files_module, + "_windows_included_cleanup_parent_attributes", + return_value=( + included_files_module._WINDOWS_FILE_ATTRIBUTE_DIRECTORY + ), + ) as inspect_attributes, + ): + binding = ( + included_files_module._WindowsIncludedCleanupParentBinding.open( + parent_path, + parent_identity, + ) + ) + binding.verify() + binding.close() + binding.close() + + kernel32.CreateFileW.assert_called_once_with( + included_files_module._windows_extended_included_path(parent_path), + included_files_module._WINDOWS_FILE_TRAVERSE + | included_files_module._WINDOWS_FILE_READ_ATTRIBUTES, + included_files_module._WINDOWS_FILE_SHARE_READ + | included_files_module._WINDOWS_FILE_SHARE_WRITE, + None, + included_files_module._WINDOWS_OPEN_EXISTING, + included_files_module._WINDOWS_FILE_FLAG_BACKUP_SEMANTICS + | included_files_module._WINDOWS_FILE_FLAG_OPEN_REPARSE_POINT, + None, + ) + share_mode = kernel32.CreateFileW.call_args.args[2] + self.assertEqual( + share_mode & included_files_module._WINDOWS_FILE_SHARE_DELETE, + 0, + ) + self.assertEqual(identify.call_count, 2) + self.assertEqual(inspect_attributes.call_count, 2) + kernel32.CloseHandle.assert_called_once_with(1234) + + def test_modeled_windows_cleanup_parent_revalidation_rejects_new_identity( + self, + ) -> None: + parent_path = os.path.join(self.godot_dir, "cleanup-parent-identity") + os.mkdir(parent_path) + parent_stat = os.lstat(parent_path) + parent_identity = parent_stat.st_dev, parent_stat.st_ino + changed_identity = parent_identity[0], parent_identity[1] + 1 + kernel32 = MagicMock() + kernel32.CreateFileW.return_value = 1234 + kernel32.GetFileType.return_value = ( + included_files_module._WINDOWS_FILE_TYPE_DISK + ) + kernel32.CloseHandle.return_value = 1 + + with ( + patch.object(included_files_module.os, "name", "nt"), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_api", + return_value=kernel32, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_identity", + side_effect=(parent_identity, changed_identity), + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_attributes", + return_value=( + included_files_module._WINDOWS_FILE_ATTRIBUTE_DIRECTORY + ), + ), + ): + binding = ( + included_files_module._WindowsIncludedCleanupParentBinding.open( + parent_path, + parent_identity, + ) + ) + try: + with self.assertRaisesRegex(OSError, "cleanup parent changed"): + binding.verify() + finally: + binding.close() + + kernel32.CloseHandle.assert_called_once_with(1234) + + def test_modeled_windows_cleanup_parent_rejects_reparse_handle(self) -> None: + parent_path = os.path.join(self.godot_dir, "cleanup-parent-reparse") + os.mkdir(parent_path) + parent_stat = os.lstat(parent_path) + parent_identity = parent_stat.st_dev, parent_stat.st_ino + kernel32 = MagicMock() + kernel32.CreateFileW.return_value = 1234 + kernel32.GetFileType.return_value = ( + included_files_module._WINDOWS_FILE_TYPE_DISK + ) + kernel32.CloseHandle.return_value = 1 + + with ( + patch.object(included_files_module.os, "name", "nt"), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_api", + return_value=kernel32, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_identity", + return_value=parent_identity, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_attributes", + return_value=( + included_files_module._WINDOWS_FILE_ATTRIBUTE_DIRECTORY + | included_files_module._WINDOWS_FILE_ATTRIBUTE_REPARSE_POINT + ), + ), + self.assertRaisesRegex(OSError, "cleanup parent changed"), + ): + included_files_module._WindowsIncludedCleanupParentBinding.open( + parent_path, + parent_identity, + ) + + kernel32.CloseHandle.assert_called_once_with(1234) + + def test_modeled_windows_cleanup_parent_close_failure_is_one_shot( + self, + ) -> None: + parent_path = os.path.join(self.godot_dir, "cleanup-parent-close") + os.mkdir(parent_path) + parent_stat = os.lstat(parent_path) + parent_identity = parent_stat.st_dev, parent_stat.st_ino + kernel32 = MagicMock() + kernel32.CreateFileW.return_value = 1234 + kernel32.GetFileType.return_value = ( + included_files_module._WINDOWS_FILE_TYPE_DISK + ) + kernel32.CloseHandle.return_value = 1 + + with ( + patch.object(included_files_module.os, "name", "nt"), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_api", + return_value=kernel32, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_identity", + return_value=parent_identity, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_attributes", + return_value=( + included_files_module._WINDOWS_FILE_ATTRIBUTE_DIRECTORY + ), + ), + ): + binding = ( + included_files_module._WindowsIncludedCleanupParentBinding.open( + parent_path, + parent_identity, + ) + ) + kernel32.CloseHandle.return_value = 0 + close_error = OSError("injected cleanup handle close failure") + with ( + patch.object( + included_files_module, + "_windows_included_transaction_error", + return_value=close_error, + ), + self.assertRaises(OSError) as raised, + ): + binding.close() + self.assertIs(raised.exception, close_error) + binding.close() + + kernel32.CloseHandle.assert_called_once_with(1234) + + def test_modeled_windows_cleanup_parent_rejects_path_replacement(self) -> None: + parent_path = os.path.join(self.godot_dir, "cleanup-parent-path") + parked_path = os.path.join(self.godot_dir, "cleanup-parent-parked") + os.mkdir(parent_path) + original_file = os.path.join(parent_path, "owned.txt") + with open(original_file, "wb") as output_file: + output_file.write(b"original parent content\n") + parent_stat = os.lstat(parent_path) + parent_identity = parent_stat.st_dev, parent_stat.st_ino + kernel32 = MagicMock() + kernel32.CreateFileW.return_value = 1234 + kernel32.GetFileType.return_value = ( + included_files_module._WINDOWS_FILE_TYPE_DISK + ) + kernel32.CloseHandle.return_value = 1 + binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None + + try: + with ( + patch.object(included_files_module.os, "name", "nt"), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_api", + return_value=kernel32, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_identity", + return_value=parent_identity, + ), + patch.object( + included_files_module, + "_windows_included_cleanup_parent_attributes", + return_value=( + included_files_module._WINDOWS_FILE_ATTRIBUTE_DIRECTORY + ), + ), + ): + binding = ( + included_files_module._WindowsIncludedCleanupParentBinding.open( + parent_path, + parent_identity, + ) + ) + os.rename(parent_path, parked_path) + os.mkdir(parent_path) + with self.assertRaisesRegex( + OSError, + "cleanup parent changed", + ): + binding.verify() + finally: + if binding is not None: + binding.close() + if os.path.isdir(parent_path): + os.rmdir(parent_path) + if os.path.isdir(parked_path): + os.rename(parked_path, parent_path) + + with open(original_file, "rb") as input_file: + self.assertEqual(input_file.read(), b"original parent content\n") + + @unittest.skipUnless(os.name == "nt", "requires native Windows semantics") + def test_native_windows_cleanup_parent_binding_blocks_relocation(self) -> None: + parent_path = os.path.join(self.godot_dir, "cleanup-parent-native") + parked_path = os.path.join(self.godot_dir, "cleanup-parent-moved") + os.mkdir(parent_path) + parent_stat = os.lstat(parent_path) + parent_identity = parent_stat.st_dev, parent_stat.st_ino + binding = ( + included_files_module._WindowsIncludedCleanupParentBinding.open( + parent_path, + parent_identity, + ) + ) + try: + with self.assertRaises(OSError): + os.rename(parent_path, parked_path) + binding.verify() + finally: + binding.close() + if os.path.isdir(parked_path): + os.rename(parked_path, parent_path) + + os.rename(parent_path, parked_path) + self.assertFalse(os.path.lexists(parent_path)) + os.rename(parked_path, parent_path) + + @unittest.skipUnless(os.name == "nt", "requires native Windows semantics") + def test_native_windows_cleanup_parent_binding_rejects_junction(self) -> None: + junction_path = os.path.join( + self.godot_dir, + "cleanup-parent-junction", + ) + target_path = self._make_native_windows_junction_target( + "cleanup-parent-binding" + ) + self._make_native_windows_junction(junction_path, target_path) + try: + junction_stat = os.lstat(junction_path) + with self.assertRaisesRegex(OSError, "cleanup parent changed"): + included_files_module._WindowsIncludedCleanupParentBinding.open( + junction_path, + (junction_stat.st_dev, junction_stat.st_ino), + ) + self._assert_native_windows_junction_sentinel(target_path) + finally: + self._remove_native_windows_junction(junction_path) + def test_windows_native_paths_use_extended_length_namespace(self) -> None: cases = { r"C:\projects\game": r"\\?\C:\projects\game", @@ -6491,6 +8143,12 @@ def publish_then_fail( *, source_parent_identity: tuple[int, int] | None = None, destination_parent_identity: tuple[int, int] | None = None, + windows_source_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: nonlocal publication_failed original_move( @@ -6499,6 +8157,10 @@ def publish_then_fail( expected_identity, source_parent_identity=source_parent_identity, destination_parent_identity=destination_parent_identity, + windows_source_parent_binding=windows_source_parent_binding, + windows_destination_parent_binding=( + windows_destination_parent_binding + ), ) if destination == final_registry_path and not publication_failed: publication_failed = True @@ -9143,6 +10805,12 @@ def fail_staged_root_publish( *, source_parent_identity: tuple[int, int] | None = None, destination_parent_identity: tuple[int, int] | None = None, + windows_source_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: if ( destination == final_root_path @@ -9155,6 +10823,10 @@ def fail_staged_root_publish( expected_identity, source_parent_identity=source_parent_identity, destination_parent_identity=destination_parent_identity, + windows_source_parent_binding=windows_source_parent_binding, + windows_destination_parent_binding=( + windows_destination_parent_binding + ), ) with patch.object( @@ -9304,6 +10976,12 @@ def publish_registry_then_fail( *, source_parent_identity: tuple[int, int] | None = None, destination_parent_identity: tuple[int, int] | None = None, + windows_source_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: nonlocal publication_failed original_move( @@ -9312,6 +10990,10 @@ def publish_registry_then_fail( expected_identity, source_parent_identity=source_parent_identity, destination_parent_identity=destination_parent_identity, + windows_source_parent_binding=windows_source_parent_binding, + windows_destination_parent_binding=( + windows_destination_parent_binding + ), ) if destination == final_registry_path and not publication_failed: publication_failed = True @@ -9389,6 +11071,12 @@ def publish_registry_then_cancel( *, source_parent_identity: tuple[int, int] | None = None, destination_parent_identity: tuple[int, int] | None = None, + windows_source_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, + windows_destination_parent_binding: ( + included_files_module._WindowsIncludedCleanupParentBinding | None + ) = None, ) -> None: nonlocal cancellation_injected original_move( @@ -9397,6 +11085,10 @@ def publish_registry_then_cancel( expected_identity, source_parent_identity=source_parent_identity, destination_parent_identity=destination_parent_identity, + windows_source_parent_binding=windows_source_parent_binding, + windows_destination_parent_binding=( + windows_destination_parent_binding + ), ) if destination == final_registry_path and not cancellation_injected: cancellation_injected = True diff --git a/tests/test_version.py b/tests/test_version.py index 0162610..8fd432c 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -11,8 +11,8 @@ class TestVersion(unittest.TestCase): - def test_release_version_is_0_7_64(self) -> None: - self.assertEqual(get_version(), "0.7.64") + def test_release_version_is_0_7_65(self) -> None: + self.assertEqual(get_version(), "0.7.65") def test_release_surfaces_match_source_version(self) -> None: version_source = (PROJECT_ROOT / "src" / "version.py").read_text(