From a63506ab2494d46e9f0f8f849324d5f7148792bc Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:11:51 -0400 Subject: [PATCH 01/59] Complete package-mode generation contract Add installed-project deployment with an explicit validated first-party application wheel while preserving source-compatible deployments. Use role-aware staging and dirty-input provenance protection, isolate source and package launches, extend artifact-aware runtime rollback and fast-path state, and support deterministic package-mode releases. --- README.md | 53 +- docs/architecture.md | 37 +- .../analysis/assessor.py | 1 + .../analysis/inventory.py | 23 +- .../analysis/metadata.py | 25 + .../analysis/resources.py | 26 +- .../analysis/runtime_assumptions.py | 41 +- .../backends/uv_managed.py | 1 + src/python_deployment_builder/cli.py | 55 +- .../generation/artifacts.py | 393 ++++++++- .../generation/generator.py | 263 +++++- .../generation/manifest.py | 32 +- .../generation/structural.py | 30 + src/python_deployment_builder/models.py | 28 + .../packaging/packager.py | 1 + .../packaging/reports.py | 15 + .../packaging/smoke.py | 6 + .../planning/planner.py | 138 ++- .../templates/windows_uv/diagnostics.py | 7 + .../templates/windows_uv/launch.py | 4 +- .../templates/windows_uv/manage.py | 23 +- .../templates/windows_uv/runtime_common.py | 18 + .../validation/runtime.py | 76 +- .../validation/static.py | 108 ++- tests/test_analysis_scope.py | 110 ++- tests/test_generation.py | 796 +++++++++++++++++- tests/test_launch_contract.py | 31 + tests/test_planning.py | 108 +++ tests/test_validation.py | 239 +++++- 29 files changed, 2544 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 0a55ac4..853dc8f 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ pdbuilder plan C:\path\to\repository --online --extra map pdbuilder generate C:\path\to\repository --dry-run pdbuilder generate C:\path\to\repository --prepare-lock --bootstrap bundled_uv pdbuilder generate C:\path\to\repository --bootstrap bundled_uv --system-certs +pdbuilder generate C:\path\to\repository --application-wheel C:\wheels\application.whl pdbuilder validate C:\staging\deployment-kit --static pdbuilder validate C:\staging\deployment-kit --runtime pdbuilder validate C:\staging\deployment-kit --runtime --dry-run @@ -140,6 +141,10 @@ Generation and `all` still require an authoritative entry point and all normal r Assessment and planning JSON use the current 1.1 output schema. Commands analyze a repository and construct current models; they do not load arbitrary historical assessment/plan JSON as workflow inputs. Deployment-kit and release manifests have separate schemas and compatibility checks. +Pre-M6.1 source-mode deployment manifests remain valid because new manifest fields have safe +defaults, and those kits can still be statically validated and packaged. Package mode was introduced +with M6.1 and deliberately requires a validated `application_artifact`; a package-mode manifest that +omits it is rejected with an explicit artifact-contract failure rather than a model `KeyError`. See [docs/architecture.md](docs/architecture.md) for component boundaries and the implementation sequence. @@ -192,11 +197,14 @@ shared; environments, logs, metadata, repair, and deletion scope remain applicat ## Generating a deployment kit -Generation uses the `DeploymentPlan` as its source of truth. The default output is a separate -staging directory containing a sanitized copy of the source application plus the deployment -layer. Git/IDE state, `.env` files, tests, prior output/cache data, and credential files are not -copied. Pointing `--output-dir` at the repository root is explicit in-place generation; unknown or -locally modified file collisions are refused. +Generation uses the `DeploymentPlan` and role-aware assessment inventory as its source of truth; +it does not broadly copy the repository. Source mode stages application source, immutable runtime +resources, and required backend metadata. Deployment support, tests, documentation, examples, +development tooling, mutable state, ignored/untracked local material, unknown files, and Python +runtime caches are not staged by default. A Git source with a recorded revision is further limited +to tracked files; ordinary directories and materialized archives remain supported through the safe +inventory path. Pointing `--output-dir` at the repository root is explicit in-place generation; +unknown or locally modified file collisions are refused. ```text Run .bat @@ -214,6 +222,7 @@ deployment\ launch.py diagnostics.py wheels\ (approved artifacts only) + application\ (validated first-party wheel in package mode) README-deployment.txt ``` @@ -231,12 +240,41 @@ using PowerShell. Neither mode modifies PATH or runs uv self-update. Windows certificate store for corporate trust roots without disabling TLS validation, changing certificate stores, or adding insecure hosts. +### Source and package modes + +Source mode does not install the project. Its authoritative metadata entry-point module must be +structurally importable from the planned staged source roots. A source-tree entry-point candidate +is diagnostic and never replaces the authoritative installed entry point. + +Package mode is selected when the installed namespace cannot be provided by the physical source +layout, or when package policy otherwise requires installation. The developer supplies an exact +first-party wheel with `--application-wheel`. PDB validates its distribution, version, wheel tags, +entry-point metadata/module, declared package data, pure-Python structure, and SHA-256 before +staging it. End-user setup synchronizes locked third-party dependencies with `--no-install-project` +and `--no-build`, installs approved exceptional dependency wheels, installs the first-party wheel +with `uv pip install --no-deps --no-build`, and runs `uv pip check`. It never builds the application +on the end-user machine. The application-wheel provenance is distinct from dependency +`--artifact` inputs and participates in deployment/stale-state fingerprints. PDB records the exact +wheel filename, distribution, version, contents policy, and SHA-256 separately from the assessed +source revision. Ordinary wheel metadata does not cryptographically prove that the wheel was built +from that revision; that relationship remains a developer-controlled build/release assertion. +Changing the exact wheel bytes changes runtime state. Renaming byte-identical wheel bytes does not +rebuild an otherwise identical environment, although the staged filename and release ZIP change. + +Existing application deployment-support files are inventory evidence, not standalone runtime +inputs. They are not copied merely because they are launchers or live at the repository root. + ### Developer preparation Every real generation runs pinned `uv lock --check`. A stale lock stops generation without rewriting it. A missing lock also stops unless `--prepare-lock` explicitly authorizes a local repository mutation; URL inputs cannot use that option. The builder then runs `uv lock --python `, checks the result, reports the changed `uv.lock`, and never commits it. +For Git inputs with a recorded revision, generation also refuses modified tracked deployment +inputs (metadata, lockfile, application source, and runtime resources) rather than labeling their +working-tree bytes as clean revision provenance. Unrelated tracked documentation and harmless +untracked files do not trigger that gate. Untracked and ignored runtime-looking files are not +silently staged; required runtime material must be tracked or supplied by the application wheel. The intended source-control workflow is to create a deployment-preparation branch, assess and plan, explicitly prepare missing metadata, generate outside the repository, validate the staged @@ -414,8 +452,9 @@ do not make live OpenAI API calls or execute target code. explicitly selected optional dependencies. A present `uv.lock` is statically traversed for the selected Windows/Python/extra graph and locked wheel/source-distribution policy. - Private GitHub repositories are out of scope for the MVP. -- Current generation is focused on the two source-mode reference applications. General package - deployment and a complete offline Python/package bundle remain future work. +- Current generation supports source-compatible projects and install-oriented projects that supply + a validated pure-Python first-party wheel. Native first-party wheels and a complete offline + Python/package bundle remain future work. - The `uv_managed` backend is pilot ready, not universally compatible with every Windows Python application. Developer-side runtime validation is not a substitute for application-specific Standard User GUI, external-service, and organizational network-policy testing. diff --git a/docs/architecture.md b/docs/architecture.md index 9bdacd3..968f0fa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -60,8 +60,10 @@ metadata, lockfiles, and ignore policy. It deliberately excludes ordinary tests, examples, deployment support, and ignored/local files unless stronger runtime evidence promotes a path. It is neither a whole-repository identity nor the generated kit integrity mechanism. Optional Git revision records source provenance, while generated-file hashes cover every staged file. -Analysis roles and source-copy policy remain separate so a path excluded from import analysis is -not automatically omitted from a source deployment. +Generation consumes the role-aware inventory: source mode normally stages only application source, +runtime resources, and required backend metadata. Package mode stages backend metadata, generated +runtime material, approved dependency artifacts, and a validated first-party application wheel. +Deployment support and unknown/local material are not silently promoted into either mode. Planning retains all safely detected blocker codes. Its single readiness state is a primary summary selected in this order: blocking assessment risk, missing authoritative entry point, selected @@ -69,6 +71,10 @@ developer artifact, missing lockfile, then lock verification. Online compatibili legacy requirement groups remains informational and cannot create a selected-artifact blocker. Assessment/plan schema 1.1 reports are current outputs rather than reloadable workflow inputs; commands recompute them from repositories. Deployment and release manifests use their own schemas. +Pre-M6.1 source-mode manifests remain validation- and packaging-compatible through optional defaults +for new fields. Package mode begins with M6.1 and requires a validated first-party application +artifact; a package-mode manifest without one is an explicit contract failure, not an implicit +backward-compatible source deployment. ## Windows uv-managed policy direction @@ -91,6 +97,16 @@ fingerprints. Development extras are excluded by default. Source-only locked pac developer-artifact requirement; assessment and planning never execute their build hooks, and an end-user environment never performs an unexpected source build. +Source mode is permitted only when the authoritative standardized entry-point module is +structurally importable from the planned source roots without installing the project. If metadata +uses an installed namespace that differs from its physical source namespace, PDB selects package +mode when package metadata/resources are install-safe; genuine source-only constraints instead +produce a typed deployment-mode conflict. Package mode requires a developer-supplied first-party +wheel. Its name, version, tags, entry point, module, package data, pure-Python contents, and hash are +validated, and its provenance is recorded separately from exceptional dependency artifacts and +from the assessed source revision. The exact supplied wheel bytes are authoritative; ordinary +wheel metadata alone is not evidence that those bytes were built from the recorded revision. + Normal launch compares schema, selected Python, pinned uv, project metadata, lockfile, selected extras, approved artifact hashes, environment path, and prior verification fingerprints. Matching state takes a quick launch-critical path. Changed state triggers a scoped rebuild at the final @@ -113,8 +129,18 @@ policy back into assessment. Configuration is strict and optional; CLI inputs re Generated Python helpers run with `-B -E -s`, not `-I`: Python avoids bytecode writes, ignores user-controlled `PYTHON*` interpreter configuration, and excludes user site-packages while retaining the script directory so the standalone helpers can import their generated siblings. -`launch.py` inserts only the manifest's source roots immediately before controlled entry-point -import; it does not rely on `PYTHONPATH`. +In source mode, `launch.py` inserts only the manifest's source roots immediately before controlled +entry-point import; it does not rely on `PYTHONPATH`. In package mode it adds no application source +root and imports the authoritative target from the managed environment after the exact first-party +wheel has been installed. + +For a Git source with a recorded revision, generation intersects role-approved staging with +tracked paths and blocks when any selected tracked deployment input differs from `HEAD`. It does +not elevate ignored, untracked, documentation, example, test, deployment-support, or mutable-state +roles merely because a filename looks executable or resource-like. Non-Git directories and safely +materialized archives use the same role inventory without requiring Git; eligible application +source and runtime resources are staged directly in source mode, while package mode stages all +application runtime content only from its validated first-party wheel. The online bootstrap's localized `certutil.exe` handling searches structurally for one 64-digit hexadecimal value rather than parsing English headings. Missing, blocked, download-failing, and @@ -144,6 +170,9 @@ TLS validation and organizational controls are never bypassed. resource and mutable-state evidence, diagnostic entry-point candidates, import contexts, existing deployment/vendor-runtime inventory, typed teaching guidance, and plans that report all safe blockers before refusing generation. +6.1. Generation contract: source-entry-point compatibility gating, explicit validated first-party + wheels for package mode, installed-target launch, application-artifact provenance, and + role-aware tracked-file staging with generic runtime-cache exclusion. The first target remains urgent: general abstractions are added only when they directly support the Windows + uv-managed deployment or a clear future backend boundary. diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index 401eeb8..027b13b 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -88,6 +88,7 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen root, metadata.project.source_roots, application_files=inventory.application_files, + project=metadata.project, ) apply_resource_roles(inventory.items, resources) apply_mutable_state_roles(inventory.items, resources, runtime.write_locations) diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index a76ddba..9c6a8c6 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -48,6 +48,16 @@ "node_modules", "venv", } +PYTHON_CACHE_DIRECTORY = re.compile(r"^__pycache__(?:\s*\(\d+\))?$", re.IGNORECASE) +CONVENTIONAL_RUNTIME_RESOURCE_KINDS = { + "assets", + "configuration", + "icons", + "profiles", + "prompts", + "schemas", + "templates", +} DOCUMENTATION_SUFFIXES = {".md", ".rst"} DEVELOPMENT_FILENAMES = { ".gitignore", @@ -124,7 +134,12 @@ def _classify( ) -> tuple[RepositoryFileRole, str]: parts = {part.lower() for part in relative.parts[:-1]} name = relative.name.lower() - if ignored or parts & LOCAL_DIRECTORIES or name.endswith((".pyc", ".pyo")): + if ( + ignored + or parts & LOCAL_DIRECTORIES + or any(PYTHON_CACHE_DIRECTORY.fullmatch(part) for part in parts) + or name.endswith((".pyc", ".pyo")) + ): return ( RepositoryFileRole.IGNORED_OR_LOCAL, "Excluded by repository ignore/local-state policy.", @@ -173,7 +188,10 @@ def inventory_repository(root: Path, source_roots: list[str]) -> InventoryResult for directory_name in sorted(directory_names): relative = relative_current / directory_name posix = relative.as_posix() - local = directory_name.lower() in LOCAL_DIRECTORIES + local = ( + directory_name.lower() in LOCAL_DIRECTORIES + or PYTHON_CACHE_DIRECTORY.fullmatch(directory_name) is not None + ) if local: items.append( RepositoryFileInventoryItem( @@ -337,6 +355,7 @@ def apply_resource_roles( resource.path.rstrip("/") for resource in resources if resource.status == FindingStatus.DETECTED + or resource.kind in CONVENTIONAL_RUNTIME_RESOURCE_KINDS } for item in items: normalized = item.path.rstrip("/") diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 022ff6f..624ae2b 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -269,6 +269,9 @@ def inspect_metadata(root: Path) -> MetadataResult: requires_python: str | None = None ruff_target: str | None = None source_roots: list[str] = [] + packages: list[str] = [] + package_directories: dict[str, str] = {} + package_data: dict[str, list[str]] = {} layout = "unknown" python_evidence: list[Evidence] = [] @@ -406,6 +409,25 @@ def inspect_metadata(root: Path) -> MetadataResult: if isinstance(name, str) and isinstance(target, str): entry_points.append(_entry_point(root, pyproject_path, name, target, "scripts")) setuptools = tool.get("setuptools") if isinstance(tool.get("setuptools"), dict) else {} + configured_packages = setuptools.get("packages") + if isinstance(configured_packages, list): + packages = [value for value in configured_packages if isinstance(value, str)] + configured_package_dirs = setuptools.get("package-dir") + if isinstance(configured_package_dirs, dict): + package_directories = { + name: path + for name, path in configured_package_dirs.items() + if isinstance(name, str) and isinstance(path, str) + } + if isinstance(package_directories.get(""), str): + source_roots = source_roots or [package_directories[""]] + configured_package_data = setuptools.get("package-data") + if isinstance(configured_package_data, dict): + package_data = { + name: [pattern for pattern in patterns if isinstance(pattern, str)] + for name, patterns in configured_package_data.items() + if isinstance(name, str) and isinstance(patterns, list) + } package_find = ( setuptools.get("packages", {}).get("find", {}) if isinstance(setuptools.get("packages"), dict) @@ -632,6 +654,9 @@ def inspect_metadata(root: Path) -> MetadataResult: build_backend=build_backend, layout=layout, source_roots=source_roots, + packages=packages, + package_directories=package_directories, + package_data=package_data, entry_points=entry_points, optional_dependency_groups=optional_groups, legacy_dependency_groups=legacy_groups, diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index b35f85b..298a12f 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import fnmatch from collections import defaultdict from pathlib import Path @@ -11,9 +12,30 @@ ConfigurationRequirement, Evidence, FindingStatus, + PackagingAssessment, ResourceRequirement, ) + +def _declared_package_data_path(relative: str, project: PackagingAssessment | None) -> bool: + """Return true when setuptools metadata installs this physical resource path.""" + + if project is None: + return False + candidate = Path(relative) + for package, patterns in project.package_data.items(): + physical = project.package_directories.get(package) + if physical is None: + base = project.package_directories.get("") + physical = str(Path(base or ".") / Path(*package.split("."))) + try: + package_relative = candidate.relative_to(Path(physical)).as_posix() + except ValueError: + continue + if any(fnmatch.fnmatchcase(package_relative, pattern) for pattern in patterns): + return True + return False + RESOURCE_DIRECTORIES = { "assets": "assets", "config": "configuration", @@ -536,6 +558,7 @@ def inspect_resources( source_roots: list[str], *, application_files: list[Path] | None = None, + project: PackagingAssessment | None = None, ) -> tuple[list[ResourceRequirement], list[ConfigurationRequirement]]: literals, access_modes, unresolved = _literal_evidence(root, source_roots, application_files) resources: list[ResourceRequirement] = [] @@ -549,7 +572,8 @@ def inspect_resources( access_mode=_merge_access(access_modes[relative]), packaging_status=( "packaged" - if exists and relative.startswith("src/") + if exists + and _declared_package_data_path(relative, project) else "repository_adjacent" if exists else "unknown" diff --git a/src/python_deployment_builder/analysis/runtime_assumptions.py b/src/python_deployment_builder/analysis/runtime_assumptions.py index 801a69e..5b2945c 100644 --- a/src/python_deployment_builder/analysis/runtime_assumptions.py +++ b/src/python_deployment_builder/analysis/runtime_assumptions.py @@ -81,7 +81,7 @@ def _write_classification(expression: str) -> tuple[str, FindingStatus]: return "user_local", FindingStatus.INFERRED if any( value in lowered - for value in ("repo_root", "_repo_root", "__file__", "request_cache", "cache") + for value in ("repo_root", "_repo_root", "__file__") ): return "project_local", FindingStatus.INFERRED if any(value in lowered for value in ("output_dir", "destination", "selected", "run_dir")): @@ -105,14 +105,41 @@ def _platforms_for(category: str, name: str) -> list[str]: return ["all"] +def _simple_function_returns(tree: ast.AST) -> dict[str, str]: + """Summarize only wrappers with one statically expressible return value.""" + + summaries: dict[str, str] = {} + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + returns = [item for item in ast.walk(node) if isinstance(item, ast.Return)] + if len(returns) == 1 and returns[0].value is not None: + summaries[node.name] = _expression(returns[0].value) + return summaries + + class _RuntimeVisitor(ast.NodeVisitor): - def __init__(self, relative: str, source_lines: list[str]) -> None: + def __init__( + self, + relative: str, + source_lines: list[str], + function_returns: dict[str, str] | None = None, + ) -> None: self.relative = relative self.lines = source_lines self.runtime: dict[tuple[str, str], list[Evidence]] = defaultdict(list) self.config: dict[str, list[Evidence]] = defaultdict(list) self.writes: dict[tuple[str, str], list[Evidence]] = defaultdict(list) self.assignments: dict[str, str] = {} + self.function_returns = function_returns or {} + + def _assigned_expression(self, value: ast.expr) -> str: + expression = _expression(value) + if isinstance(value, ast.Call): + function_name = _qualified_name(value.func).split(".")[-1] + if returned := self.function_returns.get(function_name): + expression = f"{expression} -> {returned}" + return expression def _evidence(self, node: ast.AST, detail: str) -> Evidence: line = getattr(node, "lineno", None) @@ -156,7 +183,7 @@ def visit_Name(self, node: ast.Name) -> None: # noqa: N802 ) def visit_Assign(self, node: ast.Assign) -> None: # noqa: N802 - value = _expression(node.value) + value = self._assigned_expression(node.value) for target in node.targets: if isinstance(target, ast.Name): target_name = target.id @@ -178,7 +205,7 @@ def visit_Assign(self, node: ast.Assign) -> None: # noqa: N802 def visit_AnnAssign(self, node: ast.AnnAssign) -> None: # noqa: N802 if node.value is not None: - value = _expression(node.value) + value = self._assigned_expression(node.value) if isinstance(node.target, ast.Name): self.assignments[node.target.id] = value elif isinstance(node.target, ast.Attribute): @@ -298,7 +325,11 @@ def scan_runtime_assumptions( except (OSError, SyntaxError, UnicodeError) as exc: parse_errors.append(f"{relative}: {exc}") continue - visitor = _RuntimeVisitor(relative, source.splitlines()) + visitor = _RuntimeVisitor( + relative, + source.splitlines(), + function_returns=_simple_function_returns(tree), + ) visitor.visit(tree) for key, evidence in visitor.runtime.items(): runtime[key].extend(evidence) diff --git a/src/python_deployment_builder/backends/uv_managed.py b/src/python_deployment_builder/backends/uv_managed.py index c7a1b88..152af87 100644 --- a/src/python_deployment_builder/backends/uv_managed.py +++ b/src/python_deployment_builder/backends/uv_managed.py @@ -119,6 +119,7 @@ def build_plan( "--python", rf"{paths.environment_path}\Scripts\python.exe", "--no-deps", + "--no-build", "%APPLICATION_WHEEL%", ], working_directory="%PROJECT_ROOT%", diff --git a/src/python_deployment_builder/cli.py b/src/python_deployment_builder/cli.py index 7f5768d..dbdeb61 100644 --- a/src/python_deployment_builder/cli.py +++ b/src/python_deployment_builder/cli.py @@ -116,6 +116,11 @@ def build_parser() -> argparse.ArgumentParser: metavar="DISTRIBUTION=WHEEL", help="supply an exact approved wheel for a typed developer-artifact requirement", ) + generate.add_argument( + "--application-wheel", + type=Path, + help="supply the validated first-party application wheel required by package mode", + ) generate.add_argument("--dry-run", action="store_true") validate = commands.add_parser("validate", help="validate a generated deployment kit") validate.add_argument("deployment_kit", type=Path) @@ -163,7 +168,18 @@ def build_parser() -> argparse.ArgumentParser: all_command.add_argument( "--system-certs", action=argparse.BooleanOptionalAction, default=None ) - all_command.add_argument("--artifact", action="append", default=[]) + all_command.add_argument( + "--artifact", + action="append", + default=[], + metavar="DISTRIBUTION=WHEEL", + help="supply an exact approved exceptional dependency wheel", + ) + all_command.add_argument( + "--application-wheel", + type=Path, + help="supply the validated first-party application wheel required by package mode", + ) all_command.add_argument("--version") all_command.add_argument("--runtime-validation", action="store_true") all_command.add_argument("--runtime-root", type=Path) @@ -255,6 +271,7 @@ def run_generate( system_certs: bool | None, prepare_lock: bool, artifact_values: list[str], + application_wheel: Path | None, dry_run: bool, ) -> int: with materialize_repository(repository_value) as repository: @@ -295,10 +312,28 @@ def run_generate( bootstrap_mode=settings.bootstrap, system_certs=settings.system_certs, artifact_values=artifact_values, + application_wheel=application_wheel, dry_run=dry_run, ) preview = result.preview print(f"Deployment readiness: {preview.readiness_before}") + print(f"Deployment mode: {preview.deployment_mode}") + print( + "Source roots: " + + ( + ", ".join(preview.source_roots) + if preview.source_roots + else "none (installed-project mode)" + ) + ) + if preview.application_wheel_required: + print("Application wheel: required (supply --application-wheel PATH)") + elif preview.application_artifact is not None: + print( + "Application wheel: " + f"{preview.application_artifact.filename} " + f"(SHA-256 {preview.application_artifact.sha256})" + ) print(f"Bootstrap: {preview.bootstrap_mode}") print(f"System certificates: {'enabled' if preview.system_certs else 'disabled'}") print("Developer preparation:") @@ -455,6 +490,7 @@ def run_all( bootstrap_mode: str | None, system_certs: bool | None, artifact_values: list[str], + application_wheel: Path | None, version: str | None, runtime_validation: bool, runtime_root: Path | None, @@ -487,7 +523,6 @@ def run_all( distribution_root = workflow_root / "distribution" print("ASSESS") - write_assessment_reports(assessment, reports_root) print(f" {assessment.rating.value}: {assessment.rating_summary}") print("PLAN") @@ -498,17 +533,20 @@ def run_all( selected_extras=list(settings.extras), repository_root=repository.root, ) - write_deployment_plan_reports(plan, reports_root) print(f" Readiness: {plan.readiness.state}") for blocker in plan.readiness.blockers: print(f" Blocker: {blocker}") if plan.entry_point is None: + write_assessment_reports(assessment, reports_root) + write_deployment_plan_reports(plan, reports_root) print( " Stop: declare an authoritative [project.gui-scripts] or " "[project.scripts] entry point before generation." ) return 2 if plan.risk_gate.outcome == "block": + write_assessment_reports(assessment, reports_root) + write_deployment_plan_reports(plan, reports_root) print(" Stop: planning blockers must be resolved before generation.") return 2 if plan.lockfile.status == "developer_generation_required": @@ -527,6 +565,12 @@ def run_all( f"{requested}. Supply each explicitly with --artifact DISTRIBUTION=WHEEL." ) return 2 + if plan.deployment_mode == "package" and application_wheel is None: + print( + " Stop: package mode requires a validated first-party wheel. " + "Supply it with --application-wheel PATH." + ) + return 2 print("GENERATE") generated = generate_deployment_kit( @@ -539,9 +583,12 @@ def run_all( bootstrap_mode=settings.bootstrap, system_certs=settings.system_certs, artifact_values=artifact_values, + application_wheel=application_wheel, dry_run=False, ) print(f" Deployment kit: {generated.output_directory}") + write_assessment_reports(assessment, reports_root) + write_deployment_plan_reports(plan, reports_root) print("STATIC VALIDATE") static_report = validate_static_kit(kit_root) @@ -597,6 +644,7 @@ def main(argv: list[str] | None = None) -> int: system_certs=arguments.system_certs, prepare_lock=arguments.prepare_lock, artifact_values=arguments.artifact, + application_wheel=arguments.application_wheel, dry_run=arguments.dry_run, ) if arguments.command == "validate": @@ -624,6 +672,7 @@ def main(argv: list[str] | None = None) -> int: bootstrap_mode=arguments.bootstrap, system_certs=arguments.system_certs, artifact_values=arguments.artifact, + application_wheel=arguments.application_wheel, version=arguments.version, runtime_validation=arguments.runtime_validation, runtime_root=arguments.runtime_root, diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index f0cf92e..6df2934 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -2,19 +2,32 @@ from __future__ import annotations +import configparser import csv +import fnmatch import io +import re import stat import zipfile from email.parser import BytesParser +from email.policy import default from pathlib import Path, PurePosixPath from packaging.utils import canonicalize_name, parse_wheel_filename from python_deployment_builder.generation.acquisition import PreparationError, sha256_file -from python_deployment_builder.models import ApprovedArtifact, DeploymentPlan +from python_deployment_builder.models import ( + ApplicationArtifact, + ApprovedArtifact, + DeploymentPlan, + RepositoryAssessment, +) from python_deployment_builder.planning.index import wheel_matches +FORBIDDEN_APPLICATION_TEXT = (b"powershell.exe", b"pwsh.exe", b"executionpolicy") +WINDOWS_DEVELOPER_PATH = re.compile(rb"(?i)(?:[a-z]:\\(?:users|home)\\[^\r\n\"]+)") +SECRET_MEMBER_NAMES = {".env", "credentials.json", "secrets.json"} + def parse_artifact_argument(value: str) -> tuple[str, Path]: name, separator, raw_path = value.partition("=") @@ -23,22 +36,179 @@ def parse_artifact_argument(value: str) -> tuple[str, Path]: return canonicalize_name(name.strip()), Path(raw_path.strip()).expanduser() +def _normalized_wheel_path(value: str) -> str: + if ( + not value + or "\x00" in value + or "\\" in value + or value.startswith("/") + or re.match(r"^[A-Za-z]:", value) + ): + raise PreparationError(f"Wheel contains an unsafe member: {value}") + path = PurePosixPath(value) + if not path.parts or ".." in path.parts: + raise PreparationError(f"Wheel contains an unsafe member: {value}") + return path.as_posix() + + def _safe_wheel_members(bundle: zipfile.ZipFile) -> list[zipfile.ZipInfo]: members = bundle.infolist() + seen: dict[str, str] = {} for member in members: - path = PurePosixPath(member.filename.replace("\\", "/")) + normalized = _normalized_wheel_path(member.filename) file_type = (member.external_attr >> 16) & 0o170000 if ( - path.is_absolute() - or ".." in path.parts - or member.flag_bits & 0x1 + member.flag_bits & 0x1 or file_type == stat.S_IFLNK or member.file_size > 256 * 1024 * 1024 ): raise PreparationError(f"Wheel contains an unsafe member: {member.filename}") + collision_key = normalized.rstrip("/").casefold() + if previous := seen.get(collision_key): + raise PreparationError( + "Wheel contains duplicate or conflicting archive paths: " + f"{previous}, {member.filename}" + ) + seen[collision_key] = member.filename return members +def _member_map(members: list[zipfile.ZipInfo]) -> dict[str, zipfile.ZipInfo]: + return {_normalized_wheel_path(item.filename): item for item in members} + + +def _metadata_message(data: bytes, *, label: str, wheel: Path): + message = BytesParser(policy=default).parsebytes(data) + if message.defects: + raise PreparationError(f"Malformed {label} in wheel: {wheel.name}") + return message + + +def _validate_record( + bundle: zipfile.ZipFile, + members: dict[str, zipfile.ZipInfo], + record_name: str, + wheel: Path, +) -> None: + try: + rows = list( + csv.reader( + io.StringIO(bundle.read(members[record_name]).decode("utf-8")), + strict=True, + ) + ) + except (UnicodeDecodeError, csv.Error) as exc: + raise PreparationError(f"Malformed wheel RECORD: {wheel.name}") from exc + recorded: dict[str, tuple[str, str]] = {} + for row in rows: + if len(row) != 3 or not row[0]: + raise PreparationError(f"Malformed wheel RECORD: {wheel.name}") + normalized = _normalized_wheel_path(row[0]) + key = normalized.casefold() + if key in recorded: + raise PreparationError(f"Wheel RECORD contains duplicate paths: {wheel.name}") + recorded[key] = (row[1], row[2]) + actual = { + name.casefold() + for name, member in members.items() + if not member.is_dir() + and not name.endswith((".dist-info/RECORD.jws", ".dist-info/RECORD.p7s")) + } + recorded_names = set(recorded) + if actual - recorded_names: + raise PreparationError(f"Wheel RECORD is incomplete: {wheel.name}") + if recorded_names - actual: + raise PreparationError(f"Wheel RECORD references nonexistent files: {wheel.name}") + record_row = recorded.get(record_name.casefold()) + if record_row is None or record_row != ("", ""): + raise PreparationError( + f"Wheel RECORD must record itself with a blank hash and size: {wheel.name}" + ) + + +def _dist_info_members( + members: dict[str, zipfile.ZipInfo], wheel: Path +) -> tuple[str, str, str]: + metadata_names = [ + name + for name in members + if name.endswith(".dist-info/METADATA") and len(PurePosixPath(name).parts) == 2 + ] + if len(metadata_names) != 1: + raise PreparationError( + f"Wheel must contain exactly one dist-info/METADATA file: {wheel.name}" + ) + metadata_name = metadata_names[0] + dist_info = PurePosixPath(metadata_name).parent.as_posix() + return metadata_name, f"{dist_info}/WHEEL", f"{dist_info}/RECORD" + + +def _require_core_metadata(message, *, label: str, wheel: Path) -> tuple[str, str]: + metadata_versions = message.get_all("Metadata-Version", []) + names = message.get_all("Name", []) + versions = message.get_all("Version", []) + if ( + len(metadata_versions) != 1 + or not re.fullmatch(r"\d+(?:\.\d+)+", metadata_versions[0].strip()) + or len(names) != 1 + or not names[0].strip() + or len(versions) != 1 + or not versions[0].strip() + ): + raise PreparationError(f"Malformed {label} in wheel: {wheel.name}") + return names[0].strip(), versions[0].strip() + + +def _require_wheel_metadata(message, *, wheel: Path) -> set[str]: + wheel_versions = message.get_all("Wheel-Version", []) + purelib = message.get_all("Root-Is-Purelib", []) + tags = {value.strip() for value in message.get_all("Tag", []) if value.strip()} + if ( + len(wheel_versions) != 1 + or not re.fullmatch(r"\d+(?:\.\d+)+", wheel_versions[0].strip()) + or len(purelib) != 1 + or purelib[0].strip().lower() not in {"true", "false"} + or not tags + ): + raise PreparationError(f"Malformed WHEEL metadata: {wheel.name}") + return tags + + +def _validate_application_security( + bundle: zipfile.ZipFile, + members: dict[str, zipfile.ZipInfo], + wheel: Path, +) -> None: + failures: list[str] = [] + for name, member in members.items(): + member_path = PurePosixPath(name) + if member.is_dir() or ".dist-info" in member_path.parts: + continue + lowered_name = member_path.name.lower() + if member_path.suffix.lower() == ".ps1" or lowered_name in SECRET_MEMBER_NAMES: + failures.append(name) + continue + if member_path.suffix.lower() not in {".py", ".bat", ".cmd", ".json", ".txt"}: + continue + data = bundle.read(member) + lowered = data.lower() + if ( + any(value in lowered for value in FORBIDDEN_APPLICATION_TEXT) + or WINDOWS_DEVELOPER_PATH.search(data) + or (b"setx" in lowered and b"path" in lowered) + or ( + b"program files" in lowered + and any(value in lowered for value in (b"write", b"mkdir", b"open(")) + ) + ): + failures.append(name) + if failures: + raise PreparationError( + "Application wheel content violates deployment security policy: " + + ", ".join(sorted(failures)) + ) + + def validate_approved_wheel( value: str, plan: DeploymentPlan, @@ -78,43 +248,29 @@ def validate_approved_wheel( try: with zipfile.ZipFile(path) as bundle: - members = _safe_wheel_members(bundle) - names = [item.filename for item in members] - if len(names) != len(set(names)) or bundle.testzip() is not None: - raise PreparationError( - f"Wheel archive entries are duplicated or corrupt: {path.name}" - ) - metadata_members = [ - item - for item in members - if PurePosixPath(item.filename).name == "METADATA" - and ".dist-info" in PurePosixPath(item.filename).parent.name - ] - if len(metadata_members) != 1: + members = _member_map(_safe_wheel_members(bundle)) + if bundle.testzip() is not None: raise PreparationError( - f"Wheel must contain exactly one dist-info/METADATA file: {path.name}" + f"Wheel archive entries are corrupt: {path.name}" ) - metadata_member = metadata_members[0] - dist_info = PurePosixPath(metadata_member.filename).parent - wheel_name = str(dist_info / "WHEEL") - record_name = str(dist_info / "RECORD") - if wheel_name not in names or record_name not in names: + metadata_name, wheel_name, record_name = _dist_info_members(members, path) + if wheel_name not in members or record_name not in members: raise PreparationError( f"Wheel is missing required WHEEL or RECORD metadata: {path.name}" ) - metadata = BytesParser().parsebytes(bundle.read(metadata_member)) - wheel_metadata = BytesParser().parsebytes(bundle.read(wheel_name)) - record_rows = list( - csv.reader(io.StringIO(bundle.read(record_name).decode("utf-8"))) + metadata = _metadata_message( + bundle.read(members[metadata_name]), label="METADATA", wheel=path ) - recorded_paths = {row[0] for row in record_rows if row} - if not set(names) <= recorded_paths: - raise PreparationError(f"Wheel RECORD is incomplete: {path.name}") + wheel_metadata = _metadata_message( + bundle.read(members[wheel_name]), label="WHEEL", wheel=path + ) + _validate_record(bundle, members, record_name, path) except zipfile.BadZipFile as exc: raise PreparationError(f"Malformed wheel archive: {path.name}") from exc - metadata_name = metadata.get("Name", "") - metadata_version = metadata.get("Version", "") + metadata_name, metadata_version = _require_core_metadata( + metadata, label="METADATA", wheel=path + ) if canonicalize_name(metadata_name) != requested_name: raise PreparationError( f"Wheel metadata name mismatch: expected {requested_name}, received {metadata_name}." @@ -124,7 +280,7 @@ def validate_approved_wheel( f"Wheel metadata version mismatch: expected {requirement.version}, " f"received {metadata_version}." ) - declared_tags = set(wheel_metadata.get_all("Tag", [])) + declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: raise PreparationError(f"Wheel tag metadata does not match its filename: {path.name}") @@ -148,3 +304,172 @@ def validate_artifact_set( if len(names) != len(set(names)): raise PreparationError("Each developer artifact requirement may be supplied only once.") return validated + + +def validate_application_wheel( + path: Path, + assessment: RepositoryAssessment, + plan: DeploymentPlan, +) -> tuple[ApplicationArtifact, Path]: + """Validate the explicit first-party wheel required by package mode.""" + + path = path.expanduser().resolve() + expected_name = canonicalize_name(assessment.project.distribution_name or "") + expected_version = assessment.project.version or "" + if not expected_name or not expected_version: + raise PreparationError( + "Package mode requires authoritative project distribution and version metadata." + ) + if not path.is_file() or path.suffix.lower() != ".whl": + raise PreparationError(f"Application wheel must be an existing wheel file: {path}") + try: + filename_name, filename_version, _build, filename_tags = parse_wheel_filename(path.name) + except ValueError as exc: + raise PreparationError(f"Malformed application wheel filename: {path.name}") from exc + if canonicalize_name(str(filename_name)) != expected_name: + raise PreparationError( + f"Application wheel name mismatch: expected {expected_name}, received " + f"{filename_name}." + ) + if str(filename_version) != expected_version: + raise PreparationError( + f"Application wheel version mismatch: expected {expected_version}, received " + f"{filename_version}." + ) + if not wheel_matches(path.name, plan.runtime.python_version, plan.runtime.architecture): + raise PreparationError( + f"Application wheel {path.name} is incompatible with CPython " + f"{plan.runtime.python_version} on Windows {plan.runtime.architecture}." + ) + + entry_point = plan.entry_point + if entry_point is None: + raise PreparationError("Package mode requires an authoritative application entry point.") + try: + with zipfile.ZipFile(path) as bundle: + members = _member_map(_safe_wheel_members(bundle)) + names = set(members) + if bundle.testzip() is not None: + raise PreparationError( + f"Application wheel entries are corrupt: {path.name}" + ) + metadata_name, wheel_name, record_name = _dist_info_members(members, path) + dist_info = PurePosixPath(metadata_name).parent + entry_points_name = f"{dist_info.as_posix()}/entry_points.txt" + for required in (wheel_name, record_name, entry_points_name): + if required not in names: + raise PreparationError( + f"Application wheel is missing {required}: {path.name}" + ) + metadata = _metadata_message( + bundle.read(members[metadata_name]), label="METADATA", wheel=path + ) + wheel_metadata = _metadata_message( + bundle.read(members[wheel_name]), label="WHEEL", wheel=path + ) + _validate_record(bundle, members, record_name, path) + + metadata_distribution, metadata_version = _require_core_metadata( + metadata, label="METADATA", wheel=path + ) + if canonicalize_name(metadata_distribution) != expected_name: + raise PreparationError("Application wheel METADATA distribution name is wrong.") + if metadata_version != expected_version: + raise PreparationError("Application wheel METADATA version is wrong.") + declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) + filename_tag_values = {str(item) for item in filename_tags} + if not declared_tags or not filename_tag_values <= declared_tags: + raise PreparationError( + f"Application wheel tag metadata does not match its filename: {path.name}" + ) + if wheel_metadata.get("Root-Is-Purelib", "").strip().lower() != "true": + raise PreparationError( + "Application wheels containing platform/native installation content require " + "explicit future project evidence and are not accepted by package mode." + ) + + cache_members = [ + name + for name in names + if name.lower().endswith((".pyc", ".pyo")) + or any( + re.fullmatch(r"__pycache__(?:\s*\(\d+\))?", part, re.IGNORECASE) + for part in PurePosixPath(name).parts + ) + ] + native_members = [ + name + for name in names + if PurePosixPath(name).suffix.lower() + in {".dll", ".pyd", ".so", ".dylib", ".exe", ".lib"} + ] + if cache_members: + raise PreparationError( + "Application wheel contains Python runtime cache files: " + + ", ".join(cache_members) + ) + if native_members: + raise PreparationError( + "Application wheel contains unexpected native binaries: " + + ", ".join(native_members) + ) + _validate_application_security(bundle, members, path) + + parser = configparser.ConfigParser(interpolation=None) + parser.optionxform = str + parser.read_string(bundle.read(members[entry_points_name]).decode("utf-8-sig")) + entry_group = "gui_scripts" if entry_point.kind == "gui" else "console_scripts" + installed_target = parser.get( + entry_group, + entry_point.name, + fallback="", + ).strip() + if installed_target != entry_point.target: + raise PreparationError( + "Application wheel entry point disagrees with authoritative metadata: " + f"expected {entry_point.name} = {entry_point.target}, received " + f"{installed_target or 'missing'}." + ) + module_path = PurePosixPath(*entry_point.module.split(".")) + module_candidates = { + str(module_path.with_suffix(".py")), + str(module_path / "__init__.py"), + } + if not module_candidates.intersection(names): + raise PreparationError( + "Application wheel does not contain its authoritative entry-point module: " + f"{entry_point.module}" + ) + + for package, patterns in assessment.project.package_data.items(): + if package == "*": + continue + package_prefix = PurePosixPath(*package.split(".")) + package_members = [] + for name in names: + member_path = PurePosixPath(name) + try: + package_members.append(str(member_path.relative_to(package_prefix))) + except ValueError: + continue + for pattern in patterns: + if not any(fnmatch.fnmatchcase(name, pattern) for name in package_members): + raise PreparationError( + "Application wheel is missing declared package data for " + f"{package}: {pattern}" + ) + except (zipfile.BadZipFile, UnicodeDecodeError, configparser.Error) as exc: + raise PreparationError(f"Malformed application wheel: {path.name}") from exc + + return ( + ApplicationArtifact( + distribution_name=expected_name, + version=expected_version, + filename=path.name, + sha256=sha256_file(path), + wheel_tags=sorted(str(item) for item in filename_tags), + entry_point_name=entry_point.name, + entry_point_target=entry_point.target, + ), + path, + ) diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index c566fb6..ece5b20 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -5,6 +5,8 @@ import hashlib import json import os +import re +import subprocess from pathlib import Path from packaging.utils import canonicalize_name @@ -16,8 +18,14 @@ acquire_pinned_uv, sha256_file, ) -from python_deployment_builder.generation.artifacts import validate_artifact_set -from python_deployment_builder.generation.manifest import build_deployment_manifest +from python_deployment_builder.generation.artifacts import ( + validate_application_wheel, + validate_artifact_set, +) +from python_deployment_builder.generation.manifest import ( + build_deployment_manifest, + source_roots_from_plan, +) from python_deployment_builder.generation.preparation import prepare_lockfile from python_deployment_builder.generation.structural import ( validate_rendered_files, @@ -29,61 +37,124 @@ safe_windows_label, ) from python_deployment_builder.models import ( + ApplicationArtifact, GeneratedArtifact, GenerationPreview, GenerationResult, + RepositoryFileRole, ) from python_deployment_builder.planning import create_deployment_plan GENERATED_INDEX = "deployment/generated-files.json" -EXCLUDED_DIRECTORIES = { - ".git", - ".idea", - ".cache", - ".venv", - ".pytest_cache", - ".mypy_cache", - ".ruff_cache", - "__pycache__", - "pdbuilder-output", -} -EXCLUDED_ROOT_DIRECTORIES = {".github", "deployment", "logs", "outputs", "state", "tests"} -EXCLUDED_SECRET_FILES = { - ".env", - "credentials.json", - "secrets.json", - "token.json", - ".pypirc", - "pip.ini", - "agents.md", - "codex_start_prompt.md", +PYTHON_CACHE_DIRECTORY = re.compile(r"^__pycache__(?:\s*\(\d+\))?$", re.IGNORECASE) +RUNTIME_ROLES = { + RepositoryFileRole.APPLICATION_SOURCE, + RepositoryFileRole.RUNTIME_RESOURCE, } -def _is_secret_file(path: Path) -> bool: - lowered = path.name.lower() - return lowered in EXCLUDED_SECRET_FILES or ( - lowered.startswith(".env.") and lowered != ".env.example" +def _is_runtime_cache(relative: Path) -> bool: + return relative.suffix.lower() in {".pyc", ".pyo"} or any( + PYTHON_CACHE_DIRECTORY.fullmatch(part) for part in relative.parts ) -def _source_files(repository_root: Path, *, include: bool) -> dict[str, bytes]: +def _git_tracked_paths(repository_root: Path, *, required: bool) -> set[str] | None: + repository_check = subprocess.run( + ["git", "-C", str(repository_root), "rev-parse", "--is-inside-work-tree"], + capture_output=True, + text=True, + check=False, + ) + if repository_check.returncode != 0 or repository_check.stdout.strip() != "true": + return None + result = subprocess.run( + ["git", "-C", str(repository_root), "ls-files", "-z"], + capture_output=True, + check=False, + ) + if result.returncode != 0: + if required: + raise PreparationError( + "Git revision provenance is known, but tracked deployment inputs could not " + "be enumerated. Generation stopped rather than staging local files." + ) + return None + return { + value.decode("utf-8", errors="surrogateescape").replace("\\", "/") + for value in result.stdout.split(b"\0") + if value + } + + +def _dirty_tracked_deployment_paths( + repository_root: Path, selected: set[str] +) -> list[str]: + result = subprocess.run( + ["git", "-C", str(repository_root), "diff", "--name-only", "-z", "HEAD", "--"], + capture_output=True, + check=False, + ) + if result.returncode != 0: + raise PreparationError( + "Git revision provenance is known, but tracked working-tree changes could not be " + "checked. Generation stopped rather than claiming clean-revision provenance." + ) + changed = { + value.decode("utf-8", errors="surrogateescape").replace("\\", "/") + for value in result.stdout.split(b"\0") + if value + } + return sorted(changed & selected) + + +def _staging_files( + repository_root: Path, + assessment, + plan, + *, + include: bool, + allow_missing_lock: bool = False, +) -> dict[str, bytes]: + """Stage inventory-approved runtime inputs, never a broad repository copy.""" + if not include: return {} + selected = {"pyproject.toml", "uv.lock"} + if plan.deployment_mode == "source": + selected.update( + item.path.rstrip("/") + for item in assessment.file_inventory + if item.role in RUNTIME_ROLES and not item.path.endswith("/") + ) + tracked = _git_tracked_paths( + repository_root, + required=assessment.repository.revision is not None, + ) + if tracked is not None: + selected.intersection_update(tracked) + if assessment.repository.revision is not None: + dirty = _dirty_tracked_deployment_paths(repository_root, selected) + if dirty: + raise PreparationError( + "Tracked deployment inputs differ from recorded source revision " + f"{assessment.repository.revision}: {', '.join(dirty)}. Commit or restore " + "those inputs before release-oriented generation." + ) files: dict[str, bytes] = {} - for path in sorted(repository_root.rglob("*")): - relative = path.relative_to(repository_root) - if ( - relative.parts[0] in EXCLUDED_ROOT_DIRECTORIES - or any(part in EXCLUDED_DIRECTORIES for part in relative.parts) - ): - continue - if _is_secret_file(path): + for relative_text in sorted(selected): + relative = Path(relative_text) + if _is_runtime_cache(relative): continue + path = repository_root / relative if path.is_symlink(): - raise PreparationError(f"Staging refuses repository symbolic links: {relative}") + raise PreparationError(f"Staging refuses repository symbolic links: {relative_text}") if path.is_file(): files[relative.as_posix()] = path.read_bytes() + required = {"pyproject.toml"} | (set() if allow_missing_lock else {"uv.lock"}) + missing = sorted(required - files.keys()) + if missing: + raise PreparationError("Required deployment input is missing: " + ", ".join(missing)) return files @@ -112,7 +183,12 @@ def _template_values(plan, bootstrap_mode: str, system_certs: bool) -> dict[str, } -def _planned_generated_paths(plan, bootstrap_mode: str, artifact_values: list[str]) -> list[str]: +def _planned_generated_paths( + plan, + bootstrap_mode: str, + artifact_values: list[str], + application_wheel: Path | None, +) -> list[str]: run_name, repair_name, diagnose_name = _root_names(plan.application_display_name) paths = [ run_name, @@ -133,6 +209,8 @@ def _planned_generated_paths(plan, bootstrap_mode: str, artifact_values: list[st _name, separator, raw_path = value.partition("=") if separator and raw_path: paths.append(f"deployment/wheels/{Path(raw_path).name}") + if application_wheel is not None: + paths.append(f"deployment/application/{application_wheel.name}") return sorted(set(paths)) @@ -182,6 +260,7 @@ def _render_owned_files( system_certs: bool, approved, bundled_uv: Path | None, + application_artifact: tuple[ApplicationArtifact, Path] | None = None, ) -> tuple[dict[str, bytes], object]: values = _template_values(plan, bootstrap_mode, system_certs) run_name, repair_name, diagnose_name = _root_names(plan.application_display_name) @@ -204,6 +283,9 @@ def _render_owned_files( owned["deployment/bootstrap/uv.exe"] = bundled_uv.read_bytes() for artifact, path in approved: owned[f"deployment/wheels/{artifact.filename}"] = path.read_bytes() + if application_artifact is not None: + artifact, path = application_artifact + owned[f"deployment/application/{artifact.filename}"] = path.read_bytes() referenced = [ *owned, @@ -218,6 +300,7 @@ def _render_owned_files( bootstrap_mode=bootstrap_mode, system_certs=system_certs, approved_artifacts=[item[0] for item in approved], + application_artifact=application_artifact[0] if application_artifact else None, bundled_uv_sha256=sha256_file(bundled_uv) if bundled_uv else None, referenced_files=referenced, ) @@ -262,10 +345,17 @@ def _preview( system_certs: bool, prepare_lock: bool, artifact_values: list[str], + application_wheel: Path | None, + application_artifact: ApplicationArtifact | None, staging_source_paths: list[str], ) -> GenerationPreview: paths = sorted( - set(_planned_generated_paths(plan, bootstrap_mode, artifact_values) + staging_source_paths) + set( + _planned_generated_paths( + plan, bootstrap_mode, artifact_values, application_wheel + ) + + staging_source_paths + ) ) if prepare_lock and "uv.lock" not in paths: paths.append("uv.lock") @@ -286,14 +376,32 @@ def _preview( actions.append( f"Validate an approved wheel for {requirement.package}=={requirement.version}." ) + if plan.deployment_mode == "package" and application_artifact is None: + actions.append( + "Provide --application-wheel; package mode cannot produce a deployable kit without " + "a validated first-party wheel." + ) + elif application_artifact is not None: + actions.append( + "Validated first-party application wheel " + f"{application_artifact.filename} (SHA-256 {application_artifact.sha256})." + ) return GenerationPreview( application_id=plan.application_id, + deployment_mode=plan.deployment_mode, output_directory=str(output_root), dry_run=dry_run, readiness_before=plan.readiness.state, + source_roots=( + source_roots_from_plan(plan) if plan.deployment_mode == "source" else [] + ), bootstrap_mode=bootstrap_mode, system_certs=system_certs, developer_actions=actions, + application_wheel_required=( + plan.deployment_mode == "package" and application_artifact is None + ), + application_artifact=application_artifact, files_to_create=create, files_to_replace=replace, collisions=collisions, @@ -318,6 +426,7 @@ def generate_deployment_kit( bootstrap_mode: str = "bundled_uv", system_certs: bool = False, artifact_values: list[str] | None = None, + application_wheel: Path | None = None, dry_run: bool = False, uv_cache_root: Path | None = None, ) -> GenerationResult: @@ -348,7 +457,43 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) - source_files = _source_files(repository_root, include=output_root != repository_root) + if plan.entry_point is None: + raise PreparationError( + "Deployment readiness is blocked: " + "; ".join(plan.readiness.blockers) + ) + if plan.deployment_mode_condition in { + "DEPLOYMENT_MODE_CONFLICT", + "INSTALLED_PROJECT_REQUIRED", + }: + raise PreparationError( + "Deployment mode is structurally unsafe: " + "; ".join(plan.readiness.blockers) + ) + if plan.deployment_mode != "package" and application_wheel is not None: + raise PreparationError("--application-wheel is accepted only for package deployment mode.") + application_artifact = ( + validate_application_wheel(application_wheel.resolve(), assessment, plan) + if application_wheel is not None + else None + ) + approved = validate_artifact_set(artifact_values, plan) + requirements = { + canonicalize_name(item.package) + for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) + } + supplied = {item[0].distribution_name for item in approved} + unresolved = sorted(requirements - supplied) + unavailable = [ + item.package + for item in (plan.lock_graph.artifact_findings if plan.lock_graph else []) + if item.status == "unavailable" + ] + source_files = _staging_files( + repository_root, + assessment, + plan, + include=output_root != repository_root, + allow_missing_lock=prepare_lock and plan.lockfile.status == "developer_generation_required", + ) preview = _preview( plan, output_root, @@ -357,8 +502,18 @@ def generate_deployment_kit( system_certs=system_certs, prepare_lock=prepare_lock, artifact_values=artifact_values, + application_wheel=application_wheel, + application_artifact=(application_artifact[0] if application_artifact else None), staging_source_paths=list(source_files), ) + source_generated_collisions = sorted( + set(source_files) + & set(_planned_generated_paths(plan, bootstrap_mode, artifact_values, application_wheel)) + ) + preview.collisions.extend( + f"{path} (runtime source conflicts with a generated path)" + for path in source_generated_collisions + ) if dry_run: return GenerationResult( output_directory=str(output_root), @@ -371,9 +526,10 @@ def generate_deployment_kit( "Generation output contains files not safely owned by the previous generator run: " + ", ".join(preview.collisions) ) - if plan.entry_point is None: + if plan.deployment_mode == "package" and application_artifact is None: raise PreparationError( - "Deployment readiness is blocked: " + "; ".join(plan.readiness.blockers) + "Package deployment mode requires --application-wheel with a developer-built " + "first-party wheel." ) if plan.risk_gate.outcome == "block": raise PreparationError( @@ -384,6 +540,12 @@ def generate_deployment_kit( "uv.lock is missing. Re-run generation with --prepare-lock for a local " "repository to authorize developer-side lockfile creation." ) + if unresolved or unavailable: + detail = [ + *(f"approved wheel required: {item}" for item in unresolved), + *(f"no usable artifact: {item}" for item in unavailable), + ] + raise PreparationError("Deployment readiness remains blocked: " + "; ".join(detail)) uv_executable = acquire_pinned_uv( plan.runtime.bootstrap_artifact, @@ -405,6 +567,11 @@ def generate_deployment_kit( repository_root=repository_root, ) approved = validate_artifact_set(artifact_values, plan) + application_artifact = ( + validate_application_wheel(application_wheel.resolve(), assessment, plan) + if application_wheel is not None + else None + ) requirements = { canonicalize_name(item.package) for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) @@ -423,7 +590,9 @@ def generate_deployment_kit( ] raise PreparationError("Deployment readiness remains blocked: " + "; ".join(detail)) - source_files = _source_files(repository_root, include=output_root != repository_root) + source_files = _staging_files( + repository_root, assessment, plan, include=output_root != repository_root + ) bundled_uv = uv_executable if bootstrap_mode == "bundled_uv" else None owned, manifest = _render_owned_files( plan, @@ -431,6 +600,7 @@ def generate_deployment_kit( bootstrap_mode=bootstrap_mode, system_certs=system_certs, approved=approved, + application_artifact=application_artifact, bundled_uv=bundled_uv, ) index_subjects = {**source_files, **owned} @@ -446,7 +616,7 @@ def generate_deployment_kit( files_for_validation = files secret_values = [ value - for name in manifest.configuration_presence_names + for name in manifest.configuration_secret_names if (value := os.environ.get(name)) ] structural_checks = validate_rendered_files( @@ -480,6 +650,11 @@ def generate_deployment_kit( if approved else [] ), + *( + ["Validated and copied the first-party application wheel."] + if application_artifact + else [] + ), ] return GenerationResult( output_directory=str(output_root), diff --git a/src/python_deployment_builder/generation/manifest.py b/src/python_deployment_builder/generation/manifest.py index a6c3a4b..b65bd25 100644 --- a/src/python_deployment_builder/generation/manifest.py +++ b/src/python_deployment_builder/generation/manifest.py @@ -9,7 +9,12 @@ from python_deployment_builder import __version__ from python_deployment_builder.generation.acquisition import PreparationError, sha256_file -from python_deployment_builder.models import ApprovedArtifact, DeploymentManifest, DeploymentPlan +from python_deployment_builder.models import ( + ApplicationArtifact, + ApprovedArtifact, + DeploymentManifest, + DeploymentPlan, +) def source_roots_from_plan(plan: DeploymentPlan) -> list[str]: @@ -38,6 +43,7 @@ def build_deployment_manifest( approved_artifacts: list[ApprovedArtifact], bundled_uv_sha256: str | None, referenced_files: list[str], + application_artifact: ApplicationArtifact | None = None, generated_at: datetime | None = None, ) -> DeploymentManifest: repository_root = repository_root.resolve() @@ -47,7 +53,7 @@ def build_deployment_manifest( raise PreparationError("Generation currently requires pyproject.toml.") if not lockfile.is_file(): raise PreparationError("Generation requires a prepared, current uv.lock.") - source_roots = source_roots_from_plan(plan) + source_roots = source_roots_from_plan(plan) if plan.deployment_mode == "source" else [] if plan.deployment_mode == "source" and not source_roots: raise PreparationError("Source deployment plan does not provide a runtime source root.") @@ -58,6 +64,7 @@ def build_deployment_manifest( fingerprint_payload: dict[str, object] = { "schema_version": plan.schema_version, "application_id": plan.application_id, + "deployment_mode": plan.deployment_mode, "python_version": plan.runtime.python_version, "uv_version": plan.runtime.uv_version, "uv_archive_sha256": plan.runtime.bootstrap_artifact.sha256, @@ -85,6 +92,13 @@ def build_deployment_manifest( ], "environment_path": plan.runtime.paths.environment_path, } + if application_artifact is not None: + fingerprint_payload["application_artifact"] = { + "name": application_artifact.distribution_name, + "version": application_artifact.version, + "sha256": application_artifact.sha256, + "entry_point": application_artifact.entry_point_target, + } deployment_fingerprint = _deployment_fingerprint(fingerprint_payload) return DeploymentManifest( builder_version=__version__, @@ -113,14 +127,26 @@ def build_deployment_manifest( assessment_repository_fingerprint=plan.assessment_repository_fingerprint, deployment_fingerprint=deployment_fingerprint, approved_artifacts=approved_artifacts, + application_artifact=application_artifact, external_runtimes=plan.external_runtimes, runtime_paths=plan.runtime.paths, - runtime_environment=plan.runtime.environment_variables, + runtime_environment=( + plan.runtime.environment_variables + if plan.deployment_mode == "source" + else { + key: value + for key, value in plan.runtime.environment_variables.items() + if key != "PYTHONPATH" + } + ), sync_arguments=sync_arguments, project_write_probe_required=plan.writes.requires_project_write_probe, configuration_presence_names=sorted( item.name for item in plan.configuration if item.name.isidentifier() ), + configuration_secret_names=sorted( + item.name for item in plan.configuration if item.secret and item.name.isidentifier() + ), referenced_files=sorted(referenced_files), application_version=plan.application_version, runtime_backend=plan.runtime.backend, diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index 39471cb..3e2f513 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -53,6 +53,22 @@ def validate_rendered_files( "Bundled uv.exe matches its deployment-manifest SHA-256.", ) ) + application_hash_ok = manifest.application_artifact is None or ( + ( + data := files.get( + f"deployment/application/{manifest.application_artifact.filename}" + ) + ) + is not None + and hashlib.sha256(data).hexdigest() == manifest.application_artifact.sha256 + ) + checks.append( + _check( + application_hash_ok, + "APPLICATION_ARTIFACT_FINGERPRINT", + "The first-party application artifact matches its manifest SHA-256.", + ) + ) artifact_hashes_ok = all( ( data := files.get(f"deployment/wheels/{artifact.filename}") @@ -124,6 +140,15 @@ def validate_rendered_files( if path.startswith("deployment/runtime/") and b"python_deployment_builder" in files.get(path, b"") ] + cache_paths = [ + path + for path in files + if PurePosixPath(path).suffix.lower() in {".pyc", ".pyo"} + or any( + re.fullmatch(r"__pycache__(?:\s*\(\d+\))?", part, re.IGNORECASE) + for part in PurePosixPath(path).parts + ) + ] checks.extend( [ _check(not ps1_files, "NO_PS1_FILES", f"Forbidden script files: {ps1_files or 'none'}"), @@ -157,6 +182,11 @@ def validate_rendered_files( "RUNTIME_INDEPENDENT", f"Runtime helpers importing the builder: {runtime_builder_imports or 'none'}", ), + _check( + not cache_paths, + "NO_RUNTIME_CACHES", + f"Runtime cache files staged: {cache_paths or 'none'}", + ), ] ) for path in generated_paths: diff --git a/src/python_deployment_builder/models.py b/src/python_deployment_builder/models.py index 968ad65..8fd7edc 100644 --- a/src/python_deployment_builder/models.py +++ b/src/python_deployment_builder/models.py @@ -172,6 +172,9 @@ class PackagingAssessment(StrictModel): build_backend: str | None = None layout: Literal["src", "flat", "unknown"] = "unknown" source_roots: list[str] = Field(default_factory=list) + packages: list[str] = Field(default_factory=list) + package_directories: dict[str, str] = Field(default_factory=dict) + package_data: dict[str, list[str]] = Field(default_factory=dict) entry_points: list[EntryPointAssessment] = Field(default_factory=list) optional_dependency_groups: dict[str, list[str]] = Field(default_factory=dict) legacy_dependency_groups: list[LegacyDependencyGroup] = Field(default_factory=list) @@ -499,6 +502,7 @@ class DeploymentReadiness(StrictModel): "BLOCKED_PENDING_LOCKFILE", "BLOCKED_PENDING_LOCK_VERIFICATION", "BLOCKED_PENDING_DEVELOPER_ARTIFACT", + "BLOCKED_PENDING_APPLICATION_WHEEL", "BLOCKED_PENDING_ENTRYPOINT", "BLOCKED", ] @@ -569,6 +573,13 @@ class DeploymentPlan(StrictModel): application_id: str application_display_name: str deployment_mode: Literal["source", "package", "source_resource_copy"] + deployment_mode_condition: Literal[ + "SOURCE_COMPATIBLE", + "PACKAGE_PREFERRED", + "ENTRYPOINT_REQUIRES_PACKAGE_MODE", + "DEPLOYMENT_MODE_CONFLICT", + "INSTALLED_PROJECT_REQUIRED", + ] = "PACKAGE_PREFERRED" runtime: RuntimePlan entry_point: EntrypointPlan | None = None lockfile: LockfilePlan @@ -603,6 +614,16 @@ class ApprovedArtifact(StrictModel): requirement_action: Literal["developer_wheel_required"] = "developer_wheel_required" +class ApplicationArtifact(StrictModel): + distribution_name: str + version: str + filename: str + sha256: str + wheel_tags: list[str] = Field(default_factory=list) + entry_point_name: str + entry_point_target: str + + class DeploymentManifest(StrictModel): schema_version: str = SCHEMA_VERSION builder_version: str @@ -632,12 +653,14 @@ class DeploymentManifest(StrictModel): assessment_repository_fingerprint: str deployment_fingerprint: str approved_artifacts: list[ApprovedArtifact] = Field(default_factory=list) + application_artifact: ApplicationArtifact | None = None external_runtimes: list[ExternalRuntimePlan] = Field(default_factory=list) runtime_paths: RuntimePaths runtime_environment: dict[str, str] = Field(default_factory=dict) sync_arguments: list[str] = Field(default_factory=list) project_write_probe_required: bool = False configuration_presence_names: list[str] = Field(default_factory=list) + configuration_secret_names: list[str] = Field(default_factory=list) referenced_files: list[str] = Field(default_factory=list) application_version: str | None = None runtime_backend: Literal["uv_managed"] = "uv_managed" @@ -652,13 +675,17 @@ class GeneratedArtifact(StrictModel): class GenerationPreview(StrictModel): application_id: str + deployment_mode: Literal["source", "package", "source_resource_copy"] output_directory: str dry_run: bool readiness_before: str readiness_after: str | None = None + source_roots: list[str] = Field(default_factory=list) bootstrap_mode: Literal["bundled_uv", "online_cmd"] system_certs: bool = False developer_actions: list[str] = Field(default_factory=list) + application_wheel_required: bool = False + application_artifact: ApplicationArtifact | None = None repository_files_changed: list[str] = Field(default_factory=list) files_to_create: list[str] = Field(default_factory=list) files_to_replace: list[str] = Field(default_factory=list) @@ -790,6 +817,7 @@ class ReleaseManifest(StrictModel): pyproject_sha256: str lockfile_sha256: str approved_artifacts: list[ApprovedArtifact] = Field(default_factory=list) + application_artifact: ApplicationArtifact | None = None external_runtimes: list[ExternalRuntimePlan] = Field(default_factory=list) source_revision: str | None = None assessment_repository_fingerprint: str diff --git a/src/python_deployment_builder/packaging/packager.py b/src/python_deployment_builder/packaging/packager.py index d5065e2..7198fd4 100644 --- a/src/python_deployment_builder/packaging/packager.py +++ b/src/python_deployment_builder/packaging/packager.py @@ -217,6 +217,7 @@ def package_deployment_kit( pyproject_sha256=manifest.pyproject_sha256, lockfile_sha256=manifest.lockfile_sha256, approved_artifacts=manifest.approved_artifacts, + application_artifact=manifest.application_artifact, external_runtimes=manifest.external_runtimes, source_revision=manifest.source_revision, assessment_repository_fingerprint=manifest.assessment_repository_fingerprint, diff --git a/src/python_deployment_builder/packaging/reports.py b/src/python_deployment_builder/packaging/reports.py index 4d3f9c4..2ce7a94 100644 --- a/src/python_deployment_builder/packaging/reports.py +++ b/src/python_deployment_builder/packaging/reports.py @@ -47,6 +47,21 @@ def render_release_manifest_markdown(manifest: ReleaseManifest) -> str: ) else: lines.extend(["", "No approved artifact exceptions are present."]) + if manifest.application_artifact: + artifact = manifest.application_artifact + lines.extend( + [ + "", + "### First-party application artifact", + "", + f"- `{artifact.distribution_name}=={artifact.version}` - " + f"`{artifact.filename}` - `{artifact.sha256}`", + f"- Authoritative entry point: `{artifact.entry_point_target}`", + "- Provenance: exact supplied wheel bytes are identified by SHA-256; ordinary " + "wheel metadata does not cryptographically prove a relationship to the recorded " + "source revision.", + ] + ) lines.extend(["", "## External runtimes", ""]) if manifest.external_runtimes: for runtime in manifest.external_runtimes: diff --git a/src/python_deployment_builder/packaging/smoke.py b/src/python_deployment_builder/packaging/smoke.py index d45e042..cd82f84 100644 --- a/src/python_deployment_builder/packaging/smoke.py +++ b/src/python_deployment_builder/packaging/smoke.py @@ -78,6 +78,12 @@ def render_smoke_test(manifest: DeploymentManifest, zip_filename: str) -> str: f"{item.distribution_name}=={item.version}" for item in manifest.approved_artifacts ) lines.append(f"5. Confirm approved artifacts include: {artifacts}.") + if manifest.application_artifact: + artifact = manifest.application_artifact + lines.append( + "6. Confirm the first-party application artifact is " + f"{artifact.distribution_name}=={artifact.version}." + ) lines.extend( [ "", diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 1240f7f..bf3794e 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -23,6 +23,7 @@ PlanningDecision, PythonCandidatePlan, RepositoryAssessment, + RepositoryFileRole, RiskGate, RiskSeverity, SuitabilityRating, @@ -48,18 +49,110 @@ ) -def _deployment_mode(assessment: RepositoryAssessment) -> tuple[str, str]: - adjacent = any(item.packaging_status == "repository_adjacent" for item in assessment.resources) - project_writes = any( - item.classification == "project_local" for item in assessment.write_locations +def _source_entrypoint_compatible( + assessment: RepositoryAssessment, entry_point: EntrypointPlan | None +) -> tuple[bool, list[str]]: + if entry_point is None: + return False, [] + module_path = Path(*entry_point.module.split(".")) + candidates: list[str] = [] + for root in assessment.project.source_roots or ["."]: + base = Path() if root == "." else Path(root) + candidates.extend( + [ + (base / module_path.with_suffix(".py")).as_posix(), + (base / module_path / "__init__.py").as_posix(), + ] + ) + application_paths = { + item.path + for item in assessment.file_inventory + if item.role == RepositoryFileRole.APPLICATION_SOURCE + } + return any(path in application_paths for path in candidates), candidates + + +def _deployment_mode( + assessment: RepositoryAssessment, + entry_point: EntrypointPlan | None, +) -> tuple[str, str, str, list[str]]: + runtime_resource_paths = { + item.path + for item in assessment.file_inventory + if item.role == RepositoryFileRole.RUNTIME_RESOURCE + } + adjacent = [ + item.path + for item in assessment.resources + if item.packaging_status == "repository_adjacent" + and item.path in runtime_resource_paths + and item.kind != "documentation" + ] + project_writes = [ + item.path_expression + for item in assessment.write_locations + if item.classification == "project_local" + ] + source_compatible, candidates = _source_entrypoint_compatible(assessment, entry_point) + source_constraints = [ + *(f"repository-adjacent resource: {item}" for item in adjacent), + *(f"project-local write: {item}" for item in project_writes), + ] + installable = bool( + assessment.project.distribution_name + and assessment.project.version + and assessment.project.build_backend ) - if adjacent or project_writes: + if source_constraints and source_compatible: return ( "source", - "Repository-adjacent resources or project-local writes make an extracted-source " - "layout the safest initial policy.", + "Source-only runtime requirements make an extracted-source layout necessary, and the " + "authoritative entry point is importable from the planned source roots.", + "SOURCE_COMPATIBLE", + [], + ) + if source_constraints: + return ( + "package", + "Source layout requirements conflict with an authoritative entry point that cannot " + "be imported from the planned source roots.", + "DEPLOYMENT_MODE_CONFLICT", + [ + "DEPLOYMENT_MODE_CONFLICT: " + + "; ".join([*source_constraints, f"source candidates: {', '.join(candidates)}"]) + ], + ) + if not source_compatible: + if not installable: + return ( + "package", + "The authoritative entry point requires installation, but buildable project " + "metadata is incomplete.", + "INSTALLED_PROJECT_REQUIRED", + ["INSTALLED_PROJECT_REQUIRED: buildable project metadata is incomplete"], + ) + return ( + "package", + "The authoritative entry point is not source-import compatible; install a validated " + "developer-supplied first-party wheel.", + "ENTRYPOINT_REQUIRES_PACKAGE_MODE", + [], ) - return "package", "No repository-adjacent runtime dependency requires a source layout." + if assessment.project.source_roots == ["."]: + return ( + "source", + "The authoritative entry point is directly importable from the flat repository " + "source root; preserve the extracted-source contract.", + "SOURCE_COMPATIBLE", + [], + ) + return ( + "package", + "The project has an install-oriented source layout without a source-only runtime " + "constraint; use a validated first-party wheel.", + "PACKAGE_PREFERRED", + [], + ) def _entrypoint(assessment: RepositoryAssessment) -> EntrypointPlan | None: @@ -205,6 +298,8 @@ def _readiness( lockfile: LockfilePlan, lock_graph, entry_point: EntrypointPlan | None, + deployment_mode: str, + mode_blockers: list[str], ) -> DeploymentReadiness: blockers: list[str] = [] blocker_codes: list[str] = [] @@ -218,6 +313,15 @@ def _readiness( blockers.append( "ENTRYPOINT_DECLARATION_REQUIRED: declare an authoritative standardized entry point" ) + if mode_blockers: + blocker_codes.extend(item.split(":", 1)[0] for item in mode_blockers) + blockers.extend(mode_blockers) + if deployment_mode == "package" and not mode_blockers: + blocker_codes.append("APPLICATION_WHEEL_REQUIRED") + blockers.append( + "APPLICATION_WHEEL_REQUIRED: package mode requires a validated developer-supplied " + "first-party wheel at generation time" + ) if lockfile.status == "developer_generation_required": blocker_codes.append("LOCKFILE_GENERATION_REQUIRED") blockers.append("LOCKFILE_GENERATION_REQUIRED") @@ -233,8 +337,12 @@ def _readiness( state = "BLOCKED" elif entry_point is None: state = "BLOCKED_PENDING_ENTRYPOINT" + elif mode_blockers: + state = "BLOCKED" elif lock_graph and lock_graph.artifact_findings: state = "BLOCKED_PENDING_DEVELOPER_ARTIFACT" + elif deployment_mode == "package": + state = "BLOCKED_PENDING_APPLICATION_WHEEL" elif lockfile.status == "developer_generation_required": state = "BLOCKED_PENDING_LOCKFILE" elif pending: @@ -298,8 +406,10 @@ def create_deployment_plan( python_version, python_candidates = _python_candidates(assessment, compatibility) name = assessment.project.distribution_name or assessment.repository.root_name app_id = safe_application_id(name) - mode, mode_rationale = _deployment_mode(assessment) entry_point = _entrypoint(assessment) + mode, mode_rationale, mode_condition, mode_blockers = _deployment_mode( + assessment, entry_point + ) runtime = UvManagedBackend().build_plan( app_id, python_version, @@ -447,12 +557,20 @@ def create_deployment_plan( application_id=app_id, application_display_name=name.replace("-", " ").title(), deployment_mode=mode, + deployment_mode_condition=mode_condition, runtime=runtime, entry_point=entry_point, lockfile=lockfile, lock_graph=lock_graph, risk_gate=gate, - readiness=_readiness(gate, lockfile, lock_graph, entry_point), + readiness=_readiness( + gate, + lockfile, + lock_graph, + entry_point, + mode, + mode_blockers, + ), extras=extras, selected_extras_fingerprint=fingerprint, external_runtimes=external_runtimes, diff --git a/src/python_deployment_builder/templates/windows_uv/diagnostics.py b/src/python_deployment_builder/templates/windows_uv/diagnostics.py index 9febce6..f437468 100644 --- a/src/python_deployment_builder/templates/windows_uv/diagnostics.py +++ b/src/python_deployment_builder/templates/windows_uv/diagnostics.py @@ -130,6 +130,13 @@ def report(project_root: Path) -> list[str]: f"Approved artifact: {artifact['distribution_name']}=={artifact['version']} " f"{artifact['filename']} SHA-256 {artifact['sha256']}" ) + application_artifact = manifest.get("application_artifact") + if application_artifact: + lines.append( + "Application artifact: " + f"{application_artifact['distribution_name']}=={application_artifact['version']} " + f"{application_artifact['filename']} SHA-256 {application_artifact['sha256']}" + ) for name in manifest["configuration_presence_names"]: lines.append(f"{name} present: {'yes' if bool(os.environ.get(name)) else 'no'}") if manifest.get("project_write_probe_required"): diff --git a/src/python_deployment_builder/templates/windows_uv/launch.py b/src/python_deployment_builder/templates/windows_uv/launch.py index c68827d..ec1ceb6 100644 --- a/src/python_deployment_builder/templates/windows_uv/launch.py +++ b/src/python_deployment_builder/templates/windows_uv/launch.py @@ -1,4 +1,4 @@ -"""Invoke the planned application entry point from its extracted source tree.""" +"""Invoke the authoritative entry point from staged source or the installed wheel.""" from __future__ import annotations @@ -20,6 +20,8 @@ def configure_source_paths(manifest: dict, project_root: Path) -> None: + if manifest.get("deployment_mode", "source") != "source": + return for relative in reversed(manifest["source_roots"]): path = project_root if relative == "." else project_root / relative sys.path.insert(0, str(path.resolve())) diff --git a/src/python_deployment_builder/templates/windows_uv/manage.py b/src/python_deployment_builder/templates/windows_uv/manage.py index 35dc301..cdd690b 100644 --- a/src/python_deployment_builder/templates/windows_uv/manage.py +++ b/src/python_deployment_builder/templates/windows_uv/manage.py @@ -146,13 +146,32 @@ def _promote_environment( str(wheel), ] run_logged(command, cwd=project_root, environment=runtime_env, logger=logger) - if manifest["approved_artifacts"]: + application_artifact = manifest.get("application_artifact") + if application_artifact: + application_wheel = ( + deployment_directory() / "application" / application_artifact["filename"] + ) run_logged( - [str(uv_executable), "pip", "check", "--python", str(python)], + [ + str(uv_executable), + "pip", + "install", + "--python", + str(python), + "--no-deps", + "--no-build", + str(application_wheel), + ], cwd=project_root, environment=runtime_env, logger=logger, ) + run_logged( + [str(uv_executable), "pip", "check", "--python", str(python)], + cwd=project_root, + environment=runtime_env, + logger=logger, + ) launch_check = [ str(python), diff --git a/src/python_deployment_builder/templates/windows_uv/runtime_common.py b/src/python_deployment_builder/templates/windows_uv/runtime_common.py index 2fa9365..3577384 100644 --- a/src/python_deployment_builder/templates/windows_uv/runtime_common.py +++ b/src/python_deployment_builder/templates/windows_uv/runtime_common.py @@ -119,6 +119,11 @@ def state_signature(manifest: dict, project_root: Path) -> dict[str, object]: "approved_artifact_hashes": { item["filename"]: item["sha256"] for item in manifest["approved_artifacts"] }, + "application_artifact_sha256": ( + manifest["application_artifact"]["sha256"] + if manifest.get("application_artifact") + else None + ), "environment_path": str(environment_path(manifest)), } @@ -153,6 +158,11 @@ def stale_reasons(manifest: dict, project_root: Path) -> list[str]: wheel = wheel_directory / artifact["filename"] if not wheel.is_file() or sha256_file(wheel) != artifact["sha256"]: reasons.append(f"approved artifact {artifact['filename']} changed") + application_artifact = manifest.get("application_artifact") + if application_artifact: + wheel = deployment_directory() / "application" / application_artifact["filename"] + if not wheel.is_file() or sha256_file(wheel) != application_artifact["sha256"]: + reasons.append(f"application artifact {application_artifact['filename']} changed") return reasons @@ -203,6 +213,14 @@ def verify_runtime_inputs(manifest: dict, project_root: Path) -> None: raise DeploymentRuntimeError( f"Approved artifact is missing or changed: {artifact['filename']}" ) + application_artifact = manifest.get("application_artifact") + if application_artifact: + path = deployment_directory() / "application" / application_artifact["filename"] + if not path.is_file() or sha256_file(path) != application_artifact["sha256"]: + raise DeploymentRuntimeError( + "Application artifact is missing or changed: " + f"{application_artifact['filename']}" + ) def redact(text: str) -> str: diff --git a/src/python_deployment_builder/validation/runtime.py b/src/python_deployment_builder/validation/runtime.py index ca1724c..56bf09b 100644 --- a/src/python_deployment_builder/validation/runtime.py +++ b/src/python_deployment_builder/validation/runtime.py @@ -62,10 +62,13 @@ def _runtime_environment( manifest: DeploymentManifest, local_app_data: Path ) -> dict[str, str]: environment = os.environ.copy() - environment["LOCALAPPDATA"] = str(local_app_data) environment["PDBUILDER_NO_PAUSE"] = "1" for name in manifest.configuration_presence_names: environment.pop(name, None) + # LOCALAPPDATA is controlled by the validation harness even when target + # analysis records it as a configuration read. Do not let redaction/isolation + # remove the runtime root that the generated Windows bootstrap requires. + environment["LOCALAPPDATA"] = str(local_app_data) for key, value in manifest.runtime_environment.items(): if "%PROJECT_ROOT%" not in value: environment[key] = value.replace("%LOCALAPPDATA%", str(local_app_data)) @@ -152,6 +155,10 @@ def _scenario_copy( ) shutil.copy2(kit_root / "pyproject.toml", scenario_root / "pyproject.toml") shutil.copy2(kit_root / "uv.lock", scenario_root / "uv.lock") + for artifact_directory in ("wheels", "application"): + source = kit_root / "deployment" / artifact_directory + if source.is_dir(): + shutil.copytree(source, scenario_root / "deployment" / artifact_directory) return scenario_root @@ -471,6 +478,50 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) report.final_state = ValidationFinalState.FAILED return report + if manifest.application_artifact is not None: + application_probe_environment = { + **environment, + "PDBUILDER_APPLICATION_DISTRIBUTION": ( + manifest.application_artifact.distribution_name + ), + "PDBUILDER_APPLICATION_VERSION": manifest.application_artifact.version, + "PDBUILDER_APPLICATION_MODULE": manifest.entry_point_module, + } + application_probe = ( + "import importlib.metadata as m,importlib.util,os,sys;" + "name=os.environ['PDBUILDER_APPLICATION_DISTRIBUTION'];" + "version=os.environ['PDBUILDER_APPLICATION_VERSION'];" + "module=os.environ['PDBUILDER_APPLICATION_MODULE'];" + "sys.exit(m.version(name)!=version or importlib.util.find_spec(module) is None)" + ) + installed_application, application_duration = _run( + [str(app_python), *HELPER_FLAGS, "-c", application_probe], + cwd=root, + environment=application_probe_environment, + log_handle=log, + ) + application_ok = installed_application.returncode == 0 + report.runtime_checks.append( + _check( + "APPLICATION_WHEEL_INSTALLED", + "first_run", + ( + ValidationCheckStatus.PASS + if application_ok + else ValidationCheckStatus.FAIL + ), + "The exact first-party distribution/version and authoritative module are " + "installed in the managed environment." + if application_ok + else "The first-party application wheel is not installed as declared.", + evidence=[installed_application.stderr[-1000:]], + duration=application_duration, + ) + ) + if not application_ok: + report.final_state = ValidationFinalState.FAILED + return report + imports = _selected_imports(root, manifest) import_environment = {**environment, "PDBUILDER_IMPORTS_JSON": json.dumps(imports)} import_probe = ( @@ -600,6 +651,18 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) ("deployment-fingerprint", {"deployment_fingerprint": "0" * 64}), ("selected-extras-fingerprint", {"selected_extras_fingerprint": "0" * 64}), ] + if manifest.application_artifact is not None: + scenario_values.append( + ( + "application-artifact-fingerprint", + { + "application_artifact": { + **manifest.application_artifact.model_dump(mode="json"), + "sha256": "0" * 64, + } + }, + ) + ) stale_failures: list[str] = [] for sequence, (name, changes) in enumerate(scenario_values, start=1): scenario = _scenario_copy( @@ -625,7 +688,7 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) stale_failures.append(f"{name}: exit {result.returncode}") lock_scenario = _scenario_copy( root, - scenarios / f"05-lock-fingerprint-{time.time_ns()}", + scenarios / f"{len(scenario_values) + 1:02d}-lock-fingerprint-{time.time_ns()}", manifest, {}, ) @@ -646,6 +709,9 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) ) if lock_result.returncode != SETUP_REQUIRED: stale_failures.append(f"lock-fingerprint: exit {lock_result.returncode}") + stale_subjects = "deployment and extras" + if manifest.application_artifact is not None: + stale_subjects += ", application artifact" report.runtime_checks.append( _check( "CONTROLLED_STALENESS", @@ -653,7 +719,7 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) ValidationCheckStatus.PASS if not stale_failures else ValidationCheckStatus.FAIL, - "Missing state/Python and changed deployment, extras, and lock fingerprints " + f"Missing state/Python and changed {stale_subjects}, and lock fingerprints " "all request setup." if not stale_failures else "A controlled stale state did not request setup.", @@ -666,7 +732,7 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) rollback_scenario = _scenario_copy( root, - scenarios / f"06-rollback-{time.time_ns()}", + scenarios / f"{len(scenario_values) + 2:02d}-rollback-{time.time_ns()}", manifest, { "bundled_uv_sha256": None, @@ -786,7 +852,7 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) and "Managed application Python is unavailable" in broken.stdout and all( value not in healthy.stdout + broken.stdout - for name in manifest.configuration_presence_names + for name in manifest.configuration_secret_names if (value := os.environ.get(name)) ) ) diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index dad7cd2..b5d1f23 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -8,6 +8,7 @@ import platform import re import socket +import zipfile from datetime import UTC, datetime from pathlib import Path @@ -29,6 +30,7 @@ r"(?i)(?:authorization\s*[:=]\s*bearer\s+[a-z0-9._-]{12,}|sk-[a-z0-9_-]{16,})" ) FORBIDDEN_SHELL = ("powershell.exe", "pwsh.exe", "executionpolicy") +PYTHON_CACHE_DIRECTORY = re.compile(r"^__pycache__(?:\s*\(\d+\))?$", re.IGNORECASE) TEXT_SUFFIXES = {".bat", ".cmd", ".json", ".py", ".txt"} SECRET_FILENAMES = { ".env", @@ -143,6 +145,60 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR "Application ID does not match the per-user runtime path.", ) ) + + application_artifact_failures: list[str] = [] + application_wheel: Path | None = None + if manifest.deployment_mode == "package": + if manifest.application_artifact is None: + application_artifact_failures.append("manifest application artifact is missing") + else: + application_wheel = ( + root / "deployment" / "application" / manifest.application_artifact.filename + ) + if ( + not application_wheel.is_file() + or _sha256(application_wheel) != manifest.application_artifact.sha256 + ): + application_artifact_failures.append(manifest.application_artifact.filename) + elif manifest.application_artifact is not None: + application_artifact_failures.append( + "source mode unexpectedly declares an application wheel" + ) + checks.append( + _check( + "APPLICATION_ARTIFACT_HASH", + not application_artifact_failures, + "The first-party application artifact matches its manifest SHA-256.", + "The first-party application artifact is missing, changed, or misplaced.", + evidence=application_artifact_failures, + ) + ) + package_source_paths = sorted( + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() + and path.relative_to(root).parts[0] != "deployment" + and path.name not in {"pyproject.toml", "uv.lock"} + and path.suffix.lower() != ".bat" + ) + package_isolation_ok = ( + manifest.deployment_mode != "package" + or ( + not manifest.source_roots + and "PYTHONPATH" not in manifest.runtime_environment + and not package_source_paths + ) + ) + checks.append( + _check( + "PACKAGE_SOURCE_ISOLATION", + package_isolation_ok, + "Package mode has no staged source roots or PYTHONPATH and launches the installed " + "application artifact.", + "Package mode contains staged source content or source import configuration.", + evidence=package_source_paths, + ) + ) checks.append( _check( "RUNTIME_METADATA_PRESENT", @@ -303,21 +359,37 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR ) ) module_relative = Path(*manifest.entry_point_module.split(".")) - entry_candidates = [] - candidate_roots = manifest.source_roots or [".", "src"] - for source_root in candidate_roots: - base = root if source_root == "." else root / source_root - entry_candidates.extend( - [base / module_relative.with_suffix(".py"), base / module_relative / "__init__.py"] - ) + entry_candidates: list[Path] = [] + entry_evidence: list[str] = [] + entry_present = False + if manifest.deployment_mode == "source": + for source_root in manifest.source_roots: + base = root if source_root == "." else root / source_root + entry_candidates.extend( + [ + base / module_relative.with_suffix(".py"), + base / module_relative / "__init__.py", + ] + ) + entry_present = any(path.is_file() for path in entry_candidates) + entry_evidence = [str(path.relative_to(root)) for path in entry_candidates] + elif application_wheel is not None and application_wheel.is_file(): + member_base = "/".join(manifest.entry_point_module.split(".")) + member_candidates = {f"{member_base}.py", f"{member_base}/__init__.py"} + try: + with zipfile.ZipFile(application_wheel) as bundle: + entry_present = bool(member_candidates.intersection(bundle.namelist())) + except zipfile.BadZipFile: + entry_present = False + entry_evidence = sorted(member_candidates) checks.append( _check( "ENTRY_POINT_STRUCTURE", bool(manifest.entry_point_module and manifest.entry_point_callable) - and any(path.is_file() for path in entry_candidates), - "The entry-point module is structurally present under a planned source root.", - "The entry-point module is not structurally present under a planned source root.", - evidence=[str(path.relative_to(root)) for path in entry_candidates], + and entry_present, + "The entry-point module is structurally present in its deployment mode.", + "The entry-point module is not structurally present in its deployment mode.", + evidence=entry_evidence, ) ) @@ -363,7 +435,14 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR ) ) bad_bat_structure = [] + generated_root_bats = { + Path(item).name + for item in manifest.referenced_files + if "/" not in item and item.lower().endswith(".bat") + } for name in root_bats: + if name not in generated_root_bats: + continue text = (root / name).read_text(encoding="utf-8", errors="replace").lower() if not text.startswith("@echo off") or "deployment\\bootstrap\\bootstrap.cmd" not in text: bad_bat_structure.append(name) @@ -401,7 +480,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR program_files.append(relative) if OBVIOUS_SECRET.search(text): obvious_secrets.append(relative) - for name in manifest.configuration_presence_names: + for name in manifest.configuration_secret_names: value = os.environ.get(name) if value and len(value) >= 8 and value in text: obvious_secrets.append(relative) @@ -418,7 +497,10 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR str(path.relative_to(root)) for path in root.rglob("*") if path.is_file() - and (path.suffix.lower() in {".pyc", ".pyo"} or "__pycache__" in path.parts) + and ( + path.suffix.lower() in {".pyc", ".pyo"} + or any(PYTHON_CACHE_DIRECTORY.fullmatch(part) for part in path.parts) + ) ] developer_state = [ str(path.relative_to(root)) diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index ea5f58a..3f6bba9 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -12,7 +12,7 @@ from python_deployment_builder.analysis.repository import MaterializedRepository from python_deployment_builder.cli import main from python_deployment_builder.generation.acquisition import PreparationError -from python_deployment_builder.generation.generator import _source_files, generate_deployment_kit +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit from python_deployment_builder.models import ( FindingStatus, OnlineCompatibilityAssessment, @@ -731,16 +731,114 @@ def fake_inspection(dependencies, versions, architecture): } -def test_analysis_roles_do_not_control_source_staging(tmp_path: Path) -> None: +def test_analysis_roles_control_source_staging(tmp_path: Path) -> None: _write_fingerprint_app(tmp_path) + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) - staged = _source_files(tmp_path, include=True) + staged = _staging_files(tmp_path, assessment, plan, include=True) - assert "docs/snippet.py" in staged - assert "examples/example.py" in staged + assert "app.py" in staged + assert "assets/view.html" in staged + assert "docs/snippet.py" not in staged + assert "examples/example.py" not in staged assert "tests/test_app.py" not in staged assert "deployment/helper.py" not in staged - assert "historical/old.py" in staged + assert "historical/old.py" not in staged + + +def test_installed_namespace_package_data_and_user_local_wrapper_select_package_mode( + tmp_path: Path, +) -> None: + (tmp_path / "code").mkdir() + (tmp_path / "code/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "code/view.html").write_text("\n", encoding="utf-8") + (tmp_path / "code/main.py").write_text( + """from pathlib import Path +VIEW = Path(__file__).with_name("view.html") +def user_data_path(name): + return Path.home() / ".sample" / name +def oauth_path(): + return user_data_path("oauth.json") +def save(): + cache_path = oauth_path() + cache_path.write_text("state") +def main(): + return VIEW.read_text() +""", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" +[project] +name = "mapped-app" +version = "1.2.3" +requires-python = ">=3.12" +dependencies = [] +[project.gui-scripts] +mapped-app = "installed_app.main:main" +[tool.setuptools] +packages = ["installed_app"] +package-dir = {installed_app = "code"} +[tool.setuptools.package-data] +installed_app = ["view.html"] +""", + encoding="utf-8", + ) + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment) + + view = next(item for item in assessment.resources if item.path == "code/view.html") + oauth = next( + item for item in assessment.write_locations if "oauth_path" in item.path_expression + ) + assert view.packaging_status == "packaged" + assert oauth.classification == "user_local" + assert plan.deployment_mode == "package" + assert plan.deployment_mode_condition == "ENTRYPOINT_REQUIRES_PACKAGE_MODE" + assert plan.readiness.state == "BLOCKED_PENDING_APPLICATION_WHEEL" + assert plan.entry_point.target == "installed_app.main:main" + + view.packaging_status = "repository_adjacent" + conflict = create_deployment_plan(assessment) + assert conflict.deployment_mode_condition == "DEPLOYMENT_MODE_CONFLICT" + assert conflict.readiness.state == "BLOCKED" + assert "DEPLOYMENT_MODE_CONFLICT" in conflict.readiness.blocker_codes + + +def test_cache_collision_directories_are_inventory_local_state(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + cache = tmp_path / "app" / "__pycache__ (12)" + cache.mkdir(parents=True) + (cache / "module.cpython-312.pyc").write_bytes(b"cache") + + assessment = assess_repository(_repository(tmp_path)) + cache_items = [item for item in assessment.file_inventory if "__pycache__" in item.path] + + assert cache_items + assert all(item.role == RepositoryFileRole.IGNORED_OR_LOCAL for item in cache_items) + assert all(not item.included_in_runtime_scan for item in cache_items) + + +def test_materialized_archive_uses_role_aware_staging_without_git(tmp_path: Path) -> None: + _write_fingerprint_app(tmp_path) + repository = MaterializedRepository( + root=tmp_path, + source="https://github.com/example/materialized/archive", + source_kind="github_archive", + ) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + + staged = _staging_files(tmp_path, assessment, plan, include=True) + + assert "app.py" in staged + assert "assets/view.html" in staged + assert "docs/snippet.py" not in staged + assert "deployment/helper.py" not in staged def test_pathspec_is_declared_as_a_runtime_dependency() -> None: diff --git a/tests/test_generation.py b/tests/test_generation.py index 23c111b..f592b2d 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1,6 +1,7 @@ from __future__ import annotations import csv +import hashlib import importlib.util import io import os @@ -14,17 +15,21 @@ from python_deployment_builder.analysis.assessor import assess_repository from python_deployment_builder.analysis.repository import MaterializedRepository -from python_deployment_builder.cli import build_parser +from python_deployment_builder.cli import build_parser, main from python_deployment_builder.generation.acquisition import ( PreparationError, acquire_pinned_uv, extract_verified_uv, verify_uv_version, ) -from python_deployment_builder.generation.artifacts import validate_approved_wheel +from python_deployment_builder.generation.artifacts import ( + validate_application_wheel, + validate_approved_wheel, +) from python_deployment_builder.generation.cmd import parse_certutil_sha256 from python_deployment_builder.generation.generator import ( _render_owned_files, + _staging_files, generate_deployment_kit, ) from python_deployment_builder.generation.manifest import build_deployment_manifest @@ -36,7 +41,10 @@ from python_deployment_builder.generation.structural import validate_rendered_files from python_deployment_builder.generation.templates import TEMPLATE_ROOT from python_deployment_builder.models import BootstrapArtifact +from python_deployment_builder.packaging.archive import safe_extract_zip +from python_deployment_builder.packaging.packager import package_deployment_kit from python_deployment_builder.planning.planner import create_deployment_plan +from python_deployment_builder.validation.static import validate_static_kit FIXTURES = Path(__file__).parent / "fixtures" @@ -81,6 +89,118 @@ def _make_wheel(path: Path, name: str = "proxy-tools", version: str = "0.1.0") - return wheel +def _make_application_wheel( + path: Path, + *, + name: str = "mapped-app", + version: str = "1.2.3", + package: str = "installed_app", + target: str = "installed_app.main:main", + include_cache: bool = False, +) -> Path: + normalized = name.replace("-", "_") + wheel = path / f"{normalized}-{version}-py3-none-any.whl" + dist_info = f"{normalized}-{version}.dist-info" + files = { + f"{package}/__init__.py": "", + f"{package}/main.py": "def main(): return 0\n", + f"{package}/view.html": "\n", + f"{dist_info}/METADATA": ( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n" + ), + f"{dist_info}/WHEEL": ( + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + ), + f"{dist_info}/entry_points.txt": f"[gui_scripts]\nmapped-app = {target}\n", + } + if include_cache: + files[f"{package}/__pycache__ (1)/main.pyc"] = "cache" + record_name = f"{dist_info}/RECORD" + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + for filename in files: + writer.writerow((filename, "", "")) + writer.writerow((record_name, "", "")) + files[record_name] = output.getvalue() + with zipfile.ZipFile(wheel, "w") as bundle: + for filename, data in files.items(): + bundle.writestr(filename, data) + return wheel + + +def _rewrite_application_wheel( + wheel: Path, + *, + replacements: dict[str, str | bytes] | None = None, + removals: set[str] | None = None, + additions: dict[str, str | bytes] | None = None, + recorded_paths: list[str] | None = None, +) -> Path: + with zipfile.ZipFile(wheel) as bundle: + files = { + item.filename: bundle.read(item) + for item in bundle.infolist() + if not item.filename.endswith(".dist-info/RECORD") + } + record_name = next( + item.filename + for item in bundle.infolist() + if item.filename.endswith(".dist-info/RECORD") + ) + for name in removals or set(): + files.pop(name, None) + for name, data in {**(replacements or {}), **(additions or {})}.items(): + files[name] = data.encode() if isinstance(data, str) else data + paths = list(files) if recorded_paths is None else recorded_paths + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + for filename in paths: + writer.writerow((filename, "", "")) + writer.writerow((record_name, "", "")) + files[record_name] = output.getvalue().encode() + with zipfile.ZipFile(wheel, "w") as bundle: + for filename, data in files.items(): + bundle.writestr(filename, data) + return wheel + + +def _write_mapped_project(root: Path) -> None: + (root / "code").mkdir() + (root / "code/__init__.py").write_text("", encoding="utf-8") + (root / "code/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "code/view.html").write_text("\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" +[project] +name = "mapped-app" +version = "1.2.3" +requires-python = ">=3.12" +dependencies = [] +[project.gui-scripts] +mapped-app = "installed_app.main:main" +[tool.setuptools] +packages = ["installed_app"] +package-dir = {installed_app = "code"} +[tool.setuptools.package-data] +installed_app = ["view.html"] +""", + encoding="utf-8", + ) + (root / "uv.lock").write_text( + """version = 1 +revision = 3 +requires-python = ">=3.12" +[[package]] +name = "mapped-app" +version = "1.2.3" +source = { virtual = "." } +""", + encoding="utf-8", + ) + + def _load_template_module(name: str, monkeypatch: pytest.MonkeyPatch): monkeypatch.syspath_prepend(str(TEMPLATE_ROOT)) spec = importlib.util.spec_from_file_location(f"generated_{name}", TEMPLATE_ROOT / name) @@ -101,6 +221,8 @@ def test_generate_cli_parses_repeatable_inputs() -> None: "reports", "--artifact", r"proxy-tools=C:\wheels\proxy.whl", + "--application-wheel", + r"C:\wheels\application.whl", "--bootstrap", "online_cmd", "--system-certs", @@ -110,9 +232,666 @@ def test_generate_cli_parses_repeatable_inputs() -> None: ) assert arguments.extra == ["map", "reports"] assert arguments.bootstrap == "online_cmd" + assert arguments.application_wheel == Path(r"C:\wheels\application.whl") assert arguments.system_certs and arguments.prepare_lock and arguments.dry_run +def test_application_wheel_validation_and_package_staging( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "Run Legacy.bat").write_text("legacy deployment", encoding="utf-8") + (source / "tests").mkdir() + (source / "tests/test_app.py").write_text("pass\n", encoding="utf-8") + cache = source / "code/__pycache__ (1)" + cache.mkdir() + (cache / "main.pyc").write_bytes(b"cache") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path) + + artifact, resolved = validate_application_wheel(wheel, assessment, plan) + + assert resolved == wheel.resolve() + assert artifact.distribution_name == "mapped-app" + assert artifact.version == "1.2.3" + assert artifact.entry_point_target == "installed_app.main:main" + assert plan.deployment_mode == "package" + assert _staging_files(source, assessment, plan, include=True).keys() == { + "pyproject.toml", + "uv.lock", + } + + fake_uv = tmp_path / "developer-uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + output = tmp_path / "kit" + result = generate_deployment_kit(repository, output, application_wheel=wheel) + + assert result.manifest.deployment_mode == "package" + assert result.manifest.source_roots == [] + assert result.manifest.application_artifact.sha256 == artifact.sha256 + assert (output / "deployment/application" / wheel.name).is_file() + assert not (output / "code").exists() + assert not (output / "Run Legacy.bat").exists() + assert not (output / "tests").exists() + assert not list(output.rglob("*.pyc")) + + +def test_application_wheel_rejects_wrong_target_and_runtime_cache(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + + wrong_target = _make_application_wheel(tmp_path, target="installed_app.other:main") + with pytest.raises(PreparationError, match="entry point disagrees"): + validate_application_wheel(wrong_target, assessment, plan) + wrong_target.unlink() + cached = _make_application_wheel(tmp_path, include_cache=True) + with pytest.raises(PreparationError, match="runtime cache"): + validate_application_wheel(cached, assessment, plan) + + +@pytest.mark.parametrize( + ("name", "version", "message"), + [ + ("other-app", "1.2.3", "name mismatch"), + ("mapped-app", "9.9", "version mismatch"), + ], +) +def test_application_wheel_rejects_wrong_filename_identity( + tmp_path: Path, name: str, version: str, message: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _make_application_wheel(tmp_path, name=name, version=version) + + with pytest.raises(PreparationError, match=message): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize("member", ["../escape.py", "/absolute.py", "C:/absolute.py"]) +def test_application_wheel_rejects_unsafe_archive_members( + tmp_path: Path, member: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), additions={member: "unsafe"} + ) + + with pytest.raises(PreparationError, match="unsafe member"): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_rejects_duplicate_and_case_conflicting_paths( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _make_application_wheel(tmp_path) + with zipfile.ZipFile(wheel, "a") as bundle: + bundle.writestr("INSTALLED_APP/main.py", "conflict") + + with pytest.raises(PreparationError, match="duplicate or conflicting"): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize( + ("metadata_name", "content", "message"), + [ + ( + "mapped_app-1.2.3.dist-info/METADATA", + "Metadata-Version: 2.1\nName: other-app\nVersion: 1.2.3\n\n", + "distribution name", + ), + ( + "mapped_app-1.2.3.dist-info/METADATA", + "Metadata-Version: 2.1\nName: mapped-app\nVersion: 9.9\n\n", + "version", + ), + ("mapped_app-1.2.3.dist-info/METADATA", "not metadata\n", "Malformed METADATA"), + ("mapped_app-1.2.3.dist-info/WHEEL", "not wheel metadata\n", "Malformed WHEEL"), + ], +) +def test_application_wheel_rejects_metadata_disagreement_and_malformed_metadata( + tmp_path: Path, metadata_name: str, content: str, message: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), replacements={metadata_name: content} + ) + + with pytest.raises(PreparationError, match=message): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_record_is_an_exact_file_inventory(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + + wheel = _make_application_wheel(tmp_path) + with zipfile.ZipFile(wheel) as bundle: + recorded = [ + item.filename + for item in bundle.infolist() + if not item.filename.endswith(".dist-info/RECORD") + ] + _rewrite_application_wheel(wheel, recorded_paths=[*recorded, "ghost.py"]) + with pytest.raises(PreparationError, match="nonexistent"): + validate_application_wheel(wheel, assessment, plan) + + wheel.unlink() + wheel = _make_application_wheel(tmp_path) + with zipfile.ZipFile(wheel) as bundle: + recorded = [ + item.filename + for item in bundle.infolist() + if not item.filename.endswith(".dist-info/RECORD") + ] + _rewrite_application_wheel( + wheel, + additions={"installed_app/unrecorded.txt": "unexpected"}, + recorded_paths=recorded, + ) + with pytest.raises(PreparationError, match="incomplete"): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize( + ("removals", "additions", "message"), + [ + ({"installed_app/main.py"}, {}, "entry-point module"), + ({"installed_app/view.html"}, {}, "package data"), + (set(), {"installed_app/native.dll": b"native"}, "native binaries"), + (set(), {"installed_app/module.pyo": b"cache"}, "runtime cache"), + ], +) +def test_application_wheel_rejects_missing_runtime_content_and_binary_content( + tmp_path: Path, + removals: set[str], + additions: dict[str, bytes], + message: str, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), removals=removals, additions=additions + ) + + with pytest.raises(PreparationError, match=message): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize( + ("name", "content"), + [ + ("installed_app/install.ps1", "Write-Host unsafe"), + ("installed_app/tool.py", "COMMAND = 'powershell.exe -ExecutionPolicy bypass'"), + ("installed_app/path.py", r"ROOT = 'C:\Users\developer\private'"), + ("installed_app/.env", "API_KEY=secret"), + ], +) +def test_application_wheel_cannot_bypass_deployment_security_policy( + tmp_path: Path, name: str, content: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), additions={name: content} + ) + + with pytest.raises(PreparationError, match="security policy"): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize( + "filename", + [ + "mapped_app-1.2.3-cp313-cp313-win_amd64.whl", + "mapped_app-1.2.3-cp312-cp312-win_arm64.whl", + ], +) +def test_application_wheel_rejects_incompatible_python_and_platform_tags( + tmp_path: Path, filename: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _make_application_wheel(tmp_path) + wheel = wheel.replace(tmp_path / filename) + + with pytest.raises(PreparationError, match="incompatible"): + validate_application_wheel(wheel, assessment, plan) + + +def test_package_generation_validates_application_wheel_before_acquisition_or_writes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + output = tmp_path / "kit" + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail("uv acquisition must not run"), + ) + + with pytest.raises(PreparationError, match="requires --application-wheel"): + generate_deployment_kit(repository, output) + assert not output.exists() + + wrong = _make_application_wheel(tmp_path, target="installed_app.other:main") + with pytest.raises(PreparationError, match="entry point disagrees"): + generate_deployment_kit(repository, output, application_wheel=wrong) + assert not output.exists() + + +def test_package_dry_run_reports_missing_valid_and_invalid_application_wheels( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + output = tmp_path / "kit" + + missing = generate_deployment_kit(repository, output, dry_run=True) + assert missing.preview.deployment_mode == "package" + assert missing.preview.source_roots == [] + assert missing.preview.application_wheel_required + assert missing.preview.readiness_before == "BLOCKED_PENDING_APPLICATION_WHEEL" + assert not output.exists() + + wheel = _make_application_wheel(tmp_path) + valid = generate_deployment_kit(repository, output, application_wheel=wheel, dry_run=True) + assert not valid.preview.application_wheel_required + assert valid.preview.source_roots == [] + assert valid.preview.application_artifact.sha256 + assert f"deployment/application/{wheel.name}" in valid.preview.files_to_create + assert "code/main.py" not in valid.preview.files_to_create + assert not output.exists() + + wheel.unlink() + malformed = tmp_path / wheel.name + malformed.write_bytes(b"not a zip") + with pytest.raises(PreparationError, match="Malformed application wheel"): + generate_deployment_kit( + repository, output, application_wheel=malformed, dry_run=True + ) + assert not output.exists() + + +def test_deployment_fingerprint_separates_mode_and_exact_application_wheel_bytes( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + package_plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path) + artifact, _ = validate_application_wheel(wheel, assessment, package_plan) + + def manifest(plan, application): + return build_deployment_manifest( + plan, + source, + bootstrap_mode="online_cmd", + system_certs=False, + approved_artifacts=[], + application_artifact=application, + bundled_uv_sha256=None, + referenced_files=[], + ) + + original = manifest(package_plan, artifact) + renamed_wheel = tmp_path / "mapped_app-1.2.3-1-py3-none-any.whl" + renamed_wheel.write_bytes(wheel.read_bytes()) + renamed_artifact, _ = validate_application_wheel( + renamed_wheel, assessment, package_plan + ) + renamed = manifest(package_plan, renamed_artifact) + assert renamed.deployment_fingerprint == original.deployment_fingerprint + + _rewrite_application_wheel( + wheel, additions={"installed_app/additional-runtime-data.txt": "changed bytes"} + ) + changed_artifact, _ = validate_application_wheel(wheel, assessment, package_plan) + changed = manifest(package_plan, changed_artifact) + assert changed.deployment_fingerprint != original.deployment_fingerprint + + source_plan = package_plan.model_copy(deep=True) + source_plan.deployment_mode = "source" + source_plan.runtime.environment_variables["PYTHONPATH"] = "%PROJECT_ROOT%" + source_manifest = manifest(source_plan, None) + assert source_manifest.application_artifact is None + assert source_manifest.deployment_fingerprint != original.deployment_fingerprint + + +def test_package_mode_release_is_deterministic_and_survives_extraction( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + wheel = _make_application_wheel(tmp_path) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generated = generate_deployment_kit(repository, kit, application_wheel=wheel) + + first = package_deployment_kit(kit, output_directory=tmp_path / "dist-one") + second = package_deployment_kit(kit, output_directory=tmp_path / "dist-two") + assert first.manifest.deployment_mode == "package" + assert ( + first.manifest.application_artifact.sha256 + == generated.manifest.application_artifact.sha256 + ) + assert first.manifest.deployment_fingerprint == generated.manifest.deployment_fingerprint + assert first.manifest.zip_sha256 == second.manifest.zip_sha256 + assert hashlib.sha256(Path(first.zip_path).read_bytes()).hexdigest().upper() == ( + first.manifest.zip_sha256 + ) + with zipfile.ZipFile(first.zip_path) as bundle: + names = set(bundle.namelist()) + assert f"deployment/application/{wheel.name}" in names + assert not any(name.startswith("code/") for name in names) + + extracted = tmp_path / "extracted" + safe_extract_zip(Path(first.zip_path), extracted) + assert validate_static_kit(extracted).final_state.value == "STATIC_VALID" + smoke = Path(first.smoke_test_path).read_text(encoding="utf-8") + assert "first-party application artifact is mapped-app==1.2.3" in smoke + + +def test_generate_and_all_cli_propagate_first_party_application_wheel( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + wheel = _make_application_wheel(tmp_path) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + + generate_output = tmp_path / "generated" + assert ( + main( + [ + "generate", + str(source), + "--output-dir", + str(generate_output), + "--application-wheel", + str(wheel), + "--bootstrap", + "online_cmd", + ] + ) + == 0 + ) + assert (generate_output / "deployment/application" / wheel.name).is_file() + + all_output = tmp_path / "all-output" + assert ( + main( + [ + "all", + str(source), + "--output-dir", + str(all_output), + "--application-wheel", + str(wheel), + "--bootstrap", + "online_cmd", + ] + ) + == 0 + ) + assert (all_output / "deployment-kit/deployment/application" / wheel.name).is_file() + assert list((all_output / "distribution").glob("*.zip")) + + missing_output = tmp_path / "missing-output" + assert ( + main( + [ + "all", + str(source), + "--output-dir", + str(missing_output), + "--bootstrap", + "online_cmd", + ] + ) + == 2 + ) + assert not missing_output.exists() + output_text = capsys.readouterr().out + assert "--application-wheel" in output_text + assert "Source roots: none (installed-project mode)" in output_text + + help_text = build_parser().format_help() + generate_help = build_parser()._subparsers._group_actions[0].choices["generate"].format_help() + assert "first-party application wheel" in generate_help + assert "approved wheel" in generate_help + assert help_text + +def test_git_source_staging_excludes_untracked_application_files(tmp_path: Path) -> None: + source = tmp_path / "git-source" + source.mkdir() + (source / "pyproject.toml").write_text( + """[project] +name = "tracked-app" +version = "1.0.0" +dependencies = [] +[project.scripts] +tracked-app = "app:main" +[tool.setuptools] +py-modules = ["app"] +""", + encoding="utf-8", + ) + (source / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = \">=3.11\"\n", + encoding="utf-8", + ) + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + (source / "local_helper.py").write_text("VALUE = 1\n", encoding="utf-8") + + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + staged = _staging_files(source, assessment, plan, include=True) + + assert assessment.repository.revision + assert "app.py" in staged + assert "local_helper.py" not in staged + + +def test_git_source_staging_blocks_dirty_tracked_inputs_but_ignores_unrelated_docs( + tmp_path: Path, +) -> None: + source = tmp_path / "git-source" + source.mkdir() + (source / "docs").mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='clean-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nclean-app='app:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "docs/readme.md").write_text("docs\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + (source / "docs/readme.md").write_text("unrelated docs change\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + assert "app.py" in _staging_files(source, assessment, plan, include=True) + + (source / "app.py").write_text("def main(): return 1\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + with pytest.raises(PreparationError, match="differ from recorded source revision.*app.py"): + _staging_files(source, assessment, plan, include=True) + + +def test_source_staging_includes_required_root_nested_and_adjacent_resources( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + (source / "package/resources").mkdir(parents=True) + (source / "runtime").mkdir() + (source / "package/__init__.py").write_text("", encoding="utf-8") + (source / "package/app.py").write_text( + "from pathlib import Path\n" + "HERE=Path(__file__).parent\n" + "A=(HERE/'resources/nested.json').read_text()\n" + "B=Path('root-data.json').read_text()\n" + "C=Path('runtime/adjacent.txt').read_text()\n" + "def main(): return A+B+C\n", + encoding="utf-8", + ) + (source / "package/resources/nested.json").write_text("{}", encoding="utf-8") + (source / "root-data.json").write_text("root", encoding="utf-8") + (source / "runtime/adjacent.txt").write_text("adjacent", encoding="utf-8") + (source / "pyproject.toml").write_text( + "[project]\nname='resource-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nresource-app='package.app:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + staged = _staging_files(source, assessment, plan, include=True) + + assert plan.deployment_mode == "source" + assert { + "package/__init__.py", + "package/app.py", + "package/resources/nested.json", + "root-data.json", + "runtime/adjacent.txt", + } <= staged.keys() + + +def test_source_runtime_file_cannot_collide_with_generated_path(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + launcher = source / "deployment/README-deployment.txt" + launcher.parent.mkdir() + launcher.write_text("runtime payload", encoding="utf-8") + (source / "app.py").write_text( + "from pathlib import Path\n" + "PAYLOAD=Path('deployment/README-deployment.txt').read_text()\n" + "def main(): return PAYLOAD\n", + encoding="utf-8", + ) + (source / "pyproject.toml").write_text( + "[project]\nname='collision-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\ncollision-app='app:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + preview = generate_deployment_kit(repository, tmp_path / "kit", dry_run=True).preview + + assert preview.deployment_mode == "source" + assert preview.collisions == [ + "deployment/README-deployment.txt (runtime source conflicts with a generated path)" + ] + + def test_dry_run_makes_no_output_or_lock(tmp_path: Path) -> None: repository = _repository("target_app") output = tmp_path / "kit" @@ -370,6 +1149,12 @@ def test_runtime_common_staleness_and_deletion_guards( (environment / "Scripts").mkdir(parents=True) (environment / "Scripts" / "python.exe").touch() (environment / "Scripts" / "pythonw.exe").touch() + deployment = tmp_path / "deployment" + application = deployment / "application" + application.mkdir(parents=True) + application_wheel = application / "sample-1.0-py3-none-any.whl" + application_wheel.write_bytes(b"application wheel") + monkeypatch.setattr(common, "deployment_directory", lambda: deployment) manifest = { "schema_version": "1.0", "application_id": "sample", @@ -380,6 +1165,10 @@ def test_runtime_common_staleness_and_deletion_guards( "lockfile_sha256": common.sha256_file(project / "uv.lock"), "selected_extras_fingerprint": "extras", "approved_artifacts": [], + "application_artifact": { + "filename": application_wheel.name, + "sha256": common.sha256_file(application_wheel), + }, "runtime_paths": { "application_root": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample", "environment_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\env", @@ -389,6 +1178,9 @@ def test_runtime_common_staleness_and_deletion_guards( } common.write_state(manifest, project, "now") assert common.stale_reasons(manifest, project) == [] + application_wheel.write_bytes(b"changed application wheel") + assert any("application artifact" in item for item in common.stale_reasons(manifest, project)) + application_wheel.write_bytes(b"application wheel") (project / "uv.lock").write_text("changed", encoding="utf-8") assert "lockfile_sha256 changed" in common.stale_reasons(manifest, project) with pytest.raises(common.DeploymentRuntimeError, match="unsafe"): diff --git a/tests/test_launch_contract.py b/tests/test_launch_contract.py index 9af2708..d6fd3d8 100644 --- a/tests/test_launch_contract.py +++ b/tests/test_launch_contract.py @@ -34,6 +34,7 @@ def _write_launch_fixture(tmp_path: Path, target_source: str) -> tuple[Path, Pat "entry_point_kind": "gui", "entry_point_module": "synthetic_target", "entry_point_callable": "main", + "deployment_mode": "source", "source_roots": ["."], "project_write_probe_required": False, "schema_version": "1.0", @@ -234,3 +235,33 @@ def main(): diagnostics = _run_diagnostics(project_root, launcher, local_app_data) assert diagnostics.returncode == 0, diagnostics.stderr assert f"Most recent application launch failure: {logs[0]}" in diagnostics.stdout + + +def test_package_launch_does_not_import_same_named_module_from_repository_cwd( + tmp_path: Path, +) -> None: + project_root, launcher, local_app_data = _write_launch_fixture( + tmp_path, + """ + from pathlib import Path + Path("source-sentinel.txt").write_text("source", encoding="utf-8") + def main(): return 0 + """, + ) + (launcher.parent / "synthetic_target.py").write_text( + "from pathlib import Path\n" + "Path('installed-sentinel.txt').write_text('installed', encoding='utf-8')\n" + "def main(): return 0\n", + encoding="utf-8", + ) + manifest_path = project_root / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["deployment_mode"] = "package" + manifest["source_roots"] = [] + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + result = _run_launch(project_root, launcher, local_app_data) + + assert result.returncode == 0, result.stderr + assert (project_root / "installed-sentinel.txt").read_text(encoding="utf-8") == "installed" + assert not (project_root / "source-sentinel.txt").exists() diff --git a/tests/test_planning.py b/tests/test_planning.py index f44de21..c1216c9 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -30,6 +30,114 @@ def _assess(name: str = "target_app"): ) +def _write_mode_project( + root: Path, + *, + layout: str = "flat", + target: str = "sample_app:main", + buildable: bool = True, + source_constraint: bool = False, + mapped: bool = False, +) -> None: + source = root / ("code" if mapped else "src" if layout == "src" else ".") + source.mkdir(parents=True, exist_ok=True) + module = source / ("main.py" if mapped else "sample_app.py") + module.write_text( + ( + "from pathlib import Path\nRUNTIME = Path('runtime.json')\nRUNTIME.read_text()\n" + if source_constraint + else "" + ) + + "def main(): return 0\n", + encoding="utf-8", + ) + if mapped: + (source / "__init__.py").write_text("", encoding="utf-8") + if source_constraint: + (root / "runtime.json").write_text("{}\n", encoding="utf-8") + build = ( + "[build-system]\nrequires = ['setuptools>=68']\n" + "build-backend = 'setuptools.build_meta'\n" + if buildable + else "" + ) + setuptools = "" + if mapped: + setuptools = ( + "[tool.setuptools]\npackages = ['installed_app']\n" + "package-dir = {installed_app = 'code'}\n" + ) + elif layout == "src": + setuptools = "[tool.setuptools.packages.find]\nwhere = ['src']\n" + (root / "pyproject.toml").write_text( + build + + "[project]\nname = 'sample-app'\nversion = '1.0.0'\ndependencies = []\n" + + f"[project.scripts]\nsample-app = '{target}'\n" + + setuptools, + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + +def test_complete_deployment_mode_decision_table(tmp_path: Path) -> None: + cases = { + "flat-source": dict(), + "src-constrained": dict(layout="src", source_constraint=True), + "src-install-oriented": dict(layout="src"), + "mapped-installed-namespace": dict( + mapped=True, target="installed_app.main:main" + ), + "mapped-conflict": dict( + mapped=True, + target="installed_app.main:main", + source_constraint=True, + ), + "metadata-insufficient": dict( + target="missing_app:main", + buildable=False, + ), + "metadata-over-ast": dict(target="missing_app:main"), + } + results = {} + for name, options in cases.items(): + root = tmp_path / name + root.mkdir() + _write_mode_project(root, **options) + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + results[name] = create_deployment_plan(assessment) + + assert ( + results["flat-source"].deployment_mode, + results["flat-source"].deployment_mode_condition, + ) == ( + "source", + "SOURCE_COMPATIBLE", + ) + assert ( + results["src-constrained"].deployment_mode, + results["src-constrained"].deployment_mode_condition, + ) == ("source", "SOURCE_COMPATIBLE") + assert ( + results["src-install-oriented"].deployment_mode, + results["src-install-oriented"].deployment_mode_condition, + ) == ("package", "PACKAGE_PREFERRED") + assert results["mapped-installed-namespace"].deployment_mode_condition == ( + "ENTRYPOINT_REQUIRES_PACKAGE_MODE" + ) + conflict = results["mapped-conflict"] + assert conflict.deployment_mode_condition == "DEPLOYMENT_MODE_CONFLICT" + assert conflict.readiness.blocker_codes == ["DEPLOYMENT_MODE_CONFLICT"] + insufficient = results["metadata-insufficient"] + assert insufficient.deployment_mode_condition == "INSTALLED_PROJECT_REQUIRED" + assert insufficient.readiness.blocker_codes == ["INSTALLED_PROJECT_REQUIRED"] + metadata = results["metadata-over-ast"] + assert metadata.entry_point.target == "missing_app:main" + assert metadata.deployment_mode_condition == "ENTRYPOINT_REQUIRES_PACKAGE_MODE" + assert all(plan.decisions[0].rationale for plan in results.values()) + + def test_target_plan_selects_source_gui_and_external_environment() -> None: plan = create_deployment_plan(_assess()) diff --git a/tests/test_validation.py b/tests/test_validation.py index ce5dfc6..44d8c0e 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json from pathlib import Path @@ -10,11 +11,17 @@ from python_deployment_builder.generation.generator import generate_deployment_kit from python_deployment_builder.generation.preparation import LockPreparationResult from python_deployment_builder.models import ( + DeploymentManifest, ValidationCheckStatus, ValidationFinalState, ) +from python_deployment_builder.packaging import package_deployment_kit from python_deployment_builder.reporting.json_report import write_validation_reports -from python_deployment_builder.validation.runtime import validate_runtime_kit +from python_deployment_builder.validation.runtime import ( + _runtime_environment, + _scenario_copy, + validate_runtime_kit, +) from python_deployment_builder.validation.static import validate_static_kit FIXTURES = Path(__file__).parent / "fixtures" @@ -47,6 +54,16 @@ def _status(report, code: str) -> ValidationCheckStatus: return next(item.status for item in report.static_checks if item.code == code) +def _refresh_manifest_index(kit: Path) -> None: + index_path = kit / "deployment/generated-files.json" + index = json.loads(index_path.read_text(encoding="utf-8")) + manifest = kit / "deployment/manifest.json" + for item in index["files"]: + if item["path"] == "deployment/manifest.json": + item["sha256"] = hashlib.sha256(manifest.read_bytes()).hexdigest() + index_path.write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") + + def test_validate_cli_defaults_static_and_requires_explicit_runtime() -> None: static = build_parser().parse_args(["validate", "kit"]) runtime = build_parser().parse_args( @@ -78,6 +95,27 @@ def test_static_kit_validation_and_report_serialization( assert "STATIC_VALID" in markdown_path.read_text(encoding="utf-8") +def test_pre_m61_source_manifest_defaults_remain_statically_compatible( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + kit = _kit(monkeypatch, tmp_path) + manifest_path = kit / "deployment/manifest.json" + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + payload.pop("application_artifact", None) + payload.pop("configuration_secret_names", None) + manifest_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + _refresh_manifest_index(kit) + + manifest = DeploymentManifest.model_validate(payload) + report = validate_static_kit(kit) + packaged = package_deployment_kit(kit, output_directory=tmp_path / "release") + + assert manifest.application_artifact is None + assert manifest.configuration_secret_names == [] + assert report.final_state == ValidationFinalState.STATIC_VALID + assert packaged.generated + + @pytest.mark.parametrize( ("relative", "code"), [ @@ -123,7 +161,7 @@ def test_static_validation_detects_runtime_bytecode_cache( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: kit = _kit(monkeypatch, tmp_path) - cache = kit / "deployment" / "runtime" / "__pycache__" + cache = kit / "deployment" / "runtime" / "__pycache__ (1)" cache.mkdir() (cache / "runtime_common.pyc").write_bytes(b"validation mutation") @@ -132,6 +170,44 @@ def test_static_validation_detects_runtime_bytecode_cache( assert _status(report, "NO_RUNTIME_CACHES") == ValidationCheckStatus.FAIL +def test_runtime_validation_preserves_controlled_localappdata_after_config_scrub( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + kit = _kit(monkeypatch, tmp_path) + manifest = DeploymentManifest.model_validate_json( + (kit / "deployment/manifest.json").read_text(encoding="utf-8") + ) + manifest.configuration_presence_names.append("LOCALAPPDATA") + isolated = tmp_path / "isolated-local-app-data" + + environment = _runtime_environment(manifest, isolated) + + assert environment["LOCALAPPDATA"] == str(isolated) + + +def test_runtime_scenario_copy_preserves_staged_artifacts( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + kit = _kit(monkeypatch, tmp_path) + manifest = DeploymentManifest.model_validate_json( + (kit / "deployment/manifest.json").read_text(encoding="utf-8") + ) + for directory, filename in ( + ("wheels", "approved.whl"), + ("application", "application.whl"), + ): + artifact = kit / "deployment" / directory / filename + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(directory.encode()) + + scenario = _scenario_copy(kit, tmp_path / "scenario", manifest, {}) + + assert (scenario / "deployment/wheels/approved.whl").read_bytes() == b"wheels" + assert ( + scenario / "deployment/application/application.whl" + ).read_bytes() == b"application" + + def test_runtime_dry_run_executes_nothing_and_creates_no_runtime_root( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -207,3 +283,162 @@ def controlled_failure(*args, **kwargs): assert python.read_text(encoding="utf-8") == "working" assert not (app_root / "env.previous").exists() assert not (app_root / "env.failed").exists() + + +def test_package_runtime_installs_artifacts_in_order_and_states_success_last( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import importlib.util + import sys + + template_root = ( + Path(__file__).parents[1] + / "src" + / "python_deployment_builder" + / "templates" + / "windows_uv" + ) + monkeypatch.syspath_prepend(str(template_root)) + spec = importlib.util.spec_from_file_location("ordered_manage", template_root / "manage.py") + assert spec and spec.loader + manage = importlib.util.module_from_spec(spec) + sys.modules["ordered_manage"] = manage + spec.loader.exec_module(manage) + + local = tmp_path / "LocalAppData" + monkeypatch.setenv("LOCALAPPDATA", str(local)) + project = tmp_path / "project" + project.mkdir() + deployment = tmp_path / "deployment" + (deployment / "wheels").mkdir(parents=True) + (deployment / "application").mkdir() + dependency = deployment / "wheels/dependency.whl" + application = deployment / "application/application.whl" + dependency.write_bytes(b"dependency") + application.write_bytes(b"application") + monkeypatch.setattr(manage, "deployment_directory", lambda: deployment) + app_root = local / "PythonDeploymentBuilder/apps/sample" + environment = app_root / "env" + events: list[str] = [] + manifest = { + "application_id": "sample", + "runtime_paths": { + "application_root": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample", + "environment_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\env", + "logs_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\logs", + "state_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\state", + }, + "runtime_environment": {}, + "sync_arguments": ["sync", "--locked", "--no-build"], + "approved_artifacts": [{"filename": dependency.name}], + "application_artifact": {"filename": application.name}, + } + + def record(command, **kwargs): + if command[1] == "sync": + events.append("locked-sync") + (environment / "Scripts").mkdir(parents=True) + (environment / "Scripts/python.exe").write_bytes(b"python") + (environment / "Scripts/pythonw.exe").write_bytes(b"pythonw") + elif str(dependency) in command: + events.append("dependency-artifact") + elif str(application) in command: + events.append("application-wheel") + elif "check" in command and command[1:3] == ["pip", "check"]: + events.append("pip-check") + else: + events.append("entry-point-check") + + monkeypatch.setattr(manage, "run_logged", record) + monkeypatch.setattr( + manage, + "write_state", + lambda *args: events.append("state-success"), + ) + + manage._promote_environment( + manifest, project, tmp_path / "uv.exe", manage.logging.getLogger("test-order") + ) + + assert events == [ + "locked-sync", + "dependency-artifact", + "application-wheel", + "pip-check", + "entry-point-check", + "state-success", + ] + + +def test_package_runtime_application_install_failure_restores_previous_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import importlib.util + import sys + + template_root = ( + Path(__file__).parents[1] + / "src" + / "python_deployment_builder" + / "templates" + / "windows_uv" + ) + monkeypatch.syspath_prepend(str(template_root)) + spec = importlib.util.spec_from_file_location( + "application_failure_manage", template_root / "manage.py" + ) + assert spec and spec.loader + manage = importlib.util.module_from_spec(spec) + sys.modules["application_failure_manage"] = manage + spec.loader.exec_module(manage) + + local = tmp_path / "LocalAppData" + monkeypatch.setenv("LOCALAPPDATA", str(local)) + project = tmp_path / "project" + project.mkdir() + deployment = tmp_path / "deployment" + (deployment / "application").mkdir(parents=True) + application = deployment / "application/application.whl" + application.write_bytes(b"application") + monkeypatch.setattr(manage, "deployment_directory", lambda: deployment) + app_root = local / "PythonDeploymentBuilder/apps/sample" + environment = app_root / "env" + (environment / "Scripts").mkdir(parents=True) + old_python = environment / "Scripts/python.exe" + old_python.write_bytes(b"known-good") + (environment / "Scripts/pythonw.exe").write_bytes(b"known-good") + state_writes: list[str] = [] + manifest = { + "application_id": "sample", + "runtime_paths": { + "application_root": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample", + "environment_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\env", + "logs_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\logs", + "state_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\state", + }, + "runtime_environment": {}, + "sync_arguments": ["sync"], + "approved_artifacts": [], + "application_artifact": {"filename": application.name}, + } + + def fail_application(command, **kwargs): + if command[1] == "sync": + (environment / "Scripts").mkdir(parents=True) + (environment / "Scripts/python.exe").write_bytes(b"candidate") + (environment / "Scripts/pythonw.exe").write_bytes(b"candidate") + return + raise manage.DeploymentRuntimeError("controlled application install failure") + + monkeypatch.setattr(manage, "run_logged", fail_application) + monkeypatch.setattr(manage, "write_state", lambda *args: state_writes.append("written")) + + with pytest.raises(manage.DeploymentRuntimeError, match="application install failure"): + manage._promote_environment( + manifest, project, tmp_path / "uv.exe", manage.logging.getLogger("test-app-fail") + ) + + assert old_python.read_bytes() == b"known-good" + assert not state_writes + assert not (app_root / "env.previous").exists() + assert not (app_root / "env.failed").exists() From 2aa4b0b38b76e33fadb6e1542607de5333286917 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:44:39 -0400 Subject: [PATCH 02/59] Harden M6.1 deployment provenance analysis Compare role-approved deployment inputs from both HEAD and the working tree so deleted and unstaged-renamed runtime inputs cannot escape dirty-source protection. Bound wrapper return inference to owned returns and unique callable names, preserving conservative unknown classifications for ambiguous code. --- README.md | 18 +- docs/architecture.md | 26 +-- .../analysis/runtime_assumptions.py | 24 ++- .../generation/generator.py | 105 +++++++++--- tests/test_analysis_scope.py | 74 ++++++++ tests/test_generation.py | 159 ++++++++++++++++++ 6 files changed, 361 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 853dc8f..e86be25 100644 --- a/README.md +++ b/README.md @@ -249,8 +249,10 @@ is diagnostic and never replaces the authoritative installed entry point. Package mode is selected when the installed namespace cannot be provided by the physical source layout, or when package policy otherwise requires installation. The developer supplies an exact first-party wheel with `--application-wheel`. PDB validates its distribution, version, wheel tags, -entry-point metadata/module, declared package data, pure-Python structure, and SHA-256 before -staging it. End-user setup synchronizes locked third-party dependencies with `--no-install-project` +entry-point metadata/module, explicitly named setuptools package-data mappings, pure-Python +structure, and SHA-256 before staging it. Wildcard (`"*"`) package-data mappings are retained as +metadata but are not exhaustively asserted by the M6.1 wheel validator. End-user setup synchronizes +locked third-party dependencies with `--no-install-project` and `--no-build`, installs approved exceptional dependency wheels, installs the first-party wheel with `uv pip install --no-deps --no-build`, and runs `uv pip check`. It never builds the application on the end-user machine. The application-wheel provenance is distinct from dependency @@ -270,11 +272,13 @@ Every real generation runs pinned `uv lock --check`. A stale lock stops generati rewriting it. A missing lock also stops unless `--prepare-lock` explicitly authorizes a local repository mutation; URL inputs cannot use that option. The builder then runs `uv lock --python `, checks the result, reports the changed `uv.lock`, and never commits it. -For Git inputs with a recorded revision, generation also refuses modified tracked deployment -inputs (metadata, lockfile, application source, and runtime resources) rather than labeling their -working-tree bytes as clean revision provenance. Unrelated tracked documentation and harmless -untracked files do not trigger that gate. Untracked and ignored runtime-looking files are not -silently staged; required runtime material must be tracked or supplied by the application wheel. +For Git inputs with a recorded revision, generation compares deployment inputs identified in both +the recorded `HEAD` tree and the current working-tree inventory. Modified, deleted, or unstaged- +renamed tracked metadata, lockfiles, application source, and runtime resources are refused rather +than labeling changed working-tree contents as clean revision provenance. Unrelated tracked +documentation and harmless untracked files do not trigger that gate. Untracked and ignored +runtime-looking files are not silently staged; required runtime material must be tracked or +supplied by the application wheel. The intended source-control workflow is to create a deployment-preparation branch, assess and plan, explicitly prepare missing metadata, generate outside the repository, validate the staged diff --git a/docs/architecture.md b/docs/architecture.md index 968f0fa..be4590f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -102,10 +102,12 @@ structurally importable from the planned source roots without installing the pro uses an installed namespace that differs from its physical source namespace, PDB selects package mode when package metadata/resources are install-safe; genuine source-only constraints instead produce a typed deployment-mode conflict. Package mode requires a developer-supplied first-party -wheel. Its name, version, tags, entry point, module, package data, pure-Python contents, and hash are -validated, and its provenance is recorded separately from exceptional dependency artifacts and -from the assessed source revision. The exact supplied wheel bytes are authoritative; ordinary -wheel metadata alone is not evidence that those bytes were built from the recorded revision. +wheel. Its name, version, tags, entry point, module, explicitly named setuptools package-data +mappings, pure-Python contents, and hash are validated. Wildcard (`"*"`) package-data mappings are +recorded but are not exhaustively asserted in M6.1. Wheel provenance is recorded separately from +exceptional dependency artifacts and from the assessed source revision. The exact supplied wheel +bytes are authoritative; ordinary wheel metadata alone is not evidence that those bytes were built +from the recorded revision. Normal launch compares schema, selected Python, pinned uv, project metadata, lockfile, selected extras, approved artifact hashes, environment path, and prior verification fingerprints. Matching @@ -134,13 +136,15 @@ entry-point import; it does not rely on `PYTHONPATH`. In package mode it adds no root and imports the authoritative target from the managed environment after the exact first-party wheel has been installed. -For a Git source with a recorded revision, generation intersects role-approved staging with -tracked paths and blocks when any selected tracked deployment input differs from `HEAD`. It does -not elevate ignored, untracked, documentation, example, test, deployment-support, or mutable-state -roles merely because a filename looks executable or resource-like. Non-Git directories and safely -materialized archives use the same role inventory without requiring Git; eligible application -source and runtime resources are staged directly in source mode, while package mode stages all -application runtime content only from its validated first-party wheel. +For a Git source with a recorded revision, generation intersects role-approved staging with tracked +paths and blocks when any deployment input identified by either the `HEAD` snapshot or current +working-tree inventory differs. This preserves evidence for deleted and unstaged-renamed source or +resource paths that no longer appear in the current inventory. It does not elevate ignored, +untracked, documentation, example, test, deployment-support, or mutable-state roles merely because +a filename looks executable or resource-like. Non-Git directories and safely materialized archives +use the same role inventory without requiring Git; eligible application source and runtime +resources are staged directly in source mode, while package mode stages all application runtime +content only from its validated first-party wheel. The online bootstrap's localized `certutil.exe` handling searches structurally for one 64-digit hexadecimal value rather than parsing English headings. Missing, blocked, download-failing, and diff --git a/src/python_deployment_builder/analysis/runtime_assumptions.py b/src/python_deployment_builder/analysis/runtime_assumptions.py index 5b2945c..a7c4128 100644 --- a/src/python_deployment_builder/analysis/runtime_assumptions.py +++ b/src/python_deployment_builder/analysis/runtime_assumptions.py @@ -106,15 +106,31 @@ def _platforms_for(category: str, name: str) -> list[str]: def _simple_function_returns(tree: ast.AST) -> dict[str, str]: - """Summarize only wrappers with one statically expressible return value.""" + """Summarize only unique wrappers with one return owned by that function.""" summaries: dict[str, str] = {} + definitions: dict[str, list[ast.FunctionDef | ast.AsyncFunctionDef]] = defaultdict(list) for node in ast.walk(tree): - if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + definitions[node.name].append(node) + for name, nodes in definitions.items(): + if len(nodes) != 1: continue - returns = [item for item in ast.walk(node) if isinstance(item, ast.Return)] + returns: list[ast.Return] = [] + pending: list[ast.AST] = list(nodes[0].body) + while pending: + item = pending.pop() + if isinstance(item, ast.Return): + returns.append(item) + continue + if isinstance( + item, + (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef), + ): + continue + pending.extend(ast.iter_child_nodes(item)) if len(returns) == 1 and returns[0].value is not None: - summaries[node.name] = _expression(returns[0].value) + summaries[name] = _expression(returns[0].value) return summaries diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index ece5b20..694cdc4 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -7,12 +7,17 @@ import os import re import subprocess +import tempfile from pathlib import Path from packaging.utils import canonicalize_name from python_deployment_builder.analysis import assess_repository -from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.repository import ( + MaterializedRepository, + RepositoryLoadError, + safe_extract_zip, +) from python_deployment_builder.generation.acquisition import ( PreparationError, acquire_pinned_uv, @@ -105,7 +110,79 @@ def _dirty_tracked_deployment_paths( for value in result.stdout.split(b"\0") if value } - return sorted(changed & selected) + if not changed: + return [] + with tempfile.TemporaryDirectory(prefix="pdbuilder-head-inventory-") as temporary: + temporary_root = Path(temporary) + archive = temporary_root / "repository.zip" + extracted = temporary_root / "repository" + archived = subprocess.run( + [ + "git", + "-C", + str(repository_root), + "archive", + "--format=zip", + f"--output={archive}", + "HEAD", + ], + capture_output=True, + check=False, + ) + if archived.returncode != 0: + raise PreparationError( + "Git revision provenance is known, but the recorded source revision could not " + "be inventoried. Generation stopped rather than claiming clean provenance." + ) + try: + head_root = safe_extract_zip(archive, extracted) + head_repository = MaterializedRepository( + root=head_root, + source=f"{repository_root}@HEAD", + source_kind="local", + ) + head_assessment = assess_repository(head_repository) + head_plan = create_deployment_plan( + head_assessment, repository_root=head_root + ) + head_selected = _selected_deployment_paths(head_assessment, head_plan) + except (OSError, RepositoryLoadError, ValueError) as exc: + raise PreparationError( + "Git revision provenance is known, but the recorded source revision could not " + "be inventoried. Generation stopped rather than claiming clean provenance." + ) from exc + return sorted(changed & (selected | head_selected)) + + +def _selected_deployment_paths(assessment, plan) -> set[str]: + selected = {"pyproject.toml", "uv.lock"} + if plan.deployment_mode == "source": + selected.update( + item.path.rstrip("/") + for item in assessment.file_inventory + if item.role in RUNTIME_ROLES and not item.path.endswith("/") + ) + return selected + + +def _tracked_deployment_paths(repository_root: Path, assessment, plan) -> set[str]: + selected = _selected_deployment_paths(assessment, plan) + tracked = _git_tracked_paths( + repository_root, + required=assessment.repository.revision is not None, + ) + if tracked is None: + return selected + selected.intersection_update(tracked) + if assessment.repository.revision is not None: + dirty = _dirty_tracked_deployment_paths(repository_root, selected) + if dirty: + raise PreparationError( + "Tracked deployment inputs differ from recorded source revision " + f"{assessment.repository.revision}: {', '.join(dirty)}. Commit or restore " + "those inputs before release-oriented generation." + ) + return selected def _staging_files( @@ -120,27 +197,7 @@ def _staging_files( if not include: return {} - selected = {"pyproject.toml", "uv.lock"} - if plan.deployment_mode == "source": - selected.update( - item.path.rstrip("/") - for item in assessment.file_inventory - if item.role in RUNTIME_ROLES and not item.path.endswith("/") - ) - tracked = _git_tracked_paths( - repository_root, - required=assessment.repository.revision is not None, - ) - if tracked is not None: - selected.intersection_update(tracked) - if assessment.repository.revision is not None: - dirty = _dirty_tracked_deployment_paths(repository_root, selected) - if dirty: - raise PreparationError( - "Tracked deployment inputs differ from recorded source revision " - f"{assessment.repository.revision}: {', '.join(dirty)}. Commit or restore " - "those inputs before release-oriented generation." - ) + selected = _tracked_deployment_paths(repository_root, assessment, plan) files: dict[str, bytes] = {} for relative_text in sorted(selected): relative = Path(relative_text) @@ -457,6 +514,8 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) + if assessment.repository.revision is not None: + _tracked_deployment_paths(repository_root, assessment, plan) if plan.entry_point is None: raise PreparationError( "Deployment readiness is blocked: " + "; ".join(plan.readiness.blockers) diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 3f6bba9..44519cf 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -809,6 +809,80 @@ def main(): assert "DEPLOYMENT_MODE_CONFLICT" in conflict.readiness.blocker_codes +def test_unique_write_path_wrapper_infers_user_local(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text( + """from pathlib import Path +def user_data_path(name): + return Path.home() / ".sample" / name +def oauth_path(): + return user_data_path("oauth.json") +oauth = oauth_path() +oauth.write_text("state") +""", + encoding="utf-8", + ) + + assessment = assess_repository(_repository(tmp_path)) + oauth = next( + item for item in assessment.write_locations if "oauth_path" in item.path_expression + ) + + assert oauth.classification == "user_local" + + +def test_nested_function_return_does_not_summarize_outer_wrapper(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text( + """from pathlib import Path +def user_data_path(name): + return Path.home() / ".sample" / name +def outer(): + def inner(): + return user_data_path("state.json") + do_something() +state = outer() +state.write_text("value") +""", + encoding="utf-8", + ) + + assessment = assess_repository(_repository(tmp_path)) + state = next( + item for item in assessment.write_locations if "outer" in item.path_expression + ) + + assert state.classification == "unknown" + assert state.status == FindingStatus.NEEDS_VALIDATION + + +def test_duplicate_method_names_do_not_share_return_summary(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text( + """from pathlib import Path +def user_data_path(name): + return Path.home() / ".sample" / name +class A: + def cache_path(self): + return user_data_path("state.json") +class B: + def cache_path(self): + return Path(__file__).with_name("state.json") +a_state = A().cache_path() +b_state = B().cache_path() +a_state.write_text("a") +b_state.write_text("b") +""", + encoding="utf-8", + ) + + assessment = assess_repository(_repository(tmp_path)) + cache_writes = [ + item for item in assessment.write_locations if "cache_path" in item.path_expression + ] + + assert len(cache_writes) == 2 + assert all(item.classification == "unknown" for item in cache_writes) + assert all(item.status == FindingStatus.NEEDS_VALIDATION for item in cache_writes) + + def test_cache_collision_directories_are_inventory_local_state(tmp_path: Path) -> None: (tmp_path / "app.py").write_text("def main(): return 0\n", encoding="utf-8") cache = tmp_path / "app" / "__pycache__ (12)" diff --git a/tests/test_generation.py b/tests/test_generation.py index f592b2d..e03789f 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -823,6 +823,165 @@ def test_git_source_staging_blocks_dirty_tracked_inputs_but_ignores_unrelated_do _staging_files(source, assessment, plan, include=True) +def _committed_source_fixture(tmp_path: Path) -> tuple[Path, MaterializedRepository]: + source = tmp_path / "git-source" + source.mkdir() + (source / "docs").mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='dirty-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\ndirty-app='app:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (source / "app.py").write_text( + "from pathlib import Path\n" + "STATE = Path(__file__).with_name('state.json').read_text()\n" + "def main(): return STATE\n", + encoding="utf-8", + ) + (source / "state.json").write_text("{}\n", encoding="utf-8") + (source / "docs/readme.md").write_text("documentation\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + return source, MaterializedRepository( + root=source, source=str(source), source_kind="local" + ) + + +def test_clean_git_source_fixture_stages_and_previews_normally(tmp_path: Path) -> None: + source, repository = _committed_source_fixture(tmp_path) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + staged = _staging_files(source, assessment, plan, include=True) + preview = generate_deployment_kit( + repository, + tmp_path / "kit", + bootstrap_mode="online_cmd", + dry_run=True, + ).preview + + assert {"app.py", "state.json"} <= staged.keys() + assert {"app.py", "state.json"} <= set(preview.files_to_create) + + +def test_git_source_staging_blocks_deleted_tracked_application_source( + tmp_path: Path, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + (source / "app.py").unlink() + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError) as caught: + _staging_files(source, assessment, plan, include=True) + assert assessment.repository.revision in str(caught.value) + assert "app.py" in str(caught.value) + + with pytest.raises(PreparationError, match="recorded source revision.*app.py"): + generate_deployment_kit( + repository, + tmp_path / "kit", + bootstrap_mode="online_cmd", + dry_run=True, + ) + + +def test_git_source_staging_blocks_unstaged_application_source_rename( + tmp_path: Path, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + (source / "app.py").rename(source / "renamed_app.py") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError) as caught: + _staging_files(source, assessment, plan, include=True) + assert assessment.repository.revision in str(caught.value) + assert "app.py" in str(caught.value) + + +@pytest.mark.parametrize("operation", ["modified", "deleted"]) +def test_git_source_staging_blocks_dirty_tracked_runtime_resource( + tmp_path: Path, + operation: str, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + resource = source / "state.json" + if operation == "modified": + resource.write_text('{"changed": true}\n', encoding="utf-8") + else: + resource.unlink() + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError) as caught: + _staging_files(source, assessment, plan, include=True) + assert assessment.repository.revision in str(caught.value) + assert "state.json" in str(caught.value) + + +@pytest.mark.parametrize( + ("relative", "operation"), + [ + ("pyproject.toml", "modified"), + ("pyproject.toml", "deleted"), + ("uv.lock", "modified"), + ("uv.lock", "deleted"), + ], +) +def test_git_source_staging_blocks_dirty_tracked_project_metadata( + tmp_path: Path, + relative: str, + operation: str, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + path = source / relative + if operation == "modified": + path.write_text(path.read_text(encoding="utf-8") + "\n# changed\n", encoding="utf-8") + else: + path.unlink() + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError) as caught: + _staging_files(source, assessment, plan, include=True) + assert assessment.repository.revision in str(caught.value) + assert relative in str(caught.value) + + +def test_git_source_staging_allows_deleted_unrelated_documentation( + tmp_path: Path, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + (source / "docs/readme.md").unlink() + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + staged = _staging_files(source, assessment, plan, include=True) + + assert {"app.py", "state.json"} <= staged.keys() + assert "docs/readme.md" not in staged + + +def test_git_source_staging_allows_untracked_unrelated_file(tmp_path: Path) -> None: + source, repository = _committed_source_fixture(tmp_path) + (source / "local-notes.txt").write_text("not a deployment input\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + staged = _staging_files(source, assessment, plan, include=True) + + assert {"app.py", "state.json"} <= staged.keys() + assert "local-notes.txt" not in staged + + def test_source_staging_includes_required_root_nested_and_adjacent_resources( tmp_path: Path, ) -> None: From 8b57ae648a836e8760c943df7bf9da3b4f7bce76 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:59:02 -0400 Subject: [PATCH 03/59] Make wrapper inference receiver-aware --- .../analysis/runtime_assumptions.py | 65 ++++++++++++-- tests/test_analysis_scope.py | 84 +++++++++++++++++++ 2 files changed, 143 insertions(+), 6 deletions(-) diff --git a/src/python_deployment_builder/analysis/runtime_assumptions.py b/src/python_deployment_builder/analysis/runtime_assumptions.py index a7c4128..dd9ebe3 100644 --- a/src/python_deployment_builder/analysis/runtime_assumptions.py +++ b/src/python_deployment_builder/analysis/runtime_assumptions.py @@ -106,14 +106,20 @@ def _platforms_for(category: str, name: str) -> list[str]: def _simple_function_returns(tree: ast.AST) -> dict[str, str]: - """Summarize only unique wrappers with one return owned by that function.""" + """Summarize unique module functions and direct methods by lexical identity.""" summaries: dict[str, str] = {} definitions: dict[str, list[ast.FunctionDef | ast.AsyncFunctionDef]] = defaultdict(list) - for node in ast.walk(tree): + if not isinstance(tree, ast.Module): + return summaries + for node in tree.body: if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): definitions[node.name].append(node) - for name, nodes in definitions.items(): + elif isinstance(node, ast.ClassDef): + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): + definitions[f"{node.name}.{member.name}"].append(member) + for identity, nodes in definitions.items(): if len(nodes) != 1: continue returns: list[ast.Return] = [] @@ -130,7 +136,7 @@ def _simple_function_returns(tree: ast.AST) -> dict[str, str]: continue pending.extend(ast.iter_child_nodes(item)) if len(returns) == 1 and returns[0].value is not None: - summaries[name] = _expression(returns[0].value) + summaries[identity] = _expression(returns[0].value) return summaries @@ -148,15 +154,62 @@ def __init__( self.writes: dict[tuple[str, str], list[Evidence]] = defaultdict(list) self.assignments: dict[str, str] = {} self.function_returns = function_returns or {} + self.lexical_scopes: list[tuple[str, str | None]] = [("module", None)] + + def _call_summary_identity(self, function: ast.expr) -> str | None: + if isinstance(function, ast.Name): + return function.id + if not isinstance(function, ast.Attribute) or not isinstance( + function.value, ast.Name + ): + return None + receiver = function.value.id + if receiver in {"self", "cls"}: + scope_kind, method_class = self.lexical_scopes[-1] + if scope_kind != "function" or method_class is None: + return None + return f"{method_class}.{function.attr}" + return None def _assigned_expression(self, value: ast.expr) -> str: expression = _expression(value) if isinstance(value, ast.Call): - function_name = _qualified_name(value.func).split(".")[-1] - if returned := self.function_returns.get(function_name): + identity = self._call_summary_identity(value.func) + if identity is not None and (returned := self.function_returns.get(identity)): expression = f"{expression} -> {returned}" return expression + def visit_ClassDef(self, node: ast.ClassDef) -> None: # noqa: N802 + self.lexical_scopes.append(("class", node.name)) + try: + self.generic_visit(node) + finally: + self.lexical_scopes.pop() + + def _visit_function( + self, node: ast.FunctionDef | ast.AsyncFunctionDef + ) -> None: + scope_kind, scope_name = self.lexical_scopes[-1] + method_class = scope_name if scope_kind == "class" else None + self.lexical_scopes.append(("function", method_class)) + try: + self.generic_visit(node) + finally: + self.lexical_scopes.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: # noqa: N802 + self._visit_function(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: # noqa: N802 + self._visit_function(node) + + def visit_Lambda(self, node: ast.Lambda) -> None: # noqa: N802 + self.lexical_scopes.append(("function", None)) + try: + self.generic_visit(node) + finally: + self.lexical_scopes.pop() + def _evidence(self, node: ast.AST, detail: str) -> Evidence: line = getattr(node, "lineno", None) excerpt = self.lines[line - 1].strip() if line and line <= len(self.lines) else None diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 44519cf..0a60a2a 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -830,6 +830,90 @@ def oauth_path(): assert oauth.classification == "user_local" +def test_same_class_self_method_wrapper_infers_user_local(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text( + """from pathlib import Path +def user_data_path(name): + return Path.home() / ".sample" / name +class Paths: + def oauth_path(self): + return user_data_path("oauth.json") + def save(self): + oauth = self.oauth_path() + oauth.write_text("state") +""", + encoding="utf-8", + ) + + assessment = assess_repository(_repository(tmp_path)) + oauth = next( + item for item in assessment.write_locations if "oauth_path" in item.path_expression + ) + + assert oauth.classification == "user_local" + + +def test_same_class_cls_method_wrapper_infers_user_local(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text( + """from pathlib import Path +def user_data_path(name): + return Path.home() / ".sample" / name +class Paths: + @classmethod + def oauth_path(cls): + return user_data_path("oauth.json") + @classmethod + def save(cls): + oauth = cls.oauth_path() + oauth.write_text("state") +""", + encoding="utf-8", + ) + + assessment = assess_repository(_repository(tmp_path)) + oauth = next( + item for item in assessment.write_locations if "oauth_path" in item.path_expression + ) + + assert oauth.classification == "user_local" + + +@pytest.mark.parametrize( + ("method_return", "incorrect_classification"), + [ + ('user_data_path("state.json")', "user_local"), + ('Path(__file__).with_name("state.json")', "project_local"), + ], +) +def test_unrelated_attribute_call_does_not_borrow_local_method_summary( + tmp_path: Path, + method_return: str, + incorrect_classification: str, +) -> None: + (tmp_path / "app.py").write_text( + f"""from pathlib import Path +def user_data_path(name): + return Path.home() / ".sample" / name +class LocalPaths: + def cache_path(self): + return {method_return} +external = SomeImportedClient() +state = external.cache_path() +state.write_text("value") +""", + encoding="utf-8", + ) + + assessment = assess_repository(_repository(tmp_path)) + state = next( + item for item in assessment.write_locations if "cache_path" in item.path_expression + ) + + assert state.classification != incorrect_classification + assert state.classification == "unknown" + assert state.status == FindingStatus.NEEDS_VALIDATION + + def test_nested_function_return_does_not_summarize_outer_wrapper(tmp_path: Path) -> None: (tmp_path / "app.py").write_text( """from pathlib import Path From 07cc9b0ae03e9a6c2a9461007919057f9260d3c7 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:49:37 -0400 Subject: [PATCH 04/59] Resolve M6.1 generation review findings --- .../generation/artifacts.py | 17 +- .../generation/generator.py | 89 +++++- tests/test_generation.py | 291 +++++++++++++++++- 3 files changed, 386 insertions(+), 11 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 6df2934..ea16e12 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -14,6 +14,7 @@ from pathlib import Path, PurePosixPath from packaging.utils import canonicalize_name, parse_wheel_filename +from packaging.version import InvalidVersion, Version from python_deployment_builder.generation.acquisition import PreparationError, sha256_file from python_deployment_builder.models import ( @@ -320,6 +321,12 @@ def validate_application_wheel( raise PreparationError( "Package mode requires authoritative project distribution and version metadata." ) + try: + expected_version_value = Version(expected_version) + except InvalidVersion as exc: + raise PreparationError( + f"Application authoritative project version is invalid: {expected_version!r}." + ) from exc if not path.is_file() or path.suffix.lower() != ".whl": raise PreparationError(f"Application wheel must be an existing wheel file: {path}") try: @@ -331,7 +338,7 @@ def validate_application_wheel( f"Application wheel name mismatch: expected {expected_name}, received " f"{filename_name}." ) - if str(filename_version) != expected_version: + if filename_version != expected_version_value: raise PreparationError( f"Application wheel version mismatch: expected {expected_version}, received " f"{filename_version}." @@ -374,7 +381,13 @@ def validate_application_wheel( ) if canonicalize_name(metadata_distribution) != expected_name: raise PreparationError("Application wheel METADATA distribution name is wrong.") - if metadata_version != expected_version: + try: + metadata_version_value = Version(metadata_version) + except InvalidVersion as exc: + raise PreparationError( + f"Application wheel METADATA version is invalid: {metadata_version!r}." + ) from exc + if metadata_version_value != expected_version_value: raise PreparationError("Application wheel METADATA version is wrong.") declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 694cdc4..06440ad 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -165,7 +165,13 @@ def _selected_deployment_paths(assessment, plan) -> set[str]: return selected -def _tracked_deployment_paths(repository_root: Path, assessment, plan) -> set[str]: +def _tracked_deployment_paths( + repository_root: Path, + assessment, + plan, + *, + created_lock: Path | None = None, +) -> set[str]: selected = _selected_deployment_paths(assessment, plan) tracked = _git_tracked_paths( repository_root, @@ -173,6 +179,14 @@ def _tracked_deployment_paths(repository_root: Path, assessment, plan) -> set[st ) if tracked is None: return selected + if created_lock is not None: + expected_lock = (repository_root / "uv.lock").resolve() + if created_lock.resolve() != expected_lock or not created_lock.is_file(): + raise PreparationError( + "The lockfile reported as created by --prepare-lock is not the repository " + "uv.lock. Generation stopped rather than widening untracked-file staging." + ) + tracked.add("uv.lock") selected.intersection_update(tracked) if assessment.repository.revision is not None: dirty = _dirty_tracked_deployment_paths(repository_root, selected) @@ -192,12 +206,15 @@ def _staging_files( *, include: bool, allow_missing_lock: bool = False, + created_lock: Path | None = None, ) -> dict[str, bytes]: """Stage inventory-approved runtime inputs, never a broad repository copy.""" if not include: return {} - selected = _tracked_deployment_paths(repository_root, assessment, plan) + selected = _tracked_deployment_paths( + repository_root, assessment, plan, created_lock=created_lock + ) files: dict[str, bytes] = {} for relative_text in sorted(selected): relative = Path(relative_text) @@ -309,6 +326,47 @@ def _classify_output( return create, replace, collisions +def _obsolete_owned_paths( + output_root: Path, + planned_paths: list[str] | set[str], + previous: dict[str, str], +) -> tuple[list[str], list[str]]: + obsolete: list[str] = [] + collisions: list[str] = [] + output_root = output_root.resolve() + for relative in sorted(set(previous) - set(planned_paths)): + path = output_root / Path(relative) + resolved = path.resolve() + try: + resolved.relative_to(output_root) + except ValueError: + collisions.append(f"{relative} (previous generated index contains an unsafe path)") + continue + if path.is_symlink(): + collisions.append(f"{relative} (obsolete previously generated path is a symlink)") + elif not path.exists(): + continue + elif not path.is_file(): + collisions.append( + f"{relative} (obsolete previously generated path is not a regular file)" + ) + elif sha256_file(path) != previous[relative]: + collisions.append( + f"{relative} (obsolete previously generated file was modified)" + ) + else: + obsolete.append(relative) + return obsolete, collisions + + +def _remove_obsolete_owned_files(output_root: Path, obsolete: list[str]) -> None: + output_root = output_root.resolve() + for relative in obsolete: + path = output_root / Path(relative) + path.resolve().relative_to(output_root) + path.unlink(missing_ok=True) + + def _render_owned_files( plan, repository_root: Path, @@ -419,7 +477,14 @@ def _preview( paths.sort() previous = _load_previous_index(output_root) create, replace, collisions = _classify_output(output_root, paths, previous) + obsolete, obsolete_collisions = _obsolete_owned_paths(output_root, paths, previous) + collisions.extend(obsolete_collisions) actions = ["Run pinned uv lock --check with the selected Python minor."] + if obsolete: + actions.append( + "Remove unchanged files owned by the previous generator run that are no longer " + "part of the deployment kit." + ) if plan.lockfile.status == "developer_generation_required": actions.insert( 0, @@ -530,7 +595,7 @@ def generate_deployment_kit( if plan.deployment_mode != "package" and application_wheel is not None: raise PreparationError("--application-wheel is accepted only for package deployment mode.") application_artifact = ( - validate_application_wheel(application_wheel.resolve(), assessment, plan) + validate_application_wheel(application_wheel, assessment, plan) if application_wheel is not None else None ) @@ -627,7 +692,7 @@ def generate_deployment_kit( ) approved = validate_artifact_set(artifact_values, plan) application_artifact = ( - validate_application_wheel(application_wheel.resolve(), assessment, plan) + validate_application_wheel(application_wheel, assessment, plan) if application_wheel is not None else None ) @@ -650,7 +715,11 @@ def generate_deployment_kit( raise PreparationError("Deployment readiness remains blocked: " + "; ".join(detail)) source_files = _staging_files( - repository_root, assessment, plan, include=output_root != repository_root + repository_root, + assessment, + plan, + include=output_root != repository_root, + created_lock=(lock_result.path if prepare_lock and lock_result.created else None), ) bundled_uv = uv_executable if bootstrap_mode == "bundled_uv" else None owned, manifest = _render_owned_files( @@ -684,6 +753,16 @@ def generate_deployment_kit( generated_paths=set(owned), secret_values=secret_values, ) + previous = _load_previous_index(output_root) + obsolete, obsolete_collisions = _obsolete_owned_paths( + output_root, set(files), previous + ) + if obsolete_collisions: + raise PreparationError( + "Generation output contains files not safely owned by the previous generator run: " + + ", ".join(obsolete_collisions) + ) + _remove_obsolete_owned_files(output_root, obsolete) _write_files(output_root, files) validate_written_files(output_root, manifest) artifacts = [ diff --git a/tests/test_generation.py b/tests/test_generation.py index e03789f..4f84a45 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -4,6 +4,7 @@ import hashlib import importlib.util import io +import json import os import shutil import subprocess @@ -164,25 +165,25 @@ def _rewrite_application_wheel( return wheel -def _write_mapped_project(root: Path) -> None: +def _write_mapped_project(root: Path, *, version: str = "1.2.3") -> None: (root / "code").mkdir() (root / "code/__init__.py").write_text("", encoding="utf-8") (root / "code/main.py").write_text("def main(): return 0\n", encoding="utf-8") (root / "code/view.html").write_text("\n", encoding="utf-8") (root / "pyproject.toml").write_text( - """[build-system] + f"""[build-system] requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] name = "mapped-app" -version = "1.2.3" +version = "{version}" requires-python = ">=3.12" dependencies = [] [project.gui-scripts] mapped-app = "installed_app.main:main" [tool.setuptools] packages = ["installed_app"] -package-dir = {installed_app = "code"} +package-dir = {{installed_app = "code"}} [tool.setuptools.package-data] installed_app = ["view.html"] """, @@ -329,6 +330,54 @@ def test_application_wheel_rejects_wrong_filename_identity( validate_application_wheel(wheel, assessment, plan) +@pytest.mark.parametrize( + ("project_version", "wheel_version"), + [("1.0-rc1", "1.0rc1"), ("1.0-1", "1.0.post1")], +) +def test_application_wheel_accepts_equivalent_pep440_version_spellings( + tmp_path: Path, project_version: str, wheel_version: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source, version=project_version) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _make_application_wheel(tmp_path, version=wheel_version) + + artifact, _ = validate_application_wheel(wheel, assessment, plan) + + assert artifact.version == project_version + + +def test_application_wheel_rejects_invalid_authoritative_and_metadata_versions( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _make_application_wheel(tmp_path) + + assessment.project.version = "invalid version!" + with pytest.raises(PreparationError, match="authoritative project version is invalid"): + validate_application_wheel(wheel, assessment, plan) + + assessment.project.version = "1.2.3" + _rewrite_application_wheel( + wheel, + replacements={ + "mapped_app-1.2.3.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: mapped-app\nVersion: invalid version!\n\n" + ) + }, + ) + with pytest.raises(PreparationError, match="METADATA version is invalid"): + validate_application_wheel(wheel, assessment, plan) + + @pytest.mark.parametrize("member", ["../escape.py", "/absolute.py", "C:/absolute.py"]) def test_application_wheel_rejects_unsafe_archive_members( tmp_path: Path, member: str @@ -665,6 +714,92 @@ def test_package_mode_release_is_deterministic_and_survives_extraction( assert "first-party application artifact is mapped-app==1.2.3" in smoke +def test_package_regeneration_removes_unchanged_obsolete_application_wheel( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + old_wheel = _make_application_wheel(tmp_path) + new_wheel = tmp_path / "mapped_app-1.2.3-1-py3-none-any.whl" + new_wheel.write_bytes(old_wheel.read_bytes()) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit(repository, kit, application_wheel=old_wheel) + + generate_deployment_kit(repository, kit, application_wheel=new_wheel) + + old_staged = kit / "deployment/application" / old_wheel.name + new_staged = kit / "deployment/application" / new_wheel.name + index = json.loads((kit / "deployment/generated-files.json").read_text(encoding="utf-8")) + indexed_paths = {item["path"] for item in index["files"]} + report = validate_static_kit(kit) + packaged = package_deployment_kit(kit, output_directory=tmp_path / "release") + assert not old_staged.exists() + assert new_staged.is_file() + assert f"deployment/application/{old_wheel.name}" not in indexed_paths + assert f"deployment/application/{new_wheel.name}" in indexed_paths + assert report.final_state.value == "STATIC_VALID" + assert not any( + item.code == "NO_UNINDEXED_STAGED_FILES" and item.status.value == "fail" + for item in report.static_checks + ) + assert packaged.generated + + +def test_package_regeneration_protects_modified_obsolete_application_wheel( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + old_wheel = _make_application_wheel(tmp_path) + new_wheel = tmp_path / "mapped_app-1.2.3-1-py3-none-any.whl" + new_wheel.write_bytes(old_wheel.read_bytes()) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit(repository, kit, application_wheel=old_wheel) + old_staged = kit / "deployment/application" / old_wheel.name + modified = b"developer-modified obsolete artifact" + old_staged.write_bytes(modified) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail( + "modified obsolete output must block before uv acquisition" + ), + ) + + with pytest.raises(PreparationError, match="obsolete previously generated file was modified"): + generate_deployment_kit(repository, kit, application_wheel=new_wheel) + + assert old_staged.read_bytes() == modified + assert not (kit / "deployment/application" / new_wheel.name).exists() + + def test_generate_and_all_cli_propagate_first_party_application_wheel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -787,6 +922,99 @@ def test_git_source_staging_excludes_untracked_application_files(tmp_path: Path) assert "local_helper.py" not in staged +def test_prepare_lock_stages_only_lock_created_by_current_authorized_operation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "git-source" + source.mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='lock-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nlock-app='app:main'\n", + encoding="utf-8", + ) + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + unrelated = source / "local_runtime.py" + unrelated.write_text("VALUE = 'untracked'\n", encoding="utf-8") + lock_bytes = b"version = 1\nrevision = 3\nrequires-python = \">=3.11\"\n" + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + + def create_lock(root: Path, *args, **kwargs) -> LockPreparationResult: + path = root / "uv.lock" + assert not path.exists() + path.write_bytes(lock_bytes) + return LockPreparationResult(path=path, created=True, checked=True, commands=()) + + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", create_lock + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + output = tmp_path / "external-kit" + + result = generate_deployment_kit( + repository, + output, + prepare_lock=True, + bootstrap_mode="online_cmd", + ) + + assert result.generated + assert (output / "uv.lock").read_bytes() == lock_bytes + assert not (output / unrelated.name).exists() + assert result.preview.repository_files_changed == [str(source / "uv.lock")] + index = json.loads((output / "deployment/generated-files.json").read_text(encoding="utf-8")) + lock_entry = next(item for item in index["files"] if item["path"] == "uv.lock") + assert lock_entry["sha256"] == hashlib.sha256(lock_bytes).hexdigest() + status = subprocess.run( + ["git", "-C", str(source), "status", "--short"], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + assert "?? uv.lock" in status + assert f"?? {unrelated.name}" in status + + +def test_preexisting_untracked_lock_does_not_bypass_git_staging_policy( + tmp_path: Path, +) -> None: + source = tmp_path / "git-source" + source.mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='lock-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nlock-app='app:main'\n", + encoding="utf-8", + ) + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + (source / "uv.lock").write_text("untracked lock\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match="Required deployment input is missing: uv.lock"): + _staging_files(source, assessment, plan, include=True) + + def test_git_source_staging_blocks_dirty_tracked_inputs_but_ignores_unrelated_docs( tmp_path: Path, ) -> None: @@ -1069,6 +1297,61 @@ def test_dry_run_makes_no_output_or_lock(tmp_path: Path) -> None: assert any("Create uv.lock" in item for item in result.preview.developer_actions) +def test_application_wheel_tilde_path_works_for_cli_dry_run_and_generation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + fake_home = tmp_path / "home" + wheel_directory = fake_home / "dist" + wheel_directory.mkdir(parents=True) + wheel = _make_application_wheel(wheel_directory) + monkeypatch.setenv("HOME", str(fake_home)) + monkeypatch.setenv("USERPROFILE", str(fake_home)) + tilde_wheel = Path("~/dist") / wheel.name + + assert ( + main( + [ + "generate", + str(source), + "--output-dir", + str(tmp_path / "preview-kit"), + "--application-wheel", + str(tilde_wheel), + "--bootstrap", + "online_cmd", + "--dry-run", + ] + ) + == 0 + ) + + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + output = tmp_path / "kit" + result = generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + output, + application_wheel=tilde_wheel, + bootstrap_mode="online_cmd", + ) + + assert result.generated + assert (output / "deployment/application" / wheel.name).is_file() + + def test_dry_run_reports_unowned_output_collision(tmp_path: Path) -> None: output = tmp_path / "kit" output.mkdir() From 038ab6c4d918674beb15749f55ad0504f9a0f20f Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:21:36 -0400 Subject: [PATCH 05/59] Close M6.1 validation provenance gaps Unify staged-kit and application-wheel security policy, guard Git ignore inputs as analysis provenance, and compare installed application versions with PEP 440 semantics. --- .../generation/artifacts.py | 44 +++--- .../generation/generator.py | 36 ++++- .../security_policy.py | 60 ++++++++ .../validation/runtime.py | 74 +++++++-- .../validation/static.py | 55 +++---- tests/test_generation.py | 145 ++++++++++++++++++ tests/test_validation.py | 60 ++++++++ 7 files changed, 407 insertions(+), 67 deletions(-) create mode 100644 src/python_deployment_builder/security_policy.py diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index ea16e12..1e1a153 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -6,6 +6,7 @@ import csv import fnmatch import io +import os import re import stat import zipfile @@ -24,10 +25,11 @@ RepositoryAssessment, ) from python_deployment_builder.planning.index import wheel_matches - -FORBIDDEN_APPLICATION_TEXT = (b"powershell.exe", b"pwsh.exe", b"executionpolicy") -WINDOWS_DEVELOPER_PATH = re.compile(rb"(?i)(?:[a-z]:\\(?:users|home)\\[^\r\n\"]+)") -SECRET_MEMBER_NAMES = {".env", "credentials.json", "secrets.json"} +from python_deployment_builder.security_policy import ( + TEXT_SUFFIXES, + is_secret_filename, + text_security_findings, +) def parse_artifact_argument(value: str) -> tuple[str, Path]: @@ -178,29 +180,22 @@ def _require_wheel_metadata(message, *, wheel: Path) -> set[str]: def _validate_application_security( bundle: zipfile.ZipFile, members: dict[str, zipfile.ZipInfo], - wheel: Path, + *, + configured_secret_values: tuple[str, ...] = (), ) -> None: failures: list[str] = [] for name, member in members.items(): member_path = PurePosixPath(name) - if member.is_dir() or ".dist-info" in member_path.parts: + if member.is_dir(): continue - lowered_name = member_path.name.lower() - if member_path.suffix.lower() == ".ps1" or lowered_name in SECRET_MEMBER_NAMES: + if member_path.suffix.lower() == ".ps1" or is_secret_filename(member_path.name): failures.append(name) continue - if member_path.suffix.lower() not in {".py", ".bat", ".cmd", ".json", ".txt"}: + if ".dist-info" in member_path.parts or member_path.suffix.lower() not in TEXT_SUFFIXES: continue - data = bundle.read(member) - lowered = data.lower() - if ( - any(value in lowered for value in FORBIDDEN_APPLICATION_TEXT) - or WINDOWS_DEVELOPER_PATH.search(data) - or (b"setx" in lowered and b"path" in lowered) - or ( - b"program files" in lowered - and any(value in lowered for value in (b"write", b"mkdir", b"open(")) - ) + text = bundle.read(member).decode("utf-8", errors="replace") + if text_security_findings( + text, configured_secret_values=configured_secret_values ): failures.append(name) if failures: @@ -426,7 +421,16 @@ def validate_application_wheel( "Application wheel contains unexpected native binaries: " + ", ".join(native_members) ) - _validate_application_security(bundle, members, path) + configured_secret_values = tuple( + value + for item in plan.configuration + if item.secret and (value := os.environ.get(item.name)) is not None + ) + _validate_application_security( + bundle, + members, + configured_secret_values=configured_secret_values, + ) parser = configparser.ConfigParser(interpolation=None) parser.optionxform = str diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 06440ad..fa05eeb 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -93,7 +93,7 @@ def _git_tracked_paths(repository_root: Path, *, required: bool) -> set[str] | N def _dirty_tracked_deployment_paths( - repository_root: Path, selected: set[str] + repository_root: Path, provenance_guarded: set[str] ) -> list[str]: result = subprocess.run( ["git", "-C", str(repository_root), "diff", "--name-only", "-z", "HEAD", "--"], @@ -145,13 +145,13 @@ def _dirty_tracked_deployment_paths( head_plan = create_deployment_plan( head_assessment, repository_root=head_root ) - head_selected = _selected_deployment_paths(head_assessment, head_plan) + head_guarded = _provenance_guard_paths(head_assessment, head_plan) except (OSError, RepositoryLoadError, ValueError) as exc: raise PreparationError( "Git revision provenance is known, but the recorded source revision could not " "be inventoried. Generation stopped rather than claiming clean provenance." ) from exc - return sorted(changed & (selected | head_selected)) + return sorted(changed & (provenance_guarded | head_guarded)) def _selected_deployment_paths(assessment, plan) -> set[str]: @@ -165,6 +165,21 @@ def _selected_deployment_paths(assessment, plan) -> set[str]: return selected +def _analysis_policy_paths(assessment) -> set[str]: + """Return ignore-policy files that influence inventory without staging them.""" + + return { + item.path.rstrip("/") + for item in assessment.file_inventory + if not item.path.endswith("/") + and Path(item.path).name.casefold() == ".gitignore" + } + + +def _provenance_guard_paths(assessment, plan) -> set[str]: + return _selected_deployment_paths(assessment, plan) | _analysis_policy_paths(assessment) + + def _tracked_deployment_paths( repository_root: Path, assessment, @@ -173,12 +188,21 @@ def _tracked_deployment_paths( created_lock: Path | None = None, ) -> set[str]: selected = _selected_deployment_paths(assessment, plan) + analysis_policy = _analysis_policy_paths(assessment) tracked = _git_tracked_paths( repository_root, required=assessment.repository.revision is not None, ) if tracked is None: return selected + if assessment.repository.revision is not None: + untracked_policy = sorted(analysis_policy - tracked) + if untracked_policy: + raise PreparationError( + "Untracked .gitignore analysis inputs cannot be combined with recorded source " + f"revision {assessment.repository.revision}: {', '.join(untracked_policy)}. " + "Commit or remove those policy inputs before release-oriented generation." + ) if created_lock is not None: expected_lock = (repository_root / "uv.lock").resolve() if created_lock.resolve() != expected_lock or not created_lock.is_file(): @@ -187,15 +211,17 @@ def _tracked_deployment_paths( "uv.lock. Generation stopped rather than widening untracked-file staging." ) tracked.add("uv.lock") - selected.intersection_update(tracked) if assessment.repository.revision is not None: - dirty = _dirty_tracked_deployment_paths(repository_root, selected) + dirty = _dirty_tracked_deployment_paths( + repository_root, _provenance_guard_paths(assessment, plan) + ) if dirty: raise PreparationError( "Tracked deployment inputs differ from recorded source revision " f"{assessment.repository.revision}: {', '.join(dirty)}. Commit or restore " "those inputs before release-oriented generation." ) + selected.intersection_update(tracked) return selected diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py new file mode 100644 index 0000000..6a0164e --- /dev/null +++ b/src/python_deployment_builder/security_policy.py @@ -0,0 +1,60 @@ +"""Security rules shared by staged-kit and opaque application-content validation.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable + +FORBIDDEN_SHELL = ("powershell.exe", "pwsh.exe", "executionpolicy") +WINDOWS_ABSOLUTE = re.compile(r"(?i)[a-z]:\\(?:users|home)\\[^\r\n\"]+") +OBVIOUS_SECRET = re.compile( + r"(?i)(?:authorization\s*[:=]\s*bearer\s+[a-z0-9._-]{12,}|sk-[a-z0-9_-]{16,})" +) +TEXT_SUFFIXES = frozenset({".bat", ".cmd", ".json", ".py", ".txt"}) +PROGRAM_FILES_WRITE_TOKENS = ("mkdir", "copy ", "write_text", "open(", "write") +SECRET_FILENAMES = frozenset( + {".env", "credentials.json", "secrets.json", "token.json", ".pypirc", "pip.ini"} +) + + +def is_secret_filename(filename: str) -> bool: + """Return whether a case-insensitive basename is prohibited secret material.""" + + lowered = filename.casefold() + return lowered in SECRET_FILENAMES or ( + lowered.startswith(".env.") and lowered != ".env.example" + ) + + +def text_security_findings( + text: str, *, configured_secret_values: Iterable[str] = () +) -> set[str]: + """Return the deployment security rules violated by application text.""" + + lowered = text.lower() + findings: set[str] = set() + if any(item in lowered for item in FORBIDDEN_SHELL): + findings.add("forbidden_shell") + if WINDOWS_ABSOLUTE.search(text): + findings.add("developer_path") + if "setx" in lowered and "path" in lowered: + findings.add("permanent_path") + if "program files" in lowered and any( + token in lowered for token in PROGRAM_FILES_WRITE_TOKENS + ): + findings.add("program_files_write") + if OBVIOUS_SECRET.search(text): + findings.add("obvious_secret") + if any( + value and len(value) >= 8 and value in text for value in configured_secret_values + ): + findings.add("configured_secret") + return findings + + +__all__ = [ + "FORBIDDEN_SHELL", + "TEXT_SUFFIXES", + "is_secret_filename", + "text_security_findings", +] diff --git a/src/python_deployment_builder/validation/runtime.py b/src/python_deployment_builder/validation/runtime.py index 56bf09b..17f6634 100644 --- a/src/python_deployment_builder/validation/runtime.py +++ b/src/python_deployment_builder/validation/runtime.py @@ -12,6 +12,8 @@ from datetime import UTC, datetime from pathlib import Path +from packaging.version import InvalidVersion, Version + from python_deployment_builder.analysis import assess_repository from python_deployment_builder.analysis.repository import MaterializedRepository from python_deployment_builder.generation.security import redact_secrets @@ -27,6 +29,62 @@ SETUP_REQUIRED = 20 HELPER_FLAGS = ("-B", "-E", "-s") +APPLICATION_PROBE = """\ +import importlib.metadata as metadata +import importlib.util +import json +import os + +try: + actual_version = metadata.version(os.environ["PDBUILDER_APPLICATION_DISTRIBUTION"]) + module_found = importlib.util.find_spec( + os.environ["PDBUILDER_APPLICATION_MODULE"] + ) is not None + error = None +except Exception as exc: + actual_version = None + module_found = False + error = type(exc).__name__ +print(json.dumps({ + "version": actual_version, + "module_found": module_found, + "error": error, +})) +""" + + +def _application_probe_result( + completed: subprocess.CompletedProcess[str], expected_version: str +) -> tuple[bool, list[str]]: + """Compare managed-environment observations using PDB's PEP 440 implementation.""" + + evidence: list[str] = [] + if completed.returncode != 0: + return False, [(completed.stderr or completed.stdout)[-1000:]] + try: + payload = json.loads(completed.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError): + return False, ["Managed application probe returned malformed output."] + actual_version = payload.get("version") if isinstance(payload, dict) else None + module_found = payload.get("module_found") is True if isinstance(payload, dict) else False + if not isinstance(actual_version, str): + error = payload.get("error") if isinstance(payload, dict) else None + return False, [ + f"Managed application metadata probe failed: {error or 'unknown error'}." + ] + evidence.extend( + [ + f"Expected application version: {expected_version}", + f"Installed application version: {actual_version}", + f"Authoritative module import-discoverable: {module_found}", + ] + ) + try: + version_matches = Version(actual_version) == Version(expected_version) + except InvalidVersion: + evidence.append("Application version metadata is not valid PEP 440.") + return False, evidence + return version_matches and module_found, evidence def _check( @@ -484,23 +542,17 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) "PDBUILDER_APPLICATION_DISTRIBUTION": ( manifest.application_artifact.distribution_name ), - "PDBUILDER_APPLICATION_VERSION": manifest.application_artifact.version, "PDBUILDER_APPLICATION_MODULE": manifest.entry_point_module, } - application_probe = ( - "import importlib.metadata as m,importlib.util,os,sys;" - "name=os.environ['PDBUILDER_APPLICATION_DISTRIBUTION'];" - "version=os.environ['PDBUILDER_APPLICATION_VERSION'];" - "module=os.environ['PDBUILDER_APPLICATION_MODULE'];" - "sys.exit(m.version(name)!=version or importlib.util.find_spec(module) is None)" - ) installed_application, application_duration = _run( - [str(app_python), *HELPER_FLAGS, "-c", application_probe], + [str(app_python), *HELPER_FLAGS, "-c", APPLICATION_PROBE], cwd=root, environment=application_probe_environment, log_handle=log, ) - application_ok = installed_application.returncode == 0 + application_ok, application_evidence = _application_probe_result( + installed_application, manifest.application_artifact.version + ) report.runtime_checks.append( _check( "APPLICATION_WHEEL_INSTALLED", @@ -514,7 +566,7 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) "installed in the managed environment." if application_ok else "The first-party application wheel is not installed as declared.", - evidence=[installed_application.stderr[-1000:]], + evidence=application_evidence, duration=application_duration, ) ) diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index b5d1f23..77d1a96 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -23,23 +23,15 @@ ValidationHost, ValidationReport, ) +from python_deployment_builder.security_policy import ( + FORBIDDEN_SHELL, + TEXT_SUFFIXES, + is_secret_filename, + text_security_findings, +) HEX_SHA256 = re.compile(r"^[0-9a-f]{64}$") -WINDOWS_ABSOLUTE = re.compile(r"(?i)[a-z]:\\(?:users|home)\\[^\r\n\"]+") -OBVIOUS_SECRET = re.compile( - r"(?i)(?:authorization\s*[:=]\s*bearer\s+[a-z0-9._-]{12,}|sk-[a-z0-9_-]{16,})" -) -FORBIDDEN_SHELL = ("powershell.exe", "pwsh.exe", "executionpolicy") PYTHON_CACHE_DIRECTORY = re.compile(r"^__pycache__(?:\s*\(\d+\))?$", re.IGNORECASE) -TEXT_SUFFIXES = {".bat", ".cmd", ".json", ".py", ".txt"} -SECRET_FILENAMES = { - ".env", - "credentials.json", - "secrets.json", - "token.json", - ".pypirc", - "pip.ini", -} class KitValidationError(ValueError): @@ -462,36 +454,37 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR permanent_path: list[str] = [] program_files: list[str] = [] obvious_secrets: list[str] = [] + configured_secret_values = [ + value + for name in manifest.configuration_secret_names + if (value := os.environ.get(name)) is not None + ] security_paths = [*root.glob("*.bat"), *(root / "deployment").rglob("*")] for path in security_paths: if not path.is_file() or path.suffix.lower() not in TEXT_SUFFIXES: continue relative = str(path.relative_to(root)) text = path.read_text(encoding="utf-8", errors="replace") - lowered = text.lower() - forbidden.extend(f"{relative}: {item}" for item in FORBIDDEN_SHELL if item in lowered) - if WINDOWS_ABSOLUTE.search(text): + findings = text_security_findings( + text, configured_secret_values=configured_secret_values + ) + if "forbidden_shell" in findings: + lowered = text.lower() + forbidden.extend( + f"{relative}: {item}" for item in FORBIDDEN_SHELL if item in lowered + ) + if "developer_path" in findings: developer_paths.append(relative) - if "setx" in lowered and "path" in lowered: + if "permanent_path" in findings: permanent_path.append(relative) - if "program files" in lowered and any( - token in lowered for token in ("mkdir", "copy ", "write_text", "open(") - ): + if "program_files_write" in findings: program_files.append(relative) - if OBVIOUS_SECRET.search(text): + if {"obvious_secret", "configured_secret"} & findings: obvious_secrets.append(relative) - for name in manifest.configuration_secret_names: - value = os.environ.get(name) - if value and len(value) >= 8 and value in text: - obvious_secrets.append(relative) secret_files = [ str(path.relative_to(root)) for path in root.rglob("*") - if path.is_file() - and ( - path.name.lower() in SECRET_FILENAMES - or (path.name.lower().startswith(".env.") and path.name.lower() != ".env.example") - ) + if path.is_file() and is_secret_filename(path.name) ] cache_files = [ str(path.relative_to(root)) diff --git a/tests/test_generation.py b/tests/test_generation.py index 4f84a45..fe55c94 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -519,6 +519,15 @@ def test_application_wheel_rejects_missing_runtime_content_and_binary_content( ("installed_app/tool.py", "COMMAND = 'powershell.exe -ExecutionPolicy bypass'"), ("installed_app/path.py", r"ROOT = 'C:\Users\developer\private'"), ("installed_app/.env", "API_KEY=secret"), + ("installed_app/token.json", "{}"), + ("installed_app/TOKEN.JSON", "{}"), + ("mapped_app-1.2.3.dist-info/token.json", "{}"), + ("installed_app/.pypirc", "[distutils]"), + ("installed_app/pip.ini", "[global]"), + ("installed_app/.env.production", "API_KEY=secret"), + ("installed_app/.env.local", "API_KEY=secret"), + ("installed_app/secret.py", "TOKEN = 'sk-abcdefghijklmnop'"), + ("installed_app/copy.py", "COMMAND = 'copy payload C:\\Program Files\\App'"), ], ) def test_application_wheel_cannot_bypass_deployment_security_policy( @@ -538,6 +547,47 @@ def test_application_wheel_cannot_bypass_deployment_security_policy( validate_application_wheel(wheel, assessment, plan) +def test_application_wheel_allows_environment_example_file(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/.env.example": "API_KEY=replace-me"}, + ) + + artifact, _path = validate_application_wheel(wheel, assessment, plan) + + assert artifact.filename == wheel.name + + +def test_application_wheel_rejects_configured_secret_value( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "code/main.py").write_text( + "import os\nAPI_TOKEN = os.environ['APP_API_TOKEN']\ndef main(): return 0\n", + encoding="utf-8", + ) + secret = "configured-value-that-must-not-ship" + monkeypatch.setenv("APP_API_TOKEN", secret) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/config.py": f"TOKEN = {secret!r}"}, + ) + + with pytest.raises(PreparationError, match="security policy"): + validate_application_wheel(wheel, assessment, plan) + + @pytest.mark.parametrize( "filename", [ @@ -1055,6 +1105,7 @@ def _committed_source_fixture(tmp_path: Path) -> tuple[Path, MaterializedReposit source = tmp_path / "git-source" source.mkdir() (source / "docs").mkdir() + (source / "data").mkdir() (source / "pyproject.toml").write_text( "[project]\nname='dirty-app'\nversion='1.0'\ndependencies=[]\n" "[project.scripts]\ndirty-app='app:main'\n", @@ -1069,6 +1120,9 @@ def _committed_source_fixture(tmp_path: Path) -> tuple[Path, MaterializedReposit ) (source / "state.json").write_text("{}\n", encoding="utf-8") (source / "docs/readme.md").write_text("documentation\n", encoding="utf-8") + (source / ".gitignore").write_text("# root policy\n", encoding="utf-8") + (source / "data/.gitignore").write_text("# nested policy\n", encoding="utf-8") + (source / "data/observed.txt").write_text("analysis candidate\n", encoding="utf-8") subprocess.run(["git", "init", "-q", str(source)], check=True) subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) subprocess.run( @@ -1097,6 +1151,97 @@ def test_clean_git_source_fixture_stages_and_previews_normally(tmp_path: Path) - assert {"app.py", "state.json"} <= staged.keys() assert {"app.py", "state.json"} <= set(preview.files_to_create) + assert ".gitignore" not in staged + assert "data/.gitignore" not in staged + + +@pytest.mark.parametrize("relative", [".gitignore", "data/.gitignore"]) +def test_git_source_staging_blocks_modified_tracked_analysis_policy( + tmp_path: Path, relative: str +) -> None: + source, repository = _committed_source_fixture(tmp_path) + recorded = assess_repository(repository) + policy = source / relative + policy.write_text(policy.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError) as caught: + _staging_files(source, assessment, plan, include=True) + + assert assessment.repository.revision in str(caught.value) + assert relative in str(caught.value).replace("\\", "/") + assert assessment.repository.fingerprint != recorded.repository.fingerprint + if relative == ".gitignore": + with pytest.raises(PreparationError, match="recorded source revision.*gitignore"): + generate_deployment_kit( + repository, + tmp_path / "kit", + bootstrap_mode="online_cmd", + dry_run=True, + ) + + +@pytest.mark.parametrize("operation", ["deleted", "renamed"]) +def test_git_source_staging_blocks_missing_tracked_analysis_policy( + tmp_path: Path, operation: str +) -> None: + source, repository = _committed_source_fixture(tmp_path) + policy = source / "data/.gitignore" + if operation == "deleted": + policy.unlink() + else: + policy.rename(source / "data/.gitignore-renamed") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError) as caught: + _staging_files(source, assessment, plan, include=True) + + assert assessment.repository.revision in str(caught.value) + assert "data/.gitignore" in str(caught.value).replace("\\", "/") + + +@pytest.mark.parametrize("relative", [".gitignore", "local/.gitignore"]) +def test_git_source_staging_blocks_untracked_analysis_policy( + tmp_path: Path, relative: str +) -> None: + source = tmp_path / "git-source" + source.mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='policy-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\npolicy-app='app:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + policy = source / relative + policy.parent.mkdir(parents=True, exist_ok=True) + policy.write_text("# local analysis policy\n", encoding="utf-8") + if policy.parent != source: + (policy.parent / "candidate.py").write_text("VALUE = 1\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match="Untracked .*gitignore"): + _staging_files(source, assessment, plan, include=True) + if relative == ".gitignore": + with pytest.raises(PreparationError, match="Untracked .*gitignore"): + generate_deployment_kit( + repository, + tmp_path / "kit", + bootstrap_mode="online_cmd", + dry_run=True, + ) def test_git_source_staging_blocks_deleted_tracked_application_source( diff --git a/tests/test_validation.py b/tests/test_validation.py index 44d8c0e..57397f1 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -2,6 +2,9 @@ import hashlib import json +import os +import subprocess +import sys from pathlib import Path import pytest @@ -18,6 +21,8 @@ from python_deployment_builder.packaging import package_deployment_kit from python_deployment_builder.reporting.json_report import write_validation_reports from python_deployment_builder.validation.runtime import ( + APPLICATION_PROBE, + _application_probe_result, _runtime_environment, _scenario_copy, validate_runtime_kit, @@ -223,6 +228,61 @@ def test_runtime_dry_run_executes_nothing_and_creates_no_runtime_root( ) +@pytest.mark.parametrize( + ("expected", "installed"), + [("1.0-rc1", "1.0rc1"), ("1.0-1", "1.0.post1")], +) +def test_runtime_application_probe_compares_pep440_versions_semantically( + expected: str, installed: str +) -> None: + completed = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps({"version": installed, "module_found": True, "error": None}), + stderr="", + ) + + accepted, evidence = _application_probe_result(completed, expected) + + assert accepted + assert f"Expected application version: {expected}" in evidence + assert f"Installed application version: {installed}" in evidence + + +def test_runtime_application_probe_rejects_different_or_invalid_version() -> None: + for installed in ("2.0", "not a version"): + completed = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=json.dumps({"version": installed, "module_found": True, "error": None}), + stderr="", + ) + + accepted, _evidence = _application_probe_result(completed, "1.0") + + assert not accepted + + +def test_managed_application_probe_uses_only_standard_library() -> None: + environment = { + **os.environ, + "PDBUILDER_APPLICATION_DISTRIBUTION": "distribution-that-does-not-exist", + "PDBUILDER_APPLICATION_MODULE": "json", + } + + completed = subprocess.run( + [sys.executable, "-I", "-S", "-c", APPLICATION_PROBE], + env=environment, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0 + assert json.loads(completed.stdout)["error"] == "PackageNotFoundError" + assert "packaging" not in APPLICATION_PROBE + + def test_generated_runtime_rolls_back_after_setup_failure( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From f26c0bec4121d601cef947af8dde3887971eb919 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:02:51 -0400 Subject: [PATCH 06/59] Close package-data and artifact planning gaps --- .../analysis/inventory.py | 42 +++- .../analysis/resources.py | 137 ++++++++++-- .../generation/generator.py | 91 ++++++-- .../generation/structural.py | 34 ++- .../security_policy.py | 24 ++- tests/test_analysis_scope.py | 89 ++++++++ tests/test_generation.py | 203 +++++++++++++++++- tests/test_validation.py | 13 ++ 8 files changed, 563 insertions(+), 70 deletions(-) diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index 9c6a8c6..b52b4fb 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -351,24 +351,52 @@ def apply_resource_roles( ) -> AnalysisScopeSummary: """Promote only statically supported resource paths in the inventory.""" - resource_paths = { - resource.path.rstrip("/") + by_path = { + resource.path.rstrip("/"): resource for resource in resources if resource.status == FindingStatus.DETECTED or resource.kind in CONVENTIONAL_RUNTIME_RESOURCE_KINDS } for item in items: normalized = item.path.rstrip("/") - if any( - normalized == path or normalized.startswith(path + "/") for path in resource_paths - ) and item.role not in { + matching = [ + resource + for path, resource in by_path.items() + if normalized == path or normalized.startswith(path + "/") + ] + authoritative = any( + resource.packaging_status == "packaged" for resource in matching + ) + if matching and item.role not in { RepositoryFileRole.APPLICATION_SOURCE, RepositoryFileRole.IGNORED_OR_LOCAL, RepositoryFileRole.MUTABLE_STATE_CANDIDATE, - }: + } and ( + not authoritative + or item.role + in { + RepositoryFileRole.UNKNOWN, + RepositoryFileRole.DOCUMENTATION, + RepositoryFileRole.EXAMPLE_OR_SNIPPET, + RepositoryFileRole.RUNTIME_RESOURCE, + } + ): item.role = RepositoryFileRole.RUNTIME_RESOURCE item.included_in_runtime_scan = False - item.reason = "Application source contains a static runtime reference to this path." + if authoritative: + item.reason = ( + "Authoritative setuptools package-data metadata identifies this " + "runtime resource." + ) + item.evidence.extend( + evidence + for resource in matching + if resource.packaging_status == "packaged" + for evidence in resource.evidence + if evidence not in item.evidence + ) + else: + item.reason = "Application source contains a static runtime reference to this path." return summarize_inventory(items) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 298a12f..ffac26b 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -3,9 +3,8 @@ from __future__ import annotations import ast -import fnmatch from collections import defaultdict -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from python_deployment_builder.analysis.imports import EXCLUDED_DIRECTORIES from python_deployment_builder.models import ( @@ -17,24 +16,104 @@ ) -def _declared_package_data_path(relative: str, project: PackagingAssessment | None) -> bool: - """Return true when setuptools metadata installs this physical resource path.""" +def _safe_package_data_pattern(pattern: str) -> bool: + """Return whether a setuptools package-data pattern stays under its package root.""" - if project is None: - return False - candidate = Path(relative) - for package, patterns in project.package_data.items(): - physical = project.package_directories.get(package) - if physical is None: - base = project.package_directories.get("") - physical = str(Path(base or ".") / Path(*package.split("."))) + normalized = pattern.replace("\\", "/") + path = PurePosixPath(normalized) + return bool(normalized) and not path.is_absolute() and not PureWindowsPath( + pattern + ).is_absolute() and ".." not in path.parts + + +def _known_packages(project: PackagingAssessment) -> set[str]: + """Return only package identities established by static packaging metadata.""" + + return { + package + for package in [ + *project.packages, + *(name for name in project.package_directories if name), + *(name for name in project.package_data if name != "*"), + ] + if package and package != "*" + } + + +def _physical_package_roots( + root: Path, project: PackagingAssessment, package: str +) -> list[Path]: + """Resolve a declared installed package name to existing source directories.""" + + candidates: list[Path] = [] + explicit = project.package_directories.get(package) + if explicit is not None: + candidates.append(root / explicit) + else: + base = project.package_directories.get("") + if base is not None: + candidates.append(root / base / Path(*package.split("."))) + else: + candidates.extend( + root / source_root / Path(*package.split(".")) + for source_root in project.source_roots + ) + candidates.append(root / Path(*package.split("."))) + + resolved_root = root.resolve() + roots: list[Path] = [] + for candidate in candidates: + if candidate.is_symlink() or not candidate.is_dir(): + continue try: - package_relative = candidate.relative_to(Path(physical)).as_posix() + candidate.resolve().relative_to(resolved_root) except ValueError: continue - if any(fnmatch.fnmatchcase(package_relative, pattern) for pattern in patterns): - return True - return False + if candidate not in roots: + roots.append(candidate) + return roots + + +def _declared_package_data( + root: Path, project: PackagingAssessment | None +) -> dict[str, list[Evidence]]: + """Resolve existing, safe setuptools package-data members by physical package root.""" + + if project is None: + return {} + resolved_root = root.resolve() + declared: dict[str, list[Evidence]] = defaultdict(list) + for declared_package, patterns in project.package_data.items(): + packages = _known_packages(project) if declared_package == "*" else {declared_package} + for package in packages: + for package_root in _physical_package_roots(root, project, package): + resolved_package_root = package_root.resolve() + for pattern in patterns: + if not _safe_package_data_pattern(pattern): + continue + try: + matches = package_root.glob(pattern) + except (OSError, ValueError): + continue + for candidate in matches: + if candidate.is_symlink() or not candidate.is_file(): + continue + try: + resolved = candidate.resolve() + resolved.relative_to(resolved_package_root) + relative = resolved.relative_to(resolved_root).as_posix() + except ValueError: + continue + evidence = Evidence( + file="pyproject.toml", + detail=( + "Authoritative setuptools package-data declaration " + f"{declared_package} = {pattern!r} includes this runtime resource." + ), + ) + if evidence not in declared[relative]: + declared[relative].append(evidence) + return declared RESOURCE_DIRECTORIES = { "assets": "assets", @@ -561,6 +640,7 @@ def inspect_resources( project: PackagingAssessment | None = None, ) -> tuple[list[ResourceRequirement], list[ConfigurationRequirement]]: literals, access_modes, unresolved = _literal_evidence(root, source_roots, application_files) + declared_package_data = _declared_package_data(root, project) resources: list[ResourceRequirement] = [] for relative, references in sorted(literals.items()): path = root / relative @@ -573,7 +653,7 @@ def inspect_resources( packaging_status=( "packaged" if exists - and _declared_package_data_path(relative, project) + and relative in declared_package_data else "repository_adjacent" if exists else "unknown" @@ -582,6 +662,7 @@ def inspect_resources( evidence=( [ Evidence(file=relative, detail="Referenced runtime resource exists."), + *declared_package_data.get(relative, []), *references, ] if exists @@ -590,6 +671,28 @@ def inspect_resources( ) ) + known_resources = {resource.path for resource in resources} + for relative, evidence in sorted(declared_package_data.items()): + if relative in known_resources: + continue + path = root / relative + resources.append( + ResourceRequirement( + path=relative, + kind=_kind(path), + access_mode="read", + packaging_status="packaged", + status=FindingStatus.DETECTED, + evidence=[ + Evidence( + file=relative, + detail="Declared package-data runtime resource exists.", + ), + *evidence, + ], + ) + ) + for name, kind in {**RESOURCE_DIRECTORIES, **RESOURCE_FILES}.items(): path = root / name if not path.exists() or name in literals: diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index fa05eeb..2014357 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -43,6 +43,7 @@ ) from python_deployment_builder.models import ( ApplicationArtifact, + FindingStatus, GeneratedArtifact, GenerationPreview, GenerationResult, @@ -176,6 +177,16 @@ def _analysis_policy_paths(assessment) -> set[str]: } +def _authoritative_package_data_paths(assessment) -> set[str]: + """Return source-mode runtime members selected by setuptools package-data metadata.""" + + return { + resource.path.rstrip("/") + for resource in assessment.resources + if resource.status == FindingStatus.DETECTED and resource.packaging_status == "packaged" + } + + def _provenance_guard_paths(assessment, plan) -> set[str]: return _selected_deployment_paths(assessment, plan) | _analysis_policy_paths(assessment) @@ -195,6 +206,30 @@ def _tracked_deployment_paths( ) if tracked is None: return selected + if assessment.repository.revision is not None and plan.deployment_mode == "source": + declared_package_data = _authoritative_package_data_paths(assessment) + inventory = {item.path.rstrip("/"): item for item in assessment.file_inventory} + untracked_package_data = sorted(declared_package_data - tracked) + excluded_package_data = sorted( + path + for path in declared_package_data + if path in inventory + and inventory[path].role + in { + RepositoryFileRole.IGNORED_OR_LOCAL, + RepositoryFileRole.MUTABLE_STATE_CANDIDATE, + } + ) + if untracked_package_data or excluded_package_data: + details = [ + *(f"untracked: {path}" for path in untracked_package_data), + *(f"excluded by local-state policy: {path}" for path in excluded_package_data), + ] + raise PreparationError( + "Authoritative setuptools package-data runtime resources must be tracked and " + "stageable for Git release generation: " + + ", ".join(details) + ) if assessment.repository.revision is not None: untracked_policy = sorted(analysis_policy - tracked) if untracked_policy: @@ -286,8 +321,8 @@ def _template_values(plan, bootstrap_mode: str, system_certs: bool) -> dict[str, def _planned_generated_paths( plan, bootstrap_mode: str, - artifact_values: list[str], - application_wheel: Path | None, + approved, + application_artifact: tuple[ApplicationArtifact, Path] | None, ) -> list[str]: run_name, repair_name, diagnose_name = _root_names(plan.application_display_name) paths = [ @@ -305,12 +340,10 @@ def _planned_generated_paths( ] if bootstrap_mode == "bundled_uv": paths.append("deployment/bootstrap/uv.exe") - for value in artifact_values: - _name, separator, raw_path = value.partition("=") - if separator and raw_path: - paths.append(f"deployment/wheels/{Path(raw_path).name}") - if application_wheel is not None: - paths.append(f"deployment/application/{application_wheel.name}") + for artifact, _path in approved: + paths.append(f"deployment/wheels/{artifact.filename}") + if application_artifact is not None: + paths.append(f"deployment/application/{application_artifact[0].filename}") return sorted(set(paths)) @@ -485,15 +518,14 @@ def _preview( bootstrap_mode: str, system_certs: bool, prepare_lock: bool, - artifact_values: list[str], - application_wheel: Path | None, - application_artifact: ApplicationArtifact | None, + approved, + application_artifact: tuple[ApplicationArtifact, Path] | None, staging_source_paths: list[str], ) -> GenerationPreview: paths = sorted( set( _planned_generated_paths( - plan, bootstrap_mode, artifact_values, application_wheel + plan, bootstrap_mode, approved, application_artifact ) + staging_source_paths ) @@ -524,15 +556,16 @@ def _preview( actions.append( f"Validate an approved wheel for {requirement.package}=={requirement.version}." ) - if plan.deployment_mode == "package" and application_artifact is None: + application_model = application_artifact[0] if application_artifact else None + if plan.deployment_mode == "package" and application_model is None: actions.append( "Provide --application-wheel; package mode cannot produce a deployable kit without " "a validated first-party wheel." ) - elif application_artifact is not None: + elif application_model is not None: actions.append( "Validated first-party application wheel " - f"{application_artifact.filename} (SHA-256 {application_artifact.sha256})." + f"{application_model.filename} (SHA-256 {application_model.sha256})." ) return GenerationPreview( application_id=plan.application_id, @@ -547,9 +580,9 @@ def _preview( system_certs=system_certs, developer_actions=actions, application_wheel_required=( - plan.deployment_mode == "package" and application_artifact is None + plan.deployment_mode == "package" and application_model is None ), - application_artifact=application_artifact, + application_artifact=application_model, files_to_create=create, files_to_replace=replace, collisions=collisions, @@ -651,14 +684,13 @@ def generate_deployment_kit( bootstrap_mode=bootstrap_mode, system_certs=system_certs, prepare_lock=prepare_lock, - artifact_values=artifact_values, - application_wheel=application_wheel, - application_artifact=(application_artifact[0] if application_artifact else None), + approved=approved, + application_artifact=application_artifact, staging_source_paths=list(source_files), ) source_generated_collisions = sorted( set(source_files) - & set(_planned_generated_paths(plan, bootstrap_mode, artifact_values, application_wheel)) + & set(_planned_generated_paths(plan, bootstrap_mode, approved, application_artifact)) ) preview.collisions.extend( f"{path} (runtime source conflicts with a generated path)" @@ -760,6 +792,15 @@ def generate_deployment_kit( index_subjects = {**source_files, **owned} owned[GENERATED_INDEX] = _generated_index(plan.application_id, index_subjects) files = {**source_files, **owned} + final_generated_paths = set( + _planned_generated_paths(plan, bootstrap_mode, approved, application_artifact) + ) + final_source_generated_collisions = sorted(set(source_files) & final_generated_paths) + if final_source_generated_collisions: + raise PreparationError( + "Generation output contains a runtime source conflict with a generated path: " + + ", ".join(final_source_generated_collisions) + ) if output_root == repository_root: files_for_validation = { "pyproject.toml": (repository_root / "pyproject.toml").read_bytes(), @@ -780,13 +821,17 @@ def generate_deployment_kit( secret_values=secret_values, ) previous = _load_previous_index(output_root) + _create, _replace, final_collisions = _classify_output( + output_root, sorted(files), previous + ) obsolete, obsolete_collisions = _obsolete_owned_paths( output_root, set(files), previous ) - if obsolete_collisions: + final_collisions.extend(obsolete_collisions) + if final_collisions: raise PreparationError( "Generation output contains files not safely owned by the previous generator run: " - + ", ".join(obsolete_collisions) + + ", ".join(final_collisions) ) _remove_obsolete_owned_files(output_root, obsolete) _write_files(output_root, files) diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index 3e2f513..f8c2cef 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -14,9 +14,7 @@ RiskFinding, RiskSeverity, ) - -FORBIDDEN_TEXT = ("powershell.exe", "pwsh.exe", "executionpolicy") -WINDOWS_ABSOLUTE = re.compile(rb"(?i)(?:[a-z]:\\(?:users|home)\\[^\r\n\"]+)") +from python_deployment_builder.security_policy import TEXT_SUFFIXES, text_security_findings def _check(condition: bool, code: str, description: str) -> RiskFinding: @@ -111,28 +109,22 @@ def validate_rendered_files( program_files_hits: list[str] = [] secret_hits: list[str] = [] for relative in generated_paths: - if PurePosixPath(relative).suffix.lower() not in { - ".bat", - ".cmd", - ".json", - ".py", - ".txt", - }: + if PurePosixPath(relative).suffix.lower() not in TEXT_SUFFIXES: continue - data = files.get(relative, b"") - lowered = data.lower() - for value in FORBIDDEN_TEXT: - if value.encode() in lowered: - forbidden_hits.append(f"{relative}:{value}") - if WINDOWS_ABSOLUTE.search(data): + text = files.get(relative, b"").decode("utf-8", errors="replace") + findings = text_security_findings( + text, configured_secret_values=secret_values or [] + ) + if "forbidden_shell" in findings: + forbidden_hits.append(relative) + if "developer_path" in findings: developer_path_hits.append(relative) - if b"setx" in lowered and b"path" in lowered: + if "permanent_path" in findings: permanent_path_hits.append(relative) - if b"program files" in lowered and (b"write" in lowered or b"mkdir" in lowered): + if "program_files_write" in findings: program_files_hits.append(relative) - for secret in secret_values or []: - if len(secret) >= 8 and secret.encode("utf-8") in data: - secret_hits.append(relative) + if {"obvious_secret", "configured_secret"} & findings: + secret_hits.append(relative) ps1_files = [path for path in generated_paths if PurePosixPath(path).suffix.lower() == ".ps1"] runtime_builder_imports = [ path diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index 6a0164e..2e2a2e5 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -10,7 +10,29 @@ OBVIOUS_SECRET = re.compile( r"(?i)(?:authorization\s*[:=]\s*bearer\s+[a-z0-9._-]{12,}|sk-[a-z0-9_-]{16,})" ) -TEXT_SUFFIXES = frozenset({".bat", ".cmd", ".json", ".py", ".txt"}) +TEXT_SUFFIXES = frozenset( + { + ".bat", + ".cfg", + ".cmd", + ".conf", + ".config", + ".csv", + ".htm", + ".html", + ".ini", + ".json", + ".md", + ".py", + ".rst", + ".toml", + ".tsv", + ".txt", + ".xml", + ".yaml", + ".yml", + } +) PROGRAM_FILES_WRITE_TOKENS = ("mkdir", "copy ", "write_text", "open(", "write") SECRET_FILENAMES = frozenset( {".env", "credentials.json", "secrets.json", "token.json", ".pypirc", "pip.ini"} diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 0a60a2a..f92b226 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -747,6 +747,95 @@ def test_analysis_roles_control_source_staging(tmp_path: Path) -> None: assert "historical/old.py" not in staged +@pytest.mark.parametrize( + ("layout", "package_directory", "resource_path"), + [ + ("flat", "app", "app/data/default.json"), + ("src", "src/app", "src/app/data/default.json"), + ("mapped", "code", "code/data/default.json"), + ], +) +def test_authoritative_setuptools_package_data_is_promoted_and_staged( + tmp_path: Path, layout: str, package_directory: str, resource_path: str +) -> None: + package_root = tmp_path / package_directory + (package_root / "data").mkdir(parents=True) + (package_root / "__init__.py").write_text("", encoding="utf-8") + (package_root / "main.py").write_text( + "import importlib.resources\n" + "def main():\n" + " name = 'default.json'\n" + " return importlib.resources.files('app').joinpath('data', name).read_text()\n", + encoding="utf-8", + ) + (package_root / "data/default.json").write_text('{"default": true}\n', encoding="utf-8") + setuptools = ( + "[tool.setuptools]\npackages = ['app']\n" + "package-dir = {app = 'code'}\n" + if layout == "mapped" + else "[tool.setuptools]\npackages = ['app']\n" + if layout == "flat" + else "[tool.setuptools]\npackage-dir = {'' = 'src'}\npackages = ['app']\n" + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'package-data-app'\nversion = '1.0.0'\ndependencies = []\n" + "[project.scripts]\npackage-data-app = 'app.main:main'\n" + + setuptools + + "[tool.setuptools.package-data]\napp = ['data/*.json']\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (tmp_path / "unrelated.bin").write_bytes(b"not declared package data") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + source_plan = plan.model_copy(deep=True) + source_plan.deployment_mode = "source" + staged = _staging_files(tmp_path, assessment, source_plan, include=True) + resource = next(item for item in assessment.resources if item.path == resource_path) + inventory = next(item for item in assessment.file_inventory if item.path == resource_path) + original = assessment.repository.fingerprint + + assert plan.deployment_mode == ("source" if layout == "flat" else "package") + assert resource.status == FindingStatus.DETECTED + assert resource.packaging_status == "packaged" + assert any("Authoritative setuptools package-data" in item.detail for item in resource.evidence) + assert inventory.role == RepositoryFileRole.RUNTIME_RESOURCE + assert "Authoritative setuptools package-data" in inventory.reason + assert resource_path in staged + assert "unrelated.bin" not in staged + data = tmp_path / resource_path + data.write_text('{"default": false}\n', encoding="utf-8") + assert assess_repository(_repository(tmp_path)).repository.fingerprint != original + + +def test_wildcard_setuptools_package_data_uses_known_physical_package_mapping( + tmp_path: Path, +) -> None: + package = tmp_path / "code" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text("def main(): return 0\n", encoding="utf-8") + (package / "view.html").write_text("

runtime

\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'wildcard-data-app'\nversion = '1.0.0'\ndependencies = []\n" + "[project.scripts]\nwildcard-data-app = 'app.main:main'\n" + "[tool.setuptools]\npackages = ['app']\npackage-dir = {app = 'code'}\n" + "[tool.setuptools.package-data]\n'*' = ['*.html']\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + source_plan = plan.model_copy(deep=True) + source_plan.deployment_mode = "source" + + resource = next(item for item in assessment.resources if item.path == "code/view.html") + assert resource.packaging_status == "packaged" + assert "code/view.html" in _staging_files(tmp_path, assessment, source_plan, include=True) + + def test_installed_namespace_package_data_and_user_local_wrapper_select_package_mode( tmp_path: Path, ) -> None: diff --git a/tests/test_generation.py b/tests/test_generation.py index fe55c94..b2bbb0f 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -29,6 +29,7 @@ ) from python_deployment_builder.generation.cmd import parse_certutil_sha256 from python_deployment_builder.generation.generator import ( + _planned_generated_paths, _render_owned_files, _staging_files, generate_deployment_kit, @@ -41,7 +42,11 @@ from python_deployment_builder.generation.security import redact_secrets from python_deployment_builder.generation.structural import validate_rendered_files from python_deployment_builder.generation.templates import TEMPLATE_ROOT -from python_deployment_builder.models import BootstrapArtifact +from python_deployment_builder.models import ( + ApplicationArtifact, + ApprovedArtifact, + BootstrapArtifact, +) from python_deployment_builder.packaging.archive import safe_extract_zip from python_deployment_builder.packaging.packager import package_deployment_kit from python_deployment_builder.planning.planner import create_deployment_plan @@ -527,6 +532,11 @@ def test_application_wheel_rejects_missing_runtime_content_and_binary_content( ("installed_app/.env.production", "API_KEY=secret"), ("installed_app/.env.local", "API_KEY=secret"), ("installed_app/secret.py", "TOKEN = 'sk-abcdefghijklmnop'"), + ("installed_app/settings.yaml", "api_key: sk-abcdefghijklmnop"), + ("installed_app/settings.yml", "api_key: sk-abcdefghijklmnop"), + ("installed_app/settings.toml", "api_key = 'sk-abcdefghijklmnop'"), + ("installed_app/settings.ini", "api_key = sk-abcdefghijklmnop"), + ("installed_app/settings.cfg", "api_key = sk-abcdefghijklmnop"), ("installed_app/copy.py", "COMMAND = 'copy payload C:\\Program Files\\App'"), ], ) @@ -564,6 +574,23 @@ def test_application_wheel_allows_environment_example_file(tmp_path: Path) -> No assert artifact.filename == wheel.name +def test_application_wheel_allows_non_secret_textual_configuration_file(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/settings.yaml": "theme: light\n"}, + ) + + artifact, _path = validate_application_wheel(wheel, assessment, plan) + + assert artifact.filename == wheel.name + + def test_application_wheel_rejects_configured_secret_value( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -972,6 +999,44 @@ def test_git_source_staging_excludes_untracked_application_files(tmp_path: Path) assert "local_helper.py" not in staged +def test_git_source_staging_blocks_untracked_authoritative_package_data( + tmp_path: Path, +) -> None: + source = tmp_path / "git-source" + (source / "app/data").mkdir(parents=True) + (source / "app/__init__.py").write_text("", encoding="utf-8") + (source / "app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + data = source / "app/data/default.json" + (source / "pyproject.toml").write_text( + "[project]\nname = 'package-data-app'\nversion = '1.0.0'\ndependencies = []\n" + "[project.scripts]\npackage-data-app = 'app.main:main'\n" + "[tool.setuptools]\npackages = ['app']\n" + "[tool.setuptools.package-data]\napp = ['data/*.json']\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + data.write_text("{}\n", encoding="utf-8") + unrelated = source / "untracked-runtime-looking.json" + unrelated.write_text("{}\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match="package-data runtime resources.*untracked"): + _staging_files(source, assessment, plan, include=True) + + assert data.is_file() + assert unrelated.is_file() + + def test_prepare_lock_stages_only_lock_created_by_current_authorized_operation( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -1497,6 +1562,117 @@ def test_application_wheel_tilde_path_works_for_cli_dry_run_and_generation( assert (output / "deployment/application" / wheel.name).is_file() +def _wheel_alias(alias: Path, target: Path) -> Path: + try: + alias.symlink_to(target) + except (NotImplementedError, OSError) as exc: + pytest.skip(f"wheel alias symlink is unavailable: {exc}") + return alias + + +def test_validated_artifact_models_are_the_generated_destination_authority() -> None: + plan = _plan("optional_map_app", ["map"]) + approved = ApprovedArtifact( + distribution_name="proxy-tools", + version="0.1.0", + filename="proxy_tools-0.1.0-py3-none-any.whl", + sha256="a" * 64, + ) + application = ApplicationArtifact( + distribution_name="mapped-app", + version="1.2.3", + filename="mapped_app-1.2.3-py3-none-any.whl", + sha256="b" * 64, + entry_point_name="mapped-app", + entry_point_target="installed_app.main:main", + ) + + paths = _planned_generated_paths( + plan, + "online_cmd", + [(approved, Path("reviewed-latest.whl"))], + (application, Path("latest.whl")), + ) + + assert "deployment/wheels/proxy_tools-0.1.0-py3-none-any.whl" in paths + assert "deployment/application/mapped_app-1.2.3-py3-none-any.whl" in paths + assert not any("latest.whl" in path for path in paths) + + +def test_application_wheel_preview_and_write_use_validated_filename( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + wheel = _make_application_wheel(tmp_path) + alias = _wheel_alias(tmp_path / "latest.whl", wheel) + output = tmp_path / "kit" + destination = output / "deployment/application" / wheel.name + destination.parent.mkdir(parents=True) + destination.write_bytes(b"unowned destination") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + preview = generate_deployment_kit( + repository, output, application_wheel=alias, bootstrap_mode="online_cmd", dry_run=True + ).preview + + assert f"deployment/application/{wheel.name}" in preview.collisions + assert f"deployment/application/{alias.name}" not in preview.files_to_create + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail("validated destination collision must block early"), + ) + with pytest.raises(PreparationError, match=wheel.name): + generate_deployment_kit( + repository, output, application_wheel=alias, bootstrap_mode="online_cmd" + ) + assert destination.read_bytes() == b"unowned destination" + + destination.unlink() + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + result = generate_deployment_kit( + repository, output, application_wheel=alias, bootstrap_mode="online_cmd" + ) + + assert result.generated + assert destination.is_file() + assert not (output / "deployment/application" / alias.name).exists() + + +def test_dependency_artifact_preview_uses_validated_filename(tmp_path: Path) -> None: + wheel = _make_wheel(tmp_path) + alias = _wheel_alias(tmp_path / "reviewed-latest.whl", wheel) + output = tmp_path / "kit" + destination = output / "deployment/wheels" / wheel.name + destination.parent.mkdir(parents=True) + destination.write_bytes(b"unowned destination") + + preview = generate_deployment_kit( + _repository("optional_map_app"), + output, + selected_extras=["map"], + artifact_values=[f"proxy-tools={alias}"], + bootstrap_mode="online_cmd", + dry_run=True, + ).preview + + assert f"deployment/wheels/{wheel.name}" in preview.collisions + assert f"deployment/wheels/{alias.name}" not in preview.files_to_create + assert destination.read_bytes() == b"unowned destination" + + def test_dry_run_reports_unowned_output_collision(tmp_path: Path) -> None: output = tmp_path / "kit" output.mkdir() @@ -1614,6 +1790,31 @@ def test_templates_are_thin_and_forbid_prohibited_shells(tmp_path: Path) -> None assert any(item.code == "NO_FORBIDDEN_SHELL" for item in checks) +def test_generation_structural_validation_scans_shared_textual_configuration_formats( + tmp_path: Path, +) -> None: + plan = _plan() + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + owned, manifest = _render_owned_files( + plan, + FIXTURES / "prepared_gui", + bootstrap_mode="bundled_uv", + system_certs=False, + approved=[], + bundled_uv=fake_uv, + ) + owned["deployment/runtime/settings.toml"] = b"api_key = 'sk-abcdefghijklmnop'\n" + files = { + "pyproject.toml": (FIXTURES / "prepared_gui" / "pyproject.toml").read_bytes(), + "uv.lock": (FIXTURES / "prepared_gui" / "uv.lock").read_bytes(), + **owned, + } + + with pytest.raises(PreparationError, match="NO_SECRET_VALUES"): + validate_rendered_files(files, manifest, generated_paths=set(owned), secret_values=[]) + + def test_uv_archive_rejects_traversal_and_hash_version_mismatch(tmp_path: Path) -> None: archive = tmp_path / "uv.zip" with zipfile.ZipFile(archive, "w") as bundle: diff --git a/tests/test_validation.py b/tests/test_validation.py index 57397f1..688aff9 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -162,6 +162,19 @@ def test_static_validation_detects_missing_helper_and_forbidden_shell( assert _status(report, "NO_POWERSHELL") == ValidationCheckStatus.FAIL +def test_static_validation_scans_shared_textual_configuration_formats( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + kit = _kit(monkeypatch, tmp_path) + (kit / "deployment/runtime/settings.yaml").write_text( + "api_key: sk-abcdefghijklmnop\n", encoding="utf-8" + ) + + report = validate_static_kit(kit) + + assert _status(report, "NO_SECRET_CONTENT") == ValidationCheckStatus.FAIL + + def test_static_validation_detects_runtime_bytecode_cache( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 82d111199cf107ec9d5eae705ddaa291453d1edf Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 3 Sep 2026 20:00:00 -0400 Subject: [PATCH 07/59] Harden provenance and wheel metadata checks --- .../analysis/resources.py | 70 ++++- .../generation/artifacts.py | 66 +++-- .../generation/generator.py | 20 +- tests/test_analysis_scope.py | 5 + tests/test_generation.py | 258 +++++++++++++++++- 5 files changed, 387 insertions(+), 32 deletions(-) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index ffac26b..bde18cc 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -4,6 +4,7 @@ import ast from collections import defaultdict +from dataclasses import dataclass from pathlib import Path, PurePosixPath, PureWindowsPath from python_deployment_builder.analysis.imports import EXCLUDED_DIRECTORIES @@ -16,6 +17,17 @@ ) +@dataclass(frozen=True) +class ResolvedPackageDataMember: + """A safe concrete setuptools package-data member and its wheel destination.""" + + package_name: str + pattern: str + source_path: str + installed_member_path: str + evidence: Evidence + + def _safe_package_data_pattern(pattern: str) -> bool: """Return whether a setuptools package-data pattern stays under its package root.""" @@ -74,15 +86,16 @@ def _physical_package_roots( return roots -def _declared_package_data( +def resolve_package_data_members( root: Path, project: PackagingAssessment | None -) -> dict[str, list[Evidence]]: - """Resolve existing, safe setuptools package-data members by physical package root.""" +) -> list[ResolvedPackageDataMember]: + """Resolve existing safe package-data source files and installed wheel member paths.""" if project is None: - return {} + return [] resolved_root = root.resolve() - declared: dict[str, list[Evidence]] = defaultdict(list) + resolved_members: list[ResolvedPackageDataMember] = [] + seen: set[tuple[str, str, str, str]] = set() for declared_package, patterns in project.package_data.items(): packages = _known_packages(project) if declared_package == "*" else {declared_package} for package in packages: @@ -101,9 +114,15 @@ def _declared_package_data( try: resolved = candidate.resolve() resolved.relative_to(resolved_package_root) - relative = resolved.relative_to(resolved_root).as_posix() + source_path = resolved.relative_to(resolved_root).as_posix() + package_relative = resolved.relative_to( + resolved_package_root + ).as_posix() except ValueError: continue + installed_member_path = str( + PurePosixPath(*package.split(".")) / package_relative + ) evidence = Evidence( file="pyproject.toml", detail=( @@ -111,8 +130,43 @@ def _declared_package_data( f"{declared_package} = {pattern!r} includes this runtime resource." ), ) - if evidence not in declared[relative]: - declared[relative].append(evidence) + identity = ( + package, + pattern, + source_path, + installed_member_path, + ) + if identity not in seen: + seen.add(identity) + resolved_members.append( + ResolvedPackageDataMember( + package_name=package, + pattern=pattern, + source_path=source_path, + installed_member_path=installed_member_path, + evidence=evidence, + ) + ) + return sorted( + resolved_members, + key=lambda item: ( + item.source_path, + item.installed_member_path, + item.package_name, + item.pattern, + ), + ) + + +def _declared_package_data( + root: Path, project: PackagingAssessment | None +) -> dict[str, list[Evidence]]: + """Group concrete setuptools package-data evidence by physical source path.""" + + declared: dict[str, list[Evidence]] = defaultdict(list) + for member in resolve_package_data_members(root, project): + if member.evidence not in declared[member.source_path]: + declared[member.source_path].append(member.evidence) return declared RESOURCE_DIRECTORIES = { diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 1e1a153..9dcb7e1 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -4,7 +4,6 @@ import configparser import csv -import fnmatch import io import os import re @@ -14,9 +13,11 @@ from email.policy import default from pathlib import Path, PurePosixPath +from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name, parse_wheel_filename from packaging.version import InvalidVersion, Version +from python_deployment_builder.analysis.resources import resolve_package_data_members from python_deployment_builder.generation.acquisition import PreparationError, sha256_file from python_deployment_builder.models import ( ApplicationArtifact, @@ -25,6 +26,7 @@ RepositoryAssessment, ) from python_deployment_builder.planning.index import wheel_matches +from python_deployment_builder.planning.policies import python_satisfies from python_deployment_builder.security_policy import ( TEXT_SUFFIXES, is_secret_filename, @@ -177,6 +179,28 @@ def _require_wheel_metadata(message, *, wheel: Path) -> set[str]: return tags +def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> None: + """Apply the planner's conservative minor-as-.0 policy to wheel Core Metadata.""" + + values = metadata.get_all("Requires-Python", []) + if not values: + return + if len(values) != 1 or not values[0].strip(): + raise PreparationError(f"Malformed Requires-Python metadata in wheel: {wheel.name}") + constraint = values[0].strip() + try: + SpecifierSet(constraint) + except InvalidSpecifier as exc: + raise PreparationError( + f"Malformed Requires-Python metadata in wheel: {wheel.name}" + ) from exc + if not python_satisfies(plan.runtime.python_version, constraint): + raise PreparationError( + f"Wheel Requires-Python {constraint!r} is incompatible with selected Python " + f"{plan.runtime.python_version}." + ) + + def _validate_application_security( bundle: zipfile.ZipFile, members: dict[str, zipfile.ZipInfo], @@ -276,6 +300,7 @@ def validate_approved_wheel( f"Wheel metadata version mismatch: expected {requirement.version}, " f"received {metadata_version}." ) + _validate_requires_python(metadata, plan, path) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: @@ -306,6 +331,8 @@ def validate_application_wheel( path: Path, assessment: RepositoryAssessment, plan: DeploymentPlan, + *, + repository_root: Path | None = None, ) -> tuple[ApplicationArtifact, Path]: """Validate the explicit first-party wheel required by package mode.""" @@ -384,6 +411,7 @@ def validate_application_wheel( ) from exc if metadata_version_value != expected_version_value: raise PreparationError("Application wheel METADATA version is wrong.") + _validate_requires_python(metadata, plan, path) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: @@ -458,23 +486,25 @@ def validate_application_wheel( f"{entry_point.module}" ) - for package, patterns in assessment.project.package_data.items(): - if package == "*": - continue - package_prefix = PurePosixPath(*package.split(".")) - package_members = [] - for name in names: - member_path = PurePosixPath(name) - try: - package_members.append(str(member_path.relative_to(package_prefix))) - except ValueError: - continue - for pattern in patterns: - if not any(fnmatch.fnmatchcase(name, pattern) for name in package_members): - raise PreparationError( - "Application wheel is missing declared package data for " - f"{package}: {pattern}" - ) + source_root = repository_root + if source_root is None: + candidate = Path(assessment.repository.source).expanduser() + source_root = candidate if candidate.is_dir() else None + if assessment.project.package_data and source_root is None: + raise PreparationError( + "Application wheel package-data validation requires the assessed " + "repository root." + ) + expected_members = { + member.installed_member_path + for member in resolve_package_data_members(source_root, assessment.project) + } if source_root is not None else set() + missing_members = sorted(expected_members - names) + if missing_members: + raise PreparationError( + "Application wheel is missing concrete declared package data: " + + ", ".join(missing_members) + ) except (zipfile.BadZipFile, UnicodeDecodeError, configparser.Error) as exc: raise PreparationError(f"Malformed application wheel: {path.name}") from exc diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 2014357..9b539bd 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -97,7 +97,17 @@ def _dirty_tracked_deployment_paths( repository_root: Path, provenance_guarded: set[str] ) -> list[str]: result = subprocess.run( - ["git", "-C", str(repository_root), "diff", "--name-only", "-z", "HEAD", "--"], + [ + "git", + "-C", + str(repository_root), + "diff", + "--no-renames", + "--name-only", + "-z", + "HEAD", + "--", + ], capture_output=True, check=False, ) @@ -654,7 +664,9 @@ def generate_deployment_kit( if plan.deployment_mode != "package" and application_wheel is not None: raise PreparationError("--application-wheel is accepted only for package deployment mode.") application_artifact = ( - validate_application_wheel(application_wheel, assessment, plan) + validate_application_wheel( + application_wheel, assessment, plan, repository_root=repository_root + ) if application_wheel is not None else None ) @@ -750,7 +762,9 @@ def generate_deployment_kit( ) approved = validate_artifact_set(artifact_values, plan) application_artifact = ( - validate_application_wheel(application_wheel, assessment, plan) + validate_application_wheel( + application_wheel, assessment, plan, repository_root=repository_root + ) if application_wheel is not None else None ) diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index f92b226..b577466 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -10,6 +10,7 @@ from python_deployment_builder.analysis.imports import scan_imports from python_deployment_builder.analysis.metadata import inspect_metadata from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.resources import resolve_package_data_members from python_deployment_builder.cli import main from python_deployment_builder.generation.acquisition import PreparationError from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit @@ -804,6 +805,10 @@ def test_authoritative_setuptools_package_data_is_promoted_and_staged( assert "Authoritative setuptools package-data" in inventory.reason assert resource_path in staged assert "unrelated.bin" not in staged + assert [ + (item.source_path, item.installed_member_path) + for item in resolve_package_data_members(tmp_path, assessment.project) + ] == [(resource_path, "app/data/default.json")] data = tmp_path / resource_path data.write_text('{"default": false}\n', encoding="utf-8") assert assess_repository(_repository(tmp_path)).repository.fingerprint != original diff --git a/tests/test_generation.py b/tests/test_generation.py index b2bbb0f..6d9553e 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -69,13 +69,28 @@ def _plan(name: str = "prepared_gui", extras: list[str] | None = None): ) -def _make_wheel(path: Path, name: str = "proxy-tools", version: str = "0.1.0") -> Path: +def _make_wheel( + path: Path, + name: str = "proxy-tools", + version: str = "0.1.0", + *, + requires_python: str | None = None, + requires_python_values: list[str] | None = None, +) -> Path: normalized = name.replace("-", "_") wheel = path / f"{normalized}-{version}-py3-none-any.whl" dist_info = f"{normalized}-{version}.dist-info" files = { f"{dist_info}/METADATA": ( - f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n" + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + + "".join( + f"Requires-Python: {value}\n" + for value in ( + requires_python_values + or ([requires_python] if requires_python else []) + ) + ) + + "\n" ), f"{dist_info}/WHEEL": ( "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" @@ -103,6 +118,8 @@ def _make_application_wheel( package: str = "installed_app", target: str = "installed_app.main:main", include_cache: bool = False, + requires_python: str | None = None, + requires_python_values: list[str] | None = None, ) -> Path: normalized = name.replace("-", "_") wheel = path / f"{normalized}-{version}-py3-none-any.whl" @@ -112,7 +129,15 @@ def _make_application_wheel( f"{package}/main.py": "def main(): return 0\n", f"{package}/view.html": "\n", f"{dist_info}/METADATA": ( - f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n" + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + + "".join( + f"Requires-Python: {value}\n" + for value in ( + requires_python_values + or ([requires_python] if requires_python else []) + ) + ) + + "\n" ), f"{dist_info}/WHEEL": ( "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" @@ -517,6 +542,124 @@ def test_application_wheel_rejects_missing_runtime_content_and_binary_content( validate_application_wheel(wheel, assessment, plan) +def test_application_wheel_requires_every_concrete_declared_package_data_member( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "code/data").mkdir() + (source / "code/data/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "code/data/schema.json").write_text("{}\n", encoding="utf-8") + pyproject = source / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + 'installed_app = ["view.html"]', + 'installed_app = ["view.html", "data/*.json"]', + ), + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + only_one = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/data/defaults.json": "{}\n"}, + ) + + with pytest.raises(PreparationError, match="installed_app/data/schema.json"): + validate_application_wheel(only_one, assessment, plan) + + complete_directory = tmp_path / "complete" + complete_directory.mkdir() + complete = _rewrite_application_wheel( + _make_application_wheel(complete_directory), + additions={ + "installed_app/data/defaults.json": "{}\n", + "installed_app/data/schema.json": "{}\n", + }, + ) + artifact, _ = validate_application_wheel(complete, assessment, plan) + + assert artifact.filename == complete.name + + +def test_application_wheel_resolves_wildcard_package_data_against_known_package( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "code/data").mkdir() + (source / "code/data/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "code/data/schema.json").write_text("{}\n", encoding="utf-8") + pyproject = source / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + 'installed_app = ["view.html"]', + "'*' = [\"view.html\", \"data/*.json\"]", + ), + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/data/defaults.json": "{}\n"}, + ) + + with pytest.raises(PreparationError, match="installed_app/data/schema.json"): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize( + ("requires_python", "accepted"), + [ + (None, True), + (">=3.11", True), + (">=3.12,<3.13", True), + (">=3.13", False), + ("<3.12", False), + ], +) +def test_application_wheel_requires_python_uses_selected_minor_policy( + tmp_path: Path, requires_python: str | None, accepted: bool +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path, requires_python=requires_python) + + if accepted: + assert validate_application_wheel(wheel, assessment, plan)[0].filename == wheel.name + else: + with pytest.raises(PreparationError, match="Requires-Python"): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize( + "values", + [[">=three"], [">=3.11", "<3.12"]], +) +def test_application_wheel_rejects_malformed_or_multiple_requires_python( + tmp_path: Path, values: list[str] +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path, requires_python_values=values) + + with pytest.raises(PreparationError, match="Malformed Requires-Python"): + validate_application_wheel(wheel, assessment, plan) + + @pytest.mark.parametrize( ("name", "content"), [ @@ -1345,6 +1488,78 @@ def test_git_source_staging_blocks_unstaged_application_source_rename( assert "app.py" in str(caught.value) +def _committed_runtime_rename_fixture( + tmp_path: Path, +) -> tuple[Path, MaterializedRepository]: + source, repository = _committed_source_fixture(tmp_path) + (source / "helper.py").write_text("VALUE = 'runtime input'\n", encoding="utf-8") + subprocess.run(["git", "-C", str(source), "add", "helper.py"], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "add runtime helper"], check=True) + return source, repository + + +def test_git_staged_runtime_rename_to_documentation_blocks_provenance( + tmp_path: Path, +) -> None: + source, repository = _committed_runtime_rename_fixture(tmp_path) + subprocess.run(["git", "-C", str(source), "mv", "helper.py", "docs/helper.py"], check=True) + default_changed = subprocess.run( + ["git", "-C", str(source), "diff", "--name-only", "-z", "HEAD", "--"], + capture_output=True, + check=True, + ).stdout.split(b"\0") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert b"docs/helper.py" in default_changed + assert b"helper.py" not in default_changed + with pytest.raises(PreparationError) as caught: + _staging_files(source, assessment, plan, include=True) + assert assessment.repository.revision in str(caught.value) + assert "helper.py" in str(caught.value) + + +def test_git_staged_runtime_rename_to_runtime_path_blocks_provenance( + tmp_path: Path, +) -> None: + source, repository = _committed_runtime_rename_fixture(tmp_path) + subprocess.run( + ["git", "-C", str(source), "mv", "helper.py", "renamed_helper.py"], check=True + ) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match="helper.py|renamed_helper.py"): + _staging_files(source, assessment, plan, include=True) + + +def test_git_staged_documentation_rename_remains_allowed(tmp_path: Path) -> None: + source, repository = _committed_source_fixture(tmp_path) + subprocess.run( + ["git", "-C", str(source), "mv", "docs/readme.md", "docs/renamed.md"], check=True + ) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + staged = _staging_files(source, assessment, plan, include=True) + + assert {"app.py", "state.json"} <= staged.keys() + assert "docs/renamed.md" not in staged + + +def test_git_staged_gitignore_rename_blocks_provenance(tmp_path: Path) -> None: + source, repository = _committed_source_fixture(tmp_path) + subprocess.run( + ["git", "-C", str(source), "mv", "data/.gitignore", "data/renamed.ignore"], + check=True, + ) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match="data/.gitignore"): + _staging_files(source, assessment, plan, include=True) + + @pytest.mark.parametrize("operation", ["modified", "deleted"]) def test_git_source_staging_blocks_dirty_tracked_runtime_resource( tmp_path: Path, @@ -1922,6 +2137,43 @@ def test_approved_wheel_rejects_wrong_name_version_and_metadata(tmp_path: Path) validate_approved_wheel(f"proxy-tools={wrong}", plan) +@pytest.mark.parametrize( + ("requires_python", "accepted"), + [ + (None, True), + (">=3.11", True), + (">=3.12,<3.13", True), + (">=3.13", False), + ("<3.12", False), + ], +) +def test_approved_wheel_requires_python_uses_selected_minor_policy( + tmp_path: Path, requires_python: str | None, accepted: bool +) -> None: + plan = _plan("optional_map_app", ["map"]) + wheel = _make_wheel(tmp_path, requires_python=requires_python) + + if accepted: + assert validate_approved_wheel(f"proxy-tools={wheel}", plan)[0].filename == wheel.name + else: + with pytest.raises(PreparationError, match="Requires-Python"): + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + +@pytest.mark.parametrize( + "values", + [[">=three"], [">=3.11", "<3.12"]], +) +def test_approved_wheel_rejects_malformed_or_multiple_requires_python( + tmp_path: Path, values: list[str] +) -> None: + plan = _plan("optional_map_app", ["map"]) + wheel = _make_wheel(tmp_path, requires_python_values=values) + + with pytest.raises(PreparationError, match="Malformed Requires-Python"): + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + def test_runtime_common_staleness_and_deletion_guards( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 368c17cbaa3c945fd252caf920161e2adc5ca37b Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:45:33 -0400 Subject: [PATCH 08/59] Protect selected runtime and wheel metadata inputs --- .../analysis/resources.py | 27 ++- .../generation/artifacts.py | 4 +- .../generation/generator.py | 28 ++- .../security_policy.py | 23 ++ tests/test_analysis_scope.py | 36 +++ tests/test_generation.py | 218 +++++++++++++++++- 6 files changed, 318 insertions(+), 18 deletions(-) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index bde18cc..f46ce89 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -62,15 +62,26 @@ def _physical_package_roots( if explicit is not None: candidates.append(root / explicit) else: - base = project.package_directories.get("") - if base is not None: - candidates.append(root / base / Path(*package.split("."))) + package_parts = package.split(".") + parent_mappings = [ + mapping + for mapping in project.package_directories + if mapping and (package == mapping or package.startswith(f"{mapping}.")) + ] + if parent_mappings: + parent = max(parent_mappings, key=lambda mapping: len(mapping.split("."))) + remainder = package_parts[len(parent.split(".")) :] + candidates.append(root / project.package_directories[parent] / Path(*remainder)) else: - candidates.extend( - root / source_root / Path(*package.split(".")) - for source_root in project.source_roots - ) - candidates.append(root / Path(*package.split("."))) + base = project.package_directories.get("") + if base is not None: + candidates.append(root / base / Path(*package_parts)) + else: + candidates.extend( + root / source_root / Path(*package_parts) + for source_root in project.source_roots + ) + candidates.append(root / Path(*package_parts)) resolved_root = root.resolve() roots: list[Path] = [] diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 9dcb7e1..1aa4924 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -28,8 +28,8 @@ from python_deployment_builder.planning.index import wheel_matches from python_deployment_builder.planning.policies import python_satisfies from python_deployment_builder.security_policy import ( - TEXT_SUFFIXES, is_secret_filename, + is_textual_wheel_member, text_security_findings, ) @@ -215,7 +215,7 @@ def _validate_application_security( if member_path.suffix.lower() == ".ps1" or is_secret_filename(member_path.name): failures.append(name) continue - if ".dist-info" in member_path.parts or member_path.suffix.lower() not in TEXT_SUFFIXES: + if not is_textual_wheel_member(member_path): continue text = bundle.read(member).decode("utf-8", errors="replace") if text_security_findings( diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 9b539bd..170b963 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -207,6 +207,7 @@ def _tracked_deployment_paths( plan, *, created_lock: Path | None = None, + allow_missing_lock: bool = False, ) -> set[str]: selected = _selected_deployment_paths(assessment, plan) analysis_policy = _analysis_policy_paths(assessment) @@ -256,7 +257,20 @@ def _tracked_deployment_paths( "uv.lock. Generation stopped rather than widening untracked-file staging." ) tracked.add("uv.lock") + elif allow_missing_lock and not (repository_root / "uv.lock").exists(): + # This is only the non-mutating preflight for the current explicitly authorized + # --prepare-lock operation. The post-creation pass requires its exact path. + tracked.add("uv.lock") if assessment.repository.revision is not None: + untracked_selected = sorted(selected - tracked) + if untracked_selected: + raise PreparationError( + "Selected deployment inputs must be tracked for Git release generation " + f"at recorded source revision {assessment.repository.revision}: " + + ", ".join(untracked_selected) + + ". Commit those inputs, or use only the current --prepare-lock-created " + "uv.lock exception." + ) dirty = _dirty_tracked_deployment_paths( repository_root, _provenance_guard_paths(assessment, plan) ) @@ -284,7 +298,11 @@ def _staging_files( if not include: return {} selected = _tracked_deployment_paths( - repository_root, assessment, plan, created_lock=created_lock + repository_root, + assessment, + plan, + created_lock=created_lock, + allow_missing_lock=allow_missing_lock, ) files: dict[str, bytes] = {} for relative_text in sorted(selected): @@ -649,7 +667,13 @@ def generate_deployment_kit( repository_root=repository_root, ) if assessment.repository.revision is not None: - _tracked_deployment_paths(repository_root, assessment, plan) + _tracked_deployment_paths( + repository_root, + assessment, + plan, + allow_missing_lock=prepare_lock + and plan.lockfile.status == "developer_generation_required", + ) if plan.entry_point is None: raise PreparationError( "Deployment readiness is blocked: " + "; ".join(plan.readiness.blockers) diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index 2e2a2e5..8227799 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -4,6 +4,7 @@ import re from collections.abc import Iterable +from pathlib import PurePosixPath FORBIDDEN_SHELL = ("powershell.exe", "pwsh.exe", "executionpolicy") WINDOWS_ABSOLUTE = re.compile(r"(?i)[a-z]:\\(?:users|home)\\[^\r\n\"]+") @@ -37,6 +38,18 @@ SECRET_FILENAMES = frozenset( {".env", "credentials.json", "secrets.json", "token.json", ".pypirc", "pip.ini"} ) +TEXTUAL_WHEEL_METADATA_FILENAMES = frozenset( + { + "metadata", + "wheel", + "record", + "entry_points.txt", + "top_level.txt", + "installer", + "requested", + "direct_url.json", + } +) def is_secret_filename(filename: str) -> bool: @@ -48,6 +61,15 @@ def is_secret_filename(filename: str) -> bool: ) +def is_textual_wheel_member(path: PurePosixPath) -> bool: + """Return whether a wheel member has content suitable for text security checks.""" + + return path.suffix.lower() in TEXT_SUFFIXES or ( + any(part.casefold().endswith(".dist-info") for part in path.parts) + and path.name.casefold() in TEXTUAL_WHEEL_METADATA_FILENAMES + ) + + def text_security_findings( text: str, *, configured_secret_values: Iterable[str] = () ) -> set[str]: @@ -78,5 +100,6 @@ def text_security_findings( "FORBIDDEN_SHELL", "TEXT_SUFFIXES", "is_secret_filename", + "is_textual_wheel_member", "text_security_findings", ] diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index b577466..c9025b9 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -18,6 +18,7 @@ FindingStatus, OnlineCompatibilityAssessment, OnlineIndexContext, + PackagingAssessment, RepositoryFileRole, WheelCompatibility, ) @@ -841,6 +842,41 @@ def test_wildcard_setuptools_package_data_uses_known_physical_package_mapping( assert "code/view.html" in _staging_files(tmp_path, assessment, source_plan, include=True) +@pytest.mark.parametrize( + ("package", "package_directories", "source_roots", "physical_root"), + [ + ("app", {"app": "code"}, [], "code"), + ("app.sub", {"app": "lib"}, [], "lib/sub"), + ("app.sub.deep", {"app": "lib"}, [], "lib/sub/deep"), + ("app.sub.deep", {"app": "lib", "app.sub": "special"}, [], "special/deep"), + ("app.sub", {"": "src"}, [], "src/app/sub"), + ("app", {}, ["."], "app"), + ], +) +def test_package_data_resolver_uses_longest_parent_package_dir_mapping( + tmp_path: Path, + package: str, + package_directories: dict[str, str], + source_roots: list[str], + physical_root: str, +) -> None: + resource = tmp_path / physical_root / "data/default.json" + resource.parent.mkdir(parents=True) + resource.write_text("{}\n", encoding="utf-8") + project = PackagingAssessment( + packages=[package], + package_directories=package_directories, + source_roots=source_roots, + package_data={package: ["data/*.json"]}, + ) + + resolved = resolve_package_data_members(tmp_path, project) + + assert [(item.source_path, item.installed_member_path) for item in resolved] == [ + (f"{physical_root}/data/default.json", f"{package.replace('.', '/')}/data/default.json") + ] + + def test_installed_namespace_package_data_and_user_local_wrapper_select_package_mode( tmp_path: Path, ) -> None: diff --git a/tests/test_generation.py b/tests/test_generation.py index 6d9553e..a29d31f 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -16,6 +16,7 @@ from python_deployment_builder.analysis.assessor import assess_repository from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.resources import resolve_package_data_members from python_deployment_builder.cli import build_parser, main from python_deployment_builder.generation.acquisition import ( PreparationError, @@ -31,6 +32,7 @@ from python_deployment_builder.generation.generator import ( _planned_generated_paths, _render_owned_files, + _selected_deployment_paths, _staging_files, generate_deployment_kit, ) @@ -584,6 +586,48 @@ def test_application_wheel_requires_every_concrete_declared_package_data_member( assert artifact.filename == complete.name +def test_application_wheel_requires_nested_package_data_from_parent_mapping( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + (source / "lib/sub/data").mkdir(parents=True) + (source / "lib/__init__.py").write_text("", encoding="utf-8") + (source / "lib/sub/__init__.py").write_text("", encoding="utf-8") + (source / "lib/sub/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "lib/sub/data/default.json").write_text("{}\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + "[project]\nname = 'mapped-app'\nversion = '1.2.3'\ndependencies = []\n" + "[project.gui-scripts]\nmapped-app = 'app.sub.main:main'\n" + "[tool.setuptools]\npackages = ['app', 'app.sub']\npackage-dir = {app = 'lib'}\n" + "[tool.setuptools.package-data]\n'app.sub' = ['data/*.json']\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + incomplete = _rewrite_application_wheel( + _make_application_wheel(tmp_path, package="app", target="app.sub.main:main"), + additions={"app/sub/__init__.py": "", "app/sub/main.py": "def main(): return 0\n"}, + ) + + assert [ + (item.source_path, item.installed_member_path) + for item in resolve_package_data_members(source, assessment.project) + ] == [("lib/sub/data/default.json", "app/sub/data/default.json")] + with pytest.raises(PreparationError, match="app/sub/data/default.json"): + validate_application_wheel(incomplete, assessment, plan, repository_root=source) + + complete = _rewrite_application_wheel( + incomplete, additions={"app/sub/data/default.json": "{}\n"} + ) + artifact, _path = validate_application_wheel( + complete, assessment, plan, repository_root=source + ) + + assert artifact.filename == complete.name + + def test_application_wheel_resolves_wildcard_package_data_against_known_package( tmp_path: Path, ) -> None: @@ -734,6 +778,49 @@ def test_application_wheel_allows_non_secret_textual_configuration_file(tmp_path assert artifact.filename == wheel.name +def test_application_wheel_rejects_obvious_secret_in_textual_metadata(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + replacements={ + "mapped_app-1.2.3.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: mapped-app\nVersion: 1.2.3\n\n" + "Project description: sk-abcdefghijklmnop\n" + ) + }, + ) + + with pytest.raises(PreparationError, match="security policy"): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_allows_non_secret_textual_metadata(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + replacements={ + "mapped_app-1.2.3.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: mapped-app\nVersion: 1.2.3\n\n" + "A normal project description.\n" + ) + }, + ) + + artifact, _path = validate_application_wheel(wheel, assessment, plan) + + assert artifact.filename == wheel.name + + def test_application_wheel_rejects_configured_secret_value( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -758,6 +845,37 @@ def test_application_wheel_rejects_configured_secret_value( validate_application_wheel(wheel, assessment, plan) +def test_application_wheel_rejects_configured_secret_in_metadata( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "code/main.py").write_text( + "import os\nAPI_TOKEN = os.environ['APP_API_TOKEN']\ndef main(): return 0\n", + encoding="utf-8", + ) + secret = "configured-value-that-must-not-ship" + monkeypatch.setenv("APP_API_TOKEN", secret) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + replacements={ + "mapped_app-1.2.3.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: mapped-app\nVersion: 1.2.3\n\n" + f"Operational notes: {secret}\n" + ) + }, + ) + + with pytest.raises(PreparationError, match="security policy") as caught: + validate_application_wheel(wheel, assessment, plan) + + assert secret not in str(caught.value) + + @pytest.mark.parametrize( "filename", [ @@ -1102,7 +1220,9 @@ def test_generate_and_all_cli_propagate_first_party_application_wheel( assert "approved wheel" in generate_help assert help_text -def test_git_source_staging_excludes_untracked_application_files(tmp_path: Path) -> None: +def test_git_source_staging_allows_untracked_unselected_application_like_files( + tmp_path: Path, +) -> None: source = tmp_path / "git-source" source.mkdir() (source / "pyproject.toml").write_text( @@ -1130,7 +1250,8 @@ def test_git_source_staging_excludes_untracked_application_files(tmp_path: Path) ) subprocess.run(["git", "-C", str(source), "add", "."], check=True) subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) - (source / "local_helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (source / "docs").mkdir() + (source / "docs/local_helper.py").write_text("VALUE = 1\n", encoding="utf-8") repository = MaterializedRepository(root=source, source=str(source), source_kind="local") assessment = assess_repository(repository) @@ -1139,7 +1260,89 @@ def test_git_source_staging_excludes_untracked_application_files(tmp_path: Path) assert assessment.repository.revision assert "app.py" in staged - assert "local_helper.py" not in staged + assert "docs/local_helper.py" not in staged + + +def test_git_source_staging_blocks_untracked_selected_import_and_stages_after_commit( + tmp_path: Path, +) -> None: + source = tmp_path / "git-source" + source.mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='tracked-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\ntracked-app='main:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (source / "main.py").write_text( + "from helper import VALUE\ndef main(): return VALUE\n", encoding="utf-8" + ) + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], check=True + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + helper = source / "helper.py" + helper.write_text("VALUE = 1\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert any( + item.path == "helper.py" and item.role.value == "application_source" + for item in assessment.file_inventory + ) + with pytest.raises( + PreparationError, match="Selected deployment inputs must be tracked.*helper.py" + ): + _staging_files(source, assessment, plan, include=True) + + subprocess.run(["git", "-C", str(source), "add", "helper.py"], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "track helper"], check=True) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + staged = _staging_files(source, assessment, plan, include=True) + + assert _selected_deployment_paths(assessment, plan) <= staged.keys() + assert "helper.py" in staged + + +def test_git_source_staging_blocks_untracked_selected_runtime_resource(tmp_path: Path) -> None: + source = tmp_path / "git-source" + source.mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='tracked-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\ntracked-app='main:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (source / "main.py").write_text( + "from pathlib import Path\nSTATE = Path(__file__).with_name('state.json').read_text()\n" + "def main(): return STATE\n", + encoding="utf-8", + ) + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], check=True + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + (source / "state.json").write_text("{}\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert any( + item.path == "state.json" and item.role.value == "runtime_resource" + for item in assessment.file_inventory + ) + with pytest.raises( + PreparationError, match="Selected deployment inputs must be tracked.*state.json" + ): + _staging_files(source, assessment, plan, include=True) def test_git_source_staging_blocks_untracked_authoritative_package_data( @@ -1199,7 +1402,8 @@ def test_prepare_lock_stages_only_lock_created_by_current_authorized_operation( ) subprocess.run(["git", "-C", str(source), "add", "."], check=True) subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) - unrelated = source / "local_runtime.py" + (source / "docs").mkdir() + unrelated = source / "docs/local_runtime.py" unrelated.write_text("VALUE = 'untracked'\n", encoding="utf-8") lock_bytes = b"version = 1\nrevision = 3\nrequires-python = \">=3.11\"\n" fake_uv = tmp_path / "uv.exe" @@ -1242,7 +1446,7 @@ def create_lock(root: Path, *args, **kwargs) -> LockPreparationResult: check=True, ).stdout.splitlines() assert "?? uv.lock" in status - assert f"?? {unrelated.name}" in status + assert "?? docs/" in status def test_preexisting_untracked_lock_does_not_bypass_git_staging_policy( @@ -1269,7 +1473,9 @@ def test_preexisting_untracked_lock_does_not_bypass_git_staging_policy( assessment = assess_repository(repository) plan = create_deployment_plan(assessment, repository_root=source) - with pytest.raises(PreparationError, match="Required deployment input is missing: uv.lock"): + with pytest.raises( + PreparationError, match="Selected deployment inputs must be tracked.*uv.lock" + ): _staging_files(source, assessment, plan, include=True) From bdf72564489ec476d883193b83aa5ff205047dcc Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:09:42 -0400 Subject: [PATCH 09/59] Harden provenance snapshots and wheel integrity --- .../analysis/repository.py | 39 ++++ src/python_deployment_builder/cli.py | 8 +- .../generation/artifacts.py | 44 ++++ .../generation/generator.py | 13 +- tests/test_cli.py | 93 ++++++++ tests/test_generation.py | 206 ++++++++++++++++-- tests/test_packaging.py | 6 +- tests/test_repository.py | 23 ++ 8 files changed, 398 insertions(+), 34 deletions(-) diff --git a/src/python_deployment_builder/analysis/repository.py b/src/python_deployment_builder/analysis/repository.py index 42c5ecd..a640d3c 100644 --- a/src/python_deployment_builder/analysis/repository.py +++ b/src/python_deployment_builder/analysis/repository.py @@ -104,6 +104,45 @@ def safe_extract_zip(archive: Path, destination: Path) -> Path: return destination +def materialize_git_head_snapshot(archive: Path, destination: Path) -> set[str]: + """Materialize regular files from a locally generated Git HEAD archive. + + This is deliberately separate from ``safe_extract_zip``: external archives must + reject links, while this read-only Git provenance snapshot can skip link entries + so unrelated links cannot poison analysis of regular HEAD files. + """ + + destination.mkdir(parents=True, exist_ok=True) + skipped_symlinks: set[str] = set() + with zipfile.ZipFile(archive) as bundle: + members = bundle.infolist() + if len(members) > MAX_MEMBERS: + raise RepositoryLoadError("Git HEAD archive contains too many entries.") + if sum(member.file_size for member in members) > MAX_EXTRACTED_BYTES: + raise RepositoryLoadError("Git HEAD archive is too large after extraction.") + for member in members: + if member.flag_bits & 0x1: + raise RepositoryLoadError("Encrypted Git HEAD archives are not supported.") + if member.file_size > MAX_MEMBER_BYTES: + raise RepositoryLoadError(f"Git HEAD member is too large: {member.filename}") + target = _safe_member_path(destination, member.filename) + mode = member.external_attr >> 16 + if stat.S_IFMT(mode) == stat.S_IFLNK: + skipped_symlinks.add(PurePosixPath(member.filename).as_posix()) + continue + if member.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + if stat.S_IFMT(mode) not in {0, stat.S_IFREG}: + raise RepositoryLoadError( + f"Unsupported Git HEAD archive member type: {member.filename}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + with bundle.open(member) as source, target.open("wb") as output: + shutil.copyfileobj(source, output) + return skipped_symlinks + + def _download_github_archive(owner: str, repository: str, destination: Path) -> None: url = f"https://api.github.com/repos/{owner}/{repository}/zipball" request = urllib.request.Request( diff --git a/src/python_deployment_builder/cli.py b/src/python_deployment_builder/cli.py index dbdeb61..9082f39 100644 --- a/src/python_deployment_builder/cli.py +++ b/src/python_deployment_builder/cli.py @@ -536,17 +536,15 @@ def run_all( print(f" Readiness: {plan.readiness.state}") for blocker in plan.readiness.blockers: print(f" Blocker: {blocker}") + write_assessment_reports(assessment, reports_root) + write_deployment_plan_reports(plan, reports_root) if plan.entry_point is None: - write_assessment_reports(assessment, reports_root) - write_deployment_plan_reports(plan, reports_root) print( " Stop: declare an authoritative [project.gui-scripts] or " "[project.scripts] entry point before generation." ) return 2 if plan.risk_gate.outcome == "block": - write_assessment_reports(assessment, reports_root) - write_deployment_plan_reports(plan, reports_root) print(" Stop: planning blockers must be resolved before generation.") return 2 if plan.lockfile.status == "developer_generation_required": @@ -587,8 +585,6 @@ def run_all( dry_run=False, ) print(f" Deployment kit: {generated.output_directory}") - write_assessment_reports(assessment, reports_root) - write_deployment_plan_reports(plan, reports_root) print("STATIC VALIDATE") static_report = validate_static_kit(kit_root) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 1aa4924..a071d82 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -2,8 +2,12 @@ from __future__ import annotations +import base64 +import binascii import configparser import csv +import hashlib +import hmac import io import os import re @@ -129,6 +133,46 @@ def _validate_record( raise PreparationError( f"Wheel RECORD must record itself with a blank hash and size: {wheel.name}" ) + for name, member in members.items(): + if member.is_dir() or name.endswith((".dist-info/RECORD.jws", ".dist-info/RECORD.p7s")): + continue + if name == record_name: + continue + recorded_hash, recorded_size = recorded[name.casefold()] + if not recorded_size.isascii() or not recorded_size.isdecimal(): + raise PreparationError(f"Wheel RECORD has an invalid size for {name}: {wheel.name}") + try: + expected_size = int(recorded_size) + except ValueError as exc: # pragma: no cover - guarded by isdecimal + raise PreparationError( + f"Wheel RECORD has an invalid size for {name}: {wheel.name}" + ) from exc + if expected_size != member.file_size: + raise PreparationError(f"Wheel RECORD size mismatch for {name}: {wheel.name}") + algorithm, separator, encoded_digest = recorded_hash.partition("=") + if ( + not separator + or algorithm not in {"sha256", "sha384", "sha512"} + or not encoded_digest + or not re.fullmatch(r"[A-Za-z0-9_-]+", encoded_digest) + ): + raise PreparationError(f"Wheel RECORD has an invalid hash for {name}: {wheel.name}") + try: + expected_digest = base64.b64decode( + encoded_digest + "=" * (-len(encoded_digest) % 4), + altchars=b"-_", + validate=True, + ) + except (ValueError, binascii.Error) as exc: + raise PreparationError( + f"Wheel RECORD has an invalid hash for {name}: {wheel.name}" + ) from exc + digest = hashlib.new(algorithm) + with bundle.open(member) as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + if not hmac.compare_digest(digest.digest(), expected_digest): + raise PreparationError(f"Wheel RECORD hash mismatch for {name}: {wheel.name}") def _dist_info_members( diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 170b963..e465444 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -16,7 +16,7 @@ from python_deployment_builder.analysis.repository import ( MaterializedRepository, RepositoryLoadError, - safe_extract_zip, + materialize_git_head_snapshot, ) from python_deployment_builder.generation.acquisition import ( PreparationError, @@ -146,15 +146,15 @@ def _dirty_tracked_deployment_paths( "be inventoried. Generation stopped rather than claiming clean provenance." ) try: - head_root = safe_extract_zip(archive, extracted) + head_symlinks = materialize_git_head_snapshot(archive, extracted) head_repository = MaterializedRepository( - root=head_root, + root=extracted, source=f"{repository_root}@HEAD", source_kind="local", ) head_assessment = assess_repository(head_repository) head_plan = create_deployment_plan( - head_assessment, repository_root=head_root + head_assessment, repository_root=extracted ) head_guarded = _provenance_guard_paths(head_assessment, head_plan) except (OSError, RepositoryLoadError, ValueError) as exc: @@ -162,7 +162,10 @@ def _dirty_tracked_deployment_paths( "Git revision provenance is known, but the recorded source revision could not " "be inventoried. Generation stopped rather than claiming clean provenance." ) from exc - return sorted(changed & (provenance_guarded | head_guarded)) + changed_symlinks = (changed & head_symlinks) | { + path for path in changed if (repository_root / Path(path)).is_symlink() + } + return sorted(changed & (provenance_guarded | head_guarded) | changed_symlinks) def _selected_deployment_paths(assessment, plan) -> set[str]: diff --git a/tests/test_cli.py b/tests/test_cli.py index b5921e7..28999d4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,10 +1,41 @@ from pathlib import Path +import pytest + from python_deployment_builder.cli import application_id, build_parser, main FIXTURES = Path(__file__).parent / "fixtures" +def _assert_all_reports(output: Path) -> None: + reports = output / "reports" + assert (reports / "assessment.json").is_file() + assert (reports / "assessment.md").is_file() + assert (reports / "deployment-plan.json").is_file() + assert (reports / "deployment-plan.md").is_file() + + +def _write_all_source(root: Path, *, package_mode: bool = False, lock: bool = True) -> None: + package = root / ("code" if package_mode else "app") + package.mkdir(parents=True, exist_ok=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text("def main(): return 0\n", encoding="utf-8") + mapping = ( + "[tool.setuptools]\npackages = ['installed_app']\n" + "package-dir = {installed_app = 'code'}\n" + if package_mode + else "[tool.setuptools]\npackages = ['app']\n" + ) + target = "installed_app.main:main" if package_mode else "app.main:main" + (root / "pyproject.toml").write_text( + "[project]\nname='all-report-app'\nversion='1.0'\ndependencies=[]\n" + f"[project.scripts]\nall-report-app='{target}'\n" + mapping, + encoding="utf-8", + ) + if lock: + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + def test_safe_application_id() -> None: assert application_id("Geo Map Explanation Extractor") == "geo-map-explanation-extractor" assert application_id("../../") == "python-application" @@ -46,3 +77,65 @@ def test_plan_parser_accepts_repeatable_extras() -> None: ) assert arguments.extra == ["map", "feature-two"] + + +@pytest.mark.parametrize("case", ["missing-entry", "missing-lock", "missing-wheel"]) +def test_all_persists_assessment_and_plan_reports_before_readiness_returns( + tmp_path: Path, case: str +) -> None: + source = tmp_path / "source" + source.mkdir() + if case == "missing-entry": + (source / "pyproject.toml").write_text( + "[project]\nname='no-entry'\nversion='1.0'\ndependencies=[]\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + else: + _write_all_source(source, package_mode=case == "missing-wheel", lock=case != "missing-lock") + output = tmp_path / "output" + + result = main(["all", str(source), "--output-dir", str(output)]) + + assert result == 2 + _assert_all_reports(output) + assert not (output / "deployment-kit").exists() + assert not (output / "distribution").exists() + + +def test_all_persists_reports_when_reviewed_dependency_artifact_is_missing(tmp_path: Path) -> None: + output = tmp_path / "output" + + result = main( + [ + "all", + str(FIXTURES / "optional_map_app"), + "--extra", + "map", + "--output-dir", + str(output), + ] + ) + + assert result == 2 + _assert_all_reports(output) + assert not (output / "deployment-kit").exists() + + +def test_all_persists_reports_when_risk_gate_blocks(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_all_source(source) + (source / "app/main.py").write_text( + "from pathlib import Path\n" + "Path('C:\\\\Program Files\\\\Unsafe\\\\state.json').write_text('state')\n" + "def main(): return 0\n", + encoding="utf-8", + ) + output = tmp_path / "output" + + result = main(["all", str(source), "--output-dir", str(output)]) + + assert result == 2 + _assert_all_reports(output) + assert not (output / "deployment-kit").exists() diff --git a/tests/test_generation.py b/tests/test_generation.py index a29d31f..f174a98 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import csv import hashlib import importlib.util @@ -97,15 +98,10 @@ def _make_wheel( f"{dist_info}/WHEEL": ( "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" ), - f"{normalized}/__init__.py": "", + f"{normalized}/__init__.py": "VALUE = 1\n", } record_name = f"{dist_info}/RECORD" - output = io.StringIO(newline="") - writer = csv.writer(output, lineterminator="\n") - for filename in files: - writer.writerow((filename, "", "")) - writer.writerow((record_name, "", "")) - files[record_name] = output.getvalue() + files[record_name] = _record_contents(files, record_name) with zipfile.ZipFile(wheel, "w") as bundle: for filename, data in files.items(): bundle.writestr(filename, data) @@ -149,12 +145,7 @@ def _make_application_wheel( if include_cache: files[f"{package}/__pycache__ (1)/main.pyc"] = "cache" record_name = f"{dist_info}/RECORD" - output = io.StringIO(newline="") - writer = csv.writer(output, lineterminator="\n") - for filename in files: - writer.writerow((filename, "", "")) - writer.writerow((record_name, "", "")) - files[record_name] = output.getvalue() + files[record_name] = _record_contents(files, record_name) with zipfile.ZipFile(wheel, "w") as bundle: for filename, data in files.items(): bundle.writestr(filename, data) @@ -168,6 +159,8 @@ def _rewrite_application_wheel( removals: set[str] | None = None, additions: dict[str, str | bytes] | None = None, recorded_paths: list[str] | None = None, + recalculate_record: bool = True, + record_contents: str | bytes | None = None, ) -> Path: with zipfile.ZipFile(wheel) as bundle: files = { @@ -180,23 +173,67 @@ def _rewrite_application_wheel( for item in bundle.infolist() if item.filename.endswith(".dist-info/RECORD") ) + existing_record = bundle.read(record_name) for name in removals or set(): files.pop(name, None) for name, data in {**(replacements or {}), **(additions or {})}.items(): files[name] = data.encode() if isinstance(data, str) else data - paths = list(files) if recorded_paths is None else recorded_paths - output = io.StringIO(newline="") - writer = csv.writer(output, lineterminator="\n") - for filename in paths: - writer.writerow((filename, "", "")) - writer.writerow((record_name, "", "")) - files[record_name] = output.getvalue().encode() + if record_contents is not None: + files[record_name] = ( + record_contents.encode() if isinstance(record_contents, str) else record_contents + ) + elif recalculate_record: + paths = list(files) if recorded_paths is None else recorded_paths + files[record_name] = _record_contents(files, record_name, paths).encode() + else: + files[record_name] = existing_record with zipfile.ZipFile(wheel, "w") as bundle: for filename, data in files.items(): bundle.writestr(filename, data) return wheel +def _record_contents( + files: dict[str, str | bytes], record_name: str, paths: list[str] | None = None +) -> str: + """Create a Wheel-compliant RECORD for the supplied in-memory members.""" + + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + for filename in paths or files: + data = files.get(filename) + if data is None: + writer.writerow((filename, "", "")) + continue + data_bytes = data.encode() if isinstance(data, str) else data + digest = base64.urlsafe_b64encode(hashlib.sha256(data_bytes).digest()).rstrip(b"=") + writer.writerow((filename, f"sha256={digest.decode()}", str(len(data_bytes)))) + writer.writerow((record_name, "", "")) + return output.getvalue() + + +def _record_with_member_values( + wheel: Path, member_name: str, *, digest: str | None = None, size: str | None = None +) -> str: + """Return RECORD content with one ordinary member's integrity values replaced.""" + + with zipfile.ZipFile(wheel) as bundle: + record_name = next( + item.filename + for item in bundle.infolist() + if item.filename.endswith(".dist-info/RECORD") + ) + rows = list(csv.reader(io.StringIO(bundle.read(record_name).decode("utf-8")))) + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + for row in rows: + if row[0] == member_name: + row[1] = row[1] if digest is None else digest + row[2] = row[2] if size is None else size + writer.writerow(row) + return output.getvalue() + + def _write_mapped_project(root: Path, *, version: str = "1.2.3") -> None: (root / "code").mkdir() (root / "code/__init__.py").write_text("", encoding="utf-8") @@ -515,6 +552,92 @@ def test_application_wheel_record_is_an_exact_file_inventory(tmp_path: Path) -> validate_application_wheel(wheel, assessment, plan) +@pytest.mark.parametrize( + ("wheel_kind", "member_name"), + [ + ("application", "installed_app/main.py"), + ("approved", "proxy_tools/__init__.py"), + ], +) +def test_wheel_record_rejects_stale_hash_and_size_for_all_wheel_contracts( + tmp_path: Path, wheel_kind: str, member_name: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + application_plan = create_deployment_plan(assessment) + approved_plan = _plan("optional_map_app", ["map"]) + wheel = ( + _make_application_wheel(tmp_path) + if wheel_kind == "application" + else _make_wheel(tmp_path) + ) + + stale_hash = _rewrite_application_wheel( + wheel, + replacements={ + member_name: "def main(): return 1\n" if wheel_kind == "application" else "VALUE = 2\n" + }, + recalculate_record=False, + ) + with pytest.raises(PreparationError, match="RECORD hash mismatch"): + if wheel_kind == "application": + validate_application_wheel(stale_hash, assessment, application_plan) + else: + validate_approved_wheel(f"proxy-tools={stale_hash}", approved_plan) + + wheel.unlink() + wheel = ( + _make_application_wheel(tmp_path) + if wheel_kind == "application" + else _make_wheel(tmp_path) + ) + stale_size = _rewrite_application_wheel( + wheel, + replacements={ + member_name: "def main(): return 100\n" if wheel_kind == "application" else "longer" + }, + recalculate_record=False, + ) + with pytest.raises(PreparationError, match="RECORD size mismatch"): + if wheel_kind == "application": + validate_application_wheel(stale_size, assessment, application_plan) + else: + validate_approved_wheel(f"proxy-tools={stale_size}", approved_plan) + + +@pytest.mark.parametrize( + ("digest", "size", "message"), + [ + ("not-a-digest", None, "invalid hash"), + ("sha256=!!!", None, "invalid hash"), + ("md5=abcd", None, "invalid hash"), + ("", None, "invalid hash"), + (None, "not-a-size", "invalid size"), + (None, "", "invalid size"), + ], +) +def test_application_wheel_rejects_malformed_record_integrity_values( + tmp_path: Path, digest: str | None, size: str | None, message: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + wheel = _make_application_wheel(tmp_path) + record = _record_with_member_values( + wheel, "installed_app/main.py", digest=digest, size=size + ) + _rewrite_application_wheel(wheel, record_contents=record) + + with pytest.raises(PreparationError, match=message): + validate_application_wheel(wheel, assessment, plan) + + @pytest.mark.parametrize( ("removals", "additions", "message"), [ @@ -1194,6 +1317,10 @@ def test_generate_and_all_cli_propagate_first_party_application_wheel( ) assert (all_output / "deployment-kit/deployment/application" / wheel.name).is_file() assert list((all_output / "distribution").glob("*.zip")) + assert (all_output / "reports/assessment.json").is_file() + assert (all_output / "reports/assessment.md").is_file() + assert (all_output / "reports/deployment-plan.json").is_file() + assert (all_output / "reports/deployment-plan.md").is_file() missing_output = tmp_path / "missing-output" assert ( @@ -1209,7 +1336,10 @@ def test_generate_and_all_cli_propagate_first_party_application_wheel( ) == 2 ) - assert not missing_output.exists() + assert (missing_output / "reports/assessment.json").is_file() + assert (missing_output / "reports/deployment-plan.json").is_file() + assert not (missing_output / "deployment-kit").exists() + assert not (missing_output / "distribution").exists() output_text = capsys.readouterr().out assert "--application-wheel" in output_text assert "Source roots: none (installed-project mode)" in output_text @@ -1753,6 +1883,40 @@ def test_git_staged_documentation_rename_remains_allowed(tmp_path: Path) -> None assert "docs/renamed.md" not in staged +def test_git_head_symlink_does_not_poison_unrelated_documentation_provenance( + tmp_path: Path, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + external = tmp_path / "external-content.py" + external.write_text("EXTERNAL = 'must not be read from HEAD snapshot'\n", encoding="utf-8") + link = source / "docs/unrelated-link.py" + try: + link.symlink_to(external) + except (NotImplementedError, OSError) as exc: + pytest.skip(f"Git symlink fixture is unavailable: {exc}") + subprocess.run(["git", "-C", str(source), "add", "docs/unrelated-link.py"], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "add unrelated link"], check=True) + (source / "docs/readme.md").write_text("changed documentation\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + staged = _staging_files(source, assessment, plan, include=True) + + assert {"app.py", "state.json"} <= staged.keys() + assert "docs/unrelated-link.py" not in staged + assert external.read_text(encoding="utf-8").startswith("EXTERNAL") + + replacement = tmp_path / "replacement-content.py" + replacement.write_text("EXTERNAL = 'changed'\n", encoding="utf-8") + link.unlink() + link.symlink_to(replacement) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match="docs/unrelated-link.py"): + _staging_files(source, assessment, plan, include=True) + + def test_git_staged_gitignore_rename_blocks_provenance(tmp_path: Path) -> None: source, repository = _committed_source_fixture(tmp_path) subprocess.run( diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 50c313b..0bcc7da 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import csv import hashlib import io @@ -46,8 +47,9 @@ def _make_wheel(path: Path) -> Path: record_name = f"{dist_info}/RECORD" output = io.StringIO(newline="") writer = csv.writer(output, lineterminator="\n") - for filename in files: - writer.writerow((filename, "", "")) + for filename, data in files.items(): + digest = base64.urlsafe_b64encode(hashlib.sha256(data.encode()).digest()).rstrip(b"=") + writer.writerow((filename, f"sha256={digest.decode()}", str(len(data.encode())))) writer.writerow((record_name, "", "")) files[record_name] = output.getvalue() with zipfile.ZipFile(wheel, "w") as bundle: diff --git a/tests/test_repository.py b/tests/test_repository.py index 2b1a7c5..32baa02 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -1,3 +1,4 @@ +import stat import zipfile from pathlib import Path @@ -5,6 +6,7 @@ from python_deployment_builder.analysis.repository import ( RepositoryLoadError, + materialize_git_head_snapshot, parse_public_github_url, safe_extract_zip, ) @@ -39,3 +41,24 @@ def test_safe_zip_extraction_returns_single_root(tmp_path: Path) -> None: root = safe_extract_zip(archive, tmp_path / "output") assert root.name == "owner-repo-sha" assert (root / "pyproject.toml").is_file() + + +def test_git_head_materialization_skips_links_without_weakening_external_zip_safety( + tmp_path: Path, +) -> None: + archive = tmp_path / "head.zip" + link = zipfile.ZipInfo("docs/unrelated-link") + link.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("app.py", "def main(): return 0\n") + bundle.writestr(link, "../../outside") + + with pytest.raises(RepositoryLoadError, match="symbolic links are not allowed"): + safe_extract_zip(archive, tmp_path / "external-output") + + destination = tmp_path / "head-output" + skipped = materialize_git_head_snapshot(archive, destination) + + assert skipped == {"docs/unrelated-link"} + assert (destination / "app.py").is_file() + assert not (destination / "docs/unrelated-link").exists() From 86c7065aae85d9ee84e50b99033bafc9c9c6f37f Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:37:10 -0400 Subject: [PATCH 10/59] Preserve declared entry-point groups --- src/python_deployment_builder/__init__.py | 6 +- .../analysis/metadata.py | 6 + .../generation/artifacts.py | 8 +- src/python_deployment_builder/models.py | 4 + .../planning/planner.py | 1 + .../reporting/markdown.py | 5 +- tests/test_assessment.py | 1 + tests/test_generation.py | 103 +++++++++++++++++- tests/test_metadata.py | 81 ++++++++++++++ tests/test_planning.py | 31 ++++++ 10 files changed, 236 insertions(+), 10 deletions(-) diff --git a/src/python_deployment_builder/__init__.py b/src/python_deployment_builder/__init__.py index 574c4a4..f77f119 100644 --- a/src/python_deployment_builder/__init__.py +++ b/src/python_deployment_builder/__init__.py @@ -8,5 +8,7 @@ __version__ = "0.1.0" SCHEMA_VERSION = "1.0" -ANALYSIS_SCHEMA_VERSION = "1.1" -PLANNING_SCHEMA_VERSION = "1.1" +# Analysis and planning retain defaults for older serialized forms, but their +# emitted contracts changed when entry-point packaging groups became explicit. +ANALYSIS_SCHEMA_VERSION = "1.2" +PLANNING_SCHEMA_VERSION = "1.2" diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 624ae2b..b3afad4 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -68,10 +68,12 @@ def _entry_point( ) -> EntryPointAssessment: target_lower = f"{name} {target}".lower() kind = "gui" if group == "gui-scripts" or "gui" in target_lower else "cli" + declared_group = "gui_scripts" if group == "gui-scripts" else "console_scripts" return EntryPointAssessment( name=name, target=target, kind=kind, + declared_group=declared_group, evidence=[ _evidence( root, @@ -407,6 +409,8 @@ def inspect_metadata(root: Path) -> MetadataResult: ) for name, target in poetry_scripts.items(): if isinstance(name, str) and isinstance(target, str): + # The supported string form maps to Poetry's standard + # console-script entry-point behavior. entry_points.append(_entry_point(root, pyproject_path, name, target, "scripts")) setuptools = tool.get("setuptools") if isinstance(tool.get("setuptools"), dict) else {} configured_packages = setuptools.get("packages") @@ -486,6 +490,7 @@ def inspect_metadata(root: Path) -> MetadataResult: name=name, target=target, kind="gui" if group == "gui_scripts" else "cli", + declared_group=group, evidence=[ _evidence( root, @@ -540,6 +545,7 @@ def inspect_metadata(root: Path) -> MetadataResult: name=name, target=target, kind="gui" if group == "gui_scripts" else "cli", + declared_group=group, evidence=[ _evidence( root, diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index a071d82..0ecf935 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -418,6 +418,11 @@ def validate_application_wheel( entry_point = plan.entry_point if entry_point is None: raise PreparationError("Package mode requires an authoritative application entry point.") + if entry_point.declared_group not in {"console_scripts", "gui_scripts"}: + raise PreparationError( + "Package mode requires an authoritative entry-point packaging group; " + "PDB will not infer it from launch classification." + ) try: with zipfile.ZipFile(path) as bundle: members = _member_map(_safe_wheel_members(bundle)) @@ -507,9 +512,8 @@ def validate_application_wheel( parser = configparser.ConfigParser(interpolation=None) parser.optionxform = str parser.read_string(bundle.read(members[entry_points_name]).decode("utf-8-sig")) - entry_group = "gui_scripts" if entry_point.kind == "gui" else "console_scripts" installed_target = parser.get( - entry_group, + entry_point.declared_group, entry_point.name, fallback="", ).strip() diff --git a/src/python_deployment_builder/models.py b/src/python_deployment_builder/models.py index 8fd7edc..17bdaf8 100644 --- a/src/python_deployment_builder/models.py +++ b/src/python_deployment_builder/models.py @@ -87,6 +87,9 @@ class EntryPointAssessment(StrictModel): name: str target: str kind: Literal["cli", "gui", "unknown"] + # This is the installed-wheel group declared by packaging metadata. It + # intentionally remains independent from PDB's launch/UI classification. + declared_group: Literal["console_scripts", "gui_scripts", "unknown"] = "unknown" status: FindingStatus = FindingStatus.DETECTED evidence: list[Evidence] = Field(default_factory=list) @@ -360,6 +363,7 @@ class EntrypointPlan(StrictModel): name: str target: str kind: Literal["cli", "gui", "unknown"] + declared_group: Literal["console_scripts", "gui_scripts", "unknown"] = "unknown" module: str callable: str alternatives: list[str] = Field(default_factory=list) diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index bf3794e..7deb27b 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -167,6 +167,7 @@ def _entrypoint(assessment: RepositoryAssessment) -> EntrypointPlan | None: name=chosen.name, target=chosen.target, kind=chosen.kind, + declared_group=chosen.declared_group, module=module, callable=callable_name, alternatives=[item.name for item in entries if item.name != chosen.name], diff --git a/src/python_deployment_builder/reporting/markdown.py b/src/python_deployment_builder/reporting/markdown.py index 53f6e7b..3a81b04 100644 --- a/src/python_deployment_builder/reporting/markdown.py +++ b/src/python_deployment_builder/reporting/markdown.py @@ -53,9 +53,10 @@ def render_assessment_markdown(assessment: RepositoryAssessment) -> str: "", ] if project.entry_points: - lines.extend(["| Name | Kind | Target |", "|---|---|---|"]) + lines.extend(["| Name | Declared group | Launch kind | Target |", "|---|---|---|---|"]) lines.extend( - f"| `{_escape(item.name)}` | {item.kind} | `{_escape(item.target)}` |" + f"| `{_escape(item.name)}` | `{item.declared_group}` | {item.kind} | " + f"`{_escape(item.target)}` |" for item in project.entry_points ) else: diff --git a/tests/test_assessment.py b/tests/test_assessment.py index 23f0f4b..83715ef 100644 --- a/tests/test_assessment.py +++ b/tests/test_assessment.py @@ -40,6 +40,7 @@ def test_program_files_fixture_is_red() -> None: def test_markdown_contains_human_readable_sections() -> None: rendered = render_assessment_markdown(_assess("target_app")) assert "# Static deployment assessment" in rendered + assert "Declared group" in rendered assert "## Runtime assumptions" in rendered assert "OPENAI_API_KEY" in rendered assert "REPOSITORY_ADJACENT_RESOURCES" in rendered diff --git a/tests/test_generation.py b/tests/test_generation.py index f174a98..8589ca0 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -115,6 +115,8 @@ def _make_application_wheel( version: str = "1.2.3", package: str = "installed_app", target: str = "installed_app.main:main", + entry_group: str = "gui_scripts", + entry_name: str = "mapped-app", include_cache: bool = False, requires_python: str | None = None, requires_python_values: list[str] | None = None, @@ -140,7 +142,7 @@ def _make_application_wheel( f"{dist_info}/WHEEL": ( "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" ), - f"{dist_info}/entry_points.txt": f"[gui_scripts]\nmapped-app = {target}\n", + f"{dist_info}/entry_points.txt": f"[{entry_group}]\n{entry_name} = {target}\n", } if include_cache: files[f"{package}/__pycache__ (1)/main.pyc"] = "cache" @@ -234,7 +236,14 @@ def _record_with_member_values( return output.getvalue() -def _write_mapped_project(root: Path, *, version: str = "1.2.3") -> None: +def _write_mapped_project( + root: Path, + *, + version: str = "1.2.3", + entry_group: str = "gui-scripts", + entry_name: str = "mapped-app", + target: str = "installed_app.main:main", +) -> None: (root / "code").mkdir() (root / "code/__init__.py").write_text("", encoding="utf-8") (root / "code/main.py").write_text("def main(): return 0\n", encoding="utf-8") @@ -248,8 +257,8 @@ def _write_mapped_project(root: Path, *, version: str = "1.2.3") -> None: version = "{version}" requires-python = ">=3.12" dependencies = [] -[project.gui-scripts] -mapped-app = "installed_app.main:main" +[project.{entry_group}] +{entry_name} = "{target}" [tool.setuptools] packages = ["installed_app"] package-dir = {{installed_app = "code"}} @@ -360,6 +369,92 @@ def test_application_wheel_validation_and_package_staging( assert not list(output.rglob("*.pyc")) +def test_application_wheel_uses_declared_group_not_launch_kind(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source, entry_group="scripts", entry_name="gui-tool") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + + assert plan.entry_point is not None + assert (plan.entry_point.declared_group, plan.entry_point.kind) == ( + "console_scripts", + "gui", + ) + valid = _make_application_wheel( + tmp_path, + entry_group="console_scripts", + entry_name="gui-tool", + ) + validate_application_wheel(valid, assessment, plan) + + valid.unlink() + wrong_group = _make_application_wheel( + tmp_path, + entry_group="gui_scripts", + entry_name="gui-tool", + ) + with pytest.raises(PreparationError, match="entry point disagrees"): + validate_application_wheel(wrong_group, assessment, plan) + + plan.entry_point.declared_group = "unknown" + with pytest.raises(PreparationError, match="will not infer"): + validate_application_wheel(wrong_group, assessment, plan) + + +def test_gui_scripts_wheel_must_match_gui_declared_group(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + assert plan.entry_point is not None + assert (plan.entry_point.declared_group, plan.entry_point.kind) == ("gui_scripts", "gui") + + valid = _make_application_wheel(tmp_path) + validate_application_wheel(valid, assessment, plan) + valid.unlink() + wrong_group = _make_application_wheel(tmp_path, entry_group="console_scripts") + with pytest.raises(PreparationError, match="entry point disagrees"): + validate_application_wheel(wrong_group, assessment, plan) + + +def test_poetry_string_script_uses_console_scripts_for_wheel_validation(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "code").mkdir() + (source / "code/__init__.py").write_text("", encoding="utf-8") + (source / "code/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + """[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" +[tool.poetry] +name = "mapped-app" +version = "1.2.3" +[tool.poetry.scripts] +poetry-tool = "installed_app.main:main" +""", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment) + assert plan.entry_point is not None + assert plan.deployment_mode == "package" + assert plan.entry_point.declared_group == "console_scripts" + + wheel = _make_application_wheel( + tmp_path, + entry_group="console_scripts", + entry_name="poetry-tool", + ) + validate_application_wheel(wheel, assessment, plan) + + def test_application_wheel_rejects_wrong_target_and_runtime_cache(tmp_path: Path) -> None: source = tmp_path / "source" source.mkdir() diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 525b8ea..207f4fa 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,6 +1,7 @@ from pathlib import Path from python_deployment_builder.analysis.metadata import inspect_metadata +from python_deployment_builder.models import EntryPointAssessment FIXTURES = Path(__file__).parent / "fixtures" @@ -15,6 +16,7 @@ def test_pyproject_parsing_and_entry_points() -> None: assert [(entry.name, entry.target) for entry in result.project.entry_points] == [ ("simple-cli", "simple_cli.cli:main") ] + assert result.project.entry_points[0].declared_group == "console_scripts" assert result.dependencies[0].distribution_name == "requests" @@ -51,6 +53,7 @@ def test_setup_cfg_is_parsed_statically(tmp_path: Path) -> None: assert result.python.requires_python == ">=3.10" assert result.dependencies[0].distribution_name == "Pillow" assert result.project.entry_points[0].kind == "gui" + assert result.project.entry_points[0].declared_group == "gui_scripts" def test_setup_py_literals_are_read_without_execution(tmp_path: Path) -> None: @@ -71,9 +74,87 @@ def test_setup_py_literals_are_read_without_execution(tmp_path: Path) -> None: assert result.python.requires_python == ">=3.11" assert result.dependencies[0].distribution_name == "PyYAML" assert result.project.entry_points[0].target == "literal_app:main" + assert result.project.entry_points[0].declared_group == "console_scripts" assert not marker.exists() +def test_entry_point_declared_group_is_independent_from_gui_heuristic(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + """[project] +name = "example" +version = "1.0" +[project.scripts] +gui-tool = "app:main" +tool = "app.gui:main" +[project.gui-scripts] +native-gui = "app:main" +""", + encoding="utf-8", + ) + + entries = {item.name: item for item in inspect_metadata(tmp_path).project.entry_points} + + assert (entries["gui-tool"].declared_group, entries["gui-tool"].kind) == ( + "console_scripts", + "gui", + ) + assert (entries["tool"].declared_group, entries["tool"].kind) == ( + "console_scripts", + "gui", + ) + assert (entries["native-gui"].declared_group, entries["native-gui"].kind) == ( + "gui_scripts", + "gui", + ) + + +def test_legacy_and_poetry_entry_point_groups_are_preserved(tmp_path: Path) -> None: + (tmp_path / "setup.cfg").write_text( + """[options.entry_points] +console_scripts = + console = app:main +gui_scripts = + gui = app:main +""", + encoding="utf-8", + ) + (tmp_path / "setup.py").write_text( + """from setuptools import setup +setup(entry_points={ + "console_scripts": ["literal-console = app:main"], + "gui_scripts": ["literal-gui = app:main"], +}) +""", + encoding="utf-8", + ) + poetry_root = tmp_path / "poetry" + poetry_root.mkdir() + (poetry_root / "pyproject.toml").write_text( + """[tool.poetry] +name = "poetry-example" +version = "1.0" +[tool.poetry.scripts] +poetry-tool = "app:main" +""", + encoding="utf-8", + ) + + legacy = {item.name: item for item in inspect_metadata(tmp_path).project.entry_points} + assert legacy["console"].declared_group == "console_scripts" + assert legacy["gui"].declared_group == "gui_scripts" + assert legacy["literal-console"].declared_group == "console_scripts" + assert legacy["literal-gui"].declared_group == "gui_scripts" + + poetry = inspect_metadata(poetry_root).project.entry_points + assert poetry[0].declared_group == "console_scripts" + + +def test_older_entry_point_model_forms_default_to_unknown_declared_group() -> None: + entry = EntryPointAssessment(name="tool", target="app:main", kind="cli") + + assert entry.declared_group == "unknown" + + def test_optional_dependency_markers_are_preserved_separately() -> None: result = inspect_metadata(FIXTURES / "optional_map_app") diff --git a/tests/test_planning.py b/tests/test_planning.py index c1216c9..2af7029 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -143,6 +143,7 @@ def test_target_plan_selects_source_gui_and_external_environment() -> None: assert plan.deployment_mode == "source" assert plan.entry_point.kind == "gui" + assert plan.entry_point.declared_group == "console_scripts" assert plan.runtime.backend == "uv_managed" assert plan.runtime.python_version == "3.12" assert plan.runtime.uv_version == UV_VERSION @@ -163,6 +164,36 @@ def test_target_plan_selects_source_gui_and_external_environment() -> None: assert api_key.supply_strategy == "existing_application_workflow" +def test_selected_entry_point_retains_group_from_exact_assessment_entry(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "duplicate-entry" +version = "1.0" +[project.scripts] +tool = "app.console:main" +[project.gui-scripts] +tool = "app.gui:main" +""", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + plan = create_deployment_plan( + assess_repository( + MaterializedRepository(root=tmp_path, source=str(tmp_path), source_kind="local") + ) + ) + + assert plan.entry_point is not None + assert (plan.entry_point.target, plan.entry_point.declared_group) == ( + "app.gui:main", + "gui_scripts", + ) + + def test_policy_uses_next_supported_python_when_312_is_rejected() -> None: assessment = _assess() assessment.python.requires_python = ">=3.13" From 6d02dd506644bb49cf905786513836e044583c0c Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:23:43 -0400 Subject: [PATCH 11/59] Harden package-data and wheel dependency contracts --- .../analysis/metadata.py | 20 +++ .../generation/artifacts.py | 76 ++++++++++- .../generation/generator.py | 80 ++++++----- tests/test_analysis_scope.py | 118 ++++++++++++++++ tests/test_generation.py | 127 +++++++++++++++++- 5 files changed, 388 insertions(+), 33 deletions(-) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index b3afad4..bc23cba 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -458,6 +458,7 @@ def inspect_metadata(root: Path) -> MetadataResult: if setup_cfg_path.is_file(): metadata_files.append("setup.cfg") parser = configparser.ConfigParser() + parser.optionxform = str parser.read(setup_cfg_path, encoding="utf-8") if distribution_name is None: distribution_name = parser.get("metadata", "name", fallback=None) @@ -504,6 +505,25 @@ def inspect_metadata(root: Path) -> MetadataResult: configured_where = parser.get("options.packages.find", "where", fallback="").strip() if configured_where and not source_roots: source_roots = [configured_where] + configured_package_dir = parser.get("options", "package_dir", fallback="") + if configured_package_dir: + setup_cfg_directories = { + name.strip(): path.strip() + for value in _multiline_values(configured_package_dir) + if "=" in value + for name, path in [value.split("=", 1)] + if path.strip() + } + package_directories.update(setup_cfg_directories) + if isinstance(package_directories.get(""), str): + source_roots = source_roots or [package_directories[""]] + if parser.has_section("options.package_data"): + for package, value in parser.items("options.package_data"): + patterns = _multiline_values(value) + if not patterns: + continue + existing = package_data.setdefault(package, []) + existing.extend(pattern for pattern in patterns if pattern not in existing) setup_py_path = root / "setup.py" if setup_py_path.is_file(): diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 0ecf935..558019f 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -17,6 +17,7 @@ from email.policy import default from pathlib import Path, PurePosixPath +from packaging.requirements import InvalidRequirement, Requirement from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name, parse_wheel_filename from packaging.version import InvalidVersion, Version @@ -29,7 +30,7 @@ DeploymentPlan, RepositoryAssessment, ) -from python_deployment_builder.planning.index import wheel_matches +from python_deployment_builder.planning.index import marker_applies, wheel_matches from python_deployment_builder.planning.policies import python_satisfies from python_deployment_builder.security_policy import ( is_secret_filename, @@ -371,6 +372,78 @@ def validate_artifact_set( return validated +def _application_requirement_applies(requirement: Requirement, plan: DeploymentPlan) -> bool: + """Evaluate Core Metadata markers against the planned Windows target, not this host.""" + + marker = str(requirement.marker) if requirement.marker else None + if marker is None: + return True + selected_extras = plan.lock_graph.selected_extras if plan.lock_graph else [] + return any( + marker_applies( + marker, + plan.runtime.python_version, + plan.runtime.architecture, + extra=extra, + ) + for extra in ["", *selected_extras] + ) + + +def _validate_application_requires_dist( + metadata, plan: DeploymentPlan, application_name: str +) -> None: + """Prove every applicable first-party wheel requirement is in the selected lock graph.""" + + raw_requirements = metadata.get_all("Requires-Dist", []) + if not raw_requirements: + return + graph = plan.lock_graph + if graph is None or not graph.inspected: + raise PreparationError( + "Application wheel Requires-Dist validation requires an inspected selected uv.lock " + "dependency graph." + ) + locked: dict[str, list[str]] = {} + for dependency in graph.dependencies: + locked.setdefault(canonicalize_name(dependency.name), []).append(dependency.version) + for raw in raw_requirements: + try: + requirement = Requirement(raw) + except InvalidRequirement as exc: + raise PreparationError( + f"Application wheel has malformed Requires-Dist metadata: {raw!r}." + ) from exc + if not _application_requirement_applies(requirement, plan): + continue + name = canonicalize_name(requirement.name) + if name == application_name: + continue + if requirement.url: + raise PreparationError( + "Application wheel Requires-Dist direct references are not provable against " + f"the selected locked environment: {requirement.name}." + ) + versions = locked.get(name, []) + if not versions: + raise PreparationError( + "Application wheel Requires-Dist is absent from the selected locked " + f"environment: {requirement.name}." + ) + try: + compatible = any(Version(version) in requirement.specifier for version in versions) + except InvalidVersion as exc: + raise PreparationError( + "Selected lock graph has an invalid version for application wheel " + f"Requires-Dist {requirement.name}: {versions!r}." + ) from exc + if not compatible: + raise PreparationError( + "Application wheel Requires-Dist is incompatible with the selected locked " + f"environment: {requirement}. Locked versions: {', '.join(sorted(versions))}." + ) + + def validate_application_wheel( path: Path, assessment: RepositoryAssessment, @@ -461,6 +534,7 @@ def validate_application_wheel( if metadata_version_value != expected_version_value: raise PreparationError("Application wheel METADATA version is wrong.") _validate_requires_python(metadata, plan, path) + _validate_application_requires_dist(metadata, plan, expected_name) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index e465444..a4c15ec 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -18,6 +18,7 @@ RepositoryLoadError, materialize_git_head_snapshot, ) +from python_deployment_builder.analysis.resources import resolve_package_data_members from python_deployment_builder.generation.acquisition import ( PreparationError, acquire_pinned_uv, @@ -43,7 +44,6 @@ ) from python_deployment_builder.models import ( ApplicationArtifact, - FindingStatus, GeneratedArtifact, GenerationPreview, GenerationResult, @@ -156,7 +156,7 @@ def _dirty_tracked_deployment_paths( head_plan = create_deployment_plan( head_assessment, repository_root=extracted ) - head_guarded = _provenance_guard_paths(head_assessment, head_plan) + head_guarded = _provenance_guard_paths(extracted, head_assessment, head_plan) except (OSError, RepositoryLoadError, ValueError) as exc: raise PreparationError( "Git revision provenance is known, but the recorded source revision could not " @@ -168,7 +168,18 @@ def _dirty_tracked_deployment_paths( return sorted(changed & (provenance_guarded | head_guarded) | changed_symlinks) -def _selected_deployment_paths(assessment, plan) -> set[str]: +def _authoritative_package_data_paths(repository_root: Path, assessment, plan) -> set[str]: + """Return concrete first-party package data required by source-mode staging.""" + + if plan.deployment_mode != "source": + return set() + return { + member.source_path + for member in resolve_package_data_members(repository_root, assessment.project) + } + + +def _selected_deployment_paths(repository_root: Path, assessment, plan) -> set[str]: selected = {"pyproject.toml", "uv.lock"} if plan.deployment_mode == "source": selected.update( @@ -176,6 +187,7 @@ def _selected_deployment_paths(assessment, plan) -> set[str]: for item in assessment.file_inventory if item.role in RUNTIME_ROLES and not item.path.endswith("/") ) + selected.update(_authoritative_package_data_paths(repository_root, assessment, plan)) return selected @@ -190,18 +202,10 @@ def _analysis_policy_paths(assessment) -> set[str]: } -def _authoritative_package_data_paths(assessment) -> set[str]: - """Return source-mode runtime members selected by setuptools package-data metadata.""" - - return { - resource.path.rstrip("/") - for resource in assessment.resources - if resource.status == FindingStatus.DETECTED and resource.packaging_status == "packaged" - } - - -def _provenance_guard_paths(assessment, plan) -> set[str]: - return _selected_deployment_paths(assessment, plan) | _analysis_policy_paths(assessment) +def _provenance_guard_paths(repository_root: Path, assessment, plan) -> set[str]: + return _selected_deployment_paths(repository_root, assessment, plan) | _analysis_policy_paths( + assessment + ) def _tracked_deployment_paths( @@ -212,8 +216,28 @@ def _tracked_deployment_paths( created_lock: Path | None = None, allow_missing_lock: bool = False, ) -> set[str]: - selected = _selected_deployment_paths(assessment, plan) + selected = _selected_deployment_paths(repository_root, assessment, plan) analysis_policy = _analysis_policy_paths(assessment) + declared_package_data = _authoritative_package_data_paths( + repository_root, assessment, plan + ) + inventory = {item.path.rstrip("/"): item for item in assessment.file_inventory} + excluded_package_data = sorted( + path + for path in declared_package_data + if path in inventory + and inventory[path].role + in { + RepositoryFileRole.IGNORED_OR_LOCAL, + RepositoryFileRole.MUTABLE_STATE_CANDIDATE, + } + ) + if excluded_package_data: + raise PreparationError( + "Authoritative setuptools package-data runtime resources conflict with " + "source staging policy and cannot be silently omitted: " + + ", ".join(excluded_package_data) + ) tracked = _git_tracked_paths( repository_root, required=assessment.repository.revision is not None, @@ -221,23 +245,10 @@ def _tracked_deployment_paths( if tracked is None: return selected if assessment.repository.revision is not None and plan.deployment_mode == "source": - declared_package_data = _authoritative_package_data_paths(assessment) - inventory = {item.path.rstrip("/"): item for item in assessment.file_inventory} untracked_package_data = sorted(declared_package_data - tracked) - excluded_package_data = sorted( - path - for path in declared_package_data - if path in inventory - and inventory[path].role - in { - RepositoryFileRole.IGNORED_OR_LOCAL, - RepositoryFileRole.MUTABLE_STATE_CANDIDATE, - } - ) - if untracked_package_data or excluded_package_data: + if untracked_package_data: details = [ *(f"untracked: {path}" for path in untracked_package_data), - *(f"excluded by local-state policy: {path}" for path in excluded_package_data), ] raise PreparationError( "Authoritative setuptools package-data runtime resources must be tracked and " @@ -275,7 +286,7 @@ def _tracked_deployment_paths( "uv.lock exception." ) dirty = _dirty_tracked_deployment_paths( - repository_root, _provenance_guard_paths(assessment, plan) + repository_root, _provenance_guard_paths(repository_root, assessment, plan) ) if dirty: raise PreparationError( @@ -317,6 +328,13 @@ def _staging_files( raise PreparationError(f"Staging refuses repository symbolic links: {relative_text}") if path.is_file(): files[relative.as_posix()] = path.read_bytes() + required_package_data = _authoritative_package_data_paths(repository_root, assessment, plan) + missing_package_data = sorted(required_package_data - files.keys()) + if missing_package_data: + raise PreparationError( + "Authoritative setuptools package-data runtime resources could not be staged: " + + ", ".join(missing_package_data) + ) required = {"pyproject.toml"} | (set() if allow_missing_lock else {"uv.lock"}) missing = sorted(required - files.keys()) if missing: diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index c9025b9..d48229e 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -842,6 +842,124 @@ def test_wildcard_setuptools_package_data_uses_known_physical_package_mapping( assert "code/view.html" in _staging_files(tmp_path, assessment, source_plan, include=True) +@pytest.mark.parametrize( + ("source_root", "resource_path", "installed_path"), + [ + ("", "app/data/default.json", "app/data/default.json"), + ("src", "src/app/data/default.json", "app/data/default.json"), + ], +) +def test_setup_cfg_package_data_is_authoritative_for_source_staging( + tmp_path: Path, source_root: str, resource_path: str, installed_path: str +) -> None: + package = tmp_path / source_root / "app" + (package / "data").mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text( + "import importlib.resources\n" + "def main():\n" + " name = 'default' + '.json'\n" + " return importlib.resources.files('app').joinpath('data', name).read_text()\n", + encoding="utf-8", + ) + (package / "data/default.json").write_text('{"default": true}\n', encoding="utf-8") + package_dir = "\npackage_dir =\n = src" if source_root else "" + find_where = "\n[options.packages.find]\nwhere = src" if source_root else "" + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n", + encoding="utf-8", + ) + (tmp_path / "setup.cfg").write_text( + f"""[metadata] +name = setup-cfg-data +version = 1.0 +[options] +packages = find: +python_requires = >=3.12{package_dir} +[options.entry_points] +console_scripts = + setup-cfg-data = app.main:main +[options.package_data] +app = + data/*.json + templates/*.html +* = + *.txt{find_where} +""", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + source_plan = plan.model_copy(deep=True) + source_plan.deployment_mode = "source" + resource = next(item for item in assessment.resources if item.path == resource_path) + inventory = next(item for item in assessment.file_inventory if item.path == resource_path) + members = resolve_package_data_members(tmp_path, assessment.project) + + assert assessment.project.package_data == { + "app": ["data/*.json", "templates/*.html"], + "*": ["*.txt"], + } + assert resource.packaging_status == "packaged" + assert inventory.role == RepositoryFileRole.RUNTIME_RESOURCE + assert resource_path in _staging_files(tmp_path, assessment, source_plan, include=True) + assert [(member.source_path, member.installed_member_path) for member in members] == [ + (resource_path, installed_path) + ] + + +@pytest.mark.parametrize( + ("conflict", "expected_role"), + [ + ("ignored", RepositoryFileRole.IGNORED_OR_LOCAL), + ("mutable", RepositoryFileRole.MUTABLE_STATE_CANDIDATE), + ], +) +def test_non_git_authoritative_package_data_conflicts_block_source_staging( + tmp_path: Path, conflict: str, expected_role: RepositoryFileRole +) -> None: + package = tmp_path / "app" + (package / "data").mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + main = ( + "from pathlib import Path\n" + "def main():\n" + " return Path(__file__).with_name('data').joinpath('default.json').read_text()\n" + ) + if conflict == "mutable": + main = main.replace("read_text()", "write_text('local state')") + (package / "main.py").write_text(main, encoding="utf-8") + (package / "data/default.json").write_text('{"default": true}\n', encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[project] +name = "non-git-package-data" +version = "1.0" +[project.scripts] +non-git-package-data = "app.main:main" +[tool.setuptools] +packages = ["app"] +[tool.setuptools.package-data] +app = ["data/*.json"] +""", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + if conflict == "ignored": + (tmp_path / ".gitignore").write_text("app/data/default.json\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + inventory = next( + item for item in assessment.file_inventory if item.path == "app/data/default.json" + ) + + assert inventory.role == expected_role + with pytest.raises(PreparationError, match="cannot be silently omitted"): + _staging_files(tmp_path, assessment, plan, include=True) + + @pytest.mark.parametrize( ("package", "package_directories", "source_roots", "physical_root"), [ diff --git a/tests/test_generation.py b/tests/test_generation.py index 8589ca0..ef2e7dc 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -48,7 +48,10 @@ from python_deployment_builder.models import ( ApplicationArtifact, ApprovedArtifact, + ArtifactAvailability, BootstrapArtifact, + LockedDependency, + LockGraphAssessment, ) from python_deployment_builder.packaging.archive import safe_extract_zip from python_deployment_builder.packaging.packager import package_deployment_kit @@ -120,6 +123,7 @@ def _make_application_wheel( include_cache: bool = False, requires_python: str | None = None, requires_python_values: list[str] | None = None, + requires_dist_values: list[str] | None = None, ) -> Path: normalized = name.replace("-", "_") wheel = path / f"{normalized}-{version}-py3-none-any.whl" @@ -137,6 +141,7 @@ def _make_application_wheel( or ([requires_python] if requires_python else []) ) ) + + "".join(f"Requires-Dist: {value}\n" for value in requires_dist_values or []) + "\n" ), f"{dist_info}/WHEEL": ( @@ -280,6 +285,35 @@ def _write_mapped_project( ) +def _application_plan_with_locked_dependencies( + plan, + dependencies: list[tuple[str, str]], + *, + extras: list[str] | None = None, +): + configured = plan.model_copy(deep=True) + configured.lock_graph = LockGraphAssessment( + inspected=True, + python_version=configured.runtime.python_version, + architecture=configured.runtime.architecture, + selected_extras=extras or [], + dependencies=[ + LockedDependency( + name=name, + version=version, + direct=True, + artifact=ArtifactAvailability( + compatible_wheel_available=True, + source_distribution_available=False, + policy="wheel_usable", + ), + ) + for name, version in dependencies + ], + ) + return configured + + def _load_template_module(name: str, monkeypatch: pytest.MonkeyPatch): monkeypatch.syspath_prepend(str(TEMPLATE_ROOT)) spec = importlib.util.spec_from_file_location(f"generated_{name}", TEMPLATE_ROOT / name) @@ -472,6 +506,97 @@ def test_application_wheel_rejects_wrong_target_and_runtime_cache(tmp_path: Path validate_application_wheel(cached, assessment, plan) +@pytest.mark.parametrize( + ("requires_dist", "locked", "error"), + [ + ([], [], None), + (["requests>=2"], [("Requests", "2.31.0")], None), + (["requests>=99"], [("requests", "2.31.0")], "incompatible"), + (["totally-new-package>=1"], [], "absent"), + (["windows-only>=1; sys_platform == 'win32'"], [("windows-only", "1.0")], None), + (["linux-only>=1; sys_platform == 'linux'"], [], None), + (["future-only>=1; python_version >= '3.13'"], [], None), + ], +) +def test_application_wheel_requires_dist_uses_selected_locked_target_environment( + tmp_path: Path, + requires_dist: list[str], + locked: list[tuple[str, str]], + error: str | None, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, repository_root=source), locked + ) + wheel = _make_application_wheel(tmp_path, requires_dist_values=requires_dist) + + if error is None: + validate_application_wheel(wheel, assessment, plan) + else: + with pytest.raises(PreparationError, match=error): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_requires_dist_selected_extra_controls_marker(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + base_plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel( + tmp_path, + requires_dist_values=["pywebview>=6; extra == 'map'"], + ) + + validate_application_wheel( + wheel, + assessment, + _application_plan_with_locked_dependencies(base_plan, []), + ) + validate_application_wheel( + wheel, + assessment, + _application_plan_with_locked_dependencies( + base_plan, [("pywebview", "6.0")], extras=["map"] + ), + ) + with pytest.raises(PreparationError, match="absent"): + validate_application_wheel( + wheel, + assessment, + _application_plan_with_locked_dependencies(base_plan, [], extras=["map"]), + ) + + +@pytest.mark.parametrize( + ("requires_dist", "error"), + [ + (["requests=>2"], "malformed Requires-Dist"), + (["requests @ https://example.invalid/requests.whl"], "direct references"), + ], +) +def test_application_wheel_requires_dist_rejects_unprovable_metadata( + tmp_path: Path, requires_dist: list[str], error: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, repository_root=source), [("requests", "2.31.0")] + ) + wheel = _make_application_wheel(tmp_path, requires_dist_values=requires_dist) + + with pytest.raises(PreparationError, match=error): + validate_application_wheel(wheel, assessment, plan) + + @pytest.mark.parametrize( ("name", "version", "message"), [ @@ -1530,7 +1655,7 @@ def test_git_source_staging_blocks_untracked_selected_import_and_stages_after_co plan = create_deployment_plan(assessment, repository_root=source) staged = _staging_files(source, assessment, plan, include=True) - assert _selected_deployment_paths(assessment, plan) <= staged.keys() + assert _selected_deployment_paths(source, assessment, plan) <= staged.keys() assert "helper.py" in staged From 5fa00e0a7b6d0d00c6c5d9fd85b07e9cf88b335e Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:09:34 -0400 Subject: [PATCH 12/59] Order package wheel validation after lock preparation --- .../generation/artifacts.py | 18 +- .../generation/generator.py | 31 ++- tests/test_generation.py | 247 +++++++++++++++++- 3 files changed, 285 insertions(+), 11 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 558019f..bbde58f 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -213,14 +213,20 @@ def _require_wheel_metadata(message, *, wheel: Path) -> set[str]: wheel_versions = message.get_all("Wheel-Version", []) purelib = message.get_all("Root-Is-Purelib", []) tags = {value.strip() for value in message.get_all("Tag", []) if value.strip()} + wheel_version = wheel_versions[0].strip() if len(wheel_versions) == 1 else "" if ( len(wheel_versions) != 1 - or not re.fullmatch(r"\d+(?:\.\d+)+", wheel_versions[0].strip()) + or not re.fullmatch(r"\d+(?:\.\d+)+", wheel_version) or len(purelib) != 1 or purelib[0].strip().lower() not in {"true", "false"} or not tags ): raise PreparationError(f"Malformed WHEEL metadata: {wheel.name}") + if int(wheel_version.split(".", 1)[0]) != 1: + raise PreparationError( + f"Unsupported Wheel-Version {wheel_version!r}: {wheel.name}. " + "PDB supports Wheel major version 1." + ) return tags @@ -450,8 +456,13 @@ def validate_application_wheel( plan: DeploymentPlan, *, repository_root: Path | None = None, + validate_locked_dependencies: bool = True, ) -> tuple[ApplicationArtifact, Path]: - """Validate the explicit first-party wheel required by package mode.""" + """Validate the explicit first-party wheel required by package mode. + + Structural artifact checks are always performed. Requires-Dist is evaluated only + when the caller has a current inspected lock graph to compare against. + """ path = path.expanduser().resolve() expected_name = canonicalize_name(assessment.project.distribution_name or "") @@ -534,7 +545,8 @@ def validate_application_wheel( if metadata_version_value != expected_version_value: raise PreparationError("Application wheel METADATA version is wrong.") _validate_requires_python(metadata, plan, path) - _validate_application_requires_dist(metadata, plan, expected_name) + if validate_locked_dependencies: + _validate_application_requires_dist(metadata, plan, expected_name) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index a4c15ec..5bae1b0 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -687,6 +687,17 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) + if plan.deployment_mode == "package": + try: + repository_root.relative_to(output_root) + except ValueError: + pass + else: + raise PreparationError( + "Package deployment output must be external to the application source " + "repository so the kit cannot retain application source outside the " + "validated first-party wheel." + ) if assessment.repository.revision is not None: _tracked_deployment_paths( repository_root, @@ -710,7 +721,11 @@ def generate_deployment_kit( raise PreparationError("--application-wheel is accepted only for package deployment mode.") application_artifact = ( validate_application_wheel( - application_wheel, assessment, plan, repository_root=repository_root + application_wheel, + assessment, + plan, + repository_root=repository_root, + validate_locked_dependencies=False, ) if application_wheel is not None else None @@ -727,6 +742,15 @@ def generate_deployment_kit( for item in (plan.lock_graph.artifact_findings if plan.lock_graph else []) if item.status == "unavailable" ] + if ( + not dry_run + and plan.lockfile.status == "developer_generation_required" + and not prepare_lock + ): + raise PreparationError( + "uv.lock is missing. Re-run generation with --prepare-lock for a local " + "repository to authorize developer-side lockfile creation." + ) source_files = _staging_files( repository_root, assessment, @@ -774,11 +798,6 @@ def generate_deployment_kit( raise PreparationError( "Deployment planning is blocked: " + ", ".join(plan.risk_gate.blocking_codes) ) - if plan.lockfile.status == "developer_generation_required" and not prepare_lock: - raise PreparationError( - "uv.lock is missing. Re-run generation with --prepare-lock for a local " - "repository to authorize developer-side lockfile creation." - ) if unresolved or unavailable: detail = [ *(f"approved wheel required: {item}" for item in unresolved), diff --git a/tests/test_generation.py b/tests/test_generation.py index ef2e7dc..131ebc5 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -82,6 +82,7 @@ def _make_wheel( *, requires_python: str | None = None, requires_python_values: list[str] | None = None, + wheel_version: str = "1.0", ) -> Path: normalized = name.replace("-", "_") wheel = path / f"{normalized}-{version}-py3-none-any.whl" @@ -99,7 +100,7 @@ def _make_wheel( + "\n" ), f"{dist_info}/WHEEL": ( - "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + f"Wheel-Version: {wheel_version}\nRoot-Is-Purelib: true\nTag: py3-none-any\n" ), f"{normalized}/__init__.py": "VALUE = 1\n", } @@ -124,6 +125,7 @@ def _make_application_wheel( requires_python: str | None = None, requires_python_values: list[str] | None = None, requires_dist_values: list[str] | None = None, + wheel_version: str = "1.0", ) -> Path: normalized = name.replace("-", "_") wheel = path / f"{normalized}-{version}-py3-none-any.whl" @@ -145,7 +147,7 @@ def _make_application_wheel( + "\n" ), f"{dist_info}/WHEEL": ( - "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + f"Wheel-Version: {wheel_version}\nRoot-Is-Purelib: true\nTag: py3-none-any\n" ), f"{dist_info}/entry_points.txt": f"[{entry_group}]\n{entry_name} = {target}\n", } @@ -314,6 +316,31 @@ def _application_plan_with_locked_dependencies( return configured +def _write_requests_lock(root: Path, *, version: str = "2.31.0") -> bytes: + """Write a minimal inspected uv lock graph for mapped-app -> requests.""" + + content = ( + f'''version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "mapped-app" +version = "1.2.3" +source = {{ virtual = "." }} +dependencies = [{{ name = "requests" }}] + +[[package]] +name = "requests" +version = "{version}" +source = {{ registry = "https://pypi.org/simple" }} +wheels = [{{ url = "https://example.invalid/requests-{version}-py3-none-any.whl" }}] +'''.encode() + ) + (root / "uv.lock").write_bytes(content) + return content + + def _load_template_module(name: str, monkeypatch: pytest.MonkeyPatch): monkeypatch.syspath_prepend(str(TEMPLATE_ROOT)) spec = importlib.util.spec_from_file_location(f"generated_{name}", TEMPLATE_ROOT / name) @@ -506,6 +533,51 @@ def test_application_wheel_rejects_wrong_target_and_runtime_cache(tmp_path: Path validate_application_wheel(cached, assessment, plan) +@pytest.mark.parametrize("wheel_version", ["1.0", "1.1"]) +def test_supported_wheel_major_one_versions_pass( + tmp_path: Path, wheel_version: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + + validate_application_wheel( + _make_application_wheel(tmp_path, wheel_version=wheel_version), assessment, plan + ) + + +@pytest.mark.parametrize("wheel_version", ["2.0", "broken"]) +def test_unsupported_or_malformed_wheel_version_rejects_application_wheel( + tmp_path: Path, wheel_version: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + message = "Unsupported Wheel-Version" if wheel_version == "2.0" else "Malformed WHEEL" + + with pytest.raises(PreparationError, match=message): + validate_application_wheel( + _make_application_wheel(tmp_path, wheel_version=wheel_version), assessment, plan + ) + + +def test_unsupported_wheel_version_rejects_approved_dependency_wheel(tmp_path: Path) -> None: + plan = _plan("optional_map_app", ["map"]) + + with pytest.raises(PreparationError, match="Unsupported Wheel-Version"): + validate_approved_wheel( + f"proxy-tools={_make_wheel(tmp_path, wheel_version='2.0')}", plan + ) + + @pytest.mark.parametrize( ("requires_dist", "locked", "error"), [ @@ -597,6 +669,99 @@ def test_application_wheel_requires_dist_rejects_unprovable_metadata( validate_application_wheel(wheel, assessment, plan) +def test_package_prepare_lock_reassesses_before_requires_dist_validation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "uv.lock").unlink() + pyproject = source / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + "dependencies = []", 'dependencies = ["requests>=2"]' + ), + encoding="utf-8", + ) + wheel = _make_application_wheel(tmp_path, requires_dist_values=["requests>=2"]) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + + def prepare(root: Path, *args, **kwargs) -> LockPreparationResult: + assert not (root / "uv.lock").exists() + _write_requests_lock(root) + return LockPreparationResult( + path=root / "uv.lock", created=True, checked=True, commands=() + ) + + monkeypatch.setattr("python_deployment_builder.generation.generator.prepare_lockfile", prepare) + result = generate_deployment_kit( + repository, + tmp_path / "kit", + application_wheel=wheel, + prepare_lock=True, + bootstrap_mode="online_cmd", + ) + + assert result.generated + assert (tmp_path / "kit/uv.lock").read_bytes() == (source / "uv.lock").read_bytes() + + +def test_package_wheel_requires_dist_without_prepare_lock_reports_lock_blocker( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "uv.lock").unlink() + wheel = _make_application_wheel(tmp_path, requires_dist_values=["requests>=2"]) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + with pytest.raises(PreparationError, match="uv.lock is missing"): + generate_deployment_kit(repository, tmp_path / "kit", application_wheel=wheel) + assert not (tmp_path / "kit").exists() + + +def test_package_prepare_lock_rejects_final_incompatible_requires_dist( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "uv.lock").unlink() + wheel = _make_application_wheel(tmp_path, requires_dist_values=["requests>=99"]) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: ( + _write_requests_lock(root), + LockPreparationResult(path=root / "uv.lock", created=True, checked=True, commands=()), + )[1], + ) + + with pytest.raises(PreparationError, match="incompatible"): + generate_deployment_kit( + repository, + tmp_path / "kit", + application_wheel=wheel, + prepare_lock=True, + bootstrap_mode="online_cmd", + ) + assert (source / "uv.lock").is_file() + assert not (tmp_path / "kit").exists() + + @pytest.mark.parametrize( ("name", "version", "message"), [ @@ -1265,6 +1430,84 @@ def test_package_generation_validates_application_wheel_before_acquisition_or_wr assert not output.exists() +@pytest.mark.parametrize("output_name", ["source", "."]) +def test_package_generation_rejects_output_root_that_contains_application_source( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, output_name: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + wheel = _make_application_wheel(tmp_path) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + output = source if output_name == "source" else tmp_path + original = { + path.relative_to(source).as_posix(): path.read_bytes() + for path in source.rglob("*") + if path.is_file() + } + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail("unsafe package output must block before acquisition"), + ) + + with pytest.raises(PreparationError, match="must be external"): + generate_deployment_kit( + repository, + output, + application_wheel=wheel, + bootstrap_mode="online_cmd", + ) + + assert { + path.relative_to(source).as_posix(): path.read_bytes() + for path in source.rglob("*") + if path.is_file() + } == original + assert not (source / "deployment").exists() + + +def test_package_generation_rejects_nested_output_before_writes(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + with pytest.raises(PreparationError, match="may not be nested"): + generate_deployment_kit( + repository, + source / "kit", + application_wheel=_make_application_wheel(tmp_path), + bootstrap_mode="online_cmd", + ) + assert not (source / "kit").exists() + + +def test_source_mode_in_place_generation_remains_supported( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + shutil.copytree(FIXTURES / "prepared_gui", source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + + result = generate_deployment_kit(repository, source, bootstrap_mode="online_cmd") + + assert result.generated + assert (source / "prepared_gui.py").is_file() + assert (source / "deployment/manifest.json").is_file() + + def test_package_dry_run_reports_missing_valid_and_invalid_application_wheels( tmp_path: Path, ) -> None: From 611c33cbb4565afa4f51cd30b674cf55f2d10b15 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:23:03 -0400 Subject: [PATCH 13/59] Harden preview and wheel dependency proofs --- src/python_deployment_builder/__init__.py | 4 +- .../generation/artifacts.py | 75 ++++- .../generation/generator.py | 50 +-- src/python_deployment_builder/models.py | 4 + .../planning/index.py | 80 ++++- .../planning/lockfile.py | 63 +++- tests/test_generation.py | 304 ++++++++++++++++++ 7 files changed, 526 insertions(+), 54 deletions(-) diff --git a/src/python_deployment_builder/__init__.py b/src/python_deployment_builder/__init__.py index f77f119..0f99c92 100644 --- a/src/python_deployment_builder/__init__.py +++ b/src/python_deployment_builder/__init__.py @@ -9,6 +9,6 @@ SCHEMA_VERSION = "1.0" # Analysis and planning retain defaults for older serialized forms, but their -# emitted contracts changed when entry-point packaging groups became explicit. +# emitted contracts changed when entry-point groups and lock dependency extras became explicit. ANALYSIS_SCHEMA_VERSION = "1.2" -PLANNING_SCHEMA_VERSION = "1.2" +PLANNING_SCHEMA_VERSION = "1.3" diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index bbde58f..4361401 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -30,7 +30,11 @@ DeploymentPlan, RepositoryAssessment, ) -from python_deployment_builder.planning.index import marker_applies, wheel_matches +from python_deployment_builder.planning.index import ( + TargetMarkerEnvironmentError, + target_marker_applies, + wheel_matches, +) from python_deployment_builder.planning.policies import python_satisfies from python_deployment_builder.security_policy import ( is_secret_filename, @@ -385,15 +389,66 @@ def _application_requirement_applies(requirement: Requirement, plan: DeploymentP if marker is None: return True selected_extras = plan.lock_graph.selected_extras if plan.lock_graph else [] - return any( - marker_applies( - marker, - plan.runtime.python_version, - plan.runtime.architecture, - extra=extra, + try: + return any( + target_marker_applies( + marker, + plan.runtime.python_version, + plan.runtime.architecture, + extra=extra, + ) + for extra in ["", *selected_extras] ) - for extra in ["", *selected_extras] - ) + except TargetMarkerEnvironmentError as exc: + raise PreparationError( + "Application wheel Requires-Dist marker cannot be proven for the selected " + f"target: {requirement}. {exc}" + ) from exc + + +def _validate_dependency_extra_closure(requirement: Requirement, graph) -> None: + """Prove a wheel dependency's requested extras are activated by the selected lock graph.""" + + requested = set(requirement.extras) + name = canonicalize_name(requirement.name) + candidates = [ + dependency + for dependency in graph.dependencies + if canonicalize_name(dependency.name) == name + and Version(dependency.version) in requirement.specifier + and requested <= set(dependency.requested_dependency_extras) + ] + if not candidates: + raise PreparationError( + "Application wheel Requires-Dist dependency extra cannot be proven against the " + f"selected locked environment: {requirement.name}[{','.join(sorted(requested))}]" + ) + if not any(requested <= set(item.available_dependency_extras) for item in candidates): + raise PreparationError( + "Application wheel Requires-Dist dependency extra is not declared by the locked " + f"dependency: {requirement.name}[{','.join(sorted(requested))}]" + ) + + known = {canonicalize_name(item.name) for item in graph.dependencies} + pending = [name] + visited: set[str] = set() + while pending: + parent = pending.pop() + if parent in visited: + continue + visited.add(parent) + for edge in graph.edges: + if not edge.applicable or canonicalize_name(edge.from_package) != parent: + continue + child = canonicalize_name(edge.to_package) + if child not in known: + extras = ",".join(sorted(requested)) + raise PreparationError( + "Application wheel Requires-Dist dependency extra closure is incomplete " + f"in the selected locked environment: {requirement.name}[{extras}] " + f"requires {edge.to_package}." + ) + pending.append(child) def _validate_application_requires_dist( @@ -448,6 +503,8 @@ def _validate_application_requires_dist( "Application wheel Requires-Dist is incompatible with the selected locked " f"environment: {requirement}. Locked versions: {', '.join(sorted(versions))}." ) + if requirement.extras: + _validate_dependency_extra_closure(requirement, graph) def validate_application_wheel( diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 5bae1b0..958a53f 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -59,6 +59,14 @@ } +def _allow_missing_lock_for_analysis(plan, *, dry_run: bool, prepare_lock: bool) -> bool: + """Permit a missing lock only for a non-mutating preview or authorized preparation.""" + + return plan.lockfile.status == "developer_generation_required" and ( + dry_run or prepare_lock + ) + + def _is_runtime_cache(relative: Path) -> bool: return relative.suffix.lower() in {".pyc", ".pyo"} or any( PYTHON_CACHE_DIRECTORY.fullmatch(part) for part in relative.parts @@ -263,6 +271,7 @@ def _tracked_deployment_paths( f"revision {assessment.repository.revision}: {', '.join(untracked_policy)}. " "Commit or remove those policy inputs before release-oriented generation." ) + missing_lock_is_previewed = allow_missing_lock and not (repository_root / "uv.lock").exists() if created_lock is not None: expected_lock = (repository_root / "uv.lock").resolve() if created_lock.resolve() != expected_lock or not created_lock.is_file(): @@ -271,12 +280,9 @@ def _tracked_deployment_paths( "uv.lock. Generation stopped rather than widening untracked-file staging." ) tracked.add("uv.lock") - elif allow_missing_lock and not (repository_root / "uv.lock").exists(): - # This is only the non-mutating preflight for the current explicitly authorized - # --prepare-lock operation. The post-creation pass requires its exact path. - tracked.add("uv.lock") if assessment.repository.revision is not None: - untracked_selected = sorted(selected - tracked) + selected_for_tracking = selected - ({"uv.lock"} if missing_lock_is_previewed else set()) + untracked_selected = sorted(selected_for_tracking - tracked) if untracked_selected: raise PreparationError( "Selected deployment inputs must be tracked for Git release generation " @@ -687,6 +693,9 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) + allow_missing_lock_for_analysis = _allow_missing_lock_for_analysis( + plan, dry_run=dry_run, prepare_lock=prepare_lock + ) if plan.deployment_mode == "package": try: repository_root.relative_to(output_root) @@ -698,17 +707,25 @@ def generate_deployment_kit( "repository so the kit cannot retain application source outside the " "validated first-party wheel." ) + if plan.entry_point is None: + raise PreparationError( + "Deployment readiness is blocked: " + "; ".join(plan.readiness.blockers) + ) + if ( + not dry_run + and plan.lockfile.status == "developer_generation_required" + and not prepare_lock + ): + raise PreparationError( + "uv.lock is missing. Re-run generation with --prepare-lock for a local " + "repository to authorize developer-side lockfile creation." + ) if assessment.repository.revision is not None: _tracked_deployment_paths( repository_root, assessment, plan, - allow_missing_lock=prepare_lock - and plan.lockfile.status == "developer_generation_required", - ) - if plan.entry_point is None: - raise PreparationError( - "Deployment readiness is blocked: " + "; ".join(plan.readiness.blockers) + allow_missing_lock=allow_missing_lock_for_analysis, ) if plan.deployment_mode_condition in { "DEPLOYMENT_MODE_CONFLICT", @@ -742,21 +759,12 @@ def generate_deployment_kit( for item in (plan.lock_graph.artifact_findings if plan.lock_graph else []) if item.status == "unavailable" ] - if ( - not dry_run - and plan.lockfile.status == "developer_generation_required" - and not prepare_lock - ): - raise PreparationError( - "uv.lock is missing. Re-run generation with --prepare-lock for a local " - "repository to authorize developer-side lockfile creation." - ) source_files = _staging_files( repository_root, assessment, plan, include=output_root != repository_root, - allow_missing_lock=prepare_lock and plan.lockfile.status == "developer_generation_required", + allow_missing_lock=allow_missing_lock_for_analysis, ) preview = _preview( plan, diff --git a/src/python_deployment_builder/models.py b/src/python_deployment_builder/models.py index 17bdaf8..2ad1df6 100644 --- a/src/python_deployment_builder/models.py +++ b/src/python_deployment_builder/models.py @@ -399,6 +399,8 @@ class DependencyEdge(StrictModel): marker: str | None = None applicable: bool = True selected_extra: str | None = None + requested_dependency_extras: list[str] = Field(default_factory=list) + activated_dependency_extra: str | None = None class ArtifactAvailability(StrictModel): @@ -414,6 +416,8 @@ class LockedDependency(StrictModel): direct: bool dependency_chain: list[str] = Field(default_factory=list) selected_extra: str | None = None + requested_dependency_extras: list[str] = Field(default_factory=list) + available_dependency_extras: list[str] = Field(default_factory=list) platform_relevance: Literal["applicable", "not_applicable", "unknown"] = "applicable" artifact: ArtifactAvailability diff --git a/src/python_deployment_builder/planning/index.py b/src/python_deployment_builder/planning/index.py index d42aba6..008f779 100644 --- a/src/python_deployment_builder/planning/index.py +++ b/src/python_deployment_builder/planning/index.py @@ -9,7 +9,8 @@ from urllib.parse import quote from urllib.request import Request, urlopen -from packaging.markers import InvalidMarker, Marker, default_environment +from packaging._parser import Variable +from packaging.markers import InvalidMarker, Marker from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.tags import compatible_tags, cpython_tags from packaging.utils import InvalidWheelFilename, parse_wheel_filename @@ -25,6 +26,64 @@ PYPI_JSON_BASE = "https://pypi.org/pypi" JsonFetcher = Callable[[str], dict[str, Any]] +class TargetMarkerEnvironmentError(ValueError): + """A marker requires target facts PDB does not select for M6.1.""" + + +def target_marker_environment( + python_version: str, architecture: str, *, extra: str = "" +) -> dict[str, str]: + """Return every PEP 508 marker value PDB can establish for its Windows target.""" + + full_version = f"{python_version}.0" + # platform_release and platform_version deliberately have no target values: PDB plans + # a Windows architecture and Python minor, not a specific Windows build. + return { + "implementation_name": "cpython", + "implementation_version": full_version, + "os_name": "nt", + "platform_machine": "AMD64" if architecture == "x86_64" else "ARM64", + "platform_python_implementation": "CPython", + "platform_system": "Windows", + "python_full_version": full_version, + "python_version": python_version, + "sys_platform": "win32", + "extra": extra, + } + + +def _marker_variables(value: object) -> set[str]: + """Read variable nodes from packaging's already parsed marker expression.""" + + if isinstance(value, Variable): + return {value.value} + if isinstance(value, (list, tuple)): + return set().union(*(_marker_variables(item) for item in value)) + return set() + + +def target_marker_applies( + marker: str | None, + python_version: str, + architecture: str, + *, + extra: str = "", +) -> bool: + """Evaluate a marker strictly from selected Windows target facts.""" + + if not marker: + return True + try: + parsed = Marker(marker) + except InvalidMarker as exc: + raise TargetMarkerEnvironmentError(f"Malformed environment marker: {marker!r}") from exc + environment = target_marker_environment(python_version, architecture, extra=extra) + unsupported = sorted(_marker_variables(parsed._markers) - set(environment)) + if unsupported: + raise TargetMarkerEnvironmentError( + "Target marker fields are not selected by PDB: " + ", ".join(unsupported) + ) + return parsed.evaluate(environment) def _fetch_json(url: str) -> dict[str, Any]: @@ -80,22 +139,11 @@ def marker_applies( ) -> bool: if not marker: return True - environment = default_environment() - environment.update( - { - "implementation_name": "cpython", - "os_name": "nt", - "platform_machine": "AMD64" if architecture == "x86_64" else "ARM64", - "platform_system": "Windows", - "python_full_version": f"{python_version}.0", - "python_version": python_version, - "sys_platform": "win32", - "extra": extra, - } - ) try: - return Marker(marker).evaluate(environment) - except InvalidMarker: + return target_marker_applies(marker, python_version, architecture, extra=extra) + except TargetMarkerEnvironmentError: + # Planning remains conservative for malformed or host-unknown lock markers; + # first-party wheel validation raises instead of treating them as proven. return True diff --git a/src/python_deployment_builder/planning/lockfile.py b/src/python_deployment_builder/planning/lockfile.py index 845ffc9..9ebe7f4 100644 --- a/src/python_deployment_builder/planning/lockfile.py +++ b/src/python_deployment_builder/planning/lockfile.py @@ -54,6 +54,15 @@ def _resolve_package( return candidates[0] if len(candidates) == 1 else None +def _requested_dependency_extras(edge: dict[str, object]) -> tuple[str, ...]: + """Read uv's edge-level ``extra = ["..."]`` dependency-extra request.""" + + values = edge.get("extra") + if not isinstance(values, list): + return () + return tuple(sorted({value for value in values if isinstance(value, str) and value})) + + def inspect_uv_lock( repository_root: Path, application_name: str, @@ -102,7 +111,9 @@ def inspect_uv_lock( ) root_name = str(root.get("name", application_name)) - queued: deque[tuple[dict[str, object], list[str], bool, str | None]] = deque() + queued: deque[tuple[dict[str, object], list[str], bool, str | None, tuple[str, ...]]] = ( + deque() + ) edges: list[DependencyEdge] = [] def enqueue_edges( @@ -111,6 +122,7 @@ def enqueue_edges( chain: list[str], direct: bool, selected_extra: str | None, + activated_dependency_extra: str | None = None, ) -> None: if not isinstance(values, list): return @@ -119,6 +131,7 @@ def enqueue_edges( continue applies = _edge_applies(raw_edge, python_version, architecture, selected_extra) marker = raw_edge.get("marker") + requested_extras = _requested_dependency_extras(raw_edge) edges.append( DependencyEdge( from_package=parent, @@ -126,6 +139,8 @@ def enqueue_edges( marker=marker if isinstance(marker, str) else None, applicable=applies, selected_extra=selected_extra, + requested_dependency_extras=list(requested_extras), + activated_dependency_extra=activated_dependency_extra, ) ) if not applies: @@ -133,7 +148,13 @@ def enqueue_edges( package = _resolve_package(packages, raw_edge) if package is not None: queued.append( - (package, [*chain, str(raw_edge["name"])], direct, selected_extra) + ( + package, + [*chain, str(raw_edge["name"])], + direct, + selected_extra, + requested_extras, + ) ) enqueue_edges(root_name, root.get("dependencies"), [root_name], True, None) @@ -143,13 +164,13 @@ def enqueue_edges( enqueue_edges(root_name, optional.get(extra), [root_name], True, extra) locked: dict[tuple[str, str], LockedDependency] = {} - expanded: set[tuple[str, str, str | None]] = set() + expanded: set[tuple[str, str, str | None, tuple[str, ...]]] = set() while queued: - package, chain, direct, selected_extra = queued.popleft() + package, chain, direct, selected_extra, requested_extras = queued.popleft() name = str(package.get("name", chain[-1])) version = str(package.get("version", "unversioned")) key = (canonicalize_name(name), version) - expansion_key = (*key, selected_extra) + expansion_key = (*key, selected_extra, requested_extras) wheels = [ filename for item in package.get("wheels", []) @@ -164,12 +185,20 @@ def enqueue_edges( policy = "developer_wheel_required" else: policy = "no_artifact" + optional = package.get("optional-dependencies") + available_extras = ( + sorted(extra for extra in optional if isinstance(extra, str)) + if isinstance(optional, dict) + else [] + ) candidate = LockedDependency( name=name, version=version, direct=direct, dependency_chain=chain, selected_extra=selected_extra, + requested_dependency_extras=list(requested_extras), + available_dependency_extras=available_extras, artifact=ArtifactAvailability( compatible_wheel_available=bool(wheels), matching_wheels=sorted(wheels), @@ -178,12 +207,34 @@ def enqueue_edges( ), ) existing = locked.get(key) - if existing is None or len(chain) < len(existing.dependency_chain): + if existing is None: locked[key] = candidate + else: + preferred = candidate if len(chain) < len(existing.dependency_chain) else existing + locked[key] = preferred.model_copy( + update={ + "requested_dependency_extras": sorted( + set(existing.requested_dependency_extras) | set(requested_extras) + ), + "available_dependency_extras": sorted( + set(existing.available_dependency_extras) | set(available_extras) + ), + } + ) if expansion_key in expanded: continue expanded.add(expansion_key) enqueue_edges(name, package.get("dependencies"), chain, False, selected_extra) + if isinstance(optional, dict): + for extra in requested_extras: + enqueue_edges( + name, + optional.get(extra), + chain, + False, + selected_extra, + activated_dependency_extra=extra, + ) dependencies = sorted(locked.values(), key=lambda item: (not item.direct, item.name.lower())) findings: list[ArtifactPolicyFinding] = [] diff --git a/tests/test_generation.py b/tests/test_generation.py index 131ebc5..d7b6757 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -55,6 +55,8 @@ ) from python_deployment_builder.packaging.archive import safe_extract_zip from python_deployment_builder.packaging.packager import package_deployment_kit +from python_deployment_builder.planning.index import target_marker_applies +from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.planning.planner import create_deployment_plan from python_deployment_builder.validation.static import validate_static_kit @@ -341,6 +343,76 @@ def _write_requests_lock(root: Path, *, version: str = "2.31.0") -> bytes: return content +def _write_dependency_extra_lock( + root: Path, + *, + requested_extras: list[str] | None = None, + include_bar_helper: bool = True, + include_baz_helper: bool = True, + include_transitive_helper: bool = True, + bar_marker: str | None = None, +) -> None: + """Write uv's edge ``extra`` and optional-dependency representation for foo extras.""" + + requested = ", ".join(f'"{item}"' for item in requested_extras or []) + marker = f', marker = "{bar_marker.replace(chr(34), chr(92) + chr(34))}"' if bar_marker else "" + parts = [ + '[[package]]\nname = "mapped-app"\nversion = "1.2.3"\n' + 'source = { virtual = "." }\n' + f'dependencies = [{{ name = "foo", extra = [{requested}] }}]\n', + '[[package]]\nname = "foo"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'dependencies = [{ name = "base-helper" }]\n' + 'wheels = [{ url = "https://example.invalid/foo-1.0-py3-none-any.whl" }]\n' + '\n[package.optional-dependencies]\n' + f'bar = [{{ name = "bar-helper"{marker} }}]\n' + 'baz = [{ name = "baz-helper" }]\n', + '[[package]]\nname = "base-helper"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'wheels = [{ url = "https://example.invalid/base_helper-1.0-py3-none-any.whl" }]\n', + ] + if include_bar_helper: + parts.append( + '[[package]]\nname = "bar-helper"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'dependencies = [{ name = "transitive-helper" }]\n' + 'wheels = [{ url = "https://example.invalid/bar_helper-1.0-py3-none-any.whl" }]\n' + ) + if include_baz_helper: + parts.append( + '[[package]]\nname = "baz-helper"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'wheels = [{ url = "https://example.invalid/baz_helper-1.0-py3-none-any.whl" }]\n' + ) + if include_transitive_helper: + parts.append( + '[[package]]\nname = "transitive-helper"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'wheels = [{ url = ' + '"https://example.invalid/transitive_helper-1.0-py3-none-any.whl" }]\n' + ) + (root / "uv.lock").write_text( + 'version = 1\nrevision = 3\nrequires-python = ">=3.12"\n\n' + "\n".join(parts), + encoding="utf-8", + ) + + +def _plan_with_dependency_extra_lock(source: Path, *, selected_extras: list[str] | None = None): + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + graph = inspect_uv_lock( + source, + "mapped-app", + plan.runtime.python_version, + plan.runtime.architecture, + selected_extras or [], + ) + return assessment, plan.model_copy( + update={"lock_graph": graph.model_copy(update={"selected_extras": selected_extras or []})} + ) + + def _load_template_module(name: str, monkeypatch: pytest.MonkeyPatch): monkeypatch.syspath_prepend(str(TEMPLATE_ROOT)) spec = importlib.util.spec_from_file_location(f"generated_{name}", TEMPLATE_ROOT / name) @@ -649,6 +721,7 @@ def test_application_wheel_requires_dist_selected_extra_controls_marker(tmp_path ("requires_dist", "error"), [ (["requests=>2"], "malformed Requires-Dist"), + (["helper; implementation_version <"], "malformed Requires-Dist"), (["requests @ https://example.invalid/requests.whl"], "direct references"), ], ) @@ -669,6 +742,168 @@ def test_application_wheel_requires_dist_rejects_unprovable_metadata( validate_application_wheel(wheel, assessment, plan) +@pytest.mark.parametrize( + "marker", + [ + 'python_version < "3.13"', + 'implementation_version < "3.13"', + 'platform_python_implementation == "CPython"', + 'implementation_name == "cpython"', + 'sys_platform == "win32"', + 'os_name == "nt"', + 'platform_system == "Windows"', + 'platform_machine == "AMD64"', + ], +) +def test_application_wheel_markers_use_complete_target_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, marker: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, repository_root=source), [] + ) + monkeypatch.setattr( + "packaging.markers.default_environment", + lambda: {"implementation_version": "3.13.0", "sys_platform": "linux"}, + ) + wheel = _make_application_wheel( + tmp_path, requires_dist_values=[f"helper; {marker}"] + ) + + with pytest.raises(PreparationError, match="absent"): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_target_marker_arm64_and_unprovable_field_policy( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + arm_plan = _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, architecture="arm64", repository_root=source), [] + ) + assert target_marker_applies('platform_machine == "ARM64"', "3.12", "arm64") + assert not target_marker_applies('platform_machine == "AMD64"', "3.12", "arm64") + arm_wheel = _make_application_wheel( + tmp_path, requires_dist_values=['helper; platform_machine == "ARM64"'] + ) + with pytest.raises(PreparationError, match="absent"): + validate_application_wheel(arm_wheel, assessment, arm_plan) + + with pytest.raises(PreparationError, match="cannot be proven"): + validate_application_wheel( + _make_application_wheel( + tmp_path, + requires_dist_values=['helper; platform_release == "10"'], + ), + assessment, + _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, repository_root=source), [] + ), + ) + + +def test_application_wheel_dependency_requested_extra_closure( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + + _write_dependency_extra_lock(source, requested_extras=["bar"]) + assessment, plan = _plan_with_dependency_extra_lock(source) + foo = next(item for item in plan.lock_graph.dependencies if item.name == "foo") + root_edge = next( + item + for item in plan.lock_graph.edges + if item.from_package == "mapped-app" and item.to_package == "foo" + ) + assert foo.requested_dependency_extras == ["bar"] + assert foo.available_dependency_extras == ["bar", "baz"] + assert root_edge.requested_dependency_extras == ["bar"] + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["Foo[bar]>=1"]), + assessment, + plan, + ) + + _write_dependency_extra_lock(source, requested_extras=["bar"], include_bar_helper=False) + assessment, incomplete = _plan_with_dependency_extra_lock(source) + with pytest.raises(PreparationError, match="closure is incomplete"): + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo[bar]>=1"]), + assessment, + incomplete, + ) + + +def test_application_wheel_dependency_requested_extra_variants_and_markers( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + + _write_dependency_extra_lock(source, requested_extras=["bar", "baz"]) + assessment, complete = _plan_with_dependency_extra_lock(source) + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo[bar,baz]>=1"]), + assessment, + complete, + ) + + _write_dependency_extra_lock(source, requested_extras=[] , include_bar_helper=False) + assessment, base_only = _plan_with_dependency_extra_lock(source) + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo>=1"]), + assessment, + base_only, + ) + validate_application_wheel( + _make_application_wheel( + tmp_path, requires_dist_values=['foo[bar]>=1; extra == "map"'] + ), + assessment, + base_only, + ) + + _write_dependency_extra_lock( + source, + requested_extras=["bar"], + include_bar_helper=False, + bar_marker='implementation_version < "3.13"', + ) + assessment, selected_map = _plan_with_dependency_extra_lock(source, selected_extras=["map"]) + with pytest.raises(PreparationError, match="closure is incomplete"): + validate_application_wheel( + _make_application_wheel( + tmp_path, requires_dist_values=['foo[bar]>=1; extra == "map"'] + ), + assessment, + selected_map, + ) + + _write_dependency_extra_lock( + source, requested_extras=["bar"], include_transitive_helper=False + ) + assessment, transitive_incomplete = _plan_with_dependency_extra_lock(source) + with pytest.raises(PreparationError, match="closure is incomplete"): + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo[bar]>=1"]), + assessment, + transitive_incomplete, + ) + + def test_package_prepare_lock_reassesses_before_requires_dist_validation( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -727,6 +962,75 @@ def test_package_wheel_requires_dist_without_prepare_lock_reports_lock_blocker( assert not (tmp_path / "kit").exists() +@pytest.mark.parametrize("git_backed", [False, True]) +def test_missing_lock_dry_run_is_previewable_without_staging_or_provenance_mutation( + tmp_path: Path, git_backed: bool +) -> None: + source = tmp_path / "source" + shutil.copytree(FIXTURES / "prepared_gui", source, ignore=shutil.ignore_patterns("__pycache__")) + (source / "uv.lock").unlink() + if git_backed: + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + output = tmp_path / "kit" + + result = generate_deployment_kit(repository, output, dry_run=True, bootstrap_mode="online_cmd") + + assert result.dry_run and not result.generated + assert any("--prepare-lock" in action for action in result.preview.developer_actions) + assert not (source / "uv.lock").exists() + assert not output.exists() + if git_backed: + status = subprocess.run( + ["git", "-C", str(source), "status", "--short"], + capture_output=True, + text=True, + check=True, + ) + assert status.stdout == "" + + with pytest.raises(PreparationError, match="uv.lock is missing"): + generate_deployment_kit(repository, output, bootstrap_mode="online_cmd") + + +def test_package_missing_lock_dry_run_validates_structure_but_defers_requires_dist( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "uv.lock").unlink() + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + wheel = _make_application_wheel(tmp_path, requires_dist_values=["requests>=99"]) + + preview = generate_deployment_kit( + repository, + tmp_path / "kit", + application_wheel=wheel, + dry_run=True, + bootstrap_mode="online_cmd", + ) + assert preview.preview.application_artifact is not None + assert not (source / "uv.lock").exists() + with pytest.raises(PreparationError, match="entry point disagrees"): + generate_deployment_kit( + repository, + tmp_path / "kit", + application_wheel=_make_application_wheel( + tmp_path, target="installed_app.other:main" + ), + dry_run=True, + bootstrap_mode="online_cmd", + ) + + def test_package_prepare_lock_rejects_final_incompatible_requires_dist( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From f5e3608564f27eccd60c9398d05e9029861aaa4f Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:58:06 -0400 Subject: [PATCH 14/59] Harden nested Git provenance and setup config parsing --- .../analysis/assessor.py | 14 ++- .../analysis/metadata.py | 11 +- .../generation/generator.py | 116 +++++++++++++++--- tests/test_analysis_scope.py | 58 +++++++++ tests/test_generation.py | 94 ++++++++++++++ 5 files changed, 271 insertions(+), 22 deletions(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index 027b13b..fe30ae9 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -41,9 +41,19 @@ def _git_revision(root: Path) -> str | None: - """Read a normal .git HEAD without invoking Git or following arbitrary files.""" + """Read the nearest normal enclosing Git HEAD without invoking Git.""" - git_dir = root / ".git" + # A PDB source may be a project nested within a monorepo. Its provenance is + # still the enclosing worktree's HEAD, while all later staging paths remain + # relative to this selected project root. + git_dir: Path | None = None + for candidate in (root.resolve(), *root.resolve().parents): + possible = candidate / ".git" + if possible.is_dir(): + git_dir = possible + break + if git_dir is None: + return None head_path = git_dir / "HEAD" if not head_path.is_file(): return None diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index bc23cba..5a1c275 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -458,8 +458,13 @@ def inspect_metadata(root: Path) -> MetadataResult: if setup_cfg_path.is_file(): metadata_files.append("setup.cfg") parser = configparser.ConfigParser() - parser.optionxform = str parser.read(setup_cfg_path, encoding="utf-8") + # Standard setup.cfg options are case-insensitive. Package-data option names + # are Python package identifiers, so inspect that one identifier-keyed section + # separately without changing the ordinary metadata/options parser semantics. + package_data_parser = configparser.ConfigParser() + package_data_parser.optionxform = str + package_data_parser.read(setup_cfg_path, encoding="utf-8") if distribution_name is None: distribution_name = parser.get("metadata", "name", fallback=None) project_version = parser.get("metadata", "version", fallback=None) @@ -517,8 +522,8 @@ def inspect_metadata(root: Path) -> MetadataResult: package_directories.update(setup_cfg_directories) if isinstance(package_directories.get(""), str): source_roots = source_roots or [package_directories[""]] - if parser.has_section("options.package_data"): - for package, value in parser.items("options.package_data"): + if package_data_parser.has_section("options.package_data"): + for package, value in package_data_parser.items("options.package_data"): patterns = _multiline_values(value) if not patterns: continue diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 958a53f..4269d19 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -8,7 +8,7 @@ import re import subprocess import tempfile -from pathlib import Path +from pathlib import Path, PurePosixPath from packaging.utils import canonicalize_name @@ -73,7 +73,11 @@ def _is_runtime_cache(relative: Path) -> bool: ) -def _git_tracked_paths(repository_root: Path, *, required: bool) -> set[str] | None: +def _git_worktree_context( + repository_root: Path, *, required: bool +) -> tuple[Path, PurePosixPath] | None: + """Return the enclosing worktree and selected-root prefix for Git path normalization.""" + repository_check = subprocess.run( ["git", "-C", str(repository_root), "rev-parse", "--is-inside-work-tree"], capture_output=True, @@ -82,8 +86,75 @@ def _git_tracked_paths(repository_root: Path, *, required: bool) -> set[str] | N ) if repository_check.returncode != 0 or repository_check.stdout.strip() != "true": return None + worktree = subprocess.run( + ["git", "-C", str(repository_root), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=False, + ) + if worktree.returncode != 0 or not worktree.stdout.strip(): + if required: + raise PreparationError( + "Git revision provenance is known, but the enclosing worktree could not be " + "resolved. Generation stopped rather than comparing incompatible path bases." + ) + return None + worktree_root = Path(worktree.stdout.strip()).resolve() + try: + prefix = repository_root.resolve().relative_to(worktree_root) + except ValueError as exc: + if required: + raise PreparationError( + "Git revision provenance is known, but the selected repository is not within " + "its reported worktree. Generation stopped rather than comparing ambiguous paths." + ) from exc + return None + return worktree_root, PurePosixPath(prefix.as_posix()) + + +def _repository_relative_git_paths( + values: list[bytes], *, repository_prefix: PurePosixPath +) -> set[str]: + """Normalize worktree-root Git output into selected-repository-relative paths.""" + + prefix_parts = repository_prefix.parts if repository_prefix != PurePosixPath(".") else () + normalized: set[str] = set() + for value in values: + if not value: + continue + candidate = PurePosixPath( + value.decode("utf-8", errors="surrogateescape").replace("\\", "/") + ) + if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts): + raise PreparationError( + "Git returned an unsafe repository path while checking provenance." + ) + if prefix_parts: + if candidate.parts[: len(prefix_parts)] != prefix_parts: + continue + candidate = PurePosixPath(*candidate.parts[len(prefix_parts) :]) + if candidate.parts: + normalized.add(candidate.as_posix()) + return normalized + + +def _git_tracked_paths(repository_root: Path, *, required: bool) -> set[str] | None: + context = _git_worktree_context(repository_root, required=required) + if context is None: + return None + worktree_root, repository_prefix = context + pathspec = repository_prefix.as_posix() if repository_prefix != PurePosixPath(".") else "." result = subprocess.run( - ["git", "-C", str(repository_root), "ls-files", "-z"], + [ + "git", + "-C", + str(worktree_root), + "ls-files", + "--full-name", + "-z", + "--", + pathspec, + ], capture_output=True, check=False, ) @@ -94,27 +165,31 @@ def _git_tracked_paths(repository_root: Path, *, required: bool) -> set[str] | N "be enumerated. Generation stopped rather than staging local files." ) return None - return { - value.decode("utf-8", errors="surrogateescape").replace("\\", "/") - for value in result.stdout.split(b"\0") - if value - } + return _repository_relative_git_paths( + result.stdout.split(b"\0"), repository_prefix=repository_prefix + ) def _dirty_tracked_deployment_paths( repository_root: Path, provenance_guarded: set[str] ) -> list[str]: + context = _git_worktree_context(repository_root, required=True) + if context is None: # Defensive: callers only invoke this for known Git revisions. + return [] + worktree_root, repository_prefix = context + pathspec = repository_prefix.as_posix() if repository_prefix != PurePosixPath(".") else "." result = subprocess.run( [ "git", "-C", - str(repository_root), + str(worktree_root), "diff", "--no-renames", "--name-only", "-z", "HEAD", "--", + pathspec, ], capture_output=True, check=False, @@ -124,11 +199,9 @@ def _dirty_tracked_deployment_paths( "Git revision provenance is known, but tracked working-tree changes could not be " "checked. Generation stopped rather than claiming clean-revision provenance." ) - changed = { - value.decode("utf-8", errors="surrogateescape").replace("\\", "/") - for value in result.stdout.split(b"\0") - if value - } + changed = _repository_relative_git_paths( + result.stdout.split(b"\0"), repository_prefix=repository_prefix + ) if not changed: return [] with tempfile.TemporaryDirectory(prefix="pdbuilder-head-inventory-") as temporary: @@ -139,11 +212,15 @@ def _dirty_tracked_deployment_paths( [ "git", "-C", - str(repository_root), + str(worktree_root), "archive", "--format=zip", f"--output={archive}", - "HEAD", + ( + "HEAD" + if repository_prefix == PurePosixPath(".") + else f"HEAD:{repository_prefix.as_posix()}" + ), ], capture_output=True, check=False, @@ -200,7 +277,12 @@ def _selected_deployment_paths(repository_root: Path, assessment, plan) -> set[s def _analysis_policy_paths(assessment) -> set[str]: - """Return ignore-policy files that influence inventory without staging them.""" + """Return selected-root ignore policy files that influence inventory without staging. + + Inventory deliberately reads only the source root and its descendants, not + enclosing-worktree ignore files. Those ancestor rules therefore do not become + staging or provenance inputs for a nested PDB source target. + """ return { item.path.rstrip("/") diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index d48229e..0417533 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -910,6 +910,64 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( ] +def test_setup_cfg_standard_options_are_case_insensitive_and_package_data_is_not( + tmp_path: Path, +) -> None: + package = tmp_path / "MyPackage" + (package / "Assets").mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text("def main(): return 0\n", encoding="utf-8") + (package / "Assets/default.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n", + encoding="utf-8", + ) + (tmp_path / "setup.cfg").write_text( + """[metadata] +Name = Example-App +Version = 1.2.3 +[options] +Packages = find: +Python_Requires = >=3.12 +Install_Requires = + requests>=2 +[options.entry_points] +console_scripts = + MyTool = MyPackage.main:main +[options.package_data] +MyPackage = + Assets/*.json +""", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + metadata = inspect_metadata(tmp_path) + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + source_plan = plan.model_copy(deep=True) + source_plan.deployment_mode = "source" + + assert metadata.project.distribution_name == "Example-App" + assert metadata.project.version == "1.2.3" + assert metadata.python.requires_python == ">=3.12" + assert [ + (item.distribution_name, item.declared_constraint) for item in metadata.dependencies + ] == [("requests", ">=2")] + assert [ + (item.name, item.target, item.declared_group) + for item in metadata.project.entry_points + ] == [("MyTool", "MyPackage.main:main", "console_scripts")] + assert metadata.project.package_data == {"MyPackage": ["Assets/*.json"]} + assert [ + (member.source_path, member.installed_member_path) + for member in resolve_package_data_members(tmp_path, metadata.project) + ] == [("MyPackage/Assets/default.json", "MyPackage/Assets/default.json")] + assert "MyPackage/Assets/default.json" in _staging_files( + tmp_path, assessment, source_plan, include=True + ) + + @pytest.mark.parametrize( ("conflict", "expected_role"), [ diff --git a/tests/test_generation.py b/tests/test_generation.py index d7b6757..07e45a4 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -31,6 +31,7 @@ ) from python_deployment_builder.generation.cmd import parse_certutil_sha256 from python_deployment_builder.generation.generator import ( + _git_tracked_paths, _planned_generated_paths, _render_owned_files, _selected_deployment_paths, @@ -2447,6 +2448,99 @@ def _committed_source_fixture(tmp_path: Path) -> tuple[Path, MaterializedReposit ) +def _nested_committed_source_fixture( + tmp_path: Path, +) -> tuple[Path, Path, MaterializedRepository]: + """Create a selected project beneath, rather than at, a Git worktree root.""" + + worktree = tmp_path / "monorepo" + source = worktree / "projects" / "example" + (source / "docs").mkdir(parents=True) + (source / "data").mkdir() + (worktree / "other-project").mkdir() + (source / "pyproject.toml").write_text( + "[project]\nname='nested-app'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nnested-app='app:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / ".gitignore").write_text("# selected-project policy\n", encoding="utf-8") + (source / "data/.gitignore").write_text("# nested selected-project policy\n", encoding="utf-8") + (source / "docs/readme.md").write_text("documentation\n", encoding="utf-8") + (worktree / "other-project/readme.md").write_text("sibling\n", encoding="utf-8") + (worktree / ".gitignore").write_text("# enclosing-worktree policy\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(worktree)], check=True) + subprocess.run(["git", "-C", str(worktree), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(worktree), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(worktree), "add", "."], check=True) + subprocess.run(["git", "-C", str(worktree), "commit", "-qm", "fixture"], check=True) + return worktree, source, MaterializedRepository( + root=source, source=str(source), source_kind="local" + ) + + +@pytest.mark.parametrize("operation", ["modified", "deleted", "staged_rename"]) +def test_nested_git_repository_provenance_uses_selected_root_paths( + tmp_path: Path, operation: str +) -> None: + worktree, source, repository = _nested_committed_source_fixture(tmp_path) + if operation == "modified": + (source / "app.py").write_text("def main(): return 1\n", encoding="utf-8") + elif operation == "deleted": + (source / "app.py").unlink() + else: + subprocess.run( + [ + "git", + "-C", + str(worktree), + "mv", + "projects/example/app.py", + "projects/example/docs/app.py", + ], + check=True, + ) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + tracked = _git_tracked_paths(source, required=True) + assert ("docs/app.py" if operation == "staged_rename" else "app.py") in tracked + with pytest.raises(PreparationError, match="recorded source revision.*app.py"): + _staging_files(source, assessment, plan, include=True) + + +def test_nested_git_repository_ignores_sibling_and_documentation_changes(tmp_path: Path) -> None: + worktree, source, repository = _nested_committed_source_fixture(tmp_path) + (worktree / "other-project/readme.md").write_text("changed sibling\n", encoding="utf-8") + (worktree / "other-project/untracked.py").write_text("VALUE = 1\n", encoding="utf-8") + (worktree / ".gitignore").write_text("sibling-local/\n", encoding="utf-8") + (source / "docs/readme.md").write_text("changed documentation\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + staged = _staging_files(source, assessment, plan, include=True) + + assert "app.py" in staged + assert all(not path.startswith("projects/example/") for path in staged) + + +@pytest.mark.parametrize("relative", [".gitignore", "data/.gitignore"]) +def test_nested_git_repository_guards_selected_ignore_policy( + tmp_path: Path, relative: str +) -> None: + _worktree, source, repository = _nested_committed_source_fixture(tmp_path) + (source / relative).write_text("local/\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match=f"recorded source revision.*{relative}"): + _staging_files(source, assessment, plan, include=True) + + def test_clean_git_source_fixture_stages_and_previews_normally(tmp_path: Path) -> None: source, repository = _committed_source_fixture(tmp_path) assessment = assess_repository(repository) From ddc2feed6cf0761aa550080b961032026e5038c1 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 09:39:15 -0400 Subject: [PATCH 15/59] Honor setuptools package-data exclusions --- src/python_deployment_builder/__init__.py | 2 +- .../analysis/metadata.py | 142 ++++++++++++++++-- .../analysis/resources.py | 86 ++++++++--- src/python_deployment_builder/models.py | 3 + tests/test_analysis_scope.py | 86 ++++++++++- tests/test_generation.py | 68 +++++++++ 6 files changed, 354 insertions(+), 33 deletions(-) diff --git a/src/python_deployment_builder/__init__.py b/src/python_deployment_builder/__init__.py index 0f99c92..3fa4083 100644 --- a/src/python_deployment_builder/__init__.py +++ b/src/python_deployment_builder/__init__.py @@ -10,5 +10,5 @@ SCHEMA_VERSION = "1.0" # Analysis and planning retain defaults for older serialized forms, but their # emitted contracts changed when entry-point groups and lock dependency extras became explicit. -ANALYSIS_SCHEMA_VERSION = "1.2" +ANALYSIS_SCHEMA_VERSION = "1.3" PLANNING_SCHEMA_VERSION = "1.3" diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 5a1c275..61117fe 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -161,6 +161,39 @@ def _multiline_values(value: str) -> list[str]: return [line.strip() for line in value.splitlines() if line.strip()] +def _merge_package_data_declarations( + destination: dict[str, list[str]], + evidence_by_package: dict[str, dict[str, Evidence]], + declarations: dict[str, list[str]], + evidence: Evidence, +) -> None: + """Merge literal package-data declarations while retaining their source evidence.""" + + for package, patterns in declarations.items(): + existing = destination.setdefault(package, []) + sources = evidence_by_package.setdefault(package, {}) + for pattern in patterns: + if pattern not in existing: + existing.append(pattern) + sources.setdefault(pattern, evidence) + + +def _package_data_mapping( + value: Any, *, empty_key_is_wildcard: bool = False +) -> dict[str, list[str]]: + """Read the supported literal setuptools package-data mapping shape.""" + + if not isinstance(value, dict): + return {} + return { + ("*" if empty_key_is_wildcard and package == "" else package): [ + pattern for pattern in patterns if isinstance(pattern, str) + ] + for package, patterns in value.items() + if isinstance(package, str) and isinstance(patterns, list) + } + + def _literal_setup_arguments(path: Path) -> dict[str, Any]: """Read literal setup(...) keyword values without executing setup.py.""" @@ -274,6 +307,9 @@ def inspect_metadata(root: Path) -> MetadataResult: packages: list[str] = [] package_directories: dict[str, str] = {} package_data: dict[str, list[str]] = {} + exclude_package_data: dict[str, list[str]] = {} + package_data_evidence: dict[str, dict[str, Evidence]] = {} + exclude_package_data_evidence: dict[str, dict[str, Evidence]] = {} layout = "unknown" python_evidence: list[Evidence] = [] @@ -426,12 +462,30 @@ def inspect_metadata(root: Path) -> MetadataResult: if isinstance(package_directories.get(""), str): source_roots = source_roots or [package_directories[""]] configured_package_data = setuptools.get("package-data") - if isinstance(configured_package_data, dict): - package_data = { - name: [pattern for pattern in patterns if isinstance(pattern, str)] - for name, patterns in configured_package_data.items() - if isinstance(name, str) and isinstance(patterns, list) - } + _merge_package_data_declarations( + package_data, + package_data_evidence, + _package_data_mapping(configured_package_data), + _evidence( + root, + pyproject_path, + "Authoritative setuptools package-data declaration in " + "[tool.setuptools.package-data].", + _line_number(pyproject_path, "package-data"), + ), + ) + _merge_package_data_declarations( + exclude_package_data, + exclude_package_data_evidence, + _package_data_mapping(setuptools.get("exclude-package-data")), + _evidence( + root, + pyproject_path, + "Authoritative setuptools exclude-package-data declaration in " + "[tool.setuptools.exclude-package-data].", + _line_number(pyproject_path, "exclude-package-data"), + ), + ) package_find = ( setuptools.get("packages", {}).get("find", {}) if isinstance(setuptools.get("packages"), dict) @@ -527,13 +581,43 @@ def inspect_metadata(root: Path) -> MetadataResult: patterns = _multiline_values(value) if not patterns: continue - existing = package_data.setdefault(package, []) - existing.extend(pattern for pattern in patterns if pattern not in existing) + _merge_package_data_declarations( + package_data, + package_data_evidence, + {package: patterns}, + _evidence( + root, + setup_cfg_path, + "Authoritative setuptools package-data declaration in " + "[options.package_data].", + _line_number(setup_cfg_path, package), + ), + ) + if package_data_parser.has_section("options.exclude_package_data"): + for package, value in package_data_parser.items("options.exclude_package_data"): + patterns = _multiline_values(value) + if not patterns: + continue + _merge_package_data_declarations( + exclude_package_data, + exclude_package_data_evidence, + {package: patterns}, + _evidence( + root, + setup_cfg_path, + "Authoritative setuptools exclude-package-data declaration in " + "[options.exclude_package_data].", + _line_number(setup_cfg_path, package), + ), + ) setup_py_path = root / "setup.py" if setup_py_path.is_file(): metadata_files.append("setup.py") setup_values = _literal_setup_arguments(setup_py_path) + literal_packages = setup_values.get("packages") + if isinstance(literal_packages, list) and not packages: + packages = [package for package in literal_packages if isinstance(package, str)] if distribution_name is None and isinstance(setup_values.get("name"), str): distribution_name = setup_values["name"] if project_version is None and isinstance(setup_values.get("version"), str): @@ -583,8 +667,43 @@ def inspect_metadata(root: Path) -> MetadataResult: ) ) package_dir = setup_values.get("package_dir") - if isinstance(package_dir, dict) and isinstance(package_dir.get(""), str): - source_roots = source_roots or [package_dir[""]] + if isinstance(package_dir, dict): + setup_directories = { + name: path + for name, path in package_dir.items() + if isinstance(name, str) and isinstance(path, str) + } + package_directories.update(setup_directories) + if isinstance(package_directories.get(""), str): + source_roots = source_roots or [package_directories[""]] + literal_package_data = _package_data_mapping( + setup_values.get("package_data"), empty_key_is_wildcard=True + ) + literal_exclude_package_data = _package_data_mapping( + setup_values.get("exclude_package_data"), empty_key_is_wildcard=True + ) + _merge_package_data_declarations( + package_data, + package_data_evidence, + literal_package_data, + _evidence( + root, + setup_py_path, + "Literal setup(package_data=...) value; setup.py was not executed.", + _line_number(setup_py_path, "package_data"), + ), + ) + _merge_package_data_declarations( + exclude_package_data, + exclude_package_data_evidence, + literal_exclude_package_data, + _evidence( + root, + setup_py_path, + "Literal setup(exclude_package_data=...) value; setup.py was not executed.", + _line_number(setup_py_path, "exclude_package_data"), + ), + ) requirements = _requirements_files(root) for path in requirements: @@ -688,6 +807,9 @@ def inspect_metadata(root: Path) -> MetadataResult: packages=packages, package_directories=package_directories, package_data=package_data, + exclude_package_data=exclude_package_data, + package_data_evidence=package_data_evidence, + exclude_package_data_evidence=exclude_package_data_evidence, entry_points=entry_points, optional_dependency_groups=optional_groups, legacy_dependency_groups=legacy_groups, diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index f46ce89..b0cbeb8 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -47,6 +47,7 @@ def _known_packages(project: PackagingAssessment) -> set[str]: *project.packages, *(name for name in project.package_directories if name), *(name for name in project.package_data if name != "*"), + *(name for name in project.exclude_package_data if name != "*"), ] if package and package != "*" } @@ -97,6 +98,56 @@ def _physical_package_roots( return roots +def _matching_package_data_files( + package_root: Path, pattern: str +) -> list[tuple[Path, str]]: + """Resolve safe package-relative glob matches using one matcher for include/exclude rules.""" + + if not _safe_package_data_pattern(pattern): + return [] + try: + matches = package_root.glob(pattern) + except (OSError, ValueError): + return [] + resolved_package_root = package_root.resolve() + pattern_parts = PurePosixPath(pattern.replace("\\", "/")).parts + explicitly_includes_dotfile = any(part.startswith(".") for part in pattern_parts) + resolved_matches: list[tuple[Path, str]] = [] + for candidate in matches: + if candidate.is_symlink() or not candidate.is_file(): + continue + try: + resolved = candidate.resolve() + package_relative = resolved.relative_to(resolved_package_root).as_posix() + except ValueError: + continue + # Setuptools package-data globs do not implicitly select dotfiles. Keep + # the existing pathlib matcher, but filter its broader hidden-file behavior. + if not explicitly_includes_dotfile and any( + part.startswith(".") for part in PurePosixPath(package_relative).parts + ): + continue + resolved_matches.append((resolved, package_relative)) + return resolved_matches + + +def _package_data_evidence( + project: PackagingAssessment, declared_package: str, pattern: str +) -> Evidence: + """Return the parsed declaration evidence without assuming a metadata format.""" + + return project.package_data_evidence.get(declared_package, {}).get( + pattern, + Evidence( + file=(project.metadata_files[0] if project.metadata_files else "packaging metadata"), + detail=( + "Authoritative setuptools package-data declaration " + f"{declared_package} = {pattern!r} includes this runtime resource." + ), + ), + ) + + def resolve_package_data_members( root: Path, project: PackagingAssessment | None ) -> list[ResolvedPackageDataMember]: @@ -111,36 +162,29 @@ def resolve_package_data_members( packages = _known_packages(project) if declared_package == "*" else {declared_package} for package in packages: for package_root in _physical_package_roots(root, project, package): - resolved_package_root = package_root.resolve() + exclusion_patterns = [ + *project.exclude_package_data.get(package, []), + *project.exclude_package_data.get("*", []), + ] + excluded = { + resolved + for exclusion in exclusion_patterns + for resolved, _relative in _matching_package_data_files(package_root, exclusion) + } for pattern in patterns: - if not _safe_package_data_pattern(pattern): - continue - try: - matches = package_root.glob(pattern) - except (OSError, ValueError): - continue - for candidate in matches: - if candidate.is_symlink() or not candidate.is_file(): + for resolved, package_relative in _matching_package_data_files( + package_root, pattern + ): + if resolved in excluded: continue try: - resolved = candidate.resolve() - resolved.relative_to(resolved_package_root) source_path = resolved.relative_to(resolved_root).as_posix() - package_relative = resolved.relative_to( - resolved_package_root - ).as_posix() except ValueError: continue installed_member_path = str( PurePosixPath(*package.split(".")) / package_relative ) - evidence = Evidence( - file="pyproject.toml", - detail=( - "Authoritative setuptools package-data declaration " - f"{declared_package} = {pattern!r} includes this runtime resource." - ), - ) + evidence = _package_data_evidence(project, declared_package, pattern) identity = ( package, pattern, diff --git a/src/python_deployment_builder/models.py b/src/python_deployment_builder/models.py index 2ad1df6..554f541 100644 --- a/src/python_deployment_builder/models.py +++ b/src/python_deployment_builder/models.py @@ -178,6 +178,9 @@ class PackagingAssessment(StrictModel): packages: list[str] = Field(default_factory=list) package_directories: dict[str, str] = Field(default_factory=dict) package_data: dict[str, list[str]] = Field(default_factory=dict) + exclude_package_data: dict[str, list[str]] = Field(default_factory=dict) + package_data_evidence: dict[str, dict[str, Evidence]] = Field(default_factory=dict) + exclude_package_data_evidence: dict[str, dict[str, Evidence]] = Field(default_factory=dict) entry_points: list[EntryPointAssessment] = Field(default_factory=list) optional_dependency_groups: dict[str, list[str]] = Field(default_factory=dict) legacy_dependency_groups: list[LegacyDependencyGroup] = Field(default_factory=list) diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 0417533..d685fd1 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -842,6 +842,47 @@ def test_wildcard_setuptools_package_data_uses_known_physical_package_mapping( assert "code/view.html" in _staging_files(tmp_path, assessment, source_plan, include=True) +def test_package_data_exclusions_apply_after_safe_concrete_resolution(tmp_path: Path) -> None: + for package in ("app", "other"): + data = tmp_path / package / "data" + data.mkdir(parents=True) + (tmp_path / package / "__init__.py").write_text("", encoding="utf-8") + (data / "defaults.json").write_text("{}\n", encoding="utf-8") + (data / "private.json").write_text("{}\n", encoding="utf-8") + (data / "temporary.tmp").write_text("temporary\n", encoding="utf-8") + (data / ".hidden.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[project] +name = "excluded-data-app" +version = "1.0" +[tool.setuptools] +packages = ["app", "other"] +[tool.setuptools.package-data] +"*" = ["data/*.json", "data/*.tmp"] +app = ["data/*.json"] +[tool.setuptools.exclude-package-data] +app = ["data/private.json"] +"*" = ["data/*.tmp"] +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + members = resolve_package_data_members(tmp_path, project) + paths = {member.source_path for member in members} + + assert project.exclude_package_data == { + "app": ["data/private.json"], + "*": ["data/*.tmp"], + } + assert paths == { + "app/data/defaults.json", + "other/data/defaults.json", + "other/data/private.json", + } + assert all(member.evidence.file == "pyproject.toml" for member in members) + + @pytest.mark.parametrize( ("source_root", "resource_path", "installed_path"), [ @@ -863,6 +904,7 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( encoding="utf-8", ) (package / "data/default.json").write_text('{"default": true}\n', encoding="utf-8") + (package / "data/private.json").write_text('{"private": true}\n', encoding="utf-8") package_dir = "\npackage_dir =\n = src" if source_root else "" find_where = "\n[options.packages.find]\nwhere = src" if source_root else "" (tmp_path / "pyproject.toml").write_text( @@ -885,6 +927,11 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( templates/*.html * = *.txt{find_where} +[options.exclude_package_data] +app = + data/private.json +* = + *.tmp """, encoding="utf-8", ) @@ -902,12 +949,49 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( "app": ["data/*.json", "templates/*.html"], "*": ["*.txt"], } + assert assessment.project.exclude_package_data == { + "app": ["data/private.json"], + "*": ["*.tmp"], + } assert resource.packaging_status == "packaged" assert inventory.role == RepositoryFileRole.RUNTIME_RESOURCE - assert resource_path in _staging_files(tmp_path, assessment, source_plan, include=True) + staged = _staging_files(tmp_path, assessment, source_plan, include=True) + assert resource_path in staged + assert resource_path.replace("default.json", "private.json") not in staged assert [(member.source_path, member.installed_member_path) for member in members] == [ (resource_path, installed_path) ] + assert all("private.json" not in member.source_path for member in members) + assert all(member.evidence.file == "setup.cfg" for member in members) + + +def test_literal_setup_py_package_data_and_exclusions_share_the_resolver(tmp_path: Path) -> None: + data = tmp_path / "app" / "data" + data.mkdir(parents=True) + (tmp_path / "app/__init__.py").write_text("", encoding="utf-8") + (data / "defaults.json").write_text("{}\n", encoding="utf-8") + (data / "private.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "setup.py").write_text( + """from setuptools import setup +setup( + name="literal-data", + version="1.0", + packages=["app"], + package_data={"": ["data/*.json"]}, + exclude_package_data={"": ["data/private.json"]}, +) +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.package_data == {"*": ["data/*.json"]} + assert project.exclude_package_data == {"*": ["data/private.json"]} + assert [ + (member.source_path, member.installed_member_path, member.evidence.file) + for member in resolve_package_data_members(tmp_path, project) + ] == [("app/data/defaults.json", "app/data/defaults.json", "setup.py")] def test_setup_cfg_standard_options_are_case_insensitive_and_package_data_is_not( diff --git a/tests/test_generation.py b/tests/test_generation.py index 07e45a4..a5d14bb 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1399,6 +1399,74 @@ def test_application_wheel_requires_every_concrete_declared_package_data_member( assert artifact.filename == complete.name +def test_application_wheel_and_source_staging_honor_excluded_package_data( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "code/data").mkdir() + (source / "code/data/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "code/data/private.json").write_text("{}\n", encoding="utf-8") + pyproject = source / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + 'installed_app = ["view.html"]', + 'installed_app = ["view.html", "data/*.json"]\n' + "[tool.setuptools.exclude-package-data]\n" + 'installed_app = ["data/private.json"]', + ), + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + source_plan = plan.model_copy(deep=True) + source_plan.deployment_mode = "source" + + assert { + (item.source_path, item.installed_member_path) + for item in resolve_package_data_members(source, assessment.project) + } == { + ("code/data/defaults.json", "installed_app/data/defaults.json"), + ("code/view.html", "installed_app/view.html"), + } + staged = _staging_files(source, assessment, source_plan, include=True) + assert "code/data/defaults.json" in staged + assert "code/data/private.json" not in staged + + defaults_only = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/data/defaults.json": "{}\n"}, + ) + assert validate_application_wheel( + defaults_only, assessment, plan, repository_root=source + )[0].filename == defaults_only.name + + missing_directory = tmp_path / "missing-default" + missing_directory.mkdir() + with pytest.raises(PreparationError, match="installed_app/data/defaults.json"): + validate_application_wheel( + _make_application_wheel(missing_directory), + assessment, + plan, + repository_root=source, + ) + + private_directory = tmp_path / "with-private" + private_directory.mkdir() + with_private = _rewrite_application_wheel( + _make_application_wheel(private_directory), + additions={ + "installed_app/data/defaults.json": "{}\n", + "installed_app/data/private.json": "{}\n", + }, + ) + assert validate_application_wheel( + with_private, assessment, plan, repository_root=source + )[0].filename == with_private.name + + def test_application_wheel_requires_nested_package_data_from_parent_mapping( tmp_path: Path, ) -> None: From 99a677af4739b6eef555faf4625b02b2e55b5839 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 10:24:31 -0400 Subject: [PATCH 16/59] Preserve runtime resource and marker constraints --- .../analysis/inventory.py | 28 ++- .../planning/index.py | 84 ++++++-- .../planning/planner.py | 22 ++- tests/test_analysis_scope.py | 37 ++++ tests/test_generation.py | 52 +++-- tests/test_planning.py | 181 +++++++++++++++++- 6 files changed, 362 insertions(+), 42 deletions(-) diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index b52b4fb..ca71216 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -7,7 +7,7 @@ import re from collections import Counter from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from packaging.requirements import InvalidRequirement, Requirement from pathspec import PathSpec @@ -58,6 +58,21 @@ "schemas", "templates", } + + +def resource_covers_inventory_path(resource_path: str, inventory_path: str) -> bool: + """Return whether a repository-relative resource includes an inventory member. + + Resource analysis intentionally records both concrete files and conventional + directories. Compare path *components*, not textual prefixes: ``assets`` + includes ``assets/view.html`` but never ``assets2/view.html``. + """ + + resource = PurePosixPath(resource_path.rstrip("/")) + candidate = PurePosixPath(inventory_path.rstrip("/")) + return candidate.parts[: len(resource.parts)] == resource.parts + + DOCUMENTATION_SUFFIXES = {".md", ".rst"} DEVELOPMENT_FILENAMES = { ".gitignore", @@ -351,18 +366,17 @@ def apply_resource_roles( ) -> AnalysisScopeSummary: """Promote only statically supported resource paths in the inventory.""" - by_path = { - resource.path.rstrip("/"): resource + applicable_resources = [ + resource for resource in resources if resource.status == FindingStatus.DETECTED or resource.kind in CONVENTIONAL_RUNTIME_RESOURCE_KINDS - } + ] for item in items: - normalized = item.path.rstrip("/") matching = [ resource - for path, resource in by_path.items() - if normalized == path or normalized.startswith(path + "/") + for resource in applicable_resources + if resource_covers_inventory_path(resource.path, item.path) ] authoritative = any( resource.packaging_status == "packaged" for resource in matching diff --git a/src/python_deployment_builder/planning/index.py b/src/python_deployment_builder/planning/index.py index 008f779..c8335aa 100644 --- a/src/python_deployment_builder/planning/index.py +++ b/src/python_deployment_builder/planning/index.py @@ -5,12 +5,13 @@ import json from collections.abc import Callable from datetime import UTC, datetime +from enum import StrEnum from typing import Any from urllib.parse import quote from urllib.request import Request, urlopen from packaging._parser import Variable -from packaging.markers import InvalidMarker, Marker +from packaging.markers import InvalidMarker, Marker, _evaluate_markers from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.tags import compatible_tags, cpython_tags from packaging.utils import InvalidWheelFilename, parse_wheel_filename @@ -30,22 +31,34 @@ class TargetMarkerEnvironmentError(ValueError): """A marker requires target facts PDB does not select for M6.1.""" +class TargetMarkerApplicability(StrEnum): + """Whether an environment marker can be proven for PDB's target contract.""" + + APPLIES = "applies" + DOES_NOT_APPLY = "does_not_apply" + UNPROVABLE = "unprovable" + + +_PATCH_SENSITIVE_MARKER_VARIABLES = { + "implementation_version", + "python_full_version", +} + + def target_marker_environment( python_version: str, architecture: str, *, extra: str = "" ) -> dict[str, str]: """Return every PEP 508 marker value PDB can establish for its Windows target.""" - full_version = f"{python_version}.0" - # platform_release and platform_version deliberately have no target values: PDB plans - # a Windows architecture and Python minor, not a specific Windows build. + # PDB selects a Python major/minor, not an exact patch. Deliberately omit + # patch-sensitive variables instead of fabricating ``.0``. Likewise + # platform_release and platform_version have no planned target values. return { "implementation_name": "cpython", - "implementation_version": full_version, "os_name": "nt", "platform_machine": "AMD64" if architecture == "x86_64" else "ARM64", "platform_python_implementation": "CPython", "platform_system": "Windows", - "python_full_version": full_version, "python_version": python_version, "sys_platform": "win32", "extra": extra, @@ -62,28 +75,64 @@ def _marker_variables(value: object) -> set[str]: return set() -def target_marker_applies( +def target_marker_applicability( marker: str | None, python_version: str, architecture: str, *, extra: str = "", -) -> bool: - """Evaluate a marker strictly from selected Windows target facts.""" +) -> TargetMarkerApplicability: + """Evaluate a marker without inventing unselected target facts.""" if not marker: - return True + return TargetMarkerApplicability.APPLIES try: parsed = Marker(marker) except InvalidMarker as exc: raise TargetMarkerEnvironmentError(f"Malformed environment marker: {marker!r}") from exc environment = target_marker_environment(python_version, architecture, extra=extra) - unsupported = sorted(_marker_variables(parsed._markers) - set(environment)) - if unsupported: - raise TargetMarkerEnvironmentError( - "Target marker fields are not selected by PDB: " + ", ".join(unsupported) + unprovable = sorted(_marker_variables(parsed._markers) - set(environment)) + if unprovable: + return TargetMarkerApplicability.UNPROVABLE + return ( + TargetMarkerApplicability.APPLIES + # ``Marker.evaluate`` begins from the builder host's default environment + # before applying overrides. The target environment must be complete for + # the variables we use and contain no host-derived fallback values. + if _evaluate_markers(parsed._markers, environment) + else TargetMarkerApplicability.DOES_NOT_APPLY + ) + + +def target_marker_applies( + marker: str | None, + python_version: str, + architecture: str, + *, + extra: str = "", +) -> bool: + """Strict target-marker evaluation for proofs that require certainty.""" + + applicability = target_marker_applicability( + marker, python_version, architecture, extra=extra + ) + if applicability == TargetMarkerApplicability.UNPROVABLE: + try: + variables = sorted( + _marker_variables(Marker(marker or "")._markers) + - set(target_marker_environment(python_version, architecture, extra=extra)) + ) + except InvalidMarker: # already translated by target_marker_applicability + variables = [] + patch_sensitive = sorted(set(variables) & _PATCH_SENSITIVE_MARKER_VARIABLES) + detail = ( + "patch-sensitive target facts are selected only by Python major/minor: " + + ", ".join(patch_sensitive) + if patch_sensitive + else "target marker fields are not selected by PDB: " + ", ".join(variables) ) - return parsed.evaluate(environment) + raise TargetMarkerEnvironmentError(detail) + return applicability == TargetMarkerApplicability.APPLIES def _fetch_json(url: str) -> dict[str, Any]: @@ -140,7 +189,10 @@ def marker_applies( if not marker: return True try: - return target_marker_applies(marker, python_version, architecture, extra=extra) + return ( + target_marker_applicability(marker, python_version, architecture, extra=extra) + != TargetMarkerApplicability.DOES_NOT_APPLY + ) except TargetMarkerEnvironmentError: # Planning remains conservative for malformed or host-unknown lock markers; # first-party wheel validation raises instead of treating them as proven. diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 7deb27b..1600f6d 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -10,6 +10,7 @@ from packaging.utils import canonicalize_name from python_deployment_builder import __version__ +from python_deployment_builder.analysis.inventory import resource_covers_inventory_path from python_deployment_builder.backends.uv_managed import UvManagedBackend from python_deployment_builder.models import ( ConfigurationPlan, @@ -81,13 +82,20 @@ def _deployment_mode( for item in assessment.file_inventory if item.role == RepositoryFileRole.RUNTIME_RESOURCE } - adjacent = [ - item.path - for item in assessment.resources - if item.packaging_status == "repository_adjacent" - and item.path in runtime_resource_paths - and item.kind != "documentation" - ] + # Work from the same concrete promoted inventory members used by staging. + # A conventional directory requirement (for example ``assets``) covers its + # descendants and therefore cannot be represented by package mode unless it + # is authoritative wheel-backed package data. + adjacent = sorted( + { + item_path + for resource in assessment.resources + if resource.packaging_status == "repository_adjacent" + and resource.kind != "documentation" + for item_path in runtime_resource_paths + if resource_covers_inventory_path(resource.path, item_path) + } + ) project_writes = [ item.path_expression for item in assessment.write_locations diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index d685fd1..a9176e2 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -1389,6 +1389,43 @@ def test_materialized_archive_uses_role_aware_staging_without_git(tmp_path: Path assert "deployment/helper.py" not in staged +def test_conventional_resource_directory_stages_descendants_not_similar_prefixes( + tmp_path: Path, +) -> None: + (tmp_path / "assets").mkdir() + (tmp_path / "assets2").mkdir() + (tmp_path / "assets/view.html").write_text("runtime\n", encoding="utf-8") + (tmp_path / "assets/templates").mkdir() + (tmp_path / "assets/templates/page.html").write_text("nested\n", encoding="utf-8") + (tmp_path / "assets2/view.html").write_text("unrelated\n", encoding="utf-8") + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "conventional-assets" +version = "1.0" +[project.scripts] +conventional-assets = "app.main:main" +[tool.setuptools.packages.find] +where = ["src"] +""", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + + staged = _staging_files(tmp_path, assessment, plan, include=True) + + assert plan.deployment_mode == "source" + assert {"assets/view.html", "assets/templates/page.html"} <= staged.keys() + assert "assets2/view.html" not in staged + + def test_pathspec_is_declared_as_a_runtime_dependency() -> None: root = Path(__file__).parents[1] pyproject = (root / "pyproject.toml").read_text(encoding="utf-8") diff --git a/tests/test_generation.py b/tests/test_generation.py index a5d14bb..1d4bbeb 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -744,20 +744,20 @@ def test_application_wheel_requires_dist_rejects_unprovable_metadata( @pytest.mark.parametrize( - "marker", + ("marker", "error"), [ - 'python_version < "3.13"', - 'implementation_version < "3.13"', - 'platform_python_implementation == "CPython"', - 'implementation_name == "cpython"', - 'sys_platform == "win32"', - 'os_name == "nt"', - 'platform_system == "Windows"', - 'platform_machine == "AMD64"', + ('python_version < "3.13"', "absent"), + ('implementation_version < "3.13"', "cannot be proven"), + ('platform_python_implementation == "CPython"', "absent"), + ('implementation_name == "cpython"', "absent"), + ('sys_platform == "win32"', "absent"), + ('os_name == "nt"', "absent"), + ('platform_system == "Windows"', "absent"), + ('platform_machine == "AMD64"', "absent"), ], ) def test_application_wheel_markers_use_complete_target_environment( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, marker: str + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, marker: str, error: str ) -> None: source = tmp_path / "source" source.mkdir() @@ -776,10 +776,40 @@ def test_application_wheel_markers_use_complete_target_environment( tmp_path, requires_dist_values=[f"helper; {marker}"] ) - with pytest.raises(PreparationError, match="absent"): + with pytest.raises(PreparationError, match=error): validate_application_wheel(wheel, assessment, plan) +@pytest.mark.parametrize( + "marker", + [ + 'python_full_version >= "3.12.1"', + 'implementation_version >= "3.12.1"', + ], +) +def test_application_wheel_patch_sensitive_marker_is_not_proven_for_minor_target( + tmp_path: Path, marker: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, repository_root=source), [] + ) + + with pytest.raises(PreparationError, match="patch-sensitive"): + validate_application_wheel( + _make_application_wheel( + tmp_path, requires_dist_values=[f"helper; {marker}"] + ), + assessment, + plan, + ) + + def test_application_wheel_target_marker_arm64_and_unprovable_field_policy( tmp_path: Path, ) -> None: diff --git a/tests/test_planning.py b/tests/test_planning.py index 2af7029..ee973f8 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -13,7 +13,15 @@ RuntimeRequirement, SuitabilityRating, ) -from python_deployment_builder.planning.index import inspect_dependency_wheels, marker_applies +from python_deployment_builder.planning.index import ( + TargetMarkerApplicability, + TargetMarkerEnvironmentError, + inspect_dependency_wheels, + marker_applies, + target_marker_applicability, + target_marker_applies, + target_marker_environment, +) from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.planning.planner import create_deployment_plan from python_deployment_builder.planning.platforms import windows_finding_treatments @@ -138,6 +146,123 @@ def test_complete_deployment_mode_decision_table(tmp_path: Path) -> None: assert all(plan.decisions[0].rationale for plan in results.values()) +def test_repository_adjacent_resource_directory_constrains_deployment_mode( + tmp_path: Path, +) -> None: + """A conventional directory is a source-only constraint for every descendant.""" + + source = tmp_path / "source-compatible" + (source / "src/example_app").mkdir(parents=True) + (source / "assets").mkdir() + (source / "src/example_app/__init__.py").write_text("", encoding="utf-8") + (source / "src/example_app/main.py").write_text( + "def main(): return 0\n", encoding="utf-8" + ) + (source / "assets/view.html").write_text("
view
\n", encoding="utf-8") + (source / "assets2/ignored.html").parent.mkdir() + (source / "assets2/ignored.html").write_text("ignored\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "directory-resource" +version = "1.0" +[project.scripts] +directory-resource = "example_app.main:main" +[tool.setuptools.packages.find] +where = ["src"] +""", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + roles = {item.path: item.role for item in assessment.file_inventory} + plan = create_deployment_plan(assessment, repository_root=source) + + assert roles["assets/view.html"].value == "runtime_resource" + assert roles["assets2/ignored.html"].value != "runtime_resource" + assert (plan.deployment_mode, plan.deployment_mode_condition) == ( + "source", + "SOURCE_COMPATIBLE", + ) + + +def test_repository_adjacent_resource_directory_conflicts_with_package_entrypoint( + tmp_path: Path, +) -> None: + root = tmp_path / "package-required" + (root / "src/example_app").mkdir(parents=True) + (root / "assets").mkdir() + (root / "src/example_app/__init__.py").write_text("", encoding="utf-8") + (root / "src/example_app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "assets/view.html").write_text("
view
\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "directory-resource" +version = "1.0" +[project.scripts] +directory-resource = "installed_app.main:main" +[tool.setuptools.packages.find] +where = ["src"] +""", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + plan = create_deployment_plan( + assess_repository(MaterializedRepository(root=root, source=str(root), source_kind="local")), + repository_root=root, + ) + + assert (plan.deployment_mode, plan.deployment_mode_condition) == ( + "package", + "DEPLOYMENT_MODE_CONFLICT", + ) + assert "assets/view.html" in plan.readiness.blockers[0] + + +def test_wheel_backed_package_data_resource_does_not_force_source_mode(tmp_path: Path) -> None: + root = tmp_path / "wheel-backed" + (root / "src/example_app/templates").mkdir(parents=True) + (root / "src/example_app/__init__.py").write_text("", encoding="utf-8") + (root / "src/example_app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "src/example_app/templates/view.html").write_text("view\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "wheel-backed" +version = "1.0" +[project.scripts] +wheel-backed = "example_app.main:main" +[tool.setuptools.packages.find] +where = ["src"] +[tool.setuptools.package-data] +example_app = ["templates/*.html"] +""", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + plan = create_deployment_plan( + assess_repository(MaterializedRepository(root=root, source=str(root), source_kind="local")), + repository_root=root, + ) + + assert (plan.deployment_mode, plan.deployment_mode_condition) == ( + "package", + "PACKAGE_PREFERRED", + ) + + def test_target_plan_selects_source_gui_and_external_environment() -> None: plan = create_deployment_plan(_assess()) @@ -381,6 +506,60 @@ def test_windows_environment_markers_are_applied() -> None: assert not marker_applies("sys_platform == 'linux'", "3.12", "x86_64", extra="map") +@pytest.mark.parametrize( + ("marker", "expected"), + [ + ('python_version >= "3.12"', TargetMarkerApplicability.APPLIES), + ('python_version < "3.13"', TargetMarkerApplicability.APPLIES), + ('python_version >= "3.13"', TargetMarkerApplicability.DOES_NOT_APPLY), + ('python_full_version >= "3.12.1"', TargetMarkerApplicability.UNPROVABLE), + ('python_full_version == "3.12.0"', TargetMarkerApplicability.UNPROVABLE), + ('implementation_version >= "3.12.1"', TargetMarkerApplicability.UNPROVABLE), + ], +) +def test_target_marker_applicability_does_not_fabricate_python_patch( + marker: str, expected: TargetMarkerApplicability +) -> None: + assert target_marker_applicability(marker, "3.12", "x86_64") == expected + + +def test_patch_sensitive_markers_are_conservative_for_lock_traversal() -> None: + """A possibly applicable lock edge must not vanish because 3.12.x is unknown.""" + + assert marker_applies('python_full_version >= "3.12.1"', "3.12", "x86_64") + with pytest.raises(TargetMarkerEnvironmentError, match="patch-sensitive"): + target_marker_applies('implementation_version >= "3.12.1"', "3.12", "x86_64") + environment = target_marker_environment("3.12", "x86_64") + assert "python_full_version" not in environment + assert "implementation_version" not in environment + + +def test_lock_graph_retains_dependency_edge_with_patch_sensitive_marker( + tmp_path: Path, +) -> None: + (tmp_path / "uv.lock").write_text( + """version = 1 +revision = 3 +[[package]] +name = "example" +version = "1.0" +source = { virtual = "." } +dependencies = [{ name = "helper", marker = "python_full_version >= '3.12.1'" }] +[[package]] +name = "helper" +version = "1.0" +wheels = [{ url = "https://example.invalid/helper-1.0-py3-none-any.whl" }] +""", + encoding="utf-8", + ) + + graph = inspect_uv_lock(tmp_path, "example", "3.12", "x86_64", []) + + edge = next(item for item in graph.edges if item.to_package == "helper") + assert edge.applicable + assert any(item.name == "helper" for item in graph.dependencies) + + def test_lock_graph_reports_pywebview_proxy_tools_source_only_chain() -> None: graph = inspect_uv_lock( FIXTURES / "optional_map_app", From a19d70437c36ac4ad5b644084f008500c3c560ef Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:47:13 -0400 Subject: [PATCH 17/59] Resolve setuptools package source surface --- src/python_deployment_builder/__init__.py | 2 +- .../analysis/inventory.py | 10 +- .../analysis/metadata.py | 127 +++++++++++++++++- .../analysis/resources.py | 97 +++++++++++++ .../generation/artifacts.py | 23 +++- src/python_deployment_builder/models.py | 1 + .../planning/planner.py | 36 ++++- tests/test_analysis_scope.py | 104 +++++++++++++- tests/test_generation.py | 111 +++++++++++++++ tests/test_planning.py | 69 +++++++++- 10 files changed, 567 insertions(+), 13 deletions(-) diff --git a/src/python_deployment_builder/__init__.py b/src/python_deployment_builder/__init__.py index 3fa4083..feaf0a8 100644 --- a/src/python_deployment_builder/__init__.py +++ b/src/python_deployment_builder/__init__.py @@ -10,5 +10,5 @@ SCHEMA_VERSION = "1.0" # Analysis and planning retain defaults for older serialized forms, but their # emitted contracts changed when entry-point groups and lock dependency extras became explicit. -ANALYSIS_SCHEMA_VERSION = "1.3" +ANALYSIS_SCHEMA_VERSION = "1.4" PLANNING_SCHEMA_VERSION = "1.3" diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index ca71216..93b6553 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -321,14 +321,20 @@ def promote_imported_application_files( for imported_path in _module_files(root, module): relative = imported_path.relative_to(root).as_posix() item = by_path.get(relative) - if item is None or item.role == RepositoryFileRole.APPLICATION_SOURCE: + if item is None: continue evidence = Evidence( file=relative_source, line=line, detail=f"Application source imports local module {module!r}.", ) - item.evidence.append(evidence) + if evidence not in item.evidence: + item.evidence.append(evidence) + if item.role == RepositoryFileRole.APPLICATION_SOURCE: + if imported_path not in application_files: + application_files.append(imported_path) + queued.append(imported_path) + continue if item.role == RepositoryFileRole.IGNORED_OR_LOCAL: item.reason = ( "Application source imports this ignored/local module; it remains excluded " diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 61117fe..75ea6f0 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -4,6 +4,7 @@ import ast import configparser +import fnmatch import re import tomllib from dataclasses import dataclass @@ -194,6 +195,70 @@ def _package_data_mapping( } +def _string_list(value: Any, *, default: list[str] | None = None) -> list[str]: + """Return the supported TOML/list-or-string metadata shape without coercion.""" + + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [item for item in value if isinstance(item, str)] + return list(default or []) + + +def _discover_setuptools_packages( + root: Path, + search_roots: list[str], + package_directories: dict[str, str], + include: list[str], + exclude: list[str], + namespaces: bool, +) -> list[str]: + """Statically resolve the bounded setuptools ``find`` package surface. + + This is filesystem-only metadata interpretation: it never imports modules, + follows package symlinks, or includes paths outside the assessed repository. + """ + + resolved_root = root.resolve() + discovered: set[str] = set() + includes = include or ["*"] + for configured_root in search_roots: + candidate_root = root / configured_root + if candidate_root.is_symlink() or not candidate_root.is_dir(): + continue + try: + candidate_root.resolve().relative_to(resolved_root) + except ValueError: + continue + named_prefixes = [ + name + for name, directory in package_directories.items() + if name + and (root / directory).resolve() == candidate_root.resolve() + ] + prefix = max(named_prefixes, key=lambda name: len(name.split(".")), default="") + for directory in sorted(candidate_root.rglob("*")): + if directory.is_symlink() or not directory.is_dir(): + continue + try: + relative = directory.resolve().relative_to(candidate_root.resolve()) + except ValueError: + continue + if not relative.parts or any(part == "__pycache__" for part in relative.parts): + continue + parts = (*prefix.split("."), *relative.parts) if prefix else relative.parts + if not all(part.isidentifier() for part in parts): + continue + if not namespaces and not (directory / "__init__.py").is_file(): + continue + package = ".".join(parts) + if any(fnmatch.fnmatchcase(package, pattern) for pattern in includes) and not any( + fnmatch.fnmatchcase(package, pattern) for pattern in exclude + ): + discovered.add(package) + return sorted(discovered) + + def _literal_setup_arguments(path: Path) -> dict[str, Any]: """Read literal setup(...) keyword values without executing setup.py.""" @@ -305,6 +370,7 @@ def inspect_metadata(root: Path) -> MetadataResult: ruff_target: str | None = None source_roots: list[str] = [] packages: list[str] = [] + py_modules: list[str] = [] package_directories: dict[str, str] = {} package_data: dict[str, list[str]] = {} exclude_package_data: dict[str, list[str]] = {} @@ -312,6 +378,7 @@ def inspect_metadata(root: Path) -> MetadataResult: exclude_package_data_evidence: dict[str, dict[str, Evidence]] = {} layout = "unknown" python_evidence: list[Evidence] = [] + package_discovery_rules: list[tuple[list[str], list[str], list[str], bool]] = [] pyproject_path = root / "pyproject.toml" if pyproject_path.is_file(): @@ -452,6 +519,7 @@ def inspect_metadata(root: Path) -> MetadataResult: configured_packages = setuptools.get("packages") if isinstance(configured_packages, list): packages = [value for value in configured_packages if isinstance(value, str)] + py_modules = _string_list(setuptools.get("py-modules")) configured_package_dirs = setuptools.get("package-dir") if isinstance(configured_package_dirs, dict): package_directories = { @@ -491,9 +559,17 @@ def inspect_metadata(root: Path) -> MetadataResult: if isinstance(setuptools.get("packages"), dict) else {} ) - configured_where = package_find.get("where", []) if isinstance(package_find, dict) else [] - if isinstance(configured_where, list): - source_roots = [value for value in configured_where if isinstance(value, str)] + if isinstance(setuptools.get("packages"), dict) and isinstance(package_find, dict): + configured_where = _string_list(package_find.get("where"), default=["."]) + source_roots = configured_where + package_discovery_rules.append( + ( + configured_where, + _string_list(package_find.get("include"), default=["*"]), + _string_list(package_find.get("exclude")), + package_find.get("namespaces", True) is not False, + ) + ) ruff = tool.get("ruff") if isinstance(tool.get("ruff"), dict) else {} ruff_target = ( ruff.get("target-version") if isinstance(ruff.get("target-version"), str) else None @@ -564,6 +640,13 @@ def inspect_metadata(root: Path) -> MetadataResult: configured_where = parser.get("options.packages.find", "where", fallback="").strip() if configured_where and not source_roots: source_roots = [configured_where] + configured_packages = parser.get("options", "packages", fallback="").strip() + if ( + configured_packages + and configured_packages not in {"find:", "find_namespace:"} + and not packages + ): + packages = _multiline_values(configured_packages) configured_package_dir = parser.get("options", "package_dir", fallback="") if configured_package_dir: setup_cfg_directories = { @@ -576,6 +659,25 @@ def inspect_metadata(root: Path) -> MetadataResult: package_directories.update(setup_cfg_directories) if isinstance(package_directories.get(""), str): source_roots = source_roots or [package_directories[""]] + if not py_modules: + py_modules = _multiline_values(parser.get("options", "py_modules", fallback="")) + if configured_packages in {"find:", "find_namespace:"}: + discovery_roots = _multiline_values(configured_where) + if not discovery_roots: + discovery_roots = [package_directories.get("", ".")] + package_discovery_rules.append( + ( + discovery_roots, + _multiline_values( + parser.get("options.packages.find", "include", fallback="") + ) + or ["*"], + _multiline_values( + parser.get("options.packages.find", "exclude", fallback="") + ), + configured_packages == "find_namespace:", + ) + ) if package_data_parser.has_section("options.package_data"): for package, value in package_data_parser.items("options.package_data"): patterns = _multiline_values(value) @@ -618,6 +720,10 @@ def inspect_metadata(root: Path) -> MetadataResult: literal_packages = setup_values.get("packages") if isinstance(literal_packages, list) and not packages: packages = [package for package in literal_packages if isinstance(package, str)] + if not py_modules and isinstance(setup_values.get("py_modules"), list): + py_modules = [ + module for module in setup_values["py_modules"] if isinstance(module, str) + ] if distribution_name is None and isinstance(setup_values.get("name"), str): distribution_name = setup_values["name"] if project_version is None and isinstance(setup_values.get("version"), str): @@ -773,6 +879,20 @@ def inspect_metadata(root: Path) -> MetadataResult: if not source_roots: source_roots = ["src"] if (root / "src").is_dir() else ["."] + if package_discovery_rules: + discovered_packages = { + package + for where, include, exclude, namespaces in package_discovery_rules + for package in _discover_setuptools_packages( + root, + where, + package_directories, + include, + exclude, + namespaces, + ) + } + packages = sorted({*packages, *discovered_packages}) layout = ( "src" if any(Path(value).as_posix().rstrip("/") == "src" for value in source_roots) @@ -805,6 +925,7 @@ def inspect_metadata(root: Path) -> MetadataResult: layout=layout, source_roots=source_roots, packages=packages, + py_modules=py_modules, package_directories=package_directories, package_data=package_data, exclude_package_data=exclude_package_data, diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index b0cbeb8..cb481eb 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -28,6 +28,16 @@ class ResolvedPackageDataMember: evidence: Evidence +@dataclass(frozen=True) +class ResolvedPackagedPythonSource: + """A concrete first-party Python source member expected in the wheel.""" + + source_path: str + installed_member_path: str + package_name: str | None + kind: str + + def _safe_package_data_pattern(pattern: str) -> bool: """Return whether a setuptools package-data pattern stays under its package root.""" @@ -213,6 +223,93 @@ def resolve_package_data_members( ) +def _safe_python_source(path: Path, root: Path) -> Path | None: + """Return a regular in-repository Python file without following symlinks.""" + + if path.is_symlink() or not path.is_file() or path.suffix != ".py": + return None + try: + resolved = path.resolve() + resolved.relative_to(root.resolve()) + except ValueError: + return None + return resolved + + +def _physical_py_module_candidates( + root: Path, project: PackagingAssessment, module: str +) -> list[Path]: + relative = Path(*module.split(".")).with_suffix(".py") + candidates: list[Path] = [] + base = project.package_directories.get("") + if base is not None: + candidates.append(root / base / relative) + candidates.extend(root / source_root / relative for source_root in project.source_roots) + candidates.append(root / relative) + resolved: list[Path] = [] + for candidate in candidates: + safe = _safe_python_source(candidate, root) + if safe is not None and safe not in resolved: + resolved.append(safe) + return resolved + + +def resolve_packaged_python_sources( + root: Path, project: PackagingAssessment | None +) -> list[ResolvedPackagedPythonSource]: + """Resolve the supported setuptools Python surface without importing it. + + Packages contribute only modules directly in each authoritative package root; + subpackages must be explicitly listed or discovered themselves. Standalone + modules are admitted only through authoritative ``py_modules`` metadata. + """ + + if project is None: + return [] + resolved_root = root.resolve() + resolved: list[ResolvedPackagedPythonSource] = [] + seen: set[tuple[str, str]] = set() + for package in sorted(set(project.packages)): + for package_root in _physical_package_roots(root, project, package): + for candidate in sorted(package_root.glob("*.py")): + safe = _safe_python_source(candidate, root) + if safe is None: + continue + source_path = safe.relative_to(resolved_root).as_posix() + installed = str( + PurePosixPath(*package.split(".")) / safe.name + ) + identity = (source_path, installed) + if identity not in seen: + seen.add(identity) + resolved.append( + ResolvedPackagedPythonSource( + source_path=source_path, + installed_member_path=installed, + package_name=package, + kind="package_module", + ) + ) + for module in sorted(set(project.py_modules)): + if not module or not all(part.isidentifier() for part in module.split(".")): + continue + for source in _physical_py_module_candidates(root, project, module): + source_path = source.relative_to(resolved_root).as_posix() + installed = PurePosixPath(*module.split(".")).with_suffix(".py").as_posix() + identity = (source_path, installed) + if identity not in seen: + seen.add(identity) + resolved.append( + ResolvedPackagedPythonSource( + source_path=source_path, + installed_member_path=installed, + package_name=None, + kind="py_module", + ) + ) + return sorted(resolved, key=lambda item: (item.source_path, item.installed_member_path)) + + def _declared_package_data( root: Path, project: PackagingAssessment | None ) -> dict[str, list[Evidence]]: diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 4361401..1cb9d9e 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -22,7 +22,10 @@ from packaging.utils import canonicalize_name, parse_wheel_filename from packaging.version import InvalidVersion, Version -from python_deployment_builder.analysis.resources import resolve_package_data_members +from python_deployment_builder.analysis.resources import ( + resolve_package_data_members, + resolve_packaged_python_sources, +) from python_deployment_builder.generation.acquisition import PreparationError, sha256_file from python_deployment_builder.models import ( ApplicationArtifact, @@ -681,9 +684,13 @@ def validate_application_wheel( if source_root is None: candidate = Path(assessment.repository.source).expanduser() source_root = candidate if candidate.is_dir() else None - if assessment.project.package_data and source_root is None: + if ( + assessment.project.package_data + or assessment.project.packages + or assessment.project.py_modules + ) and source_root is None: raise PreparationError( - "Application wheel package-data validation requires the assessed " + "Application wheel packaging-surface validation requires the assessed " "repository root." ) expected_members = { @@ -696,6 +703,16 @@ def validate_application_wheel( "Application wheel is missing concrete declared package data: " + ", ".join(missing_members) ) + expected_python_members = { + member.installed_member_path + for member in resolve_packaged_python_sources(source_root, assessment.project) + } if source_root is not None else set() + missing_python_members = sorted(expected_python_members - names) + if missing_python_members: + raise PreparationError( + "Application wheel is missing authoritative first-party Python source: " + + ", ".join(missing_python_members) + ) except (zipfile.BadZipFile, UnicodeDecodeError, configparser.Error) as exc: raise PreparationError(f"Malformed application wheel: {path.name}") from exc diff --git a/src/python_deployment_builder/models.py b/src/python_deployment_builder/models.py index 554f541..d66cf7d 100644 --- a/src/python_deployment_builder/models.py +++ b/src/python_deployment_builder/models.py @@ -176,6 +176,7 @@ class PackagingAssessment(StrictModel): layout: Literal["src", "flat", "unknown"] = "unknown" source_roots: list[str] = Field(default_factory=list) packages: list[str] = Field(default_factory=list) + py_modules: list[str] = Field(default_factory=list) package_directories: dict[str, str] = Field(default_factory=dict) package_data: dict[str, list[str]] = Field(default_factory=dict) exclude_package_data: dict[str, list[str]] = Field(default_factory=dict) diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 1600f6d..7df62b0 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -11,6 +11,7 @@ from python_deployment_builder import __version__ from python_deployment_builder.analysis.inventory import resource_covers_inventory_path +from python_deployment_builder.analysis.resources import resolve_packaged_python_sources from python_deployment_builder.backends.uv_managed import UvManagedBackend from python_deployment_builder.models import ( ConfigurationPlan, @@ -76,6 +77,7 @@ def _source_entrypoint_compatible( def _deployment_mode( assessment: RepositoryAssessment, entry_point: EntrypointPlan | None, + repository_root: Path | None, ) -> tuple[str, str, str, list[str]]: runtime_resource_paths = { item.path @@ -101,9 +103,33 @@ def _deployment_mode( for item in assessment.write_locations if item.classification == "project_local" ] + analysis_root = repository_root + if analysis_root is None and assessment.repository.source_kind == "local": + candidate = Path(assessment.repository.source).expanduser() + analysis_root = candidate if candidate.is_dir() else None + wheel_backed_python = { + member.source_path + for member in resolve_packaged_python_sources(analysis_root, assessment.project) + } if analysis_root is not None else set() + source_only_python = sorted( + item.path + for item in assessment.file_inventory + if item.role == RepositoryFileRole.APPLICATION_SOURCE + and item.path not in wheel_backed_python + # The inventory includes conventional top-level launch scripts. They + # are not necessarily part of the authoritative installed surface + # (SimpleGeorefGUI retains one for direct developer use). Constrain + # package mode only when static import analysis proves the Python file + # is required by production source. + and any( + evidence.detail.startswith("Application source imports local module") + for evidence in item.evidence + ) + ) source_compatible, candidates = _source_entrypoint_compatible(assessment, entry_point) source_constraints = [ *(f"repository-adjacent resource: {item}" for item in adjacent), + *(f"source-only Python module: {item}" for item in source_only_python), *(f"project-local write: {item}" for item in project_writes), ] installable = bool( @@ -120,6 +146,14 @@ def _deployment_mode( [], ) if source_constraints: + if not installable: + return ( + "package", + "The authoritative entry point requires installation, but buildable project " + "metadata is incomplete.", + "INSTALLED_PROJECT_REQUIRED", + ["INSTALLED_PROJECT_REQUIRED: buildable project metadata is incomplete"], + ) return ( "package", "Source layout requirements conflict with an authoritative entry point that cannot " @@ -417,7 +451,7 @@ def create_deployment_plan( app_id = safe_application_id(name) entry_point = _entrypoint(assessment) mode, mode_rationale, mode_condition, mode_blockers = _deployment_mode( - assessment, entry_point + assessment, entry_point, repository_root ) runtime = UvManagedBackend().build_plan( app_id, diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index a9176e2..7cd3e0a 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -10,7 +10,10 @@ from python_deployment_builder.analysis.imports import scan_imports from python_deployment_builder.analysis.metadata import inspect_metadata from python_deployment_builder.analysis.repository import MaterializedRepository -from python_deployment_builder.analysis.resources import resolve_package_data_members +from python_deployment_builder.analysis.resources import ( + resolve_package_data_members, + resolve_packaged_python_sources, +) from python_deployment_builder.cli import main from python_deployment_builder.generation.acquisition import PreparationError from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit @@ -842,6 +845,55 @@ def test_wildcard_setuptools_package_data_uses_known_physical_package_mapping( assert "code/view.html" in _staging_files(tmp_path, assessment, source_plan, include=True) +@pytest.mark.parametrize( + ("namespaces", "expected"), + [ + (True, {"example_app", "example_app.data", "example_app.namespace"}), + (False, {"example_app"}), + ], +) +def test_pyproject_find_discovers_packages_for_wildcard_package_data( + tmp_path: Path, namespaces: bool, expected: set[str] +) -> None: + app = tmp_path / "src/example_app" + (app / "data").mkdir(parents=True) + (app / "namespace").mkdir() + (app / "__init__.py").write_text("", encoding="utf-8") + (app / "main.py").write_text("def main(): return 0\n", encoding="utf-8") + (app / "data/defaults.json").write_text("{}\n", encoding="utf-8") + (app / "tests").mkdir() + (app / "tests/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + f'''[project] +name = "discovered-data" +version = "1.0" +[tool.setuptools.packages.find] +where = ["src"] +include = ["example_app*"] +exclude = ["example_app.tests*"] +namespaces = {str(namespaces).lower()} +[tool.setuptools.package-data] +"*" = ["data/*.json"] +''', + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + members = resolve_package_data_members(tmp_path, project) + + assert set(project.packages) == expected + assert [(item.source_path, item.installed_member_path) for item in members] == [ + ("src/example_app/data/defaults.json", "example_app/data/defaults.json") + ] + assert { + (item.source_path, item.installed_member_path) + for item in resolve_packaged_python_sources(tmp_path, project) + } == { + ("src/example_app/__init__.py", "example_app/__init__.py"), + ("src/example_app/main.py", "example_app/main.py"), + } + + def test_package_data_exclusions_apply_after_safe_concrete_resolution(tmp_path: Path) -> None: for package in ("app", "other"): data = tmp_path / package / "data" @@ -905,6 +957,7 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( ) (package / "data/default.json").write_text('{"default": true}\n', encoding="utf-8") (package / "data/private.json").write_text('{"private": true}\n', encoding="utf-8") + (tmp_path / source_root / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") package_dir = "\npackage_dir =\n = src" if source_root else "" find_where = "\n[options.packages.find]\nwhere = src" if source_root else "" (tmp_path / "pyproject.toml").write_text( @@ -917,6 +970,8 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( version = 1.0 [options] packages = find: +py_modules = + helper python_requires = >=3.12{package_dir} [options.entry_points] console_scripts = @@ -953,6 +1008,8 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( "app": ["data/private.json"], "*": ["*.tmp"], } + assert assessment.project.packages == ["app"] + assert assessment.project.py_modules == ["helper"] assert resource.packaging_status == "packaged" assert inventory.role == RepositoryFileRole.RUNTIME_RESOURCE staged = _staging_files(tmp_path, assessment, source_plan, include=True) @@ -965,6 +1022,41 @@ def test_setup_cfg_package_data_is_authoritative_for_source_staging( assert all(member.evidence.file == "setup.cfg" for member in members) +def test_setup_cfg_find_uses_global_package_dir_and_filters(tmp_path: Path) -> None: + (tmp_path / "src/app/tests").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/module.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "src/app/tests/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "setup.cfg").write_text( + """[metadata] +Name = setup-discovered +Version = 1.0 +[options] +packages = find: +package_dir = + = src +[options.packages.find] +include = + app* +exclude = + app.tests* +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == ["app"] + assert project.source_roots == ["src"] + assert { + (item.source_path, item.installed_member_path) + for item in resolve_packaged_python_sources(tmp_path, project) + } == { + ("src/app/__init__.py", "app/__init__.py"), + ("src/app/module.py", "app/module.py"), + } + + def test_literal_setup_py_package_data_and_exclusions_share_the_resolver(tmp_path: Path) -> None: data = tmp_path / "app" / "data" data.mkdir(parents=True) @@ -977,21 +1069,31 @@ def test_literal_setup_py_package_data_and_exclusions_share_the_resolver(tmp_pat name="literal-data", version="1.0", packages=["app"], + py_modules=["helper"], package_data={"": ["data/*.json"]}, exclude_package_data={"": ["data/private.json"]}, ) """, encoding="utf-8", ) + (tmp_path / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") project = inspect_metadata(tmp_path).project assert project.package_data == {"*": ["data/*.json"]} assert project.exclude_package_data == {"*": ["data/private.json"]} + assert project.py_modules == ["helper"] assert [ (member.source_path, member.installed_member_path, member.evidence.file) for member in resolve_package_data_members(tmp_path, project) ] == [("app/data/defaults.json", "app/data/defaults.json", "setup.py")] + assert [ + (item.source_path, item.installed_member_path) + for item in resolve_packaged_python_sources(tmp_path, project) + ] == [ + ("app/__init__.py", "app/__init__.py"), + ("helper.py", "helper.py"), + ] def test_setup_cfg_standard_options_are_case_insensitive_and_package_data_is_not( diff --git a/tests/test_generation.py b/tests/test_generation.py index 1d4bbeb..eaaee54 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1568,6 +1568,117 @@ def test_application_wheel_resolves_wildcard_package_data_against_known_package( validate_application_wheel(wheel, assessment, plan) +def test_discovered_package_wildcard_data_is_required_in_application_wheel( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + (source / "src/example_app/data").mkdir(parents=True) + (source / "src/example_app/__init__.py").write_text("", encoding="utf-8") + (source / "src/example_app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "src/example_app/data/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "example-app" +version = "1.0" +[project.scripts] +example-app = "example_app.main:main" +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false +[tool.setuptools.package-data] +"*" = ["data/*.json"] +""", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + incomplete = _make_application_wheel( + tmp_path, + name="example-app", + version="1.0", + package="example_app", + target="example_app.main:main", + entry_group="console_scripts", + entry_name="example-app", + ) + + assert assessment.project.packages == ["example_app"] + with pytest.raises(PreparationError, match="example_app/data/defaults.json"): + validate_application_wheel(incomplete, assessment, plan, repository_root=source) + + complete = _rewrite_application_wheel( + incomplete, additions={"example_app/data/defaults.json": "{}\n"} + ) + assert validate_application_wheel(complete, assessment, plan, repository_root=source)[0] + + +@pytest.mark.parametrize( + ("py_modules", "missing_member"), + [(False, "app/util.py"), (True, "helper.py")], +) +def test_application_wheel_requires_authoritative_python_source_surface( + tmp_path: Path, py_modules: bool, missing_member: str +) -> None: + source = tmp_path / "source" + (source / "src/app").mkdir(parents=True) + (source / "src/app/__init__.py").write_text("", encoding="utf-8") + (source / "src/app/main.py").write_text( + "from . import util\ndef main(): return util.VALUE\n", encoding="utf-8" + ) + (source / "src/app/util.py").write_text("VALUE = 1\n", encoding="utf-8") + (source / "src/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + py_modules_text = "[tool.setuptools]\npy-modules = [\"helper\"]\n" if py_modules else "" + (source / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "source-surface" +version = "1.0" +[project.scripts] +source-surface = "app.main:main" +""" + + py_modules_text + + """[tool.setuptools.packages.find] +where = ["src"] +namespaces = false +""", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel( + tmp_path, + name="source-surface", + version="1.0", + package="app", + target="app.main:main", + entry_group="console_scripts", + entry_name="source-surface", + ) + + with pytest.raises(PreparationError, match=missing_member): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + complete = _rewrite_application_wheel( + wheel, + additions={ + "app/util.py": "VALUE = 1\n", + **({"helper.py": "VALUE = 1\n"} if py_modules else {}), + }, + ) + assert validate_application_wheel(complete, assessment, plan, repository_root=source)[0] + + @pytest.mark.parametrize( ("requires_python", "accepted"), [ diff --git a/tests/test_planning.py b/tests/test_planning.py index ee973f8..35df605 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -47,9 +47,13 @@ def _write_mode_project( source_constraint: bool = False, mapped: bool = False, ) -> None: - source = root / ("code" if mapped else "src" if layout == "src" else ".") + source = root / ( + "code" if mapped else "src/sample_app" if layout == "src" else "." + ) source.mkdir(parents=True, exist_ok=True) - module = source / ("main.py" if mapped else "sample_app.py") + module = source / ( + "main.py" if mapped else "__init__.py" if layout == "src" else "sample_app.py" + ) module.write_text( ( "from pathlib import Path\nRUNTIME = Path('runtime.json')\nRUNTIME.read_text()\n" @@ -263,6 +267,67 @@ def test_wheel_backed_package_data_resource_does_not_force_source_mode(tmp_path: ) +@pytest.mark.parametrize( + ("target", "py_modules", "expected"), + [ + ("app.main:main", False, ("source", "SOURCE_COMPATIBLE")), + ("installed_app.main:main", False, ("package", "DEPLOYMENT_MODE_CONFLICT")), + ("app.main:main", True, ("package", "PACKAGE_PREFERRED")), + ], +) +def test_promoted_standalone_source_requires_authoritative_wheel_membership( + tmp_path: Path, + target: str, + py_modules: bool, + expected: tuple[str, str], +) -> None: + root = tmp_path / "standalone-helper" + (root / "src/app").mkdir(parents=True) + (root / "src/app/__init__.py").write_text("", encoding="utf-8") + (root / "src/app/main.py").write_text( + "import helper\ndef main(): return helper.VALUE\n", encoding="utf-8" + ) + (root / "src/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + py_modules_text = "[tool.setuptools]\npy-modules = [\"helper\"]\n" if py_modules else "" + (root / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +[project] +name = "standalone-helper" +version = "1.0" +[project.scripts] +standalone-helper = """ + + repr(target) + + "\n" + + py_modules_text + + """[tool.setuptools.packages.find] +where = ["src"] +namespaces = false +""", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=root) + + assert any( + item.path == "src/helper.py" and item.role.value == "application_source" + for item in assessment.file_inventory + ) + assert (plan.deployment_mode, plan.deployment_mode_condition) == expected + if py_modules: + assert assessment.project.py_modules == ["helper"] + assert not any("src/helper.py" in item for item in plan.readiness.blockers) + else: + assert any("src/helper.py" in item for item in plan.readiness.blockers) == ( + expected[1] == "DEPLOYMENT_MODE_CONFLICT" + ) + + def test_target_plan_selects_source_gui_and_external_environment() -> None: plan = create_deployment_plan(_assess()) From 69f42ef06a7ff33a4072808fd4aad0c5393cbfda Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:19:32 -0400 Subject: [PATCH 18/59] Harden runtime precision and Git identity --- .../analysis/assessor.py | 46 ++--- .../generation/artifacts.py | 30 ++- .../planning/external_runtimes.py | 7 +- .../planning/extras.py | 24 ++- .../planning/index.py | 22 ++- .../planning/lockfile.py | 28 ++- .../planning/planner.py | 32 +++- .../planning/policies.py | 177 +++++++++++++++++- tests/test_assessment.py | 81 ++++++-- tests/test_generation.py | 31 +++ tests/test_packaging.py | 22 ++- tests/test_planning.py | 56 +++++- 12 files changed, 469 insertions(+), 87 deletions(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index fe30ae9..db28974 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import subprocess from datetime import UTC, datetime from pathlib import Path @@ -41,38 +42,25 @@ def _git_revision(root: Path) -> str | None: - """Read the nearest normal enclosing Git HEAD without invoking Git.""" + """Read the selected root's Git HEAD using Git's own repository semantics.""" - # A PDB source may be a project nested within a monorepo. Its provenance is - # still the enclosing worktree's HEAD, while all later staging paths remain - # relative to this selected project root. - git_dir: Path | None = None - for candidate in (root.resolve(), *root.resolve().parents): - possible = candidate / ".git" - if possible.is_dir(): - git_dir = possible - break - if git_dir is None: - return None - head_path = git_dir / "HEAD" - if not head_path.is_file(): - return None + # ``.git`` may be a directory, an indirection file for a linked worktree, + # or a submodule gitdir reference. Git plumbing preserves that identity + # while still reporting the enclosing worktree's revision for a selected + # nested project directory. try: - head = head_path.read_text(encoding="ascii").strip() - if head.startswith("ref: "): - ref = head.removeprefix("ref: ") - if not ref.startswith("refs/") or not re.fullmatch(r"[A-Za-z0-9_./-]+", ref): - return None - if ".." in Path(ref).parts: - return None - ref_path = git_dir / ref - value = ref_path.read_text(encoding="ascii").strip() if ref_path.is_file() else "" - else: - value = head - valid_object_id = re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})", value) - return value.lower() if valid_object_id else None - except OSError: + result = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--verify", "HEAD"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): return None + value = result.stdout.strip() + valid_object_id = re.fullmatch(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})", value) + return value.lower() if result.returncode == 0 and valid_object_id else None def assess_repository(repository: MaterializedRepository) -> RepositoryAssessment: diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 1cb9d9e..3988b6e 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -18,7 +18,7 @@ from pathlib import Path, PurePosixPath from packaging.requirements import InvalidRequirement, Requirement -from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.specifiers import InvalidSpecifier from packaging.utils import canonicalize_name, parse_wheel_filename from packaging.version import InvalidVersion, Version @@ -38,7 +38,10 @@ target_marker_applies, wheel_matches, ) -from python_deployment_builder.planning.policies import python_satisfies +from python_deployment_builder.planning.policies import ( + MinorPythonCompatibility, + minor_python_compatibility, +) from python_deployment_builder.security_policy import ( is_secret_filename, is_textual_wheel_member, @@ -238,7 +241,7 @@ def _require_wheel_metadata(message, *, wheel: Path) -> set[str]: def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> None: - """Apply the planner's conservative minor-as-.0 policy to wheel Core Metadata.""" + """Require a precision-safe Requires-Python proof for a minor-only runtime.""" values = metadata.get_all("Requires-Python", []) if not values: @@ -247,16 +250,21 @@ def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> No raise PreparationError(f"Malformed Requires-Python metadata in wheel: {wheel.name}") constraint = values[0].strip() try: - SpecifierSet(constraint) - except InvalidSpecifier as exc: + compatibility = minor_python_compatibility(plan.runtime.python_version, constraint) + except (InvalidSpecifier, ValueError) as exc: raise PreparationError( f"Malformed Requires-Python metadata in wheel: {wheel.name}" ) from exc - if not python_satisfies(plan.runtime.python_version, constraint): + if compatibility == MinorPythonCompatibility.INCOMPATIBLE: raise PreparationError( f"Wheel Requires-Python {constraint!r} is incompatible with selected Python " f"{plan.runtime.python_version}." ) + if compatibility == MinorPythonCompatibility.UNPROVABLE: + raise PreparationError( + f"Wheel Requires-Python {constraint!r} cannot be proven for minor-only selected " + f"Python {plan.runtime.python_version}." + ) def _validate_application_security( @@ -412,21 +420,25 @@ def _application_requirement_applies(requirement: Requirement, plan: DeploymentP def _validate_dependency_extra_closure(requirement: Requirement, graph) -> None: """Prove a wheel dependency's requested extras are activated by the selected lock graph.""" - requested = set(requirement.extras) + requested = {canonicalize_name(extra) for extra in requirement.extras} name = canonicalize_name(requirement.name) candidates = [ dependency for dependency in graph.dependencies if canonicalize_name(dependency.name) == name and Version(dependency.version) in requirement.specifier - and requested <= set(dependency.requested_dependency_extras) + and requested + <= {canonicalize_name(extra) for extra in dependency.requested_dependency_extras} ] if not candidates: raise PreparationError( "Application wheel Requires-Dist dependency extra cannot be proven against the " f"selected locked environment: {requirement.name}[{','.join(sorted(requested))}]" ) - if not any(requested <= set(item.available_dependency_extras) for item in candidates): + if not any( + requested <= {canonicalize_name(extra) for extra in item.available_dependency_extras} + for item in candidates + ): raise PreparationError( "Application wheel Requires-Dist dependency extra is not declared by the locked " f"dependency: {requirement.name}[{','.join(sorted(requested))}]" diff --git a/src/python_deployment_builder/planning/external_runtimes.py b/src/python_deployment_builder/planning/external_runtimes.py index 5095b0b..14b1b64 100644 --- a/src/python_deployment_builder/planning/external_runtimes.py +++ b/src/python_deployment_builder/planning/external_runtimes.py @@ -37,13 +37,18 @@ class ExternalRuntimeRule: def external_runtime_requirements( dependencies: list[DependencyAssessment], selected_extras: list[str] ) -> list[ExternalRuntimePlan]: + selected_extra_names = {canonicalize_name(name) for name in selected_extras} by_name = {canonicalize_name(item.distribution_name): item for item in dependencies} results: list[ExternalRuntimePlan] = [] for rule in RULES: dependency = by_name.get(canonicalize_name(rule.distribution)) if dependency is None: continue - feature = dependency.group if dependency.group in selected_extras else None + feature = ( + dependency.group + if canonicalize_name(dependency.group) in selected_extra_names + else None + ) results.append( ExternalRuntimePlan( name=rule.name, diff --git a/src/python_deployment_builder/planning/extras.py b/src/python_deployment_builder/planning/extras.py index 497686d..241bb3a 100644 --- a/src/python_deployment_builder/planning/extras.py +++ b/src/python_deployment_builder/planning/extras.py @@ -16,15 +16,25 @@ def validate_selected_extras( assessment: RepositoryAssessment, selected_extras: list[str] ) -> list[str]: - available = set(assessment.project.optional_dependency_groups) - selected = list(dict.fromkeys(selected_extras)) - unknown = sorted(set(selected) - available) + available = { + canonicalize_name(name): name for name in assessment.project.optional_dependency_groups + } + selected: list[str] = [] + seen: set[str] = set() + for item in selected_extras: + canonical = canonicalize_name(item) + if canonical not in seen: + selected.append(item) + seen.add(canonical) + unknown = sorted( + item for item in selected if canonicalize_name(item) not in available + ) if unknown: raise ValueError( "Unknown optional dependency extra(s): " + ", ".join(unknown) + ". Available extras: " - + (", ".join(sorted(available)) or "none") + + (", ".join(sorted(available.values())) or "none") ) return selected @@ -36,8 +46,9 @@ def selected_dependencies( architecture: str, ) -> list[DependencyAssessment]: dependencies: list[DependencyAssessment] = [] + selected_names = {canonicalize_name(name) for name in selected_extras} for dependency in assessment.dependencies: - if dependency.group == "runtime" or dependency.group in selected_extras: + if dependency.group == "runtime" or canonicalize_name(dependency.group) in selected_names: extra = dependency.group if dependency.group != "runtime" else "" if marker_applies( dependency.environment_marker, @@ -74,9 +85,10 @@ def build_extra_plans( architecture: str, ) -> list[OptionalExtraPlan]: recommended = _recommended_groups(assessment) + selected_names = {canonicalize_name(item) for item in selected_extras} plans: list[OptionalExtraPlan] = [] for name in assessment.project.optional_dependency_groups: - selected = name in selected_extras + selected = canonicalize_name(name) in selected_names dependencies = [ ExtraDependencyPlan( distribution_name=item.distribution_name, diff --git a/src/python_deployment_builder/planning/index.py b/src/python_deployment_builder/planning/index.py index c8335aa..c787762 100644 --- a/src/python_deployment_builder/planning/index.py +++ b/src/python_deployment_builder/planning/index.py @@ -14,7 +14,7 @@ from packaging.markers import InvalidMarker, Marker, _evaluate_markers from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.tags import compatible_tags, cpython_tags -from packaging.utils import InvalidWheelFilename, parse_wheel_filename +from packaging.utils import InvalidWheelFilename, canonicalize_name, parse_wheel_filename from packaging.version import InvalidVersion, Version from python_deployment_builder.models import ( @@ -24,6 +24,10 @@ OnlineIndexContext, WheelCompatibility, ) +from python_deployment_builder.planning.policies import ( + MinorPythonCompatibility, + minor_python_compatibility, +) PYPI_JSON_BASE = "https://pypi.org/pypi" JsonFetcher = Callable[[str], dict[str, Any]] @@ -61,7 +65,10 @@ def target_marker_environment( "platform_system": "Windows", "python_version": python_version, "sys_platform": "win32", - "extra": extra, + # Packaging 26's private evaluator expects an already PEP-685 + # normalized environment value. Normalize here rather than depending + # on version-specific private-evaluator behavior. + "extra": canonicalize_name(extra) if extra else "", } @@ -171,12 +178,17 @@ def _release_version( def _supports_python(requires_python: str | None, python_version: str) -> bool: + """Return true only for a precision-safe published-wheel compatibility proof.""" + if not requires_python: return True try: - return f"{python_version}.0" in SpecifierSet(requires_python) - except InvalidSpecifier: - return True + return ( + minor_python_compatibility(python_version, requires_python) + == MinorPythonCompatibility.COMPATIBLE + ) + except (InvalidSpecifier, ValueError): + return False def marker_applies( diff --git a/src/python_deployment_builder/planning/lockfile.py b/src/python_deployment_builder/planning/lockfile.py index 9ebe7f4..3444e15 100644 --- a/src/python_deployment_builder/planning/lockfile.py +++ b/src/python_deployment_builder/planning/lockfile.py @@ -63,6 +63,24 @@ def _requested_dependency_extras(edge: dict[str, object]) -> tuple[str, ...]: return tuple(sorted({value for value in values if isinstance(value, str) and value})) +def _optional_dependencies_for_extra( + optional: object, extra: str +) -> object: + """Return an optional-dependency group using PEP-685 extra identity.""" + + if not isinstance(optional, dict): + return None + canonical = canonicalize_name(extra) + return next( + ( + values + for name, values in optional.items() + if isinstance(name, str) and canonicalize_name(name) == canonical + ), + None, + ) + + def inspect_uv_lock( repository_root: Path, application_name: str, @@ -161,7 +179,13 @@ def enqueue_edges( optional = root.get("optional-dependencies") if isinstance(optional, dict): for extra in selected_extras: - enqueue_edges(root_name, optional.get(extra), [root_name], True, extra) + enqueue_edges( + root_name, + _optional_dependencies_for_extra(optional, extra), + [root_name], + True, + extra, + ) locked: dict[tuple[str, str], LockedDependency] = {} expanded: set[tuple[str, str, str | None, tuple[str, ...]]] = set() @@ -229,7 +253,7 @@ def enqueue_edges( for extra in requested_extras: enqueue_edges( name, - optional.get(extra), + _optional_dependencies_for_extra(optional, extra), chain, False, selected_extra, diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 7df62b0..503b244 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -7,6 +7,7 @@ from datetime import UTC, datetime from pathlib import Path +from packaging.specifiers import InvalidSpecifier from packaging.utils import canonicalize_name from python_deployment_builder import __version__ @@ -45,8 +46,9 @@ from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.planning.platforms import windows_finding_treatments from python_deployment_builder.planning.policies import ( + MinorPythonCompatibility, candidate_python_versions, - python_satisfies, + minor_python_compatibility, safe_application_id, ) @@ -246,13 +248,26 @@ def _python_candidates( ) -> tuple[str, list[PythonCandidatePlan]]: candidates: list[PythonCandidatePlan] = [] for version in candidate_python_versions(assessment): - satisfies = python_satisfies(version, assessment.python.requires_python) + try: + precision = minor_python_compatibility( + version, assessment.python.requires_python + ) + except (InvalidSpecifier, ValueError): + precision = MinorPythonCompatibility.INCOMPATIBLE + satisfies = precision == MinorPythonCompatibility.COMPATIBLE compatibility = "viable" if satisfies else "incompatible" - rationale = ( - "Satisfies declared Python metadata." - if satisfies - else "Does not satisfy the declared requires-python constraint." - ) + rationale = { + MinorPythonCompatibility.COMPATIBLE: "Satisfies declared Python metadata.", + MinorPythonCompatibility.INCOMPATIBLE: ( + "Does not satisfy the declared requires-python constraint." + ), + MinorPythonCompatibility.UNPROVABLE: ( + "Cannot prove patch-sensitive requires-python metadata for a minor-only " + "managed runtime." + ), + }[precision] + if precision == MinorPythonCompatibility.UNPROVABLE: + compatibility = "unverified" if satisfies and online: checked = [ item @@ -412,12 +427,13 @@ def create_deployment_plan( """Plan only: no target code, builds, lock updates, or environment mutations occur.""" selected_extras = validate_selected_extras(assessment, selected_extras or []) + selected_extra_names = {canonicalize_name(name) for name in selected_extras} has_authoritative_entrypoint = bool(assessment.project.entry_points) selected_inspection_dependencies = [ item for item in assessment.dependencies if item.group == "runtime" - or item.group in selected_extras + or canonicalize_name(item.group) in selected_extra_names ] informational_inspection_dependencies: list[DependencyAssessment] = [] if not has_authoritative_entrypoint: diff --git a/src/python_deployment_builder/planning/policies.py b/src/python_deployment_builder/planning/policies.py index e80d4a3..7a1c2db 100644 --- a/src/python_deployment_builder/planning/policies.py +++ b/src/python_deployment_builder/planning/policies.py @@ -3,25 +3,194 @@ from __future__ import annotations import re +from enum import StrEnum from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import InvalidVersion, Version from python_deployment_builder.models import RepositoryAssessment PYTHON_POLICY_ORDER = ("3.12", "3.13", "3.11", "3.14") +class MinorPythonCompatibility(StrEnum): + """Whether a requirement is provable for a minor-only managed runtime.""" + + COMPATIBLE = "compatible" + INCOMPATIBLE = "incompatible" + UNPROVABLE = "unprovable" + + def safe_application_id(value: str) -> str: normalized = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") return normalized[:64] or "python-application" -def python_satisfies(version: str, requires_python: str | None) -> bool: +def _minor_bounds(version: str) -> tuple[Version, Version]: + """Return the closed/open patch interval represented by a Python minor.""" + + match = re.fullmatch(r"(\d+)\.(\d+)", version) + if match is None: + raise ValueError(f"Python policy version must be major.minor: {version!r}") + major, minor = (int(value) for value in match.groups()) + return Version(f"{major}.{minor}.0"), Version(f"{major}.{minor + 1}.0") + + +def _combine_minor_results( + values: list[MinorPythonCompatibility], +) -> MinorPythonCompatibility: + if MinorPythonCompatibility.INCOMPATIBLE in values: + return MinorPythonCompatibility.INCOMPATIBLE + if MinorPythonCompatibility.UNPROVABLE in values: + return MinorPythonCompatibility.UNPROVABLE + return MinorPythonCompatibility.COMPATIBLE + + +def _range_result( + operator: str, boundary: Version, low: Version, high: Version +) -> MinorPythonCompatibility: + """Classify a simple ordered comparison over every possible patch release.""" + + if operator == ">=": + return ( + MinorPythonCompatibility.COMPATIBLE + if boundary <= low + else MinorPythonCompatibility.INCOMPATIBLE + if boundary >= high + else MinorPythonCompatibility.UNPROVABLE + ) + if operator == ">": + return ( + MinorPythonCompatibility.COMPATIBLE + if boundary < low + else MinorPythonCompatibility.INCOMPATIBLE + if boundary >= high + else MinorPythonCompatibility.UNPROVABLE + ) + if operator == "<": + return ( + MinorPythonCompatibility.COMPATIBLE + if boundary >= high + else MinorPythonCompatibility.INCOMPATIBLE + if boundary <= low + else MinorPythonCompatibility.UNPROVABLE + ) + if operator == "<=": + return ( + MinorPythonCompatibility.COMPATIBLE + if boundary >= high + else MinorPythonCompatibility.INCOMPATIBLE + if boundary < low + else MinorPythonCompatibility.UNPROVABLE + ) + raise ValueError(f"Unsupported range operator: {operator}") + + +def _wildcard_result( + operator: str, raw_version: str, version: str +) -> MinorPythonCompatibility: + prefix = raw_version.removesuffix(".*").split(".") + minor_parts = version.split(".") + matches = minor_parts[: len(prefix)] == prefix + if operator == "==": + return ( + MinorPythonCompatibility.COMPATIBLE + if matches + else MinorPythonCompatibility.INCOMPATIBLE + ) + if operator == "!=": + return ( + MinorPythonCompatibility.INCOMPATIBLE + if matches + else MinorPythonCompatibility.COMPATIBLE + ) + return MinorPythonCompatibility.UNPROVABLE + + +def _compatible_upper_bound(version: Version) -> Version | None: + """Return the PEP 440 compatible-release upper bound for a plain release.""" + + if version.pre or version.post is not None or version.dev is not None or version.local: + return None + release = version.release + if len(release) < 2: + return None + prefix = list(release[:-1]) + prefix[-1] += 1 + return Version(".".join(str(value) for value in prefix)) + + +def minor_python_compatibility( + version: str, requires_python: str | None +) -> MinorPythonCompatibility: + """Evaluate a Python specifier without pretending a selected minor has patch ``.0``. + + PDB provisions a major/minor runtime. A result is compatible only when + every possible patch release of that minor satisfies the requirement; + incompatible only when none can satisfy it. All other forms are kept + deliberately unprovable. + """ + if not requires_python: - return True + return MinorPythonCompatibility.COMPATIBLE + low, high = _minor_bounds(version) + specifiers = SpecifierSet(requires_python) + results: list[MinorPythonCompatibility] = [] + for specifier in specifiers: + operator = specifier.operator + raw = specifier.version + if raw.endswith(".*"): + results.append(_wildcard_result(operator, raw, version)) + continue + try: + boundary = Version(raw) + except InvalidVersion: + results.append(MinorPythonCompatibility.UNPROVABLE) + continue + if operator in {">=", ">", "<", "<="}: + results.append(_range_result(operator, boundary, low, high)) + elif operator == "~=": + upper = _compatible_upper_bound(boundary) + results.append( + MinorPythonCompatibility.UNPROVABLE + if upper is None + else _combine_minor_results( + [ + _range_result(">=", boundary, low, high), + _range_result("<", upper, low, high), + ] + ) + ) + elif operator in {"==", "!="}: + exact = _range_result(">=", boundary, low, high) + # An exact match is only invariant when it is outside the entire + # selected-minor interval. Otherwise a patch fact is required. + if operator == "==": + results.append( + MinorPythonCompatibility.INCOMPATIBLE + if exact == MinorPythonCompatibility.INCOMPATIBLE + else MinorPythonCompatibility.UNPROVABLE + ) + else: + results.append( + MinorPythonCompatibility.COMPATIBLE + if exact == MinorPythonCompatibility.INCOMPATIBLE + else MinorPythonCompatibility.UNPROVABLE + ) + else: + results.append(MinorPythonCompatibility.UNPROVABLE) + return _combine_minor_results(results) + + +def python_satisfies(version: str, requires_python: str | None) -> bool: + """Return true only for a compatibility proof, preserving the legacy API.""" + try: - return f"{version}.0" in SpecifierSet(requires_python) - except InvalidSpecifier: + return ( + minor_python_compatibility(version, requires_python) + == MinorPythonCompatibility.COMPATIBLE + ) + except (InvalidSpecifier, ValueError): return False diff --git a/tests/test_assessment.py b/tests/test_assessment.py index 83715ef..d25d7cf 100644 --- a/tests/test_assessment.py +++ b/tests/test_assessment.py @@ -1,6 +1,7 @@ +import subprocess from pathlib import Path -from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.assessor import _git_revision, assess_repository from python_deployment_builder.analysis.repository import MaterializedRepository from python_deployment_builder.models import SuitabilityRating from python_deployment_builder.reporting.markdown import render_assessment_markdown @@ -46,18 +47,68 @@ def test_markdown_contains_human_readable_sections() -> None: assert "REPOSITORY_ADJACENT_RESOURCES" in rendered -def test_git_revision_requires_a_full_hex_object_id(tmp_path: Path) -> None: - (tmp_path / "pyproject.toml").write_text( - '[project]\nname = "revision-test"\nversion = "1.0.0"\n', - encoding="utf-8", +def _git(path: Path, *arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(path), *arguments], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _committed_repository(root: Path, *, name: str = "revision-test") -> str: + root.mkdir(parents=True, exist_ok=True) + (root / "pyproject.toml").write_text( + f'[project]\nname = "{name}"\nversion = "1.0.0"\n', encoding="utf-8" ) - git = tmp_path / ".git" - (git / "refs" / "heads").mkdir(parents=True) - (git / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii") - ref = git / "refs" / "heads" / "main" - ref.write_text("a" * 40 + "\n", encoding="ascii") - repository = MaterializedRepository(root=tmp_path, source=str(tmp_path), source_kind="local") - assert assess_repository(repository).repository.revision == "a" * 40 - - ref.write_text("not-a-commit\n", encoding="ascii") - assert assess_repository(repository).repository.revision is None + _git(root.parent, "init", "-q", str(root)) + _git(root, "config", "user.name", "PDB Test") + _git(root, "config", "user.email", "pdb@example.invalid") + _git(root, "add", ".") + _git(root, "commit", "-qm", "fixture") + return _git(root, "rev-parse", "HEAD") + + +def test_git_revision_uses_selected_repository_git_identity(tmp_path: Path) -> None: + normal = tmp_path / "normal" + normal_head = _committed_repository(normal) + assert _git_revision(normal) == normal_head + + nested = normal / "projects" / "nested" + nested.mkdir(parents=True) + assert _git_revision(nested) == normal_head + + child = tmp_path / "child" + child_head = _committed_repository(child, name="child") + superproject = tmp_path / "superproject" + _committed_repository(superproject, name="superproject") + _git( + superproject, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + str(child), + "app-submodule", + ) + _git(superproject, "commit", "-qm", "add submodule") + assert _git(superproject, "rev-parse", "HEAD") != child_head + assert (superproject / "app-submodule/.git").is_file() + assert _git_revision(superproject / "app-submodule") == child_head + + linked = tmp_path / "linked-worktree" + _git(normal, "worktree", "add", "-q", "-b", "linked", str(linked)) + assert (linked / ".git").is_file() + linked_head = _git(linked, "rev-parse", "HEAD") + assert _git_revision(linked) == linked_head + _git(linked, "checkout", "--detach", "-q") + assert _git_revision(linked) == linked_head + assert _git_revision(tmp_path / "not-git") is None + + +def test_assessment_revision_agrees_with_generation_git_identity(tmp_path: Path) -> None: + root = tmp_path / "revision-project" + head = _committed_repository(root) + repository = MaterializedRepository(root=root, source=str(root), source_kind="local") + + assert assess_repository(repository).repository.revision == head diff --git a/tests/test_generation.py b/tests/test_generation.py index eaaee54..f45e448 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -718,6 +718,31 @@ def test_application_wheel_requires_dist_selected_extra_controls_marker(tmp_path ) +@pytest.mark.parametrize("selected_extra", ["feature_one", "feature-one", "feature.one"]) +def test_application_wheel_extra_marker_normalizes_selected_extra_identity( + tmp_path: Path, selected_extra: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, repository_root=source), + [("helper", "1.0")], + extras=[selected_extra], + ) + + validate_application_wheel( + _make_application_wheel( + tmp_path, requires_dist_values=['helper>=1; extra == "feature_one"'] + ), + assessment, + plan, + ) + + @pytest.mark.parametrize( ("requires_dist", "error"), [ @@ -1685,8 +1710,11 @@ def test_application_wheel_requires_authoritative_python_source_surface( (None, True), (">=3.11", True), (">=3.12,<3.13", True), + ("==3.12.*", True), (">=3.13", False), ("<3.12", False), + (">=3.12.1", False), + ("<3.12.1", False), ], ) def test_application_wheel_requires_python_uses_selected_minor_policy( @@ -3583,8 +3611,11 @@ def test_approved_wheel_rejects_wrong_name_version_and_metadata(tmp_path: Path) (None, True), (">=3.11", True), (">=3.12,<3.13", True), + ("==3.12.*", True), (">=3.13", False), ("<3.12", False), + (">=3.12.1", False), + ("<3.12.1", False), ], ) def test_approved_wheel_requires_python_uses_selected_minor_policy( diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 0bcc7da..217bc9b 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -7,6 +7,7 @@ import json import os import shutil +import subprocess import zipfile from pathlib import Path @@ -313,12 +314,20 @@ def test_source_provenance_remains_frozen_after_repository_advances( ) -> None: source = tmp_path / "source" shutil.copytree(FIXTURES / "prepared_gui", source) - git = source / ".git" - (git / "refs" / "heads").mkdir(parents=True) - (git / "HEAD").write_text("ref: refs/heads/main\n", encoding="ascii") - ref = git / "refs" / "heads" / "main" - old_revision = "a" * 40 - ref.write_text(old_revision + "\n", encoding="ascii") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + old_revision = subprocess.run( + ["git", "-C", str(source), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() fake_uv = tmp_path / "uv.exe" fake_uv.write_bytes(b"verified uv") monkeypatch.setattr( @@ -339,7 +348,6 @@ def test_source_provenance_remains_frozen_after_repository_advances( recorded = json.loads( (kit / "deployment" / "manifest.json").read_text(encoding="utf-8") ) - ref.write_text("b" * 40 + "\n", encoding="ascii") (source / "prepared_gui.py").write_text("advanced source", encoding="utf-8") result = package_deployment_kit(kit, output_directory=tmp_path / "dist") diff --git a/tests/test_planning.py b/tests/test_planning.py index 35df605..2e80a5f 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -25,7 +25,11 @@ from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.planning.planner import create_deployment_plan from python_deployment_builder.planning.platforms import windows_finding_treatments -from python_deployment_builder.planning.policies import safe_application_id +from python_deployment_builder.planning.policies import ( + MinorPythonCompatibility, + minor_python_compatibility, + safe_application_id, +) from python_deployment_builder.reporting.markdown import render_deployment_plan_markdown FIXTURES = Path(__file__).parent / "fixtures" @@ -393,6 +397,18 @@ def test_policy_uses_next_supported_python_when_312_is_rejected() -> None: assert plan.runtime.python_version == "3.13" +def test_policy_skips_patch_unprovable_candidate_for_minor_only_runtime() -> None: + assessment = _assess() + assessment.python.requires_python = ">=3.12.1" + + plan = create_deployment_plan(assessment) + + candidate_312 = next(item for item in plan.python_candidates if item.version == "3.12") + assert candidate_312.compatibility == "unverified" + assert not candidate_312.satisfies_requires_python + assert plan.runtime.python_version == "3.13" + + def test_blocking_assessment_gates_generation_policy() -> None: assessment = _assess() assessment.rating = SuitabilityRating.RED @@ -571,6 +587,44 @@ def test_windows_environment_markers_are_applied() -> None: assert not marker_applies("sys_platform == 'linux'", "3.12", "x86_64", extra="map") +@pytest.mark.parametrize( + ("constraint", "expected"), + [ + (">=3.12", MinorPythonCompatibility.COMPATIBLE), + ("<3.13", MinorPythonCompatibility.COMPATIBLE), + (">=3.13", MinorPythonCompatibility.INCOMPATIBLE), + ("<3.12", MinorPythonCompatibility.INCOMPATIBLE), + ("==3.12.*", MinorPythonCompatibility.COMPATIBLE), + (">=3.12.1", MinorPythonCompatibility.UNPROVABLE), + ("<3.12.1", MinorPythonCompatibility.UNPROVABLE), + ("==3.12.0", MinorPythonCompatibility.UNPROVABLE), + ("!=3.12.5", MinorPythonCompatibility.UNPROVABLE), + ("~=3.12.1", MinorPythonCompatibility.UNPROVABLE), + ], +) +def test_minor_python_compatibility_does_not_fabricate_patch_precision( + constraint: str, expected: MinorPythonCompatibility +) -> None: + assert minor_python_compatibility("3.12", constraint) == expected + + +@pytest.mark.parametrize("selected", ["feature_one", "feature-one", "feature.one"]) +def test_target_marker_environment_normalizes_selected_extra(selected: str) -> None: + assert ( + target_marker_applicability( + 'extra == "feature_one"', "3.12", "x86_64", extra=selected + ) + == TargetMarkerApplicability.APPLIES + ) + assert target_marker_environment("3.12", "x86_64", extra=selected)["extra"] == "feature-one" + assert ( + target_marker_applicability( + 'extra == "feature_one"', "3.12", "x86_64", extra="other" + ) + == TargetMarkerApplicability.DOES_NOT_APPLY + ) + + @pytest.mark.parametrize( ("marker", "expected"), [ From 9b88c90228fb86f695d91bf1c2ff09b21e5ede56 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:14:59 -0400 Subject: [PATCH 19/59] Validate wheel dist-info identity --- .../generation/artifacts.py | 80 +++++++++++++++++ tests/test_generation.py | 85 ++++++++++++++++++- 2 files changed, 163 insertions(+), 2 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 3988b6e..db0aa15 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -186,9 +186,56 @@ def _validate_record( raise PreparationError(f"Wheel RECORD hash mismatch for {name}: {wheel.name}") +def _identity_directory_matches( + directory: str, + *, + suffix: str, + expected_name: str, + expected_version: Version, + wheel: Path, +) -> None: + """Require an installed metadata directory to identify this wheel. + + Wheel writers normally use one normalized ``name-version`` separator, but + consumers must tolerate historical spellings that retain punctuation in + the distribution name. Try every separator and accept only one semantic + interpretation rather than assuming a dash cannot occur in either part. + """ + + if not directory.endswith(suffix): # pragma: no cover - caller invariant + raise PreparationError(f"Malformed wheel identity directory: {wheel.name}") + stem = directory[: -len(suffix)] + candidates: list[tuple[str, Version]] = [] + for index, character in enumerate(stem): + if character != "-": + continue + candidate_name = stem[:index] + candidate_version = stem[index + 1 :] + if not candidate_name or not candidate_version: + continue + try: + parsed_version = Version(candidate_version) + except InvalidVersion: + continue + if ( + canonicalize_name(candidate_name) == expected_name + and parsed_version == expected_version + ): + candidates.append((candidate_name, parsed_version)) + if len(candidates) != 1: + raise PreparationError( + f"Wheel {suffix} directory identity does not match its filename: {wheel.name}" + ) + + def _dist_info_members( members: dict[str, zipfile.ZipInfo], wheel: Path ) -> tuple[str, str, str]: + try: + filename_name, filename_version, _build, _tags = parse_wheel_filename(wheel.name) + except ValueError as exc: # pragma: no cover - validated by callers first + raise PreparationError(f"Malformed wheel filename: {wheel.name}") from exc + expected_name = canonicalize_name(str(filename_name)) metadata_names = [ name for name in members @@ -200,6 +247,39 @@ def _dist_info_members( ) metadata_name = metadata_names[0] dist_info = PurePosixPath(metadata_name).parent.as_posix() + dist_info_roots = { + path.parts[0] + for name in members + if (path := PurePosixPath(name)).parts and path.parts[0].endswith(".dist-info") + } + if dist_info_roots != {dist_info}: + raise PreparationError( + f"Wheel must contain exactly one distribution dist-info directory: {wheel.name}" + ) + _identity_directory_matches( + dist_info, + suffix=".dist-info", + expected_name=expected_name, + expected_version=filename_version, + wheel=wheel, + ) + data_roots = { + path.parts[0] + for name in members + if (path := PurePosixPath(name)).parts and path.parts[0].endswith(".data") + } + if len(data_roots) > 1: + raise PreparationError( + f"Wheel contains multiple distribution data directories: {wheel.name}" + ) + for data_root in data_roots: + _identity_directory_matches( + data_root, + suffix=".data", + expected_name=expected_name, + expected_version=filename_version, + wheel=wheel, + ) return metadata_name, f"{dist_info}/WHEEL", f"{dist_info}/RECORD" diff --git a/tests/test_generation.py b/tests/test_generation.py index f45e448..8303988 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -83,13 +83,14 @@ def _make_wheel( name: str = "proxy-tools", version: str = "0.1.0", *, + dist_info: str | None = None, requires_python: str | None = None, requires_python_values: list[str] | None = None, wheel_version: str = "1.0", ) -> Path: normalized = name.replace("-", "_") wheel = path / f"{normalized}-{version}-py3-none-any.whl" - dist_info = f"{normalized}-{version}.dist-info" + dist_info = dist_info or f"{normalized}-{version}.dist-info" files = { f"{dist_info}/METADATA": ( f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" @@ -129,10 +130,11 @@ def _make_application_wheel( requires_python_values: list[str] | None = None, requires_dist_values: list[str] | None = None, wheel_version: str = "1.0", + dist_info: str | None = None, ) -> Path: normalized = name.replace("-", "_") wheel = path / f"{normalized}-{version}-py3-none-any.whl" - dist_info = f"{normalized}-{version}.dist-info" + dist_info = dist_info or f"{normalized}-{version}.dist-info" files = { f"{package}/__init__.py": "", f"{package}/main.py": "def main(): return 0\n", @@ -651,6 +653,85 @@ def test_unsupported_wheel_version_rejects_approved_dependency_wheel(tmp_path: P ) +@pytest.mark.parametrize( + ("dist_info", "accepted"), + [ + ("mapped_app-1.2.3.dist-info", True), + ("Mapped_App-1.2.3.dist-info", True), + ("wrong_name-1.2.3.dist-info", False), + ("mapped_app-2.0.dist-info", False), + ("wrong_name-2.0.dist-info", False), + ("mapped_app-not-a-version.dist-info", False), + ], +) +def test_application_wheel_dist_info_identity_matches_filename( + tmp_path: Path, dist_info: str, accepted: bool +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path, dist_info=dist_info) + + if accepted: + assert validate_application_wheel(wheel, assessment, plan)[0].filename == wheel.name + else: + with pytest.raises(PreparationError, match="directory identity"): + validate_application_wheel(wheel, assessment, plan) + + +def test_dist_info_namespace_and_data_identity_are_checked_before_record(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={ + "mapped_app-1.2.3.dist-info/licenses/LICENSE": "license\n", + "mapped_app-1.2.3.dist-info/sboms/source.json": "{}\n", + "other-1.0.dist-info/licenses/LICENSE": "other license\n", + }, + ) + with pytest.raises(PreparationError, match="exactly one distribution dist-info"): + validate_application_wheel(wheel, assessment, plan) + + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"wrong_name-1.2.3.data/purelib/unused.txt": "unused\n"}, + ) + with pytest.raises(PreparationError, match=".data directory identity"): + validate_application_wheel(wheel, assessment, plan) + + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={ + "mapped_app-1.2.3.dist-info/licenses/LICENSE": "license\n", + "mapped_app-1.2.3.dist-info/sboms/source.json": "{}\n", + "mapped_app-1.2.3.data/purelib/unused.txt": "unused\n", + }, + ) + assert validate_application_wheel(wheel, assessment, plan)[0].filename == wheel.name + + +def test_approved_wheel_uses_shared_dist_info_identity_validation(tmp_path: Path) -> None: + plan = _plan("optional_map_app", ["map"]) + wrong = _make_wheel(tmp_path, dist_info="wrong_name-0.1.0.dist-info") + + with pytest.raises(PreparationError, match="directory identity"): + validate_approved_wheel(f"proxy-tools={wrong}", plan) + + historical = _make_wheel(tmp_path, dist_info="Proxy_Tools-0.1.0.dist-info") + assert validate_approved_wheel(f"proxy-tools={historical}", plan)[0].filename == historical.name + + @pytest.mark.parametrize( ("requires_dist", "locked", "error"), [ From 44bdf3e348c39958ec3b3ba305168fae930d1b7b Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:09:55 -0400 Subject: [PATCH 20/59] Complete packaging surface and wheel security checks --- .../analysis/metadata.py | 53 +++++++++++ .../generation/artifacts.py | 5 +- .../planning/extras.py | 18 +++- .../security_policy.py | 21 ++++- .../validation/runtime.py | 5 +- tests/test_analysis_scope.py | 94 +++++++++++++++++++ tests/test_generation.py | 87 +++++++++++++++++ tests/test_planning.py | 27 ++++++ tests/test_validation.py | 19 ++++ 9 files changed, 318 insertions(+), 11 deletions(-) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 75ea6f0..c2ec51f 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -32,6 +32,31 @@ class MetadataResult: dependencies: list[DependencyAssessment] +# Setuptools' flat-layout auto-discovery deliberately avoids conventional +# development directories. Keep this bounded static subset explicit rather +# than treating every repository directory as application packaging surface. +_DEFAULT_FLAT_DISCOVERY_EXCLUDES = [ + "build", + "build.*", + "dist", + "dist.*", + "docs", + "docs.*", + "example", + "example.*", + "examples", + "examples.*", + "test", + "test.*", + "tests", + "tests.*", + "venv", + "venv.*", + ".venv", + ".venv.*", +] + + def _evidence(root: Path, path: Path, detail: str, line: int | None = None) -> Evidence: return Evidence(file=path.relative_to(root).as_posix(), line=line, detail=detail) @@ -879,6 +904,34 @@ def inspect_metadata(root: Path) -> MetadataResult: if not source_roots: source_roots = ["src"] if (root / "src").is_dir() else ["."] + # Setuptools' ordinary automatic discovery applies when its build backend + # is selected but source metadata has not selected packages, find rules, or + # standalone modules. This makes the resolved existing ``packages`` model + # the single source surface for downstream planning and wheel validation. + if ( + isinstance(build_backend, str) + and build_backend.startswith("setuptools.") + and not packages + and not py_modules + and not package_discovery_rules + ): + automatic_root = ( + source_roots[0] + if len(source_roots) == 1 + else "src" + if (root / "src").is_dir() + else "." + ) + package_discovery_rules.append( + ( + [automatic_root], + ["*"], + [] if automatic_root == "src" else _DEFAULT_FLAT_DISCOVERY_EXCLUDES, + # Modern setuptools automatic discovery recognizes implicit + # namespaces; explicit find configuration can still disable it. + True, + ) + ) if package_discovery_rules: discovered_packages = { package diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index db0aa15..3e59eea 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -361,9 +361,10 @@ def _validate_application_security( if member_path.suffix.lower() == ".ps1" or is_secret_filename(member_path.name): failures.append(name) continue - if not is_textual_wheel_member(member_path): + content = bundle.read(member) + if not is_textual_wheel_member(member_path, content): continue - text = bundle.read(member).decode("utf-8", errors="replace") + text = content.decode("utf-8-sig", errors="replace") if text_security_findings( text, configured_secret_values=configured_secret_values ): diff --git a/src/python_deployment_builder/planning/extras.py b/src/python_deployment_builder/planning/extras.py index 241bb3a..9c5d0be 100644 --- a/src/python_deployment_builder/planning/extras.py +++ b/src/python_deployment_builder/planning/extras.py @@ -16,15 +16,25 @@ def validate_selected_extras( assessment: RepositoryAssessment, selected_extras: list[str] ) -> list[str]: - available = { - canonicalize_name(name): name for name in assessment.project.optional_dependency_groups - } + available: dict[str, str] = {} + collisions: set[str] = set() + for declared in assessment.project.optional_dependency_groups: + canonical = canonicalize_name(declared) + if canonical in available and available[canonical] != declared: + collisions.add(canonical) + else: + available[canonical] = declared + if collisions: + raise ValueError( + "Optional dependency extra declarations collide after PEP-685 normalization: " + + ", ".join(sorted(collisions)) + ) selected: list[str] = [] seen: set[str] = set() for item in selected_extras: canonical = canonicalize_name(item) if canonical not in seen: - selected.append(item) + selected.append(available.get(canonical, item)) seen.add(canonical) unknown = sorted( item for item in selected if canonicalize_name(item) not in available diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index 8227799..0be9ebb 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -61,13 +61,26 @@ def is_secret_filename(filename: str) -> bool: ) -def is_textual_wheel_member(path: PurePosixPath) -> bool: - """Return whether a wheel member has content suitable for text security checks.""" +def is_textual_wheel_member(path: PurePosixPath, content: bytes | None = None) -> bool: + """Return whether a wheel member has content suitable for text security checks. - return path.suffix.lower() in TEXT_SUFFIXES or ( + Extensionless application resources are common. They are scanned only after + strict UTF-8 and control-byte checks establish that they are text, so an + arbitrary binary payload is never decoded with replacement characters. + """ + + if path.suffix.lower() in TEXT_SUFFIXES or ( any(part.casefold().endswith(".dist-info") for part in path.parts) and path.name.casefold() in TEXTUAL_WHEEL_METADATA_FILENAMES - ) + ): + return True + if path.suffix or content is None or b"\x00" in content: + return False + try: + text = content.decode("utf-8-sig") + except UnicodeDecodeError: + return False + return not any(ord(character) < 32 and character not in "\t\n\r" for character in text) def text_security_findings( diff --git a/src/python_deployment_builder/validation/runtime.py b/src/python_deployment_builder/validation/runtime.py index 17f6634..3da3f31 100644 --- a/src/python_deployment_builder/validation/runtime.py +++ b/src/python_deployment_builder/validation/runtime.py @@ -12,6 +12,7 @@ from datetime import UTC, datetime from pathlib import Path +from packaging.utils import canonicalize_name from packaging.version import InvalidVersion, Version from python_deployment_builder.analysis import assess_repository @@ -235,10 +236,12 @@ def _selected_imports(kit_root: Path, manifest: DeploymentManifest) -> list[str] ) assessment = assess_repository(repository) selected_groups = {"runtime", *manifest.selected_extras} + selected_group_names = {canonicalize_name(group) for group in selected_groups} imports = { name for dependency in assessment.dependencies - if dependency.group in selected_groups + if dependency.group == "runtime" + or canonicalize_name(dependency.group) in selected_group_names for name in dependency.import_names } if any( diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 7cd3e0a..5fbed7d 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -894,6 +894,100 @@ def test_pyproject_find_discovers_packages_for_wildcard_package_data( } +def test_setuptools_default_discovery_defines_python_and_wildcard_data_surface( + tmp_path: Path, +) -> None: + app = tmp_path / "src/example_app" + (app / "data").mkdir(parents=True) + (app / "__init__.py").write_text("", encoding="utf-8") + (app / "main.py").write_text("from . import helpers\n", encoding="utf-8") + (app / "helpers.py").write_text("VALUE = 1\n", encoding="utf-8") + (app / "data/defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "src/other_app").mkdir() + (tmp_path / "src/other_app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/namespace_pkg/child").mkdir(parents=True) + (tmp_path / "src/namespace_pkg/child/module.py").write_text("VALUE = 2\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "example-app" +version = "1.0" +[project.scripts] +example = "example_app.main:main" +[tool.setuptools.package-data] +"*" = ["data/*.json"] +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert {"example_app", "other_app", "namespace_pkg", "namespace_pkg.child"} <= set( + project.packages + ) + assert { + (item.source_path, item.installed_member_path) + for item in resolve_packaged_python_sources(tmp_path, project) + } >= { + ("src/example_app/__init__.py", "example_app/__init__.py"), + ("src/example_app/main.py", "example_app/main.py"), + ("src/example_app/helpers.py", "example_app/helpers.py"), + } + assert [ + (item.source_path, item.installed_member_path) + for item in resolve_package_data_members(tmp_path, project) + ] == [("src/example_app/data/defaults.json", "example_app/data/defaults.json")] + + +def test_setuptools_default_flat_discovery_excludes_development_directories(tmp_path: Path) -> None: + (tmp_path / "example_app").mkdir() + (tmp_path / "tests").mkdir() + (tmp_path / "docs").mkdir() + for directory in ("example_app", "tests", "docs"): + (tmp_path / directory / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "example-app" +version = "1.0" +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert "example_app" in project.packages + assert "tests" not in project.packages + assert "docs" not in project.packages + + +def test_explicit_py_modules_prevents_default_package_auto_discovery(tmp_path: Path) -> None: + (tmp_path / "src/example_app").mkdir(parents=True) + (tmp_path / "src/example_app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "example-app" +version = "1.0" +[tool.setuptools] +py-modules = ["helper"] +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == [] + assert project.py_modules == ["helper"] + + def test_package_data_exclusions_apply_after_safe_concrete_resolution(tmp_path: Path) -> None: for package in ("app", "other"): data = tmp_path / package / "data" diff --git a/tests/test_generation.py b/tests/test_generation.py index 8303988..5b5957a 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1724,6 +1724,69 @@ def test_discovered_package_wildcard_data_is_required_in_application_wheel( assert validate_application_wheel(complete, assessment, plan, repository_root=source)[0] +def test_default_discovered_python_surface_and_wildcard_data_are_required_in_wheel( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + (source / "src/example_app/data").mkdir(parents=True) + (source / "src/example_app/__init__.py").write_text("", encoding="utf-8") + (source / "src/example_app/main.py").write_text("from . import helpers\n", encoding="utf-8") + (source / "src/example_app/helpers.py").write_text("VALUE = 1\n", encoding="utf-8") + (source / "src/example_app/data/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "example-app" +version = "1.0" +[project.scripts] +example = "example_app.main:main" +[tool.setuptools.package-data] +"*" = ["data/*.json"] +""", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + incomplete = _make_application_wheel( + tmp_path, + name="example-app", + version="1.0", + package="example_app", + target="example_app.main:main", + entry_group="console_scripts", + entry_name="example", + ) + + assert "example_app" in assessment.project.packages + with pytest.raises(PreparationError, match="example_app/data/defaults.json"): + validate_application_wheel(incomplete, assessment, plan, repository_root=source) + + with pytest.raises(PreparationError, match="example_app/helpers.py"): + validate_application_wheel( + _rewrite_application_wheel( + incomplete, + additions={"example_app/data/defaults.json": "{}\n"}, + ), + assessment, + plan, + repository_root=source, + ) + + complete = _rewrite_application_wheel( + incomplete, + additions={ + "example_app/data/defaults.json": "{}\n", + "example_app/helpers.py": "VALUE = 1\n", + }, + ) + assert validate_application_wheel(complete, assessment, plan, repository_root=source)[0] + + @pytest.mark.parametrize( ("py_modules", "missing_member"), [(False, "app/util.py"), (True, "helper.py")], @@ -1855,6 +1918,7 @@ def test_application_wheel_rejects_malformed_or_multiple_requires_python( ("installed_app/settings.toml", "api_key = 'sk-abcdefghijklmnop'"), ("installed_app/settings.ini", "api_key = sk-abcdefghijklmnop"), ("installed_app/settings.cfg", "api_key = sk-abcdefghijklmnop"), + ("installed_app/token", "api_key = sk-abcdefghijklmnop"), ("installed_app/copy.py", "COMMAND = 'copy payload C:\\Program Files\\App'"), ], ) @@ -1909,6 +1973,29 @@ def test_application_wheel_allows_non_secret_textual_configuration_file(tmp_path assert artifact.filename == wheel.name +def test_application_wheel_extensionless_text_and_binary_resources_are_classified_safely( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment) + benign = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/scripts/tool": "#!/usr/bin/env python\nprint('ok')\n"}, + ) + assert validate_application_wheel(benign, assessment, plan)[0].filename == benign.name + + binary = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/blob": b"\x00\xffsk-abcdefghijklmnop"}, + ) + assert validate_application_wheel(binary, assessment, plan)[0].filename == binary.name + + def test_application_wheel_rejects_obvious_secret_in_textual_metadata(tmp_path: Path) -> None: source = tmp_path / "source" source.mkdir() diff --git a/tests/test_planning.py b/tests/test_planning.py index 2e80a5f..0863e64 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -582,6 +582,33 @@ def test_unknown_selected_extra_is_rejected() -> None: create_deployment_plan(_assess("optional_map_app"), selected_extras=["missing"]) +@pytest.mark.parametrize("requested", ["Foo_Bar", "foo-bar", "foo.bar"]) +def test_selected_extra_resolves_to_authoritative_declared_group(requested: str) -> None: + assessment = _assess("optional_map_app") + dependency = DependencyAssessment( + distribution_name="requests", + declared_constraint=">=2", + group="Foo_Bar", + import_names=["requests"], + ) + assessment.project.optional_dependency_groups = {"Foo_Bar": ["requests>=2"]} + assessment.dependencies = [dependency] + + plan = create_deployment_plan(assessment, selected_extras=[requested]) + + assert plan.runtime.selected_extras == ["Foo_Bar"] + assert plan.extras[0].selected + assert "Foo_Bar" in plan.runtime.sync_command.arguments + + +def test_canonically_colliding_optional_extra_declarations_fail() -> None: + assessment = _assess("optional_map_app") + assessment.project.optional_dependency_groups = {"Foo_Bar": [], "foo-bar": []} + + with pytest.raises(ValueError, match="collide after PEP-685"): + create_deployment_plan(assessment, selected_extras=["foo-bar"]) + + def test_windows_environment_markers_are_applied() -> None: assert marker_applies("sys_platform == 'win32'", "3.12", "x86_64", extra="map") assert not marker_applies("sys_platform == 'linux'", "3.12", "x86_64", extra="map") diff --git a/tests/test_validation.py b/tests/test_validation.py index 688aff9..d2cd61c 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -6,6 +6,7 @@ import subprocess import sys from pathlib import Path +from types import SimpleNamespace import pytest @@ -25,6 +26,7 @@ _application_probe_result, _runtime_environment, _scenario_copy, + _selected_imports, validate_runtime_kit, ) from python_deployment_builder.validation.static import validate_static_kit @@ -59,6 +61,23 @@ def _status(report, code: str) -> ValidationCheckStatus: return next(item.status for item in report.static_checks if item.code == code) +def test_runtime_import_probe_compares_selected_extra_groups_semantically( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + assessment = SimpleNamespace( + dependencies=[SimpleNamespace(group="Foo_Bar", import_names=["feature_probe"])], + runtime_requirements=[], + ) + monkeypatch.setattr( + "python_deployment_builder.validation.runtime.assess_repository", + lambda _: assessment, + ) + + imports = _selected_imports(tmp_path, SimpleNamespace(selected_extras=["foo-bar"])) + + assert imports == ["feature_probe"] + + def _refresh_manifest_index(kit: Path) -> None: index_path = kit / "deployment/generated-files.json" index = json.loads(index_path.read_text(encoding="utf-8")) From 7a440ee283cd6c6f2cb093be21a078b6d6384e8d Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:05:55 -0400 Subject: [PATCH 21/59] Fix minor exclusions and setuptools discovery --- .../analysis/metadata.py | 90 +++++++++- .../planning/policies.py | 16 +- tests/test_analysis_scope.py | 167 ++++++++++++++++++ tests/test_generation.py | 19 ++ tests/test_planning.py | 19 ++ 5 files changed, 303 insertions(+), 8 deletions(-) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index c2ec51f..fcc8578 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -274,8 +274,22 @@ def _discover_setuptools_packages( parts = (*prefix.split("."), *relative.parts) if prefix else relative.parts if not all(part.isidentifier() for part in parts): continue - if not namespaces and not (directory / "__init__.py").is_file(): - continue + if not namespaces: + # ``find_packages()`` cannot discover a child through a + # non-package parent. Checking only this directory's + # initializer would incorrectly turn ``container/sub`` into + # ``container.sub`` when ``container`` is not a package. + package_directories_in_path = [ + candidate_root / Path(*relative.parts[: index + 1]) + for index in range(len(relative.parts)) + ] + if any( + initializer.is_symlink() or not initializer.is_file() + for initializer in ( + item / "__init__.py" for item in package_directories_in_path + ) + ): + continue package = ".".join(parts) if any(fnmatch.fnmatchcase(package, pattern) for pattern in includes) and not any( fnmatch.fnmatchcase(package, pattern) for pattern in exclude @@ -284,6 +298,48 @@ def _discover_setuptools_packages( return sorted(discovered) +def _discover_setuptools_py_modules( + root: Path, + search_roots: list[str], + *, + excluded_modules: list[str] | None = None, +) -> list[str]: + """Resolve safe top-level modules for bounded setuptools auto-discovery. + + Setuptools discovers standalone modules from the configured source root, + not by recursively treating every Python file as a module. This shares + the same filesystem-only safety boundary as package discovery. + """ + + resolved_root = root.resolve() + discovered: set[str] = set() + exclusions = excluded_modules or [] + for configured_root in search_roots: + candidate_root = root / configured_root + if candidate_root.is_symlink() or not candidate_root.is_dir(): + continue + try: + candidate_root.resolve().relative_to(resolved_root) + except ValueError: + continue + for candidate in sorted(candidate_root.glob("*.py")): + if candidate.is_symlink() or not candidate.is_file(): + continue + try: + candidate.resolve().relative_to(candidate_root.resolve()) + except ValueError: + continue + module = candidate.stem + if ( + module == "__init__" + or not module.isidentifier() + or any(fnmatch.fnmatchcase(module, pattern) for pattern in exclusions) + ): + continue + discovered.add(module) + return sorted(discovered) + + def _literal_setup_arguments(path: Path) -> dict[str, Any]: """Read literal setup(...) keyword values without executing setup.py.""" @@ -404,6 +460,8 @@ def inspect_metadata(root: Path) -> MetadataResult: layout = "unknown" python_evidence: list[Evidence] = [] package_discovery_rules: list[tuple[list[str], list[str], list[str], bool]] = [] + automatic_setuptools_root: str | None = None + automatic_setuptools_flat_surface_ambiguous = False pyproject_path = root / "pyproject.toml" if pyproject_path.is_file(): @@ -922,6 +980,7 @@ def inspect_metadata(root: Path) -> MetadataResult: if (root / "src").is_dir() else "." ) + automatic_setuptools_root = automatic_root package_discovery_rules.append( ( [automatic_root], @@ -945,7 +1004,34 @@ def inspect_metadata(root: Path) -> MetadataResult: namespaces, ) } + if automatic_setuptools_root == ".": + top_level_packages = {package.split(".", 1)[0] for package in discovered_packages} + if len(top_level_packages) > 1: + # Setuptools rejects implicit flat layouts with multiple + # top-level packages rather than building an arbitrary subset. + # Leave the packaging surface unresolved so downstream source + # constraints remain conservative until metadata is explicit. + discovered_packages = set() + automatic_setuptools_flat_surface_ambiguous = True packages = sorted({*packages, *discovered_packages}) + if automatic_setuptools_root is not None and ( + automatic_setuptools_root == "src" or not packages + ) and not automatic_setuptools_flat_surface_ambiguous: + # Setuptools' default source-layout finder discovers top-level modules + # as well as packages. Its flat-layout finder selects a package + # surface in preference to loose modules, so only a package-free flat + # layout contributes automatic py_modules. Explicit configuration + # never reaches this branch. + discovered_modules = _discover_setuptools_py_modules( + root, + [automatic_setuptools_root], + ) + if automatic_setuptools_root == "." and len(discovered_modules) > 1: + # Mirroring the bounded flat package policy above prevents an + # undeclared multi-module distribution from becoming a fabricated + # wheel surface. + discovered_modules = [] + py_modules = discovered_modules layout = ( "src" if any(Path(value).as_posix().rstrip("/") == "src" for value in source_roots) diff --git a/src/python_deployment_builder/planning/policies.py b/src/python_deployment_builder/planning/policies.py index 7a1c2db..495a5ae 100644 --- a/src/python_deployment_builder/planning/policies.py +++ b/src/python_deployment_builder/planning/policies.py @@ -162,20 +162,24 @@ def minor_python_compatibility( ) ) elif operator in {"==", "!="}: - exact = _range_result(">=", boundary, low, high) - # An exact match is only invariant when it is outside the entire - # selected-minor interval. Otherwise a patch fact is required. if operator == "==": + exact = _range_result(">=", boundary, low, high) + # An exact match is only invariant when it is outside the entire + # selected-minor interval. Otherwise a patch fact is required. results.append( MinorPythonCompatibility.INCOMPATIBLE if exact == MinorPythonCompatibility.INCOMPATIBLE else MinorPythonCompatibility.UNPROVABLE ) else: + # Unlike equality, an exact exclusion is certainly satisfied + # whenever its excluded point is outside this minor's complete + # [low, high) patch interval. A point inside that interval + # needs an exact patch fact and therefore remains unprovable. results.append( - MinorPythonCompatibility.COMPATIBLE - if exact == MinorPythonCompatibility.INCOMPATIBLE - else MinorPythonCompatibility.UNPROVABLE + MinorPythonCompatibility.UNPROVABLE + if low <= boundary < high + else MinorPythonCompatibility.COMPATIBLE ) else: results.append(MinorPythonCompatibility.UNPROVABLE) diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 5fbed7d..9d13ca3 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -903,6 +903,7 @@ def test_setuptools_default_discovery_defines_python_and_wildcard_data_surface( (app / "main.py").write_text("from . import helpers\n", encoding="utf-8") (app / "helpers.py").write_text("VALUE = 1\n", encoding="utf-8") (app / "data/defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "src/helper.py").write_text("VALUE = 3\n", encoding="utf-8") (tmp_path / "src/other_app").mkdir() (tmp_path / "src/other_app/__init__.py").write_text("", encoding="utf-8") (tmp_path / "src/namespace_pkg/child").mkdir(parents=True) @@ -927,6 +928,7 @@ def test_setuptools_default_discovery_defines_python_and_wildcard_data_surface( assert {"example_app", "other_app", "namespace_pkg", "namespace_pkg.child"} <= set( project.packages ) + assert project.py_modules == ["helper"] assert { (item.source_path, item.installed_member_path) for item in resolve_packaged_python_sources(tmp_path, project) @@ -934,6 +936,7 @@ def test_setuptools_default_discovery_defines_python_and_wildcard_data_surface( ("src/example_app/__init__.py", "example_app/__init__.py"), ("src/example_app/main.py", "example_app/main.py"), ("src/example_app/helpers.py", "example_app/helpers.py"), + ("src/helper.py", "helper.py"), } assert [ (item.source_path, item.installed_member_path) @@ -965,6 +968,143 @@ def test_setuptools_default_flat_discovery_excludes_development_directories(tmp_ assert "docs" not in project.packages +def test_setuptools_default_flat_single_module_defines_python_surface(tmp_path: Path) -> None: + (tmp_path / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "flat-single-module" +version = "1.0" +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == [] + assert project.py_modules == ["helper"] + + +def test_setuptools_default_flat_package_surface_omits_loose_module(tmp_path: Path) -> None: + (tmp_path / "example_app").mkdir() + (tmp_path / "example_app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "flat-package-module" +version = "1.0" +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == ["example_app"] + assert project.py_modules == [] + + +def test_setuptools_default_flat_multi_package_surface_remains_unresolved(tmp_path: Path) -> None: + for package in ("one", "two"): + (tmp_path / package).mkdir() + (tmp_path / package / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "flat-multi-package" +version = "1.0" +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == [] + assert project.py_modules == [] + + +def test_setuptools_default_flat_multi_module_surface_remains_unresolved(tmp_path: Path) -> None: + for module in ("one", "two"): + (tmp_path / f"{module}.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "flat-multi-module" +version = "1.0" +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == [] + assert project.py_modules == [] + + +@pytest.mark.parametrize( + ("namespaces", "parent_initialized", "expected"), + [ + (False, False, set()), + (False, True, {"container", "container.sub"}), + (True, False, {"container", "container.sub"}), + ], +) +def test_setuptools_find_respects_non_namespace_ancestor_continuity( + tmp_path: Path, + namespaces: bool, + parent_initialized: bool, + expected: set[str], +) -> None: + child = tmp_path / "src/container/sub" + child.mkdir(parents=True) + (child / "__init__.py").write_text("", encoding="utf-8") + if parent_initialized: + (tmp_path / "src/container/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + f'''[project] +name = "ancestor-continuity" +version = "1.0" +[tool.setuptools.packages.find] +where = ["src"] +namespaces = {str(namespaces).lower()} +''', + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert set(project.packages) == expected + + +def test_non_namespace_find_rejects_deep_descendant_below_missing_parent(tmp_path: Path) -> None: + child = tmp_path / "src/container/intermediate/sub" + child.mkdir(parents=True) + (child / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/container/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + """[project] +name = "deep-ancestor-continuity" +version = "1.0" +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == ["container"] + + def test_explicit_py_modules_prevents_default_package_auto_discovery(tmp_path: Path) -> None: (tmp_path / "src/example_app").mkdir(parents=True) (tmp_path / "src/example_app/__init__.py").write_text("", encoding="utf-8") @@ -1151,6 +1291,33 @@ def test_setup_cfg_find_uses_global_package_dir_and_filters(tmp_path: Path) -> N } +@pytest.mark.parametrize( + ("finder", "expected"), + [("find:", set()), ("find_namespace:", {"container", "container.sub"})], +) +def test_setup_cfg_finder_preserves_its_namespace_policy( + tmp_path: Path, finder: str, expected: set[str] +) -> None: + child = tmp_path / "src/container/sub" + child.mkdir(parents=True) + (child / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / "setup.cfg").write_text( + f"""[metadata] +name = setup-finder-policy +version = 1.0 +[options] +packages = {finder} +package_dir = + = src +""", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert set(project.packages) == expected + + def test_literal_setup_py_package_data_and_exclusions_share_the_resolver(tmp_path: Path) -> None: data = tmp_path / "app" / "data" data.mkdir(parents=True) diff --git a/tests/test_generation.py b/tests/test_generation.py index 5b5957a..12641f8 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1733,6 +1733,7 @@ def test_default_discovered_python_surface_and_wildcard_data_are_required_in_whe (source / "src/example_app/main.py").write_text("from . import helpers\n", encoding="utf-8") (source / "src/example_app/helpers.py").write_text("VALUE = 1\n", encoding="utf-8") (source / "src/example_app/data/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "src/helper.py").write_text("VALUE = 2\n", encoding="utf-8") (source / "pyproject.toml").write_text( """[build-system] requires = ["setuptools>=68"] @@ -1763,6 +1764,7 @@ def test_default_discovered_python_surface_and_wildcard_data_are_required_in_whe ) assert "example_app" in assessment.project.packages + assert assessment.project.py_modules == ["helper"] with pytest.raises(PreparationError, match="example_app/data/defaults.json"): validate_application_wheel(incomplete, assessment, plan, repository_root=source) @@ -1777,11 +1779,26 @@ def test_default_discovered_python_surface_and_wildcard_data_are_required_in_whe repository_root=source, ) + with pytest.raises(PreparationError, match="helper.py"): + validate_application_wheel( + _rewrite_application_wheel( + incomplete, + additions={ + "example_app/data/defaults.json": "{}\n", + "example_app/helpers.py": "VALUE = 1\n", + }, + ), + assessment, + plan, + repository_root=source, + ) + complete = _rewrite_application_wheel( incomplete, additions={ "example_app/data/defaults.json": "{}\n", "example_app/helpers.py": "VALUE = 1\n", + "helper.py": "VALUE = 2\n", }, ) assert validate_application_wheel(complete, assessment, plan, repository_root=source)[0] @@ -1855,6 +1872,7 @@ def test_application_wheel_requires_authoritative_python_source_surface( (">=3.11", True), (">=3.12,<3.13", True), ("==3.12.*", True), + (">=3.9,!=3.9.0", True), (">=3.13", False), ("<3.12", False), (">=3.12.1", False), @@ -3780,6 +3798,7 @@ def test_approved_wheel_rejects_wrong_name_version_and_metadata(tmp_path: Path) (">=3.11", True), (">=3.12,<3.13", True), ("==3.12.*", True), + (">=3.9,!=3.9.0", True), (">=3.13", False), ("<3.12", False), (">=3.12.1", False), diff --git a/tests/test_planning.py b/tests/test_planning.py index 0863e64..ba9dc84 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -409,6 +409,18 @@ def test_policy_skips_patch_unprovable_candidate_for_minor_only_runtime() -> Non assert plan.runtime.python_version == "3.13" +def test_policy_accepts_exact_exclusion_outside_selected_minor() -> None: + assessment = _assess() + assessment.python.requires_python = ">=3.9,!=3.9.0" + + plan = create_deployment_plan(assessment) + + candidate_312 = next(item for item in plan.python_candidates if item.version == "3.12") + assert candidate_312.satisfies_requires_python + assert candidate_312.compatibility != "incompatible" + assert plan.runtime.python_version == "3.12" + + def test_blocking_assessment_gates_generation_policy() -> None: assessment = _assess() assessment.rating = SuitabilityRating.RED @@ -625,7 +637,14 @@ def test_windows_environment_markers_are_applied() -> None: (">=3.12.1", MinorPythonCompatibility.UNPROVABLE), ("<3.12.1", MinorPythonCompatibility.UNPROVABLE), ("==3.12.0", MinorPythonCompatibility.UNPROVABLE), + ("!=3.9.0", MinorPythonCompatibility.COMPATIBLE), + ("!=3.11.99", MinorPythonCompatibility.COMPATIBLE), + ("!=3.13.0", MinorPythonCompatibility.COMPATIBLE), + ("!=3.14.1", MinorPythonCompatibility.COMPATIBLE), ("!=3.12.5", MinorPythonCompatibility.UNPROVABLE), + (">=3.9,!=3.9.0", MinorPythonCompatibility.COMPATIBLE), + (">=3.9,!=3.12.1", MinorPythonCompatibility.UNPROVABLE), + (">=3.13,!=3.9.0", MinorPythonCompatibility.INCOMPATIBLE), ("~=3.12.1", MinorPythonCompatibility.UNPROVABLE), ], ) From d11b470e027614668a49c57a8e36d1cfbdf234ba Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:50:21 -0400 Subject: [PATCH 22/59] Harden staged content and wheel surfaces --- .../generation/artifacts.py | 72 +++++++++++- .../generation/structural.py | 22 +++- .../security_policy.py | 9 +- .../validation/static.py | 28 ++++- tests/test_generation.py | 107 ++++++++++++++++++ 5 files changed, 221 insertions(+), 17 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 3e59eea..1000fcc 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -283,6 +283,49 @@ def _dist_info_members( return metadata_name, f"{dist_info}/WHEEL", f"{dist_info}/RECORD" +def installed_wheel_member_paths( + members: dict[str, zipfile.ZipInfo], wheel: Path +) -> set[str]: + """Return logical site-packages members after valid wheel relocation. + + ``purelib`` members are relocated by installers into site-packages. Other + data schemes are deliberately not application import/package-data surface. + The caller has already validated the wheel's ``.data`` identity. + """ + + installed: dict[str, str] = {} + for name, member in members.items(): + if member.is_dir(): + continue + path = PurePosixPath(name) + if path.parts[0].endswith(".dist-info"): + continue + destination: PurePosixPath | None = path + if path.parts[0].endswith(".data"): + if len(path.parts) < 3 or path.parts[1] != "purelib": + # scripts/headers/data have no site-packages application + # representation; platlib is outside the pure-Python contract. + if len(path.parts) >= 2 and path.parts[1] == "platlib": + raise PreparationError( + "Application wheel uses .data/platlib, which is not accepted by the " + f"pure-Python package-mode contract: {wheel.name}" + ) + destination = None + else: + destination = PurePosixPath(*path.parts[2:]) + if destination is None or not destination.parts: + continue + normalized = destination.as_posix() + key = normalized.casefold() + if previous := installed.get(key): + raise PreparationError( + "Wheel contains colliding installed member paths: " + f"{previous}, {name}" + ) + installed[key] = name + return set(installed) + + def _require_core_metadata(message, *, label: str, wheel: Path) -> tuple[str, str]: metadata_versions = message.get_all("Metadata-Version", []) names = message.get_all("Name", []) @@ -548,7 +591,7 @@ def _validate_dependency_extra_closure(requirement: Requirement, graph) -> None: def _validate_application_requires_dist( - metadata, plan: DeploymentPlan, application_name: str + metadata, plan: DeploymentPlan, application_name: str, application_version: Version ) -> None: """Prove every applicable first-party wheel requirement is in the selected lock graph.""" @@ -575,6 +618,22 @@ def _validate_application_requires_dist( continue name = canonicalize_name(requirement.name) if name == application_name: + if requirement.url: + raise PreparationError( + "Application wheel self Requires-Dist direct references are not " + "statically provable: " + f"{requirement.name}." + ) + if requirement.extras: + raise PreparationError( + "Application wheel self Requires-Dist extras are not statically provable: " + f"{requirement.name}[{','.join(sorted(requirement.extras))}]." + ) + if application_version not in requirement.specifier: + raise PreparationError( + "Application wheel self Requires-Dist is incompatible with its own version: " + f"{requirement}. Application version: {application_version}." + ) continue if requirement.url: raise PreparationError( @@ -670,6 +729,7 @@ def validate_application_wheel( ) metadata_name, wheel_name, record_name = _dist_info_members(members, path) dist_info = PurePosixPath(metadata_name).parent + installed_names = installed_wheel_member_paths(members, path) entry_points_name = f"{dist_info.as_posix()}/entry_points.txt" for required in (wheel_name, record_name, entry_points_name): if required not in names: @@ -699,7 +759,9 @@ def validate_application_wheel( raise PreparationError("Application wheel METADATA version is wrong.") _validate_requires_python(metadata, plan, path) if validate_locked_dependencies: - _validate_application_requires_dist(metadata, plan, expected_name) + _validate_application_requires_dist( + metadata, plan, expected_name, expected_version_value + ) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: @@ -767,7 +829,7 @@ def validate_application_wheel( str(module_path.with_suffix(".py")), str(module_path / "__init__.py"), } - if not module_candidates.intersection(names): + if not module_candidates.intersection(installed_names): raise PreparationError( "Application wheel does not contain its authoritative entry-point module: " f"{entry_point.module}" @@ -790,7 +852,7 @@ def validate_application_wheel( member.installed_member_path for member in resolve_package_data_members(source_root, assessment.project) } if source_root is not None else set() - missing_members = sorted(expected_members - names) + missing_members = sorted(expected_members - installed_names) if missing_members: raise PreparationError( "Application wheel is missing concrete declared package data: " @@ -800,7 +862,7 @@ def validate_application_wheel( member.installed_member_path for member in resolve_packaged_python_sources(source_root, assessment.project) } if source_root is not None else set() - missing_python_members = sorted(expected_python_members - names) + missing_python_members = sorted(expected_python_members - installed_names) if missing_python_members: raise PreparationError( "Application wheel is missing authoritative first-party Python source: " diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index f8c2cef..521e8a8 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -14,7 +14,11 @@ RiskFinding, RiskSeverity, ) -from python_deployment_builder.security_policy import TEXT_SUFFIXES, text_security_findings +from python_deployment_builder.security_policy import ( + is_secret_filename, + is_textual_content, + text_security_findings, +) def _check(condition: bool, code: str, description: str) -> RiskFinding: @@ -108,10 +112,18 @@ def validate_rendered_files( permanent_path_hits: list[str] = [] program_files_hits: list[str] = [] secret_hits: list[str] = [] - for relative in generated_paths: - if PurePosixPath(relative).suffix.lower() not in TEXT_SUFFIXES: + # Every intentionally staged file is release content. Wheels remain + # opaque here because their member-level validator owns their security scan. + for relative, content in files.items(): + path = PurePosixPath(relative) + if path.suffix.lower() == ".whl": + continue + if is_secret_filename(path.name): + secret_hits.append(relative) + continue + if not is_textual_content(path, content): continue - text = files.get(relative, b"").decode("utf-8", errors="replace") + text = content.decode("utf-8-sig") findings = text_security_findings( text, configured_secret_values=secret_values or [] ) @@ -125,7 +137,7 @@ def validate_rendered_files( program_files_hits.append(relative) if {"obvious_secret", "configured_secret"} & findings: secret_hits.append(relative) - ps1_files = [path for path in generated_paths if PurePosixPath(path).suffix.lower() == ".ps1"] + ps1_files = [path for path in files if PurePosixPath(path).suffix.lower() == ".ps1"] runtime_builder_imports = [ path for path in generated_paths diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index 0be9ebb..fe11f3d 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -61,7 +61,7 @@ def is_secret_filename(filename: str) -> bool: ) -def is_textual_wheel_member(path: PurePosixPath, content: bytes | None = None) -> bool: +def is_textual_content(path: PurePosixPath, content: bytes | None = None) -> bool: """Return whether a wheel member has content suitable for text security checks. Extensionless application resources are common. They are scanned only after @@ -83,6 +83,12 @@ def is_textual_wheel_member(path: PurePosixPath, content: bytes | None = None) - return not any(ord(character) < 32 and character not in "\t\n\r" for character in text) +def is_textual_wheel_member(path: PurePosixPath, content: bytes | None = None) -> bool: + """Compatibility name for wheel callers of the shared text policy.""" + + return is_textual_content(path, content) + + def text_security_findings( text: str, *, configured_secret_values: Iterable[str] = () ) -> set[str]: @@ -112,6 +118,7 @@ def text_security_findings( __all__ = [ "FORBIDDEN_SHELL", "TEXT_SUFFIXES", + "is_textual_content", "is_secret_filename", "is_textual_wheel_member", "text_security_findings", diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 77d1a96..fe5df08 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -10,10 +10,11 @@ import socket import zipfile from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath from pydantic import ValidationError +from python_deployment_builder.generation.artifacts import installed_wheel_member_paths from python_deployment_builder.models import ( DeploymentManifest, ManualValidationItem, @@ -25,8 +26,8 @@ ) from python_deployment_builder.security_policy import ( FORBIDDEN_SHELL, - TEXT_SUFFIXES, is_secret_filename, + is_textual_content, text_security_findings, ) @@ -370,7 +371,15 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR member_candidates = {f"{member_base}.py", f"{member_base}/__init__.py"} try: with zipfile.ZipFile(application_wheel) as bundle: - entry_present = bool(member_candidates.intersection(bundle.namelist())) + members = { + PurePosixPath(member.filename).as_posix(): member + for member in bundle.infolist() + } + entry_present = bool( + member_candidates.intersection( + installed_wheel_member_paths(members, application_wheel) + ) + ) except zipfile.BadZipFile: entry_present = False entry_evidence = sorted(member_candidates) @@ -459,12 +468,19 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR for name in manifest.configuration_secret_names if (value := os.environ.get(name)) is not None ] - security_paths = [*root.glob("*.bat"), *(root / "deployment").rglob("*")] + security_paths = { + *(_safe_kit_path(root, relative) for relative in indexed_paths), + *root.glob("*.bat"), + *(root / "deployment").rglob("*"), + } for path in security_paths: - if not path.is_file() or path.suffix.lower() not in TEXT_SUFFIXES: + if path is None or not path.is_file() or path.suffix.lower() == ".whl": continue relative = str(path.relative_to(root)) - text = path.read_text(encoding="utf-8", errors="replace") + content = path.read_bytes() + if not is_textual_content(Path(relative), content): + continue + text = content.decode("utf-8-sig") findings = text_security_findings( text, configured_secret_values=configured_secret_values ) diff --git a/tests/test_generation.py b/tests/test_generation.py index 12641f8..a707b57 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -849,6 +849,84 @@ def test_application_wheel_requires_dist_rejects_unprovable_metadata( validate_application_wheel(wheel, assessment, plan) +@pytest.mark.parametrize( + ("requirement", "accepted"), + [ + ("mapped-app>=1", True), + ("mapped-app==1.2.3", True), + ("mapped_app==1.2.3", True), + ("mapped-app>=2", False), + ("mapped-app!=1.2.3", False), + ("mapped-app>=2; python_version >= '3.13'", True), + ("mapped-app @ https://example.invalid/mapped-app.whl", False), + ("mapped-app[map]>=1", False), + ], +) +def test_application_wheel_self_requires_dist_is_checked_against_application_version( + tmp_path: Path, requirement: str, accepted: bool +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_locked_dependencies( + create_deployment_plan(assessment, repository_root=source), [] + ) + wheel = _make_application_wheel(tmp_path, requires_dist_values=[requirement]) + + if accepted: + assert validate_application_wheel(wheel, assessment, plan)[0] + else: + with pytest.raises(PreparationError, match="self Requires-Dist"): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_surface_accepts_data_purelib_members(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path) + data_root = "mapped_app-1.2.3.data/purelib/installed_app" + relocated = _rewrite_application_wheel( + wheel, + removals={ + "installed_app/__init__.py", + "installed_app/main.py", + "installed_app/view.html", + }, + additions={ + f"{data_root}/__init__.py": "", + f"{data_root}/main.py": "def main(): return 0\n", + f"{data_root}/view.html": "\n", + }, + ) + + assert validate_application_wheel(relocated, assessment, plan, repository_root=source)[0] + + +def test_application_wheel_rejects_colliding_purelib_installed_member(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"mapped_app-1.2.3.data/purelib/installed_app/main.py": "def main(): return 0\n"}, + ) + + with pytest.raises(PreparationError, match="colliding installed member"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + @pytest.mark.parametrize( ("marker", "error"), [ @@ -3585,6 +3663,35 @@ def test_generation_writes_structurally_valid_kit_and_protects_edits( assert any("previously generated file was modified" in item for item in preview.collisions) +def test_source_mode_staged_secret_fails_before_writes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + shutil.copytree(FIXTURES / "prepared_gui", source) + (source / "prepared_gui.py").write_text( + "API_KEY = 'sk-abcdefghijklmnop'\ndef main(): return 0\n", encoding="utf-8" + ) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + + with pytest.raises(PreparationError, match="NO_SECRET_VALUES"): + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + ) + assert not (tmp_path / "kit").exists() + + def test_manifest_renders_flat_source_system_certs_and_selected_extra(tmp_path: Path) -> None: plan = _plan("optional_map_app", ["map"]) wheel = _make_wheel(tmp_path) From 576760e4959c0a041f3a706fbcbeef7a90a02936 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:12:15 -0400 Subject: [PATCH 23/59] Scan unknown textual resource extensions --- .../security_policy.py | 29 ++++++++--- tests/test_generation.py | 51 ++++++++++++++++++- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index fe11f3d..4c37a6b 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -34,6 +34,13 @@ ".yml", } ) +KNOWN_BINARY_SUFFIXES = frozenset( + { + ".dll", ".dylib", ".exe", ".gif", ".ico", ".jpeg", ".jpg", ".lib", + ".pdf", ".png", ".pyd", ".pyc", ".pyo", ".so", ".tif", ".tiff", + ".webp", ".whl", ".zip", + } +) PROGRAM_FILES_WRITE_TOKENS = ("mkdir", "copy ", "write_text", "open(", "write") SECRET_FILENAMES = frozenset( {".env", "credentials.json", "secrets.json", "token.json", ".pypirc", "pip.ini"} @@ -61,6 +68,18 @@ def is_secret_filename(filename: str) -> bool: ) +def is_probably_utf8_text(content: bytes) -> bool: + """Classify unknown bytes without decoding binary content with replacement.""" + + if b"\x00" in content: + return False + try: + text = content.decode("utf-8-sig") + except UnicodeDecodeError: + return False + return not any(ord(character) < 32 and character not in "\t\n\r" for character in text) + + def is_textual_content(path: PurePosixPath, content: bytes | None = None) -> bool: """Return whether a wheel member has content suitable for text security checks. @@ -74,13 +93,9 @@ def is_textual_content(path: PurePosixPath, content: bytes | None = None) -> boo and path.name.casefold() in TEXTUAL_WHEEL_METADATA_FILENAMES ): return True - if path.suffix or content is None or b"\x00" in content: - return False - try: - text = content.decode("utf-8-sig") - except UnicodeDecodeError: + if path.suffix.lower() in KNOWN_BINARY_SUFFIXES or content is None: return False - return not any(ord(character) < 32 and character not in "\t\n\r" for character in text) + return is_probably_utf8_text(content) def is_textual_wheel_member(path: PurePosixPath, content: bytes | None = None) -> bool: @@ -117,9 +132,11 @@ def text_security_findings( __all__ = [ "FORBIDDEN_SHELL", + "KNOWN_BINARY_SUFFIXES", "TEXT_SUFFIXES", "is_textual_content", "is_secret_filename", + "is_probably_utf8_text", "is_textual_wheel_member", "text_security_findings", ] diff --git a/tests/test_generation.py b/tests/test_generation.py index a707b57..bf6fcaa 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -11,7 +11,7 @@ import subprocess import sys import zipfile -from pathlib import Path +from pathlib import Path, PurePosixPath import pytest @@ -59,6 +59,7 @@ from python_deployment_builder.planning.index import target_marker_applies from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.planning.planner import create_deployment_plan +from python_deployment_builder.security_policy import is_textual_content from python_deployment_builder.validation.static import validate_static_kit FIXTURES = Path(__file__).parent / "fixtures" @@ -3791,6 +3792,54 @@ def test_generation_structural_validation_scans_shared_textual_configuration_for validate_rendered_files(files, manifest, generated_paths=set(owned), secret_values=[]) +@pytest.mark.parametrize("suffix", [".sql", ".j2", ".properties", ".mytemplate"]) +def test_staged_unknown_text_extensions_receive_security_scanning( + tmp_path: Path, suffix: str +) -> None: + plan = _plan() + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + owned, manifest = _render_owned_files( + plan, FIXTURES / "prepared_gui", bootstrap_mode="bundled_uv", + system_certs=False, approved=[], bundled_uv=fake_uv, + ) + files = { + "pyproject.toml": (FIXTURES / "prepared_gui" / "pyproject.toml").read_bytes(), + "uv.lock": (FIXTURES / "prepared_gui" / "uv.lock").read_bytes(), + f"app/query{suffix}": b"credential=sk-abcdefghijklmnop\n", + **owned, + } + + with pytest.raises(PreparationError, match="NO_SECRET_VALUES"): + validate_rendered_files(files, manifest, generated_paths=set(owned), secret_values=[]) + + +@pytest.mark.parametrize("suffix", [".sql", ".j2", ".properties", ".custom"]) +def test_application_wheel_unknown_text_extensions_receive_security_scanning( + tmp_path: Path, suffix: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={f"installed_app/query{suffix}": "credential=sk-abcdefghijklmnop\n"}, + ) + + with pytest.raises(PreparationError, match="security policy"): + validate_application_wheel(wheel, assessment, plan) + + +def test_unknown_and_known_binary_content_do_not_be_text_classified() -> None: + assert not is_textual_content(PurePosixPath("app/resource.custom"), b"\x00\xff\x01") + assert not is_textual_content(PurePosixPath("app/image.png"), b"plain utf8 bytes") + assert is_textual_content(PurePosixPath("app/query.custom"), b"select 1\n") + + def test_uv_archive_rejects_traversal_and_hash_version_mismatch(tmp_path: Path) -> None: archive = tmp_path / "uv.zip" with zipfile.ZipFile(archive, "w") as bundle: From 30defc8d3bfe1f8ee933d6e57179ef907bf0b76c Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:43:42 -0400 Subject: [PATCH 24/59] Harden environment and wheel path validation --- .../generation/artifacts.py | 35 +++++++++++++++++-- .../generation/manifest.py | 7 ++-- .../security_policy.py | 7 ++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 1000fcc..43434b0 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -48,6 +48,10 @@ text_security_findings, ) +MAX_WHEEL_MEMBERS = 10_000 +MAX_WHEEL_MEMBER_SIZE = 256 * 1024 * 1024 +MAX_WHEEL_TOTAL_UNCOMPRESSED_SIZE = 512 * 1024 * 1024 + def parse_artifact_argument(value: str) -> tuple[str, Path]: name, separator, raw_path = value.partition("=") @@ -73,14 +77,20 @@ def _normalized_wheel_path(value: str) -> str: def _safe_wheel_members(bundle: zipfile.ZipFile) -> list[zipfile.ZipInfo]: members = bundle.infolist() + if len(members) > MAX_WHEEL_MEMBERS: + raise PreparationError(f"Wheel contains too many archive members: {len(members)}") + total_size = sum(member.file_size for member in members) + if total_size > MAX_WHEEL_TOTAL_UNCOMPRESSED_SIZE: + raise PreparationError("Wheel exceeds the maximum expanded archive size.") seen: dict[str, str] = {} + regular_paths: dict[str, str] = {} for member in members: normalized = _normalized_wheel_path(member.filename) file_type = (member.external_attr >> 16) & 0o170000 if ( member.flag_bits & 0x1 or file_type == stat.S_IFLNK - or member.file_size > 256 * 1024 * 1024 + or member.file_size > MAX_WHEEL_MEMBER_SIZE ): raise PreparationError(f"Wheel contains an unsafe member: {member.filename}") collision_key = normalized.rstrip("/").casefold() @@ -90,6 +100,25 @@ def _safe_wheel_members(bundle: zipfile.ZipFile) -> list[zipfile.ZipInfo]: f"{previous}, {member.filename}" ) seen[collision_key] = member.filename + if member.is_dir(): + continue + parts = PurePosixPath(normalized).parts + for index in range(1, len(parts)): + ancestor = "/".join(parts[:index]).casefold() + if ancestor in regular_paths: + raise PreparationError( + "Wheel contains a regular-file ancestor collision: " + f"{regular_paths[ancestor]}, {member.filename}" + ) + for existing_key, existing_name in regular_paths.items(): + if collision_key.startswith(existing_key + "/") or existing_key.startswith( + collision_key + "/" + ): + raise PreparationError( + "Wheel contains a regular-file ancestor collision: " + f"{existing_name}, {member.filename}" + ) + regular_paths[collision_key] = member.filename return members @@ -322,8 +351,8 @@ def installed_wheel_member_paths( "Wheel contains colliding installed member paths: " f"{previous}, {name}" ) - installed[key] = name - return set(installed) + installed[key] = normalized + return set(installed.values()) def _require_core_metadata(message, *, label: str, wheel: Path) -> tuple[str, str]: diff --git a/src/python_deployment_builder/generation/manifest.py b/src/python_deployment_builder/generation/manifest.py index b65bd25..d3de986 100644 --- a/src/python_deployment_builder/generation/manifest.py +++ b/src/python_deployment_builder/generation/manifest.py @@ -15,6 +15,7 @@ DeploymentManifest, DeploymentPlan, ) +from python_deployment_builder.security_policy import is_valid_environment_name def source_roots_from_plan(plan: DeploymentPlan) -> list[str]: @@ -142,10 +143,12 @@ def build_deployment_manifest( sync_arguments=sync_arguments, project_write_probe_required=plan.writes.requires_project_write_probe, configuration_presence_names=sorted( - item.name for item in plan.configuration if item.name.isidentifier() + item.name for item in plan.configuration if is_valid_environment_name(item.name) ), configuration_secret_names=sorted( - item.name for item in plan.configuration if item.secret and item.name.isidentifier() + item.name + for item in plan.configuration + if item.secret and is_valid_environment_name(item.name) ), referenced_files=sorted(referenced_files), application_version=plan.application_version, diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index 4c37a6b..97ba73c 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -68,6 +68,12 @@ def is_secret_filename(filename: str) -> bool: ) +def is_valid_environment_name(name: str) -> bool: + """Return whether a name is safe as an ordinary Windows process variable.""" + + return bool(name) and "\x00" not in name and "=" not in name and not name.startswith("=") + + def is_probably_utf8_text(content: bytes) -> bool: """Classify unknown bytes without decoding binary content with replacement.""" @@ -136,6 +142,7 @@ def text_security_findings( "TEXT_SUFFIXES", "is_textual_content", "is_secret_filename", + "is_valid_environment_name", "is_probably_utf8_text", "is_textual_wheel_member", "text_security_findings", From 9425631e739980be9f039a6f7810126442fa0598 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:59:35 -0400 Subject: [PATCH 25/59] Block unsupported uv workspace generation --- .../analysis/assessor.py | 37 +++++++ .../analysis/metadata.py | 30 +++++- .../generation/generator.py | 37 +++++++ tests/test_cli.py | 36 +++++++ tests/test_generation.py | 102 ++++++++++++++++++ 5 files changed, 241 insertions(+), 1 deletion(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index db28974..eff2917 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -135,6 +135,43 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen runtime.write_locations, configuration, ) + if metadata.uv_workspace: + risks.append( + RiskFinding( + code="UV_WORKSPACE_UNSUPPORTED", + title="uv workspace deployment is not supported", + severity=RiskSeverity.BLOCKING, + status=FindingStatus.DETECTED, + description=( + "This project declares a uv workspace. M6.1 standalone deployment does " + "not preserve or install uv workspace members, while locked workspace " + "validation depends on their metadata." + ), + recommendation=( + "Generate a standalone non-workspace project or wait for workspace-aware " + "deployment support." + ), + evidence=metadata.uv_workspace_evidence, + ) + ) + elif metadata.uv_workspace_source: + risks.append( + RiskFinding( + code="UV_WORKSPACE_SOURCE_UNSUPPORTED", + title="uv workspace source is declared without workspace support", + severity=RiskSeverity.BLOCKING, + status=FindingStatus.DETECTED, + description=( + "A uv source is marked workspace=true, but this project does not declare " + "a supported standalone workspace contract." + ), + recommendation=( + "Use a standalone dependency source or define workspace-aware deployment " + "in a future milestone." + ), + evidence=metadata.uv_workspace_evidence, + ) + ) unusual_scope_imports = [ item for item in imports.observations diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index fcc8578..57aca13 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -7,7 +7,7 @@ import fnmatch import re import tomllib -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -30,6 +30,11 @@ class MetadataResult: project: PackagingAssessment python: PythonRequirementAssessment dependencies: list[DependencyAssessment] + # Assessment inputs rather than persisted packaging fields: an M6.1 + # standalone kit cannot safely represent a uv workspace. + uv_workspace: bool = False + uv_workspace_source: bool = False + uv_workspace_evidence: list[Evidence] = field(default_factory=list) # Setuptools' flat-layout auto-discovery deliberately avoids conventional @@ -462,6 +467,9 @@ def inspect_metadata(root: Path) -> MetadataResult: package_discovery_rules: list[tuple[list[str], list[str], list[str], bool]] = [] automatic_setuptools_root: str | None = None automatic_setuptools_flat_surface_ambiguous = False + uv_workspace = False + uv_workspace_source = False + uv_workspace_evidence: list[Evidence] = [] pyproject_path = root / "pyproject.toml" if pyproject_path.is_file(): @@ -549,6 +557,23 @@ def inspect_metadata(root: Path) -> MetadataResult: if isinstance(name, str) and isinstance(target, str) ) tool = document.get("tool") if isinstance(document.get("tool"), dict) else {} + uv = tool.get("uv") if isinstance(tool.get("uv"), dict) else {} + uv_workspace = isinstance(uv.get("workspace"), dict) + uv_sources = uv.get("sources") if isinstance(uv.get("sources"), dict) else {} + uv_workspace_source = any( + isinstance(source, dict) and source.get("workspace") is True + for source in uv_sources.values() + ) + if uv_workspace or uv_workspace_source: + table = "[tool.uv.workspace]" if uv_workspace else "[tool.uv.sources]" + uv_workspace_evidence.append( + _evidence( + root, + pyproject_path, + f"Declared in {table}.", + _line_number(pyproject_path, table.removeprefix("[").removesuffix("]")), + ) + ) poetry = tool.get("poetry") if isinstance(tool.get("poetry"), dict) else {} if poetry and not project: distribution_name = ( @@ -1084,4 +1109,7 @@ def inspect_metadata(root: Path) -> MetadataResult: evidence=python_evidence, ), dependencies=_merge_dependencies(dependencies), + uv_workspace=uv_workspace, + uv_workspace_source=uv_workspace_source, + uv_workspace_evidence=uv_workspace_evidence, ) diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 4269d19..24a38d6 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -775,6 +775,43 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) + # A workspace root cannot be reduced to this M6.1 kit's root + # ``pyproject.toml`` + ``uv.lock`` representation: uv still resolves + # member metadata under --no-install-project. Report the typed blocker + # before staging, artifact work, or an explicitly authorized lock update. + workspace_blockers = { + code + for code in plan.risk_gate.blocking_codes + if code in {"UV_WORKSPACE_UNSUPPORTED", "UV_WORKSPACE_SOURCE_UNSUPPORTED"} + } + if workspace_blockers: + workspace_code = sorted(workspace_blockers)[0] + if dry_run: + preview = _preview( + plan, + output_root, + dry_run=True, + bootstrap_mode=bootstrap_mode, + system_certs=system_certs, + prepare_lock=prepare_lock, + approved=[], + application_artifact=None, + staging_source_paths=[], + ) + preview.developer_actions.insert( + 0, + f"Stop: {workspace_code} prevents standalone release generation.", + ) + return GenerationResult( + output_directory=str(output_root), + dry_run=True, + generated=False, + preview=preview, + ) + raise PreparationError( + f"Deployment planning is blocked: {workspace_code}. " + "M6.1 standalone deployment does not preserve or install uv workspace members." + ) allow_missing_lock_for_analysis = _allow_missing_lock_for_analysis( plan, dry_run=dry_run, prepare_lock=prepare_lock ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 28999d4..ddc7ec5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -139,3 +139,39 @@ def test_all_persists_reports_when_risk_gate_blocks(tmp_path: Path) -> None: assert result == 2 _assert_all_reports(output) assert not (output / "deployment-kit").exists() + + +def test_uv_workspace_assess_plan_and_all_report_a_typed_blocker(tmp_path: Path) -> None: + """A workspace is diagnosable, but M6.1 must not stage a partial workspace.""" + + source = tmp_path / "workspace-root" + source.mkdir() + _write_all_source(source) + (source / "packages/unrelated/src/unrelated").mkdir(parents=True) + (source / "packages/unrelated/pyproject.toml").write_text( + "[project]\nname='unrelated'\nversion='1.0'\n", encoding="utf-8" + ) + (source / "packages/unrelated/src/unrelated/__init__.py").write_text( + "", encoding="utf-8" + ) + with (source / "pyproject.toml").open("a", encoding="utf-8") as handle: + handle.write("\n[tool.uv.workspace]\nmembers=['packages/*']\nexclude=['packages/none']\n") + + assess_output = tmp_path / "assess" + plan_output = tmp_path / "plan" + all_output = tmp_path / "all" + assert main(["assess", str(source), "--output-dir", str(assess_output)]) == 0 + assert main(["plan", str(source), "--output-dir", str(plan_output)]) == 1 + assert main(["all", str(source), "--output-dir", str(all_output)]) == 2 + + assert "UV_WORKSPACE_UNSUPPORTED" in (assess_output / "assessment.json").read_text( + encoding="utf-8" + ) + assert "UV_WORKSPACE_UNSUPPORTED" in (plan_output / "deployment-plan.json").read_text( + encoding="utf-8" + ) + _assert_all_reports(all_output) + assert "UV_WORKSPACE_UNSUPPORTED" in (all_output / "reports/deployment-plan.json").read_text( + encoding="utf-8" + ) + assert not (all_output / "deployment-kit").exists() diff --git a/tests/test_generation.py b/tests/test_generation.py index bf6fcaa..2d5df8a 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -506,6 +506,108 @@ def test_application_wheel_validation_and_package_staging( assert not list(output.rglob("*.pyc")) +def _workspace_repository( + tmp_path: Path, *, package_mode: bool, lock: bool +) -> MaterializedRepository: + root = tmp_path / ("package-workspace" if package_mode else "source-workspace") + root.mkdir() + if package_mode: + _write_mapped_project(root) + else: + (root / "app").mkdir() + (root / "app/__init__.py").write_text("", encoding="utf-8") + (root / "app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + "[project]\nname='workspace-root'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nworkspace-root='app.main:main'\n" + "[tool.setuptools]\npackages=['app']\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (root / "packages/unrelated/src/unrelated").mkdir(parents=True) + (root / "packages/unrelated/pyproject.toml").write_text( + "[project]\nname='unrelated'\nversion='1.0'\n", encoding="utf-8" + ) + (root / "packages/unrelated/src/unrelated/__init__.py").write_text("", encoding="utf-8") + with (root / "pyproject.toml").open("a", encoding="utf-8") as handle: + handle.write("\n[tool.uv.workspace]\nmembers=['packages/*']\n") + if not lock: + (root / "uv.lock").unlink(missing_ok=True) + return MaterializedRepository(root=root, source=str(root), source_kind="local") + + +@pytest.mark.parametrize( + ("package_mode", "expected_mode"), [(False, "source"), (True, "package")] +) +def test_uv_workspace_blocks_both_modes_before_staging_or_lock_mutation( + tmp_path: Path, package_mode: bool, expected_mode: str +) -> None: + repository = _workspace_repository(tmp_path, package_mode=package_mode, lock=False) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=repository.root) + output = tmp_path / "kit" + + assert plan.deployment_mode == expected_mode + assert plan.risk_gate.blocking_codes == ["UV_WORKSPACE_UNSUPPORTED"] + assert "UV_WORKSPACE_UNSUPPORTED" in plan.readiness.blocker_codes + with pytest.raises(PreparationError, match="UV_WORKSPACE_UNSUPPORTED"): + generate_deployment_kit( + repository, output, prepare_lock=True, bootstrap_mode="online_cmd" + ) + assert not (repository.root / "uv.lock").exists() + assert not output.exists() + + preview = generate_deployment_kit( + repository, output, dry_run=True, bootstrap_mode="online_cmd" + ).preview + assert preview.readiness_before == "BLOCKED" + assert any("UV_WORKSPACE_UNSUPPORTED" in action for action in preview.developer_actions) + assert not (repository.root / "uv.lock").exists() + assert not output.exists() + + +def test_workspace_source_without_workspace_table_is_a_typed_structural_blocker( + tmp_path: Path, +) -> None: + root = tmp_path / "workspace-source" + root.mkdir() + (root / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + "[project]\nname='workspace-source'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nworkspace-source='app:main'\n" + "[tool.uv.sources]\nlocal = { workspace = true }\n", + encoding="utf-8", + ) + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + + assert [item.code for item in assessment.risks if item.severity.value == "blocking"] == [ + "UV_WORKSPACE_SOURCE_UNSUPPORTED" + ] + + +def test_setuptools_multi_package_project_without_uv_workspace_is_not_blocked( + tmp_path: Path, +) -> None: + root = tmp_path / "ordinary-multi-package" + for package in ("app", "support"): + (root / package).mkdir(parents=True) + (root / package / "__init__.py").write_text("", encoding="utf-8") + (root / "app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + "[project]\nname='ordinary-multi-package'\nversion='1.0'\ndependencies=[]\n" + "[project.scripts]\nordinary-multi-package='app.main:main'\n" + "[tool.setuptools]\npackages=['app', 'support']\n", + encoding="utf-8", + ) + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + + assert not any(item.code.startswith("UV_WORKSPACE") for item in assessment.risks) + + def test_application_wheel_uses_declared_group_not_launch_kind(tmp_path: Path) -> None: source = tmp_path / "source" source.mkdir() From c979773feabdeae0b60690451be1fd29c33d19b2 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:59:55 -0400 Subject: [PATCH 26/59] Require proven package surface for package mode --- .../analysis/metadata.py | 7 + .../analysis/resources.py | 16 + .../generation/artifacts.py | 92 ++++-- .../planning/planner.py | 33 ++- tests/test_generation.py | 275 +++++++++++++++++- tests/test_planning.py | 61 ++++ 6 files changed, 451 insertions(+), 33 deletions(-) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 57aca13..3391213 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -624,6 +624,13 @@ def inspect_metadata(root: Path) -> MetadataResult: # console-script entry-point behavior. entry_points.append(_entry_point(root, pyproject_path, name, target, "scripts")) setuptools = tool.get("setuptools") if isinstance(tool.get("setuptools"), dict) else {} + if build_backend is None and setuptools: + # A project that supplies setuptools' own pyproject configuration + # but omits [build-system] follows the conventional setuptools + # legacy PEP 517 fallback. Record that supported backend explicitly + # so the same authoritative surface resolver serves this form as + # explicit setuptools.build_meta projects. + build_backend = "setuptools.build_meta:__legacy__" configured_packages = setuptools.get("packages") if isinstance(configured_packages, list): packages = [value for value in configured_packages if isinstance(value, str)] diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index cb481eb..37fa46f 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -310,6 +310,22 @@ def resolve_packaged_python_sources( return sorted(resolved, key=lambda item: (item.source_path, item.installed_member_path)) +def package_surface_resolved(project: PackagingAssessment | None) -> bool: + """Whether M6.1 has an authoritative Python wheel-surface model. + + A build backend establishes only that a project might be buildable. The + package/source resolver is deliberately a bounded static setuptools model; + it must not silently stand in for Hatchling, Poetry, or arbitrary PEP 517 + backend discovery. + """ + + return bool( + project + and project.build_backend + and project.build_backend.partition(":")[0] == "setuptools.build_meta" + ) + + def _declared_package_data( root: Path, project: PackagingAssessment | None ) -> dict[str, list[Evidence]]: diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 43434b0..e0e4692 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -23,6 +23,7 @@ from packaging.version import InvalidVersion, Version from python_deployment_builder.analysis.resources import ( + package_surface_resolved, resolve_package_data_members, resolve_packaged_python_sources, ) @@ -570,31 +571,49 @@ def _application_requirement_applies(requirement: Requirement, plan: DeploymentP ) from exc -def _validate_dependency_extra_closure(requirement: Requirement, graph) -> None: +def _target_possible_dependencies(graph, canonical_name: str): + """Return all selected-target lock candidates for one distribution.""" + + return sorted( + ( + dependency + for dependency in graph.dependencies + if canonicalize_name(dependency.name) == canonical_name + ), + key=lambda dependency: (Version(dependency.version), dependency.version), + ) + + +def _validate_dependency_extra_closure(requirement: Requirement, graph, candidates) -> None: """Prove a wheel dependency's requested extras are activated by the selected lock graph.""" requested = {canonicalize_name(extra) for extra in requirement.extras} name = canonicalize_name(requirement.name) - candidates = [ - dependency - for dependency in graph.dependencies - if canonicalize_name(dependency.name) == name - and Version(dependency.version) in requirement.specifier - and requested + without_activation = [ + dependency.version + for dependency in candidates + if not requested <= {canonicalize_name(extra) for extra in dependency.requested_dependency_extras} ] - if not candidates: + if without_activation: raise PreparationError( - "Application wheel Requires-Dist dependency extra cannot be proven against the " - f"selected locked environment: {requirement.name}[{','.join(sorted(requested))}]" + "Application wheel Requires-Dist dependency extra is not activated for every " + "target-possible locked version: " + f"{requirement.name}[{','.join(sorted(requested))}]. Missing activation: " + + ", ".join(sorted(set(without_activation))) ) - if not any( - requested <= {canonicalize_name(extra) for extra in item.available_dependency_extras} - for item in candidates - ): + without_declaration = [ + dependency.version + for dependency in candidates + if not requested + <= {canonicalize_name(extra) for extra in dependency.available_dependency_extras} + ] + if without_declaration: raise PreparationError( - "Application wheel Requires-Dist dependency extra is not declared by the locked " - f"dependency: {requirement.name}[{','.join(sorted(requested))}]" + "Application wheel Requires-Dist dependency extra is not declared by every " + "target-possible locked version: " + f"{requirement.name}[{','.join(sorted(requested))}]. Missing declaration: " + + ", ".join(sorted(set(without_declaration))) ) known = {canonicalize_name(item.name) for item in graph.dependencies} @@ -633,9 +652,6 @@ def _validate_application_requires_dist( "Application wheel Requires-Dist validation requires an inspected selected uv.lock " "dependency graph." ) - locked: dict[str, list[str]] = {} - for dependency in graph.dependencies: - locked.setdefault(canonicalize_name(dependency.name), []).append(dependency.version) for raw in raw_requirements: try: requirement = Requirement(raw) @@ -669,26 +685,34 @@ def _validate_application_requires_dist( "Application wheel Requires-Dist direct references are not provable against " f"the selected locked environment: {requirement.name}." ) - versions = locked.get(name, []) - if not versions: - raise PreparationError( - "Application wheel Requires-Dist is absent from the selected locked " - f"environment: {requirement.name}." - ) try: - compatible = any(Version(version) in requirement.specifier for version in versions) + candidates = _target_possible_dependencies(graph, name) except InvalidVersion as exc: raise PreparationError( "Selected lock graph has an invalid version for application wheel " - f"Requires-Dist {requirement.name}: {versions!r}." + f"Requires-Dist {requirement.name}." ) from exc - if not compatible: + if not candidates: raise PreparationError( - "Application wheel Requires-Dist is incompatible with the selected locked " - f"environment: {requirement}. Locked versions: {', '.join(sorted(versions))}." + "Application wheel Requires-Dist is absent from the selected locked " + f"environment: {requirement.name}." + ) + versions = sorted({candidate.version for candidate in candidates}) + incompatible = sorted( + { + candidate.version + for candidate in candidates + if Version(candidate.version) not in requirement.specifier + } + ) + if incompatible: + raise PreparationError( + "Application wheel Requires-Dist cannot be proven for every target-possible " + f"locked version: {requirement}. Possible versions: {', '.join(versions)}. " + f"incompatible possible versions: {', '.join(incompatible)}." ) if requirement.extras: - _validate_dependency_extra_closure(requirement, graph) + _validate_dependency_extra_closure(requirement, graph, candidates) def validate_application_wheel( @@ -712,6 +736,12 @@ def validate_application_wheel( raise PreparationError( "Package mode requires authoritative project distribution and version metadata." ) + if not package_surface_resolved(assessment.project): + backend = assessment.project.build_backend or "no build backend" + raise PreparationError( + "Application wheel validation requires an authoritative Python packaging-surface " + f"model; {backend} is not modeled by M6.1." + ) try: expected_version_value = Version(expected_version) except InvalidVersion as exc: diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 503b244..c98224b 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -12,7 +12,10 @@ from python_deployment_builder import __version__ from python_deployment_builder.analysis.inventory import resource_covers_inventory_path -from python_deployment_builder.analysis.resources import resolve_packaged_python_sources +from python_deployment_builder.analysis.resources import ( + package_surface_resolved, + resolve_packaged_python_sources, +) from python_deployment_builder.backends.uv_managed import UvManagedBackend from python_deployment_builder.models import ( ConfigurationPlan, @@ -139,6 +142,20 @@ def _deployment_mode( and assessment.project.version and assessment.project.build_backend ) + surface_resolved = package_surface_resolved(assessment.project) + backend = assessment.project.build_backend or "no build backend" + + def unresolved_surface_result() -> tuple[str, str, str, list[str]]: + return ( + "package", + "The authoritative entry point requires installation, but M6.1 does not model " + f"the first-party Python packaging surface for {backend}.", + "INSTALLED_PROJECT_REQUIRED", + [ + "PACKAGING_SURFACE_UNRESOLVED: package mode requires an authoritative " + f"Python packaging-surface model, but {backend} is not modeled by M6.1." + ], + ) if source_constraints and source_compatible: return ( "source", @@ -156,6 +173,8 @@ def _deployment_mode( "INSTALLED_PROJECT_REQUIRED", ["INSTALLED_PROJECT_REQUIRED: buildable project metadata is incomplete"], ) + if not surface_resolved: + return unresolved_surface_result() return ( "package", "Source layout requirements conflict with an authoritative entry point that cannot " @@ -175,6 +194,8 @@ def _deployment_mode( "INSTALLED_PROJECT_REQUIRED", ["INSTALLED_PROJECT_REQUIRED: buildable project metadata is incomplete"], ) + if not surface_resolved: + return unresolved_surface_result() return ( "package", "The authoritative entry point is not source-import compatible; install a validated " @@ -190,6 +211,16 @@ def _deployment_mode( "SOURCE_COMPATIBLE", [], ) + if not surface_resolved: + return ( + "source", + "The project uses " + f"{backend}, whose installed Python packaging surface is not modeled by M6.1. " + "The authoritative entry point is source-import compatible, so source deployment " + "preserves the statically understood runtime surface.", + "SOURCE_COMPATIBLE", + [], + ) return ( "package", "The project has an install-oriented source layout without a source-only runtime " diff --git a/tests/test_generation.py b/tests/test_generation.py index 2d5df8a..bce8d62 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -51,6 +51,7 @@ ApprovedArtifact, ArtifactAvailability, BootstrapArtifact, + DependencyEdge, LockedDependency, LockGraphAssessment, ) @@ -322,6 +323,39 @@ def _application_plan_with_locked_dependencies( return configured +def _application_plan_with_target_possible_dependencies( + plan, + versions: list[str], + *, + requested_extras: list[str] | None = None, + available_extras: dict[str, list[str]] | None = None, + edges=None, +): + configured = plan.model_copy(deep=True) + configured.lock_graph = LockGraphAssessment( + inspected=True, + python_version=configured.runtime.python_version, + architecture=configured.runtime.architecture, + dependencies=[ + LockedDependency( + name="foo", + version=version, + direct=True, + requested_dependency_extras=requested_extras or [], + available_dependency_extras=(available_extras or {}).get(version, []), + artifact=ArtifactAvailability( + compatible_wheel_available=True, + source_distribution_available=False, + policy="wheel_usable", + ), + ) + for version in versions + ], + edges=edges or [], + ) + return configured + + def _write_requests_lock(root: Path, *, version: str = "2.31.0") -> bytes: """Write a minimal inspected uv lock graph for mapped-app -> requests.""" @@ -684,6 +718,8 @@ def test_poetry_string_script_uses_console_scripts_for_wheel_validation(tmp_path plan = create_deployment_plan(assessment) assert plan.entry_point is not None assert plan.deployment_mode == "package" + assert plan.deployment_mode_condition == "INSTALLED_PROJECT_REQUIRED" + assert plan.readiness.blocker_codes == ["PACKAGING_SURFACE_UNRESOLVED"] assert plan.entry_point.declared_group == "console_scripts" wheel = _make_application_wheel( @@ -691,7 +727,8 @@ def test_poetry_string_script_uses_console_scripts_for_wheel_validation(tmp_path entry_group="console_scripts", entry_name="poetry-tool", ) - validate_application_wheel(wheel, assessment, plan) + with pytest.raises(PreparationError, match="packaging-surface model"): + validate_application_wheel(wheel, assessment, plan) def test_application_wheel_rejects_wrong_target_and_runtime_cache(tmp_path: Path) -> None: @@ -952,6 +989,241 @@ def test_application_wheel_requires_dist_rejects_unprovable_metadata( validate_application_wheel(wheel, assessment, plan) +@pytest.mark.parametrize( + ("requirement", "accepted"), + [ + ("foo>=1", True), + ("foo<3", True), + ("foo>=2", False), + ("foo==2", False), + ("foo!=1", False), + ("foo", True), + ], +) +def test_application_wheel_requires_dist_proves_every_target_possible_version( + tmp_path: Path, requirement: str, accepted: bool +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_target_possible_dependencies( + create_deployment_plan(assessment, repository_root=source), ["1.0", "2.0"] + ) + wheel = _make_application_wheel(tmp_path, requires_dist_values=[requirement]) + + if accepted: + assert validate_application_wheel(wheel, assessment, plan)[0] + else: + with pytest.raises(PreparationError, match="every target-possible.*1.0"): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_requires_dist_ignores_definitely_pruned_versions( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + # Lock inspection is responsible for pruning definitely inapplicable marker + # branches. The application proof sees only the retained 2.0 candidate. + plan = _application_plan_with_target_possible_dependencies( + create_deployment_plan(assessment, repository_root=source), ["2.0"] + ) + + assert validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo>=2"]), assessment, plan + )[0] + + +def test_application_wheel_requires_dist_proves_every_patch_marker_possible_lock_branch( + tmp_path: Path, +) -> None: + """Minor-only marker uncertainty retains both uv lock branches for proof.""" + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "uv.lock").write_text( + """version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "mapped-app" +version = "1.2.3" +source = { virtual = "." } +dependencies = [ + { name = "foo", version = "1.0", marker = "python_full_version < '3.12.1'" }, + { name = "foo", version = "2.0", marker = "python_full_version >= '3.12.1'" }, +] + +[[package]] +name = "foo" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +wheels = [{ url = "https://example.invalid/foo-1.0-py3-none-any.whl" }] + +[[package]] +name = "foo" +version = "2.0" +source = { registry = "https://pypi.org/simple" } +wheels = [{ url = "https://example.invalid/foo-2.0-py3-none-any.whl" }] +""", + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + graph = inspect_uv_lock( + source, + "mapped-app", + plan.runtime.python_version, + plan.runtime.architecture, + [], + ) + assert [(item.name, item.version) for item in graph.dependencies] == [ + ("foo", "1.0"), + ("foo", "2.0"), + ] + configured = plan.model_copy(update={"lock_graph": graph}) + + assert validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo>=1"]), + assessment, + configured, + )[0] + with pytest.raises(PreparationError, match="every target-possible.*1.0"): + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo>=2"]), + assessment, + configured, + ) + + +def test_application_wheel_requires_dist_rejects_invalid_target_possible_version( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_target_possible_dependencies( + create_deployment_plan(assessment, repository_root=source), ["1.0", "not-a-version"] + ) + + with pytest.raises(PreparationError, match="invalid version"): + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo>=1"]), assessment, plan + ) + + +def test_application_wheel_dependency_extra_requires_every_candidate_activation_and_declaration( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + base = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path, requires_dist_values=["foo[bar]>=1"]) + + complete = _application_plan_with_target_possible_dependencies( + base, + ["1.0", "2.0"], + requested_extras=["bar"], + available_extras={"1.0": ["bar"], "2.0": ["bar"]}, + ) + assert validate_application_wheel(wheel, assessment, complete)[0] + + missing_activation = _application_plan_with_target_possible_dependencies( + base, + ["1.0", "2.0"], + requested_extras=[], + available_extras={"1.0": ["bar"], "2.0": ["bar"]}, + ) + with pytest.raises(PreparationError, match="not activated.*1.0.*2.0"): + validate_application_wheel(wheel, assessment, missing_activation) + + missing_declaration = _application_plan_with_target_possible_dependencies( + base, + ["1.0", "2.0"], + requested_extras=["bar"], + available_extras={"1.0": [], "2.0": ["bar"]}, + ) + with pytest.raises(PreparationError, match="not declared.*1.0"): + validate_application_wheel(wheel, assessment, missing_declaration) + + +def test_application_wheel_dependency_extra_closure_remains_conservative_for_candidates( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_target_possible_dependencies( + create_deployment_plan(assessment, repository_root=source), + ["1.0", "2.0"], + requested_extras=["bar"], + available_extras={"1.0": ["bar"], "2.0": ["bar"]}, + edges=[DependencyEdge(from_package="foo", to_package="bar-helper")], + ) + + with pytest.raises(PreparationError, match="closure is incomplete"): + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo[bar]>=1"]), + assessment, + plan, + ) + + +def test_application_wheel_rejects_unresolved_hatchling_surface_before_entry_module_only_proof( + tmp_path: Path, +) -> None: + source = tmp_path / "hatchling-source" + (source / "src/demo_app").mkdir(parents=True) + (source / "src/demo_app/__init__.py").write_text("", encoding="utf-8") + (source / "src/demo_app/main.py").write_text( + "def main():\n from . import helper\n return helper.run()\n", encoding="utf-8" + ) + (source / "src/demo_app/helper.py").write_text("def run(): return 0\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['hatchling']\nbuild-backend='hatchling.build'\n" + "[project]\nname='demo-app'\nversion='1.0'\n" + "[project.scripts]\ndemo='demo_app.main:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version=1\nrevision=3\n", encoding="utf-8") + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source).model_copy(deep=True) + plan.deployment_mode = "package" # Simulate a stale/manual programmatic plan. + stale_wheel = _make_application_wheel( + tmp_path, + name="demo-app", + version="1.0", + package="demo_app", + target="demo_app.main:main", + entry_group="console_scripts", + entry_name="demo", + ) + + with pytest.raises(PreparationError, match="hatchling.build is not modeled"): + validate_application_wheel(stale_wheel, assessment, plan, repository_root=source) + + @pytest.mark.parametrize( ("requirement", "accepted"), [ @@ -1794,6 +2066,7 @@ def test_application_wheel_requires_nested_package_data_from_parent_mapping( (source / "lib/sub/main.py").write_text("def main(): return 0\n", encoding="utf-8") (source / "lib/sub/data/default.json").write_text("{}\n", encoding="utf-8") (source / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" "[project]\nname = 'mapped-app'\nversion = '1.2.3'\ndependencies = []\n" "[project.gui-scripts]\nmapped-app = 'app.sub.main:main'\n" "[tool.setuptools]\npackages = ['app', 'app.sub']\npackage-dir = {app = 'lib'}\n" diff --git a/tests/test_planning.py b/tests/test_planning.py index ba9dc84..30cb0d5 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -95,6 +95,67 @@ def _write_mode_project( (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") +def _write_unresolved_backend_project(root: Path, *, backend: str, target: str) -> None: + (root / "src/demo_app").mkdir(parents=True) + (root / "src/demo_app/__init__.py").write_text("", encoding="utf-8") + (root / "src/demo_app/main.py").write_text( + "def main():\n from . import helper\n return helper.run()\n", encoding="utf-8" + ) + (root / "src/demo_app/helper.py").write_text("def run(): return 0\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + "[build-system]\n" + f"requires = ['{backend.split('.')[0]}']\n" + f"build-backend = '{backend}'\n" + "[project]\nname = 'demo-app'\nversion = '1.0'\n" + f"[project.scripts]\ndemo = '{target}'\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + +@pytest.mark.parametrize("backend", ["hatchling.build", "poetry.core.masonry.api"]) +def test_unresolved_backend_src_entrypoint_preserves_source_mode( + tmp_path: Path, backend: str +) -> None: + root = tmp_path / backend.replace(".", "-") + _write_unresolved_backend_project(root, backend=backend, target="demo_app.main:main") + + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=root) + + assert assessment.project.packages == [] + assert assessment.project.py_modules == [] + assert (plan.deployment_mode, plan.deployment_mode_condition) == ( + "source", + "SOURCE_COMPATIBLE", + ) + assert backend in plan.decisions[0].rationale + + +@pytest.mark.parametrize( + "backend", ["hatchling.build", "poetry.core.masonry.api", "example.backend"] +) +def test_unresolved_backend_installed_entrypoint_is_blocked( + tmp_path: Path, backend: str +) -> None: + root = tmp_path / backend.replace(".", "-") + _write_unresolved_backend_project(root, backend=backend, target="installed_app.main:main") + + plan = create_deployment_plan( + assess_repository(MaterializedRepository(root=root, source=str(root), source_kind="local")), + repository_root=root, + ) + + assert (plan.deployment_mode, plan.deployment_mode_condition) == ( + "package", + "INSTALLED_PROJECT_REQUIRED", + ) + assert plan.readiness.state == "BLOCKED" + assert plan.readiness.blocker_codes == ["PACKAGING_SURFACE_UNRESOLVED"] + + def test_complete_deployment_mode_decision_table(tmp_path: Path) -> None: cases = { "flat-source": dict(), From 476c2fc5f34051c480fa0cf2c39ddd676c896233 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:51:31 -0400 Subject: [PATCH 27/59] Block ambiguous developer artifact forks --- .../analysis/assessor.py | 7 +- .../analysis/inventory.py | 37 +++- .../generation/artifacts.py | 17 +- .../generation/generator.py | 12 +- .../planning/lockfile.py | 34 ++++ .../planning/planner.py | 6 +- tests/test_generation.py | 160 ++++++++++++++++++ tests/test_planning.py | 62 +++++++ 8 files changed, 313 insertions(+), 22 deletions(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index eff2917..aeaa4eb 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -69,7 +69,12 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen root = repository.root metadata = inspect_metadata(root) inventory = inventory_repository(root, metadata.project.source_roots) - promote_imported_application_files(root, inventory.items, inventory.application_files) + promote_imported_application_files( + root, + inventory.items, + inventory.application_files, + metadata.project.source_roots, + ) imports = scan_imports( root, metadata.project.source_roots, diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index 93b6553..f97c2f8 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -286,21 +286,40 @@ def _imported_modules(tree: ast.AST) -> list[tuple[str, int]]: return modules -def _module_files(root: Path, module: str) -> list[Path]: +def _module_files(root: Path, module: str, source_roots: list[str]) -> list[Path]: + """Resolve a simple absolute import beneath every safe configured source root. + + This deliberately mirrors the packaging/inventory source-root model instead + of assuming only the repository root and ``src``. It is not an import + system emulator: candidates are ordinary module files or package + initializers, never imported or executed. + """ + relative = Path(*module.split(".")) - candidates = [ - root / relative.with_suffix(".py"), - root / relative / "__init__.py", - root / "src" / relative.with_suffix(".py"), - root / "src" / relative / "__init__.py", - ] - return [path for path in candidates if path.is_file()] + resolved_root = root.resolve() + candidates: list[Path] = [] + for source_root in source_roots: + candidate_root = root / source_root + if candidate_root.is_symlink() or not candidate_root.is_dir(): + continue + try: + candidate_root.resolve().relative_to(resolved_root) + except ValueError: + continue + candidates.extend( + ( + candidate_root / relative.with_suffix(".py"), + candidate_root / relative / "__init__.py", + ) + ) + return sorted({path for path in candidates if path.is_file()}) def promote_imported_application_files( root: Path, items: list[RepositoryFileInventoryItem], application_files: list[Path], + source_roots: list[str], ) -> None: """Promote non-ignored Python modules imported by production source.""" @@ -318,7 +337,7 @@ def promote_imported_application_files( except (OSError, SyntaxError, UnicodeError): continue for module, line in _imported_modules(tree): - for imported_path in _module_files(root, module): + for imported_path in _module_files(root, module, source_roots): relative = imported_path.relative_to(root).as_posix() item = by_path.get(relative) if item is None: diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index e0e4692..b44bab8 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -466,15 +466,22 @@ def validate_approved_wheel( f"Artifact name mismatch: option requested {requested_name}, filename contains " f"{filename_name}." ) - requirements = { - canonicalize_name(item.package): item + requirements = [ + item for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) - } - requirement = requirements.get(requested_name) - if requirement is None: + if canonicalize_name(item.package) == requested_name + ] + versions = {item.version for item in requirements} + if not requirements: raise PreparationError( f"No developer-wheel requirement exists for {requested_name} in this deployment plan." ) + if len(versions) != 1: + raise PreparationError( + "Developer artifact substitution is ambiguous for target-possible locked versions: " + f"{requested_name} ({', '.join(sorted(versions))})." + ) + requirement = requirements[0] if str(filename_version) != requirement.version: raise PreparationError( f"Artifact version mismatch for {requested_name}: expected {requirement.version}, " diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 24a38d6..e8467e1 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -868,10 +868,10 @@ def generate_deployment_kit( ) approved = validate_artifact_set(artifact_values, plan) requirements = { - canonicalize_name(item.package) + (canonicalize_name(item.package), item.version) for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) } - supplied = {item[0].distribution_name for item in approved} + supplied = {(item[0].distribution_name, item[0].version) for item in approved} unresolved = sorted(requirements - supplied) unavailable = [ item.package @@ -927,7 +927,7 @@ def generate_deployment_kit( ) if unresolved or unavailable: detail = [ - *(f"approved wheel required: {item}" for item in unresolved), + *(f"approved wheel required: {name}=={version}" for name, version in unresolved), *(f"no usable artifact: {item}" for item in unavailable), ] raise PreparationError("Deployment readiness remains blocked: " + "; ".join(detail)) @@ -960,10 +960,10 @@ def generate_deployment_kit( else None ) requirements = { - canonicalize_name(item.package) + (canonicalize_name(item.package), item.version) for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) } - supplied = {item[0].distribution_name for item in approved} + supplied = {(item[0].distribution_name, item[0].version) for item in approved} unresolved = sorted(requirements - supplied) unavailable = [ item.package @@ -972,7 +972,7 @@ def generate_deployment_kit( ] if unresolved or unavailable: detail = [ - *(f"approved wheel required: {item}" for item in unresolved), + *(f"approved wheel required: {name}=={version}" for name, version in unresolved), *(f"no usable artifact: {item}" for item in unavailable), ] raise PreparationError("Deployment readiness remains blocked: " + "; ".join(detail)) diff --git a/src/python_deployment_builder/planning/lockfile.py b/src/python_deployment_builder/planning/lockfile.py index 3444e15..40ed15a 100644 --- a/src/python_deployment_builder/planning/lockfile.py +++ b/src/python_deployment_builder/planning/lockfile.py @@ -263,9 +263,43 @@ def enqueue_edges( dependencies = sorted(locked.values(), key=lambda item: (not item.direct, item.name.lower())) findings: list[ArtifactPolicyFinding] = [] requirements: list[DeploymentArtifactRequirement] = [] + developer_artifact_versions: dict[str, set[str]] = {} + for dependency in dependencies: + if ( + dependency.artifact.policy == "developer_wheel_required" + and dependency.artifact.source_distribution_available + ): + developer_artifact_versions.setdefault(canonicalize_name(dependency.name), set()).add( + dependency.version + ) + artifact_forks = { + package: versions + for package, versions in developer_artifact_versions.items() + if len(versions) > 1 + } for dependency in dependencies: if dependency.artifact.policy == "wheel_usable": continue + canonical_name = canonicalize_name(dependency.name) + if canonical_name in artifact_forks: + versions = ", ".join(sorted(artifact_forks[canonical_name])) + findings.append( + ArtifactPolicyFinding( + code="MULTI_VERSION_ARTIFACT_FORK_UNSUPPORTED", + package=dependency.name, + version=dependency.version, + status="unavailable", + dependency_chain=dependency.dependency_chain, + selected_extra=dependency.selected_extra, + description=( + "The selected target leaves multiple possible locked versions of " + f"{dependency.name} that require developer-supplied wheels ({versions}). " + "PDB cannot replace uv's conditional version selection with one " + "unconditional reviewed artifact." + ), + ) + ) + continue status = ( "developer_artifact_required" if dependency.artifact.source_distribution_available diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index c98224b..d83d604 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -419,7 +419,11 @@ def _readiness( if lock_graph and lock_graph.artifact_findings: blocker_codes.extend(item.code for item in lock_graph.artifact_findings) blockers.extend( - f"DEVELOPER_ARTIFACT_REQUIRED:{item.package}=={item.version}" + ( + f"{item.code}: {item.description}" + if item.code == "MULTI_VERSION_ARTIFACT_FORK_UNSUPPORTED" + else f"DEVELOPER_ARTIFACT_REQUIRED:{item.package}=={item.version}" + ) for item in lock_graph.artifact_findings ) if assessment_gate.outcome == "block": diff --git a/tests/test_generation.py b/tests/test_generation.py index bce8d62..34679d3 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -28,6 +28,7 @@ from python_deployment_builder.generation.artifacts import ( validate_application_wheel, validate_approved_wheel, + validate_artifact_set, ) from python_deployment_builder.generation.cmd import parse_certutil_sha256 from python_deployment_builder.generation.generator import ( @@ -52,6 +53,7 @@ ArtifactAvailability, BootstrapArtifact, DependencyEdge, + DeploymentArtifactRequirement, LockedDependency, LockGraphAssessment, ) @@ -435,6 +437,30 @@ def _write_dependency_extra_lock( ) +def _write_developer_artifact_fork_lock(root: Path, *, markers: bool = True) -> None: + """Write target-possible source-only foo branches for artifact-fork tests.""" + first_marker, second_marker = ( + ", marker = \"python_full_version < '3.12.1'\"", + ", marker = \"python_full_version >= '3.12.1'\"" + ) if markers else ("", "") + (root / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = \">=3.12\"\n\n" + "[[package]]\nname = \"mapped-app\"\nversion = \"1.2.3\"\n" + "source = { virtual = \".\" }\n" + "dependencies = [\n" + f' {{ name = "foo", version = "1.0"{first_marker} }},\n' + f' {{ name = "foo", version = "2.0"{second_marker} }},\n' + "]\n\n" + "[[package]]\nname = \"foo\"\nversion = \"1.0\"\n" + "source = { registry = \"https://pypi.org/simple\" }\n" + "sdist = { url = \"https://example.invalid/foo-1.0.tar.gz\" }\n\n" + "[[package]]\nname = \"foo\"\nversion = \"2.0\"\n" + "source = { registry = \"https://pypi.org/simple\" }\n" + "sdist = { url = \"https://example.invalid/foo-2.0.tar.gz\" }\n", + encoding="utf-8", + ) + + def _plan_with_dependency_extra_lock(source: Path, *, selected_extras: list[str] | None = None): repository = MaterializedRepository(root=source, source=str(source), source_kind="local") assessment = assess_repository(repository) @@ -1105,6 +1131,140 @@ def test_application_wheel_requires_dist_proves_every_patch_marker_possible_lock ) +def test_lock_inspection_blocks_multi_version_developer_artifact_fork( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + _write_developer_artifact_fork_lock(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert [(item.name, item.version) for item in plan.lock_graph.dependencies] == [ + ("foo", "1.0"), + ("foo", "2.0"), + ] + assert plan.lock_graph.artifact_requirements == [] + findings = plan.lock_graph.artifact_findings + assert {item.code for item in findings} == {"MULTI_VERSION_ARTIFACT_FORK_UNSUPPORTED"} + assert {item.version for item in findings} == {"1.0", "2.0"} + assert "MULTI_VERSION_ARTIFACT_FORK_UNSUPPORTED" in plan.readiness.blocker_codes + + supplied: list[str] = [] + for version in ("1.0", "2.0"): + wheel = _make_wheel(tmp_path, name="foo", version=version) + supplied.append(f"foo={wheel}") + with pytest.raises(PreparationError, match="No developer-wheel requirement"): + validate_approved_wheel(f"foo={wheel}", plan) + with pytest.raises(PreparationError, match="No developer-wheel requirement"): + validate_artifact_set(supplied, plan) + output = tmp_path / "kit" + with pytest.raises(PreparationError, match="no usable artifact: foo"): + generate_deployment_kit( + repository, + output, + application_wheel=_make_application_wheel(tmp_path), + ) + assert not output.exists() + + +def test_lock_inspection_keeps_unambiguous_developer_artifact_requirements( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "uv.lock").write_text( + """version = 1 +revision = 3 +[[package]] +name = "mapped-app" +version = "1.2.3" +source = { virtual = "." } +dependencies = [ + { name = "foo", version = "1.0", marker = "python_version < '3.12'" }, + { name = "foo", version = "2.0", marker = "python_version >= '3.12'" }, +] +[[package]] +name = "foo" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://example.invalid/foo-1.0.tar.gz" } +[[package]] +name = "foo" +version = "2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://example.invalid/foo-2.0.tar.gz" } +""", + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + plan = create_deployment_plan(assess_repository(repository), repository_root=source) + + assert [(item.package, item.version) for item in plan.lock_graph.artifact_requirements] == [ + ("foo", "2.0") + ] + wheel = _make_wheel(tmp_path, name="foo", version="2.0") + assert validate_approved_wheel(f"foo={wheel}", plan)[0].version == "2.0" + + +def test_lock_inspection_collapses_duplicate_same_version_artifact_paths(tmp_path: Path) -> None: + (tmp_path / "uv.lock").write_text( + """version = 1 +revision = 3 +[[package]] +name = "example" +version = "1.0" +source = { virtual = "." } +dependencies = [ + { name = "foo", version = "1.0", marker = "python_full_version < '3.12.1'" }, + { name = "foo", version = "1.0", marker = "python_full_version >= '3.12.1'" }, +] +[[package]] +name = "foo" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://example.invalid/foo-1.0.tar.gz" } +""", + encoding="utf-8", + ) + + graph = inspect_uv_lock(tmp_path, "example", "3.12", "x86_64", []) + + assert [(item.package, item.version) for item in graph.artifact_requirements] == [ + ("foo", "1.0") + ] + assert {item.code for item in graph.artifact_findings} == {"SOURCE_ONLY_LOCKED_DEPENDENCY"} + + +def test_approved_wheel_rejects_programmatic_multi_version_requirement(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + plan = create_deployment_plan(assess_repository(repository), repository_root=source) + ambiguous = plan.model_copy(deep=True) + ambiguous.lock_graph = LockGraphAssessment( + inspected=True, + python_version=ambiguous.runtime.python_version, + architecture=ambiguous.runtime.architecture, + artifact_requirements=[ + DeploymentArtifactRequirement( + package="foo", version="1.0", action="developer_wheel_required", reason="test" + ), + DeploymentArtifactRequirement( + package="foo", version="2.0", action="developer_wheel_required", reason="test" + ), + ], + ) + + with pytest.raises(PreparationError, match="ambiguous.*1.0, 2.0"): + wheel = _make_wheel(tmp_path, name="foo", version="1.0") + validate_approved_wheel(f"foo={wheel}", ambiguous) + + def test_application_wheel_requires_dist_rejects_invalid_target_possible_version( tmp_path: Path, ) -> None: diff --git a/tests/test_planning.py b/tests/test_planning.py index 30cb0d5..035594b 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -3,6 +3,7 @@ import pytest from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.inventory import _module_files from python_deployment_builder.analysis.repository import MaterializedRepository from python_deployment_builder.backends.uv_managed import UV_VERSION, UvManagedBackend from python_deployment_builder.models import ( @@ -113,6 +114,67 @@ def _write_unresolved_backend_project(root: Path, *, backend: str, target: str) (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") +def _write_custom_source_root_project(root: Path, *, declare_helper: bool = False) -> None: + (root / "lib/app").mkdir(parents=True, exist_ok=True) + (root / "lib/app/__init__.py").write_text("", encoding="utf-8") + (root / "lib/app/main.py").write_text( + "import helper\n\ndef main():\n return helper.value()\n", encoding="utf-8" + ) + (root / "lib/helper.py").write_text("def value(): return 1\n", encoding="utf-8") + helper = "py-modules = ['helper']\n" if declare_helper else "" + (root / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools>=68']\n" + "build-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'custom-root-app'\nversion = '1.0'\n" + "[project.scripts]\ncustom-root = 'app.main:main'\n" + "[tool.setuptools]\npackage-dir = {'' = 'lib'}\n" + + helper + + "[tool.setuptools.packages.find]\nwhere = ['lib']\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + +def test_import_promotion_uses_authoritative_custom_source_roots(tmp_path: Path) -> None: + root = tmp_path / "custom-root" + root.mkdir() + _write_custom_source_root_project(root) + + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + helper = next(item for item in assessment.file_inventory if item.path == "lib/helper.py") + assert assessment.project.source_roots == ["lib"] + assert any("Application source imports local module" in item.detail for item in helper.evidence) + assert create_deployment_plan(assessment, repository_root=root).deployment_mode == "source" + + _write_custom_source_root_project(root, declare_helper=True) + declared = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + assert declared.project.py_modules == ["helper"] + assert create_deployment_plan(declared, repository_root=root).deployment_mode == "package" + + +def test_module_file_resolution_searches_all_safe_configured_roots(tmp_path: Path) -> None: + (tmp_path / "lib/foo").mkdir(parents=True) + (tmp_path / "python/foo/bar").mkdir(parents=True) + (tmp_path / "lib/helper.py").write_text("", encoding="utf-8") + (tmp_path / "python/helper.py").write_text("", encoding="utf-8") + (tmp_path / "lib/foo/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "python/foo/bar/__init__.py").write_text("", encoding="utf-8") + outside = tmp_path.parent / "outside" + outside.mkdir(exist_ok=True) + (outside / "escape.py").write_text("", encoding="utf-8") + + assert [path.relative_to(tmp_path).as_posix() for path in _module_files( + tmp_path, "helper", ["lib", "python", "missing", "../outside"] + )] == ["lib/helper.py", "python/helper.py"] + assert [path.relative_to(tmp_path).as_posix() for path in _module_files( + tmp_path, "foo.bar", ["lib", "python"] + )] == ["python/foo/bar/__init__.py"] + + @pytest.mark.parametrize("backend", ["hatchling.build", "poetry.core.masonry.api"]) def test_unresolved_backend_src_entrypoint_preserves_source_mode( tmp_path: Path, backend: str From e8ae407dac96ca63d61d6c75ef7bc1ac9be61fd9 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:07:46 -0400 Subject: [PATCH 28/59] Block sparse Git worktrees before generation --- .../analysis/assessor.py | 28 ++++ .../analysis/repository.py | 85 ++++++++++ .../generation/generator.py | 39 +++++ tests/test_cli.py | 43 ++++++ tests/test_generation.py | 146 +++++++++++++++++- 5 files changed, 340 insertions(+), 1 deletion(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index aeaa4eb..300153b 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -26,12 +26,14 @@ from python_deployment_builder.analysis.metadata import inspect_metadata from python_deployment_builder.analysis.repository import ( MaterializedRepository, + git_skip_worktree_paths, repository_fingerprint, ) from python_deployment_builder.analysis.resources import inspect_resources from python_deployment_builder.analysis.risks import build_risks, rate_suitability from python_deployment_builder.analysis.runtime_assumptions import scan_runtime_assumptions from python_deployment_builder.models import ( + Evidence, FindingStatus, RepositoryAssessment, RepositoryFileRole, @@ -140,6 +142,32 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen runtime.write_locations, configuration, ) + skip_worktree_paths = git_skip_worktree_paths(root) + if skip_worktree_paths: + displayed = skip_worktree_paths[:10] + risks.append( + RiskFinding( + code="SPARSE_WORKTREE_UNSUPPORTED", + title="Sparse Git working tree cannot represent a complete release source", + severity=RiskSeverity.BLOCKING, + status=FindingStatus.DETECTED, + description=( + f"The Git index marks {len(skip_worktree_paths)} tracked path(s) as " + "skip-worktree, so the current filesystem may not completely represent " + "the recorded HEAD revision." + ), + recommendation=( + "Populate the full repository working tree before generating a release kit." + ), + evidence=[ + Evidence( + file=path, + detail="Git index marks this tracked path skip-worktree.", + ) + for path in displayed + ], + ) + ) if metadata.uv_workspace: risks.append( RiskFinding( diff --git a/src/python_deployment_builder/analysis/repository.py b/src/python_deployment_builder/analysis/repository.py index a640d3c..f2fc8b7 100644 --- a/src/python_deployment_builder/analysis/repository.py +++ b/src/python_deployment_builder/analysis/repository.py @@ -5,6 +5,7 @@ import hashlib import shutil import stat +import subprocess import tempfile import urllib.request import zipfile @@ -31,6 +32,90 @@ class MaterializedRepository: source_kind: str +def git_skip_worktree_paths(repository_root: Path) -> list[str]: + """Return skip-worktree paths within the selected repository scope. + + Git reports paths from the enclosing worktree root even when PDB is invoked + for a nested project. Keep the selected-root boundary explicit so an + unrelated sparse path in a monorepo cannot block the selected project. + ``ls-files -t -z`` is deliberately used instead of sparse-checkout + configuration: the index ``S`` status is the authoritative indication that + the working tree may omit a tracked path. + """ + + try: + inside = subprocess.run( + ["git", "-C", str(repository_root), "rev-parse", "--is-inside-work-tree"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + if inside.returncode != 0 or inside.stdout.strip() != "true": + return [] + top_level = subprocess.run( + ["git", "-C", str(repository_root), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): + return [] + if top_level.returncode != 0 or not top_level.stdout.strip(): + return [] + + worktree_root = Path(top_level.stdout.strip()).resolve() + try: + selected_relative = repository_root.resolve().relative_to(worktree_root) + selected_prefix = PurePosixPath(selected_relative.as_posix()) + except ValueError: + return [] + pathspec = selected_prefix.as_posix() if selected_prefix != PurePosixPath(".") else "." + try: + listed = subprocess.run( + [ + "git", + "-C", + str(worktree_root), + "ls-files", + "--full-name", + "-t", + "-z", + "--", + pathspec, + ], + capture_output=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): + return [] + if listed.returncode != 0: + return [] + + prefix_parts = selected_prefix.parts if selected_prefix != PurePosixPath(".") else () + paths: list[str] = [] + for record in listed.stdout.split(b"\0"): + # ``-t`` records are exactly ````. Split only the + # fixed prefix; path bytes can legitimately contain spaces, tabs, and + # newlines, and ``-z`` is the record delimiter. + if len(record) < 3 or record[:1] != b"S" or record[1:2] != b" ": + continue + candidate = PurePosixPath( + record[2:].decode("utf-8", errors="surrogateescape").replace("\\", "/") + ) + if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts): + continue + if prefix_parts: + if candidate.parts[: len(prefix_parts)] != prefix_parts: + continue + candidate = PurePosixPath(*candidate.parts[len(prefix_parts) :]) + if candidate.parts: + paths.append(candidate.as_posix()) + return sorted(set(paths)) + + def parse_public_github_url(value: str) -> tuple[str, str] | None: """Return owner/repository for a public GitHub repository URL.""" diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index e8467e1..0579f98 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -16,6 +16,7 @@ from python_deployment_builder.analysis.repository import ( MaterializedRepository, RepositoryLoadError, + git_skip_worktree_paths, materialize_git_head_snapshot, ) from python_deployment_builder.analysis.resources import resolve_package_data_members @@ -775,6 +776,44 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) + # A skip-worktree index bit means the filesystem PDB assessed may omit a + # tracked part of HEAD. Block both deployment modes before staging, lock + # preparation, artifact work, or output mutation rather than claiming the + # recorded revision represents a complete release surface. + skip_worktree_paths = git_skip_worktree_paths(repository_root) + if skip_worktree_paths: + sparse_code = "SPARSE_WORKTREE_UNSUPPORTED" + if dry_run: + preview = _preview( + plan, + output_root, + dry_run=True, + bootstrap_mode=bootstrap_mode, + system_certs=system_certs, + prepare_lock=prepare_lock, + approved=[], + application_artifact=None, + staging_source_paths=[], + ) + preview.developer_actions.insert( + 0, + f"Stop: {sparse_code} prevents release generation while Git index paths " + "are marked skip-worktree.", + ) + return GenerationResult( + output_directory=str(output_root), + dry_run=True, + generated=False, + preview=preview, + ) + representative = ", ".join(skip_worktree_paths[:10]) + extra_count = len(skip_worktree_paths) - 10 + suffix = "" if extra_count <= 0 else f" (and {extra_count} more)" + raise PreparationError( + f"Deployment planning is blocked: {sparse_code}. The Git index marks tracked " + "paths skip-worktree, so the current filesystem may not completely represent " + f"HEAD: {representative}{suffix}. Populate the full working tree before generation." + ) # A workspace root cannot be reduced to this M6.1 kit's root # ``pyproject.toml`` + ``uv.lock`` representation: uv still resolves # member metadata under --no-install-project. Report the typed blocker diff --git a/tests/test_cli.py b/tests/test_cli.py index ddc7ec5..e088825 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,4 @@ +import subprocess from pathlib import Path import pytest @@ -175,3 +176,45 @@ def test_uv_workspace_assess_plan_and_all_report_a_typed_blocker(tmp_path: Path) encoding="utf-8" ) assert not (all_output / "deployment-kit").exists() + + +def test_sparse_worktree_assess_plan_and_all_persist_blocker_reports(tmp_path: Path) -> None: + source = tmp_path / "sparse-source" + source.mkdir() + _write_all_source(source) + (source / "lazy helper.py").write_text("VALUE = 1\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], check=True + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + subprocess.run( + [ + "git", + "-C", + str(source), + "update-index", + "--skip-worktree", + "--", + "lazy helper.py", + ], + check=True, + ) + (source / "lazy helper.py").unlink() + + assess_output = tmp_path / "assess" + plan_output = tmp_path / "plan" + all_output = tmp_path / "all" + assert main(["assess", str(source), "--output-dir", str(assess_output)]) == 0 + assert main(["plan", str(source), "--output-dir", str(plan_output)]) == 1 + assert main(["all", str(source), "--output-dir", str(all_output)]) == 2 + + for report in ( + assess_output / "assessment.json", + plan_output / "deployment-plan.json", + all_output / "reports/deployment-plan.json", + ): + assert "SPARSE_WORKTREE_UNSUPPORTED" in report.read_text(encoding="utf-8") + assert not (all_output / "deployment-kit").exists() diff --git a/tests/test_generation.py b/tests/test_generation.py index 34679d3..2689e6b 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -16,7 +16,10 @@ import pytest from python_deployment_builder.analysis.assessor import assess_repository -from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.repository import ( + MaterializedRepository, + git_skip_worktree_paths, +) from python_deployment_builder.analysis.resources import resolve_package_data_members from python_deployment_builder.cli import build_parser, main from python_deployment_builder.generation.acquisition import ( @@ -3597,6 +3600,147 @@ def test_clean_git_source_fixture_stages_and_previews_normally(tmp_path: Path) - assert "data/.gitignore" not in staged +def _mark_skip_worktree(source: Path, relative: str) -> None: + subprocess.run( + ["git", "-C", str(source), "update-index", "--skip-worktree", "--", relative], + check=True, + ) + (source / relative).unlink() + + +def test_skip_worktree_blocks_source_generation_before_staging_or_lock_mutation( + tmp_path: Path, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + lazy_helper = source / "lazy_helper.py" + lazy_helper.write_text("VALUE = 'deferred runtime helper'\n", encoding="utf-8") + (source / "app.py").write_text( + "import importlib\n" + "def main(): return importlib.import_module('lazy_helper').VALUE\n", + encoding="utf-8", + ) + subprocess.run(["git", "-C", str(source), "add", "app.py", "lazy_helper.py"], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "add lazy helper"], check=True) + _mark_skip_worktree(source, "lazy_helper.py") + (source / "uv.lock").unlink() + + tags = subprocess.run( + ["git", "-C", str(source), "ls-files", "-t", "-z"], + capture_output=True, + check=True, + ).stdout + assert b"S lazy_helper.py\0" in tags + tracked = subprocess.run( + ["git", "-C", str(source), "ls-files", "--", "lazy_helper.py"], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + dirty = subprocess.run( + ["git", "-C", str(source), "diff", "HEAD", "--name-only", "--", "lazy_helper.py"], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + assert tracked == ["lazy_helper.py"] + assert dirty == [] + assert not lazy_helper.exists() + assert git_skip_worktree_paths(source) == ["lazy_helper.py"] + + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + output = tmp_path / "kit" + + assert "SPARSE_WORKTREE_UNSUPPORTED" in [item.code for item in assessment.risks] + assert "SPARSE_WORKTREE_UNSUPPORTED" in plan.risk_gate.blocking_codes + with pytest.raises(PreparationError, match="SPARSE_WORKTREE_UNSUPPORTED"): + generate_deployment_kit( + repository, + output, + prepare_lock=True, + bootstrap_mode="online_cmd", + ) + assert not output.exists() + assert not (source / "uv.lock").exists() + + preview = generate_deployment_kit( + repository, output, dry_run=True, bootstrap_mode="online_cmd" + ).preview + assert preview.readiness_before == "BLOCKED" + assert any("SPARSE_WORKTREE_UNSUPPORTED" in action for action in preview.developer_actions) + assert not output.exists() + + +def test_skip_worktree_blocks_package_generation_before_wheel_validation(tmp_path: Path) -> None: + source = tmp_path / "package-source" + source.mkdir() + _write_mapped_project(source) + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + _mark_skip_worktree(source, "code/view.html") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + plan = create_deployment_plan(assess_repository(repository), repository_root=source) + + assert plan.deployment_mode == "package" + with pytest.raises(PreparationError, match="SPARSE_WORKTREE_UNSUPPORTED"): + generate_deployment_kit(repository, tmp_path / "kit", bootstrap_mode="online_cmd") + + +def test_skip_worktree_paths_use_nul_delimited_git_records(tmp_path: Path) -> None: + source, _repository = _committed_source_fixture(tmp_path) + unusual = source / "data" / "space and unicode ü.txt" + unusual.write_text("tracked unusual filename\n", encoding="utf-8") + subprocess.run( + ["git", "-C", str(source), "add", "data/space and unicode ü.txt"], check=True + ) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "add unusual path"], check=True) + + _mark_skip_worktree(source, "data/space and unicode ü.txt") + + assert git_skip_worktree_paths(source) == ["data/space and unicode ü.txt"] + + +def test_skip_worktree_scope_and_index_state_are_read_from_git_not_sparse_config( + tmp_path: Path, +) -> None: + worktree, source, repository = _nested_committed_source_fixture(tmp_path) + outside = worktree / "other-project" / "sparse note.txt" + outside.write_text("outside sparse input\n", encoding="utf-8") + subprocess.run(["git", "-C", str(worktree), "add", str(outside)], check=True) + subprocess.run(["git", "-C", str(worktree), "commit", "-qm", "add outside note"], check=True) + _mark_skip_worktree(worktree, "other-project/sparse note.txt") + + assert git_skip_worktree_paths(source) == [] + assert not any( + item.code == "SPARSE_WORKTREE_UNSUPPORTED" + for item in assess_repository(repository).risks + ) + + subprocess.run( + ["git", "-C", str(worktree), "config", "core.sparseCheckout", "true"], check=True + ) + # Configuration alone is not the release-surface test; only index ``S`` + # entries in the selected path domain are relevant. + assert git_skip_worktree_paths(source) == [] + + +def test_assume_unchanged_is_not_misclassified_as_skip_worktree(tmp_path: Path) -> None: + source, _repository = _committed_source_fixture(tmp_path) + subprocess.run( + ["git", "-C", str(source), "update-index", "--assume-unchanged", "docs/readme.md"], + check=True, + ) + + assert git_skip_worktree_paths(source) == [] + + @pytest.mark.parametrize("relative", [".gitignore", "data/.gitignore"]) def test_git_source_staging_blocks_modified_tracked_analysis_policy( tmp_path: Path, relative: str From a72ffdbe96ee521d18ada616447592bf118df800 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:54:09 -0400 Subject: [PATCH 29/59] Fail closed for text encodings and dynamic setup surfaces --- .../analysis/assessor.py | 18 +++ .../analysis/metadata.py | 100 +++++++++++- .../analysis/resources.py | 14 +- .../generation/artifacts.py | 13 +- .../generation/structural.py | 19 ++- .../planning/planner.py | 4 +- .../security_policy.py | 46 +++++- .../validation/static.py | 18 ++- tests/test_generation.py | 148 ++++++++++++++++++ tests/test_metadata.py | 95 ++++++++++- tests/test_validation.py | 23 +++ 11 files changed, 479 insertions(+), 19 deletions(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index 300153b..2fc65eb 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -205,6 +205,24 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen evidence=metadata.uv_workspace_evidence, ) ) + if metadata.setuptools_surface_unresolved: + risks.append( + RiskFinding( + code="PACKAGING_SURFACE_UNRESOLVED", + title="Setuptools packaging surface requires static resolution", + severity=RiskSeverity.WARNING, + status=FindingStatus.NEEDS_VALIDATION, + description=( + "setup.py declares packaging-surface configuration that PDB cannot " + "statically resolve without executing target code." + ), + recommendation=( + "Use literal setuptools package configuration or retain source deployment; " + "package mode requires an authoritative static surface." + ), + evidence=metadata.setuptools_surface_evidence, + ) + ) unusual_scope_imports = [ item for item in imports.observations diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 3391213..1034d79 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -35,6 +35,38 @@ class MetadataResult: uv_workspace: bool = False uv_workspace_source: bool = False uv_workspace_evidence: list[Evidence] = field(default_factory=list) + setuptools_surface_unresolved: bool = False + setuptools_surface_evidence: list[Evidence] = field(default_factory=list) + + +_SETUP_SURFACE_FIELDS = frozenset( + {"packages", "py_modules", "package_dir", "package_data", "exclude_package_data"} +) + + +@dataclass(frozen=True) +class SetupCallInspection: + """Non-executing setup() inspection with literal-resolution provenance.""" + + literal_values: dict[str, Any] = field(default_factory=dict) + present_keywords: frozenset[str] = frozenset() + unresolved_keywords: frozenset[str] = frozenset() + has_kwargs_expansion: bool = False + parse_failed: bool = False + + @property + def surface_unresolved(self) -> bool: + return bool( + self.parse_failed + or self.has_kwargs_expansion + or self.unresolved_keywords & _SETUP_SURFACE_FIELDS + ) + + @property + def package_selection_present(self) -> bool: + return self.has_kwargs_expansion or bool( + self.present_keywords & {"packages", "py_modules"} + ) # Setuptools' flat-layout auto-discovery deliberately avoids conventional @@ -345,13 +377,13 @@ def _discover_setuptools_py_modules( return sorted(discovered) -def _literal_setup_arguments(path: Path) -> dict[str, Any]: - """Read literal setup(...) keyword values without executing setup.py.""" +def inspect_setup_call(path: Path) -> SetupCallInspection: + """Inspect setup() literals without executing or evaluating target code.""" try: tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path)) except (OSError, SyntaxError, UnicodeError): - return {} + return SetupCallInspection(parse_failed=True) for node in ast.walk(tree): if not isinstance(node, ast.Call): continue @@ -361,15 +393,33 @@ def _literal_setup_arguments(path: Path) -> dict[str, Any]: if name != "setup": continue values: dict[str, Any] = {} + present: set[str] = set() + unresolved: set[str] = set() + has_kwargs_expansion = False for keyword in node.keywords: if keyword.arg is None: + has_kwargs_expansion = True continue + present.add(keyword.arg) try: values[keyword.arg] = ast.literal_eval(keyword.value) except (ValueError, TypeError): - continue - return values - return {} + if keyword.arg in _SETUP_SURFACE_FIELDS: + unresolved.add(keyword.arg) + return SetupCallInspection( + literal_values=values, + present_keywords=frozenset(present), + unresolved_keywords=frozenset(unresolved), + has_kwargs_expansion=has_kwargs_expansion, + ) + return SetupCallInspection() + + +def setup_py_surface_resolved(root: Path) -> bool: + """Whether a local setup.py leaves PDB's modeled surface statically known.""" + + path = root / "setup.py" + return not path.is_file() or not inspect_setup_call(path).surface_unresolved def _literal_module_attribute(root: Path, attribute: str) -> str | None: @@ -467,6 +517,9 @@ def inspect_metadata(root: Path) -> MetadataResult: package_discovery_rules: list[tuple[list[str], list[str], list[str], bool]] = [] automatic_setuptools_root: str | None = None automatic_setuptools_flat_surface_ambiguous = False + setuptools_package_selection_configured = False + setuptools_surface_unresolved = False + setuptools_surface_evidence: list[Evidence] = [] uv_workspace = False uv_workspace_source = False uv_workspace_evidence: list[Evidence] = [] @@ -632,6 +685,8 @@ def inspect_metadata(root: Path) -> MetadataResult: # explicit setuptools.build_meta projects. build_backend = "setuptools.build_meta:__legacy__" configured_packages = setuptools.get("packages") + if "packages" in setuptools or "py-modules" in setuptools: + setuptools_package_selection_configured = True if isinstance(configured_packages, list): packages = [value for value in configured_packages if isinstance(value, str)] py_modules = _string_list(setuptools.get("py-modules")) @@ -756,6 +811,8 @@ def inspect_metadata(root: Path) -> MetadataResult: if configured_where and not source_roots: source_roots = [configured_where] configured_packages = parser.get("options", "packages", fallback="").strip() + if configured_packages or parser.has_option("options", "py_modules"): + setuptools_package_selection_configured = True if ( configured_packages and configured_packages not in {"find:", "find_namespace:"} @@ -831,7 +888,32 @@ def inspect_metadata(root: Path) -> MetadataResult: setup_py_path = root / "setup.py" if setup_py_path.is_file(): metadata_files.append("setup.py") - setup_values = _literal_setup_arguments(setup_py_path) + setup_inspection = inspect_setup_call(setup_py_path) + setup_values = setup_inspection.literal_values + setuptools_package_selection_configured = ( + setuptools_package_selection_configured + or setup_inspection.package_selection_present + ) + setuptools_surface_unresolved = setup_inspection.surface_unresolved + if setuptools_surface_unresolved: + unresolved = sorted( + setup_inspection.unresolved_keywords & _SETUP_SURFACE_FIELDS + ) + detail = ( + "setup() expands **kwargs, so modeled packaging-surface fields cannot be " + "statically established." + if setup_inspection.has_kwargs_expansion + else "setup() has nonliteral packaging-surface field(s): " + + ", ".join(unresolved or ["setup.py parse failure"]) + ) + setuptools_surface_evidence.append( + _evidence( + root, + setup_py_path, + detail, + _line_number(setup_py_path, "setup("), + ) + ) literal_packages = setup_values.get("packages") if isinstance(literal_packages, list) and not packages: packages = [package for package in literal_packages if isinstance(package, str)] @@ -1004,6 +1086,8 @@ def inspect_metadata(root: Path) -> MetadataResult: and not packages and not py_modules and not package_discovery_rules + and not setuptools_package_selection_configured + and not setuptools_surface_unresolved ): automatic_root = ( source_roots[0] @@ -1119,4 +1203,6 @@ def inspect_metadata(root: Path) -> MetadataResult: uv_workspace=uv_workspace, uv_workspace_source=uv_workspace_source, uv_workspace_evidence=uv_workspace_evidence, + setuptools_surface_unresolved=setuptools_surface_unresolved, + setuptools_surface_evidence=setuptools_surface_evidence, ) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 37fa46f..0c2ecdf 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -310,7 +310,9 @@ def resolve_packaged_python_sources( return sorted(resolved, key=lambda item: (item.source_path, item.installed_member_path)) -def package_surface_resolved(project: PackagingAssessment | None) -> bool: +def package_surface_resolved( + project: PackagingAssessment | None, repository_root: Path | None = None +) -> bool: """Whether M6.1 has an authoritative Python wheel-surface model. A build backend establishes only that a project might be buildable. The @@ -319,11 +321,19 @@ def package_surface_resolved(project: PackagingAssessment | None) -> bool: backend discovery. """ - return bool( + resolved_backend = bool( project and project.build_backend and project.build_backend.partition(":")[0] == "setuptools.build_meta" ) + if not resolved_backend or repository_root is None: + return resolved_backend + # setup.py fields are not persisted in PackagingAssessment. Re-inspect + # locally when planning or validating a wheel so a dynamic selector cannot + # bypass the source-surface authority contract through a stale/manual plan. + from python_deployment_builder.analysis.metadata import setup_py_surface_resolved + + return setup_py_surface_resolved(repository_root) def _declared_package_data( diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index b44bab8..6ab1785 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -44,6 +44,8 @@ minor_python_compatibility, ) from python_deployment_builder.security_policy import ( + TextContentEncodingError, + decode_security_text, is_secret_filename, is_textual_wheel_member, text_security_findings, @@ -437,7 +439,12 @@ def _validate_application_security( content = bundle.read(member) if not is_textual_wheel_member(member_path, content): continue - text = content.decode("utf-8-sig", errors="replace") + try: + text = decode_security_text(member_path, content) + except TextContentEncodingError as exc: + raise PreparationError(str(exc)) from exc + if text is None: + continue if text_security_findings( text, configured_secret_values=configured_secret_values ): @@ -743,7 +750,9 @@ def validate_application_wheel( raise PreparationError( "Package mode requires authoritative project distribution and version metadata." ) - if not package_surface_resolved(assessment.project): + if not package_surface_resolved(assessment.project, repository_root) or any( + item.code == "PACKAGING_SURFACE_UNRESOLVED" for item in assessment.risks + ): backend = assessment.project.build_backend or "no build backend" raise PreparationError( "Application wheel validation requires an authoritative Python packaging-surface " diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index 521e8a8..c1756b1 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -15,6 +15,8 @@ RiskSeverity, ) from python_deployment_builder.security_policy import ( + TextContentEncodingError, + decode_security_text, is_secret_filename, is_textual_content, text_security_findings, @@ -112,6 +114,7 @@ def validate_rendered_files( permanent_path_hits: list[str] = [] program_files_hits: list[str] = [] secret_hits: list[str] = [] + undecodable_text: list[str] = [] # Every intentionally staged file is release content. Wheels remain # opaque here because their member-level validator owns their security scan. for relative, content in files.items(): @@ -123,7 +126,13 @@ def validate_rendered_files( continue if not is_textual_content(path, content): continue - text = content.decode("utf-8-sig") + try: + text = decode_security_text(path, content) + except TextContentEncodingError: + undecodable_text.append(relative) + continue + if text is None: + continue findings = text_security_findings( text, configured_secret_values=secret_values or [] ) @@ -176,6 +185,14 @@ def validate_rendered_files( "NO_PROGRAM_FILES_WRITES", f"Program Files write targets: {program_files_hits or 'none'}", ), + _check( + not undecodable_text, + "TEXT_SECURITY_DECODABLE", + "All staged textual content is valid UTF-8/UTF-8-SIG for security scanning." + if not undecodable_text + else "Textual content cannot be security-scanned as UTF-8: " + f"{undecodable_text}", + ), _check( not secret_hits, "NO_SECRET_VALUES", diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index d83d604..726f454 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -142,7 +142,9 @@ def _deployment_mode( and assessment.project.version and assessment.project.build_backend ) - surface_resolved = package_surface_resolved(assessment.project) + surface_resolved = package_surface_resolved(assessment.project, analysis_root) and not any( + item.code == "PACKAGING_SURFACE_UNRESOLVED" for item in assessment.risks + ) backend = assessment.project.build_backend or "no build backend" def unresolved_surface_result() -> tuple[str, str, str, list[str]]: diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index 97ba73c..ce8c775 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -59,6 +59,24 @@ ) +class TextContentEncodingError(ValueError): + """A known text member cannot be security-scanned under PDB's UTF-8 policy.""" + + def __init__(self, path: PurePosixPath) -> None: + super().__init__( + "TEXT_CONTENT_ENCODING_UNSUPPORTED: " + f"{path.as_posix()} is classified as text but is not valid UTF-8/UTF-8-SIG, " + "so deployment security scanning cannot prove its contents safe." + ) + + +def _known_text_path(path: PurePosixPath) -> bool: + return path.suffix.lower() in TEXT_SUFFIXES or ( + any(part.casefold().endswith(".dist-info") for part in path.parts) + and path.name.casefold() in TEXTUAL_WHEEL_METADATA_FILENAMES + ) + + def is_secret_filename(filename: str) -> bool: """Return whether a case-insensitive basename is prohibited secret material.""" @@ -86,6 +104,27 @@ def is_probably_utf8_text(content: bytes) -> bool: return not any(ord(character) < 32 and character not in "\t\n\r" for character in text) +def decode_security_text(path: PurePosixPath, content: bytes) -> str | None: + """Decode security-scannable text or return ``None`` for binary content. + + Known textual paths must be strict UTF-8 so a failed scan cannot be + misreported as safe. Unknown extensions retain the bounded content-based + classifier: invalid UTF-8 remains opaque binary rather than an error. + """ + + known_text = _known_text_path(path) + if path.suffix.lower() in KNOWN_BINARY_SUFFIXES: + return None + if not known_text and not is_probably_utf8_text(content): + return None + try: + return content.decode("utf-8-sig") + except UnicodeDecodeError as exc: + if known_text: + raise TextContentEncodingError(path) from exc + return None + + def is_textual_content(path: PurePosixPath, content: bytes | None = None) -> bool: """Return whether a wheel member has content suitable for text security checks. @@ -94,10 +133,7 @@ def is_textual_content(path: PurePosixPath, content: bytes | None = None) -> boo arbitrary binary payload is never decoded with replacement characters. """ - if path.suffix.lower() in TEXT_SUFFIXES or ( - any(part.casefold().endswith(".dist-info") for part in path.parts) - and path.name.casefold() in TEXTUAL_WHEEL_METADATA_FILENAMES - ): + if _known_text_path(path): return True if path.suffix.lower() in KNOWN_BINARY_SUFFIXES or content is None: return False @@ -140,6 +176,8 @@ def text_security_findings( "FORBIDDEN_SHELL", "KNOWN_BINARY_SUFFIXES", "TEXT_SUFFIXES", + "TextContentEncodingError", + "decode_security_text", "is_textual_content", "is_secret_filename", "is_valid_environment_name", diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index fe5df08..ae20df9 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -26,6 +26,8 @@ ) from python_deployment_builder.security_policy import ( FORBIDDEN_SHELL, + TextContentEncodingError, + decode_security_text, is_secret_filename, is_textual_content, text_security_findings, @@ -463,6 +465,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR permanent_path: list[str] = [] program_files: list[str] = [] obvious_secrets: list[str] = [] + undecodable_text: list[str] = [] configured_secret_values = [ value for name in manifest.configuration_secret_names @@ -480,7 +483,13 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR content = path.read_bytes() if not is_textual_content(Path(relative), content): continue - text = content.decode("utf-8-sig") + try: + text = decode_security_text(PurePosixPath(relative), content) + except TextContentEncodingError: + undecodable_text.append(relative) + continue + if text is None: + continue findings = text_security_findings( text, configured_secret_values=configured_secret_values ) @@ -553,6 +562,13 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR "A Program Files write target is present.", evidence=program_files, ), + _check( + "TEXT_SECURITY_DECODABLE", + not undecodable_text, + "All staged textual content is valid UTF-8/UTF-8-SIG for security scanning.", + "Textual content cannot be security-scanned as UTF-8.", + evidence=undecodable_text, + ), _check( "NO_SECRET_CONTENT", not secret_files and not obvious_secrets, diff --git a/tests/test_generation.py b/tests/test_generation.py index 2689e6b..f987520 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -4519,6 +4519,154 @@ def test_unknown_and_known_binary_content_do_not_be_text_classified() -> None: assert is_textual_content(PurePosixPath("app/query.custom"), b"select 1\n") +@pytest.mark.parametrize("suffix", [".txt", ".cfg", ".py"]) +def test_known_text_with_non_utf8_content_fails_closed_before_write( + tmp_path: Path, suffix: str +) -> None: + plan = _plan() + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + owned, manifest = _render_owned_files( + plan, + FIXTURES / "prepared_gui", + bootstrap_mode="bundled_uv", + system_certs=False, + approved=[], + bundled_uv=fake_uv, + ) + files = { + "pyproject.toml": (FIXTURES / "prepared_gui" / "pyproject.toml").read_bytes(), + "uv.lock": (FIXTURES / "prepared_gui" / "uv.lock").read_bytes(), + f"app/resources/legacy{suffix}": b"legacy \x93Windows-1252\x94 text\n", + **owned, + } + + with pytest.raises(PreparationError, match="TEXT_SECURITY_DECODABLE"): + validate_rendered_files(files, manifest, generated_paths=set(owned), secret_values=[]) + + +def test_utf8_and_utf8_sig_known_text_remain_security_scannable() -> None: + assert is_textual_content(PurePosixPath("app/resource.txt"), b"plain UTF-8\n") + assert is_textual_content( + PurePosixPath("app/resource.txt"), b"\xef\xbb\xbfplain UTF-8 with BOM\n" + ) + + +def test_authoritative_source_package_data_non_utf8_fails_before_output_write( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + (source / "app/resources").mkdir(parents=True) + (source / "app/__init__.py").write_text("", encoding="utf-8") + (source / "app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "app/resources/legacy.txt").write_bytes(b"legacy \x93Windows-1252\x94 text\n") + (source / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'legacy-app'\nversion = '1.0'\ndependencies = []\n" + "[project.scripts]\nlegacy = 'app.main:main'\n" + "[tool.setuptools]\npackages = ['app']\n" + "[tool.setuptools.package-data]\napp = ['resources/legacy.txt']\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + with pytest.raises(PreparationError, match="TEXT_SECURITY_DECODABLE"): + generate_deployment_kit(repository, tmp_path / "kit", bootstrap_mode="online_cmd") + assert not (tmp_path / "kit").exists() + + +def test_application_wheel_known_text_with_non_utf8_content_fails_closed(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/legacy.txt": b"legacy \x93Windows-1252\x94 text\n"}, + ) + + with pytest.raises(PreparationError, match="TEXT_CONTENT_ENCODING_UNSUPPORTED"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + +def _dynamic_setup_selector_project(root: Path, *, installed_only: bool = False) -> None: + (root / "src/app/tests").mkdir(parents=True) + (root / "src/app/__init__.py").write_text("", encoding="utf-8") + (root / "src/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "src/app/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (root / "src/app/tests/__init__.py").write_text("", encoding="utf-8") + (root / "src/app/tests/test_internal.py").write_text("VALUE = 1\n", encoding="utf-8") + target = "installed_app.main:main" if installed_only else "app.main:main" + (root / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'demo-app'\nversion = '1.0'\ndependencies = []\n" + f"[project.scripts]\ndemo = '{target}'\n", + encoding="utf-8", + ) + (root / "setup.py").write_text( + "from setuptools import find_packages, setup\n" + "setup(name='demo-app', version='1.0', package_dir={'': 'src'}, " + "packages=find_packages(where='src', exclude=['app.tests']), " + "entry_points={'console_scripts': ['demo=app.main:main']})\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + +def test_dynamic_setup_selector_falls_back_to_source_and_blocks_wheel_bypass( + tmp_path: Path, +) -> None: + source = tmp_path / "dynamic-source" + _dynamic_setup_selector_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert assessment.project.packages == [] + assert "PACKAGING_SURFACE_UNRESOLVED" in [item.code for item in assessment.risks] + assert plan.deployment_mode == "source" + assert plan.deployment_mode_condition == "SOURCE_COMPATIBLE" + wheel = _make_application_wheel( + tmp_path, + name="demo-app", + version="1.0", + package="app", + target="app.main:main", + entry_group="console_scripts", + entry_name="demo", + ) + with pytest.raises(PreparationError, match="authoritative Python packaging-surface"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + +def test_dynamic_setup_selector_blocks_installed_only_entry_point(tmp_path: Path) -> None: + source = tmp_path / "dynamic-installed" + _dynamic_setup_selector_project(source, installed_only=True) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert plan.deployment_mode == "package" + assert plan.deployment_mode_condition == "INSTALLED_PROJECT_REQUIRED" + assert "PACKAGING_SURFACE_UNRESOLVED" in plan.readiness.blocker_codes + + def test_uv_archive_rejects_traversal_and_hash_version_mismatch(tmp_path: Path) -> None: archive = tmp_path / "uv.zip" with zipfile.ZipFile(archive, "w") as bundle: diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 207f4fa..2bf4bbd 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,6 +1,9 @@ +import subprocess +import sys +import zipfile from pathlib import Path -from python_deployment_builder.analysis.metadata import inspect_metadata +from python_deployment_builder.analysis.metadata import inspect_metadata, inspect_setup_call from python_deployment_builder.models import EntryPointAssessment FIXTURES = Path(__file__).parent / "fixtures" @@ -78,6 +81,96 @@ def test_setup_py_literals_are_read_without_execution(tmp_path: Path) -> None: assert not marker.exists() +def test_setup_call_inspection_distinguishes_absent_literal_and_unresolved_surface_fields( + tmp_path: Path, +) -> None: + absent = tmp_path / "absent.py" + absent.write_text("from setuptools import setup\nsetup(name='demo')\n", encoding="utf-8") + literal = tmp_path / "literal.py" + literal.write_text( + "from setuptools import setup\nsetup(packages=['app'], py_modules=['helper'])\n", + encoding="utf-8", + ) + dynamic = tmp_path / "dynamic.py" + dynamic.write_text( + "from setuptools import find_packages, setup\n" + "setup(packages=find_packages(where='src'), package_data=get_data(), **options)\n", + encoding="utf-8", + ) + + absent_result = inspect_setup_call(absent) + literal_result = inspect_setup_call(literal) + dynamic_result = inspect_setup_call(dynamic) + + assert not absent_result.package_selection_present + assert literal_result.literal_values["packages"] == ["app"] + assert not literal_result.surface_unresolved + assert dynamic_result.present_keywords >= {"packages", "package_data"} + assert dynamic_result.unresolved_keywords >= {"packages", "package_data"} + assert dynamic_result.has_kwargs_expansion + assert dynamic_result.surface_unresolved + + +def test_dynamic_setup_package_selector_does_not_trigger_automatic_discovery( + tmp_path: Path, +) -> None: + (tmp_path / "src/app/tests").mkdir(parents=True) + for relative in ("src/app/__init__.py", "src/app/main.py", "src/app/tests/__init__.py"): + (tmp_path / relative).write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='demo-app'\nversion='1.0'\n", + encoding="utf-8", + ) + (tmp_path / "setup.py").write_text( + "from setuptools import find_packages, setup\n" + "setup(package_dir={'': 'src'}, packages=find_packages(where='src', " + "exclude=['app.tests']))\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.project.packages == [] + assert result.setuptools_surface_unresolved + assert result.setuptools_surface_evidence + + +def test_setuptools_find_packages_exclude_disposable_wheel_evidence(tmp_path: Path) -> None: + """Confirm the dynamic selector's real wheel surface without using it in PDB.""" + + (tmp_path / "src/app/tests").mkdir(parents=True) + for relative in ( + "src/app/__init__.py", + "src/app/main.py", + "src/app/tests/__init__.py", + "src/app/tests/test_internal.py", + ): + (tmp_path / relative).write_text("", encoding="utf-8") + (tmp_path / "setup.py").write_text( + "from setuptools import find_packages, setup\n" + "setup(name='demo-app', version='1.0', package_dir={'': 'src'}, " + "packages=find_packages(where='src', exclude=['app.tests']))\n", + encoding="utf-8", + ) + dist = tmp_path / "dist" + + subprocess.run( + [sys.executable, "setup.py", "bdist_wheel", "--dist-dir", str(dist)], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + wheel = next(dist.glob("demo_app-1.0-*.whl")) + with zipfile.ZipFile(wheel) as bundle: + members = set(bundle.namelist()) + + assert "app/__init__.py" in members + assert "app/main.py" in members + assert "app/tests/__init__.py" not in members + + def test_entry_point_declared_group_is_independent_from_gui_heuristic(tmp_path: Path) -> None: (tmp_path / "pyproject.toml").write_text( """[project] diff --git a/tests/test_validation.py b/tests/test_validation.py index d2cd61c..8adf3a1 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -194,6 +194,29 @@ def test_static_validation_scans_shared_textual_configuration_formats( assert _status(report, "NO_SECRET_CONTENT") == ValidationCheckStatus.FAIL +def test_static_validation_reports_non_utf8_known_text_without_crashing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + kit = _kit(monkeypatch, tmp_path) + legacy = kit / "app" / "resources" / "legacy.cfg" + legacy.parent.mkdir(parents=True) + legacy.write_bytes(b"legacy \x93Windows-1252\x94 text\n") + index_path = kit / "deployment/generated-files.json" + index = json.loads(index_path.read_text(encoding="utf-8")) + index["files"].append( + { + "path": "app/resources/legacy.cfg", + "sha256": hashlib.sha256(legacy.read_bytes()).hexdigest(), + } + ) + index_path.write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") + + report = validate_static_kit(kit) + + assert report.final_state == ValidationFinalState.FAILED + assert _status(report, "TEXT_SECURITY_DECODABLE") == ValidationCheckStatus.FAIL + + def test_static_validation_detects_runtime_bytecode_cache( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 28abe37478b8d5c3839956a15ead09f6f4a0e80f Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:05:02 -0400 Subject: [PATCH 30/59] Block escaping setuptools roots and mirror finder exclusions --- .../analysis/assessor.py | 19 ++ .../analysis/metadata.py | 134 ++++++++++- .../analysis/resources.py | 9 +- .../generation/generator.py | 32 +++ tests/test_generation.py | 55 +++++ tests/test_metadata.py | 223 +++++++++++++++++- tests/test_planning.py | 46 ++++ 7 files changed, 512 insertions(+), 6 deletions(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index 2fc65eb..20ee6c5 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -223,6 +223,25 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen evidence=metadata.setuptools_surface_evidence, ) ) + if metadata.setuptools_external_packaging_roots: + risks.append( + RiskFinding( + code="EXTERNAL_PACKAGING_ROOT_UNSUPPORTED", + title="Setuptools packaging root escapes the assessed repository", + severity=RiskSeverity.BLOCKING, + status=FindingStatus.DETECTED, + description=( + "The project declares first-party setuptools packaging content outside " + "the assessed repository boundary. PDB cannot inspect, stage, or prove " + "that external source as part of a standalone release." + ), + recommendation=( + "Move the first-party package root into the assessed repository or use " + "a future workspace-aware deployment workflow." + ), + evidence=metadata.setuptools_external_packaging_root_evidence, + ) + ) unusual_scope_imports = [ item for item in imports.observations diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 1034d79..5653a37 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -8,7 +8,7 @@ import re import tomllib from dataclasses import dataclass, field -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import Any from packaging.requirements import InvalidRequirement, Requirement @@ -37,6 +37,8 @@ class MetadataResult: uv_workspace_evidence: list[Evidence] = field(default_factory=list) setuptools_surface_unresolved: bool = False setuptools_surface_evidence: list[Evidence] = field(default_factory=list) + setuptools_external_packaging_roots: list[str] = field(default_factory=list) + setuptools_external_packaging_root_evidence: list[Evidence] = field(default_factory=list) _SETUP_SURFACE_FIELDS = frozenset( @@ -93,6 +95,54 @@ def package_selection_present(self) -> bool: ".venv.*", ] +# Verified against setuptools 79.0.1's PackageFinder and +# PEP420PackageFinder. These are finder-level exclusions, applied before +# user include/exclude filters; unlike flat-layout defaults they cannot be +# re-enabled by an include pattern. ModuleFinder has no ALWAYS_EXCLUDE set. +_SETUPTOOLS_PACKAGE_FINDER_ALWAYS_EXCLUDES = ("ez_setup", "*__pycache__") + + +@dataclass(frozen=True) +class PackagingRootInspection: + """Safety status for a declared physical setuptools packaging root.""" + + declared_root: str + status: str + normalized_root: str | None = None + + +def inspect_setuptools_packaging_root( + repository_root: Path, declared_root: str +) -> PackagingRootInspection: + """Classify a metadata root without following it outside repository scope. + + PDB's first-party surface and staging boundary is the assessed repository. + An in-repository missing path is harmless (setuptools simply finds no + members there), but absolute, escaping, or symlink-rooted declarations + cannot be treated as absent authoritative surface. + """ + + root = repository_root.resolve() + declared = Path(declared_root) + if declared.is_absolute() or PureWindowsPath(declared_root).is_absolute(): + return PackagingRootInspection(declared_root, "UNSAFE") + try: + candidate = (repository_root / declared).resolve() + relative = candidate.relative_to(root) + except (OSError, RuntimeError, ValueError): + return PackagingRootInspection(declared_root, "UNSAFE") + # A symlinked root is not a stable source-root representation for staging; + # in particular, a root symlink could be retargeted after assessment. + lexical_candidate = repository_root / declared + if lexical_candidate.is_symlink(): + return PackagingRootInspection(declared_root, "UNSAFE") + normalized = relative.as_posix() or "." + return PackagingRootInspection( + declared_root, + "SAFE" if lexical_candidate.exists() else "MISSING_SAFE", + normalized, + ) + def _evidence(root: Path, path: Path, detail: str, line: int | None = None) -> Evidence: return Evidence(file=path.relative_to(root).as_posix(), line=line, detail=detail) @@ -285,7 +335,10 @@ def _discover_setuptools_packages( discovered: set[str] = set() includes = include or ["*"] for configured_root in search_roots: - candidate_root = root / configured_root + root_inspection = inspect_setuptools_packaging_root(root, configured_root) + if root_inspection.status == "UNSAFE" or root_inspection.normalized_root is None: + continue + candidate_root = root / root_inspection.normalized_root if candidate_root.is_symlink() or not candidate_root.is_dir(): continue try: @@ -328,6 +381,14 @@ def _discover_setuptools_packages( ): continue package = ".".join(parts) + # Setuptools' PackageFinder and PEP420PackageFinder compose these + # unconditional exclusions before user include/exclude filters. + # User ``include = [\"ez_setup*\"]`` cannot re-enable them. + if any( + fnmatch.fnmatchcase(package, pattern) + for pattern in _SETUPTOOLS_PACKAGE_FINDER_ALWAYS_EXCLUDES + ): + continue if any(fnmatch.fnmatchcase(package, pattern) for pattern in includes) and not any( fnmatch.fnmatchcase(package, pattern) for pattern in exclude ): @@ -352,7 +413,10 @@ def _discover_setuptools_py_modules( discovered: set[str] = set() exclusions = excluded_modules or [] for configured_root in search_roots: - candidate_root = root / configured_root + root_inspection = inspect_setuptools_packaging_root(root, configured_root) + if root_inspection.status == "UNSAFE" or root_inspection.normalized_root is None: + continue + candidate_root = root / root_inspection.normalized_root if candidate_root.is_symlink() or not candidate_root.is_dir(): continue try: @@ -422,6 +486,12 @@ def setup_py_surface_resolved(root: Path) -> bool: return not path.is_file() or not inspect_setup_call(path).surface_unresolved +def setuptools_packaging_roots_safe(root: Path) -> bool: + """Whether all declared authoritative setuptools roots stay in scope.""" + + return not inspect_metadata(root).setuptools_external_packaging_roots + + def _literal_module_attribute(root: Path, attribute: str) -> str | None: """Resolve a setuptools dynamic version attr only when it is a string literal.""" @@ -520,6 +590,8 @@ def inspect_metadata(root: Path) -> MetadataResult: setuptools_package_selection_configured = False setuptools_surface_unresolved = False setuptools_surface_evidence: list[Evidence] = [] + setuptools_external_packaging_roots: list[str] = [] + setuptools_external_packaging_root_evidence: list[Evidence] = [] uv_workspace = False uv_workspace_source = False uv_workspace_evidence: list[Evidence] = [] @@ -1076,6 +1148,60 @@ def inspect_metadata(root: Path) -> MetadataResult: if not source_roots: source_roots = ["src"] if (root / "src").is_dir() else ["."] + + # Every source/search root that contributes to setuptools' authoritative + # first-party surface must be representable inside the assessed repository. + # Retaining only a safe subset while silently dropping another declared + # root would make that subset look authoritative when it is not. + root_metadata_path = next( + ( + root / name + for name in ("pyproject.toml", "setup.cfg", "setup.py") + if (root / name).is_file() + ), + root / "pyproject.toml", + ) + root_inspections: dict[str, PackagingRootInspection] = {} + + def inspect_declared_root(value: str) -> PackagingRootInspection: + inspection = root_inspections.get(value) + if inspection is None: + inspection = inspect_setuptools_packaging_root(root, value) + root_inspections[value] = inspection + if inspection.status == "UNSAFE": + setuptools_external_packaging_roots.append(value) + setuptools_external_packaging_root_evidence.append( + _evidence( + root, + root_metadata_path, + "Authoritative setuptools packaging root escapes the assessed " + f"repository boundary: {value!r}.", + _line_number(root_metadata_path, value), + ) + ) + return inspection + + def safe_roots(values: list[str]) -> list[str]: + result: list[str] = [] + for value in values: + inspection = inspect_declared_root(value) + if inspection.status == "UNSAFE" or inspection.normalized_root is None: + continue + if inspection.normalized_root not in result: + result.append(inspection.normalized_root) + return result + + source_roots = safe_roots(source_roots) + package_discovery_rules = [ + (safe_roots(where), include, exclude, namespaces) + for where, include, exclude, namespaces in package_discovery_rules + ] + package_directories = { + package: inspection.normalized_root + for package, directory in package_directories.items() + for inspection in [inspect_declared_root(directory)] + if inspection.status != "UNSAFE" and inspection.normalized_root is not None + } # Setuptools' ordinary automatic discovery applies when its build backend # is selected but source metadata has not selected packages, find rules, or # standalone modules. This makes the resolved existing ``packages`` model @@ -1205,4 +1331,6 @@ def inspect_metadata(root: Path) -> MetadataResult: uv_workspace_evidence=uv_workspace_evidence, setuptools_surface_unresolved=setuptools_surface_unresolved, setuptools_surface_evidence=setuptools_surface_evidence, + setuptools_external_packaging_roots=sorted(set(setuptools_external_packaging_roots)), + setuptools_external_packaging_root_evidence=setuptools_external_packaging_root_evidence, ) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 0c2ecdf..c576d3e 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -331,9 +331,14 @@ def package_surface_resolved( # setup.py fields are not persisted in PackagingAssessment. Re-inspect # locally when planning or validating a wheel so a dynamic selector cannot # bypass the source-surface authority contract through a stale/manual plan. - from python_deployment_builder.analysis.metadata import setup_py_surface_resolved + from python_deployment_builder.analysis.metadata import ( + setup_py_surface_resolved, + setuptools_packaging_roots_safe, + ) - return setup_py_surface_resolved(repository_root) + return setup_py_surface_resolved(repository_root) and setuptools_packaging_roots_safe( + repository_root + ) def _declared_package_data( diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 0579f98..5788185 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -851,6 +851,38 @@ def generate_deployment_kit( f"Deployment planning is blocked: {workspace_code}. " "M6.1 standalone deployment does not preserve or install uv workspace members." ) + # An escaping setuptools root is outside both the source/provenance + # boundary and this kit's standalone staging model. Stop before lock + # preparation, application-wheel work, artifact work, or output writes. + external_root_code = "EXTERNAL_PACKAGING_ROOT_UNSUPPORTED" + if external_root_code in plan.risk_gate.blocking_codes: + if dry_run: + preview = _preview( + plan, + output_root, + dry_run=True, + bootstrap_mode=bootstrap_mode, + system_certs=system_certs, + prepare_lock=prepare_lock, + approved=[], + application_artifact=None, + staging_source_paths=[], + ) + preview.developer_actions.insert( + 0, + f"Stop: {external_root_code} prevents standalone release generation.", + ) + return GenerationResult( + output_directory=str(output_root), + dry_run=True, + generated=False, + preview=preview, + ) + raise PreparationError( + f"Deployment planning is blocked: {external_root_code}. " + "Authoritative setuptools packaging roots must remain inside the assessed " + "repository." + ) allow_missing_lock_for_analysis = _allow_missing_lock_for_analysis( plan, dry_run=dry_run, prepare_lock=prepare_lock ) diff --git a/tests/test_generation.py b/tests/test_generation.py index f987520..736db65 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -4667,6 +4667,61 @@ def test_dynamic_setup_selector_blocks_installed_only_entry_point(tmp_path: Path assert "PACKAGING_SURFACE_UNRESOLVED" in plan.readiness.blocker_codes +@pytest.mark.parametrize( + ("target", "mode"), + [("app.main:main", "source"), ("installed_app.main:main", "package")], +) +def test_external_packaging_root_blocks_generation_before_lock_or_output_mutation( + tmp_path: Path, target: str, mode: str +) -> None: + source = tmp_path / "source" + shared = tmp_path / "shared" + (source / "src/app").mkdir(parents=True) + (shared / "helper").mkdir(parents=True) + (source / "src/app/__init__.py").write_text("", encoding="utf-8") + (source / "src/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (shared / "helper/__init__.py").write_text("", encoding="utf-8") + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='external-root-app'\nversion='1.0'\n" + f"[project.scripts]\nexternal-root='{target}'\n" + "[tool.setuptools.packages.find]\nwhere=['src', '../shared']\n", + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + output = tmp_path / "kit" + assert plan.deployment_mode == mode + + preview = generate_deployment_kit( + repository, output, dry_run=True, prepare_lock=True, bootstrap_mode="online_cmd" + ) + assert not preview.generated + assert any( + "EXTERNAL_PACKAGING_ROOT_UNSUPPORTED" in item + for item in preview.preview.developer_actions + ) + with pytest.raises(PreparationError, match="EXTERNAL_PACKAGING_ROOT_UNSUPPORTED"): + generate_deployment_kit( + repository, output, prepare_lock=True, bootstrap_mode="online_cmd" + ) + assert not output.exists() + assert not (source / "uv.lock").exists() + + wheel = _make_application_wheel( + tmp_path, + name="external-root-app", + version="1.0", + package="app", + target="app.main:main", + entry_group="console_scripts", + entry_name="external-root", + ) + with pytest.raises(PreparationError, match="authoritative Python packaging-surface"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + def test_uv_archive_rejects_traversal_and_hash_version_mismatch(tmp_path: Path) -> None: archive = tmp_path / "uv.zip" with zipfile.ZipFile(archive, "w") as bundle: diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 2bf4bbd..a9b6702 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -3,7 +3,12 @@ import zipfile from pathlib import Path -from python_deployment_builder.analysis.metadata import inspect_metadata, inspect_setup_call +from python_deployment_builder.analysis.metadata import ( + inspect_metadata, + inspect_setup_call, + inspect_setuptools_packaging_root, +) +from python_deployment_builder.analysis.resources import resolve_package_data_members from python_deployment_builder.models import EntryPointAssessment FIXTURES = Path(__file__).parent / "fixtures" @@ -171,6 +176,222 @@ def test_setuptools_find_packages_exclude_disposable_wheel_evidence(tmp_path: Pa assert "app/tests/__init__.py" not in members +def test_setuptools_package_roots_outside_repository_remain_explicitly_unresolved( + tmp_path: Path, +) -> None: + shared = tmp_path.parent / f"{tmp_path.name}-shared" + (tmp_path / "src/app").mkdir(parents=True) + (shared / "helper").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (shared / "helper/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='external-root-app'\nversion='1.0'\n" + "[tool.setuptools.packages.find]\n" + f"where=['src', '../{shared.name}']\n", + encoding="utf-8", + ) + (tmp_path / "setup.py").write_text( + "from setuptools import setup\nsetup()\n", encoding="utf-8" + ) + + # Disposable build evidence: setuptools treats both ``where`` entries as + # build-time package roots, even though PDB must not inspect the sibling. + subprocess.run( + [sys.executable, "setup.py", "bdist_wheel", "--dist-dir", str(tmp_path / "dist")], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + wheel = next((tmp_path / "dist").glob("external_root_app-1.0-*.whl")) + with zipfile.ZipFile(wheel) as bundle: + members = set(bundle.namelist()) + + result = inspect_metadata(tmp_path) + + assert "app/__init__.py" in members + assert "helper/__init__.py" in members + assert result.project.packages == ["app"] + assert result.project.source_roots == ["src"] + assert result.setuptools_external_packaging_roots == [f"../{shared.name}"] + assert result.setuptools_external_packaging_root_evidence + + +def test_setuptools_packaging_root_validator_distinguishes_safe_missing_and_unsafe( + tmp_path: Path, +) -> None: + (tmp_path / "src").mkdir() + + assert inspect_setuptools_packaging_root(tmp_path, "src").status == "SAFE" + assert inspect_setuptools_packaging_root(tmp_path, "missing").status == "MISSING_SAFE" + assert inspect_setuptools_packaging_root(tmp_path, "../shared").status == "UNSAFE" + assert ( + inspect_setuptools_packaging_root(tmp_path, str(tmp_path.parent / "shared")).status + == "UNSAFE" + ) + + +def test_symlinked_setuptools_root_is_not_an_authoritative_repository_root( + tmp_path: Path, +) -> None: + outside = tmp_path.parent / f"{tmp_path.name}-outside" + outside.mkdir() + link = tmp_path / "linked-root" + try: + link.symlink_to(outside, target_is_directory=True) + except OSError: + # Some Windows developer environments disallow symlink creation; the + # resolver's ordinary outside-root regression remains deterministic. + return + + assert inspect_setuptools_packaging_root(tmp_path, "linked-root").status == "UNSAFE" + + +def test_multiple_safe_setuptools_find_roots_remain_authoritative(tmp_path: Path) -> None: + for relative in ("src/app/__init__.py", "plugins/plugin/__init__.py"): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='multi-root'\nversion='1.0'\n" + "[tool.setuptools.packages.find]\nwhere=['src', 'plugins']\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.project.packages == ["app", "plugin"] + assert not result.setuptools_external_packaging_roots + + +def test_external_setuptools_package_dir_is_not_silently_treated_as_in_repository( + tmp_path: Path, +) -> None: + external = f"../{tmp_path.name}-shared" + cases = { + "pyproject.toml": ( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='demo'\nversion='1.0'\n" + f"[tool.setuptools]\npackage-dir={{''='{external}'}}\n" + ), + "setup.cfg": ( + "[metadata]\nname=demo\nversion=1.0\n[options]\n" + f"package_dir=\n = {external}\n" + ), + "setup.py": ( + "from setuptools import setup\n" + f"setup(name='demo', version='1.0', package_dir={{'': '{external}'}})\n" + ), + } + for name, content in cases.items(): + root = tmp_path / name.replace(".", "-") + root.mkdir() + (root / name).write_text(content, encoding="utf-8") + + result = inspect_metadata(root) + + assert result.setuptools_external_packaging_roots == [external] + + +def test_setup_cfg_external_find_where_is_not_discarded(tmp_path: Path) -> None: + external = f"../{tmp_path.name}-shared" + (tmp_path / "setup.cfg").write_text( + "[metadata]\nname=demo\nversion=1.0\n[options]\npackages=find:\n" + f"[options.packages.find]\nwhere=\n {external}\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.setuptools_external_packaging_roots == [external] + + +def test_setuptools_finder_unconditional_exclusions_match_disposable_wheel_evidence( + tmp_path: Path, +) -> None: + """PackageFinder 79.0.1 excludes ez_setup even without user exclusions.""" + + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/ez_setup").mkdir() + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/main.py").write_text("", encoding="utf-8") + (tmp_path / "src/ez_setup/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='finder-demo'\nversion='1.0'\n", + encoding="utf-8", + ) + (tmp_path / "setup.py").write_text( + "from setuptools import setup\nsetup(name='finder-demo', version='1.0')\n", + encoding="utf-8", + ) + + subprocess.run( + [sys.executable, "setup.py", "bdist_wheel", "--dist-dir", str(tmp_path / "dist")], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + wheel = next((tmp_path / "dist").glob("finder_demo-1.0-*.whl")) + with zipfile.ZipFile(wheel) as bundle: + members = set(bundle.namelist()) + + result = inspect_metadata(tmp_path) + assert "app/__init__.py" in members + assert "ez_setup/__init__.py" not in members + assert result.project.packages == ["app"] + + +def test_setuptools_finder_unconditional_exclusions_precede_user_include_and_package_data( + tmp_path: Path, +) -> None: + (tmp_path / "src/app/data").mkdir(parents=True) + (tmp_path / "src/ez_setup/data").mkdir(parents=True) + for relative in ("src/app/__init__.py", "src/ez_setup/__init__.py"): + (tmp_path / relative).write_text("", encoding="utf-8") + (tmp_path / "src/app/data/defaults.json").write_text("{}", encoding="utf-8") + (tmp_path / "src/ez_setup/data/ignored.json").write_text("{}", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='finder-demo'\nversion='1.0'\n" + "[tool.setuptools.packages.find]\nwhere=['src']\ninclude=['app*', 'ez_setup*']\n" + "namespaces=false\n" + "[tool.setuptools.package-data]\n'*'=['data/*.json']\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + members = resolve_package_data_members(tmp_path, result.project) + + assert result.project.packages == ["app"] + assert [(item.source_path, item.installed_member_path) for item in members] == [ + ("src/app/data/defaults.json", "app/data/defaults.json") + ] + + +def test_setuptools_unconditional_package_exclusions_apply_to_namespace_and_regular_finders( + tmp_path: Path, +) -> None: + for namespaces, expected in (("true", ["app"]), ("false", ["app"])): + project = tmp_path / namespaces + project.mkdir() + for relative in ("src/app/__init__.py", "src/ez_setup/__init__.py"): + destination = project / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text("", encoding="utf-8") + (project / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='finder-demo'\nversion='1.0'\n" + "[tool.setuptools.packages.find]\nwhere=['src']\n" + f"namespaces={namespaces}\n", + encoding="utf-8", + ) + assert inspect_metadata(project).project.packages == expected + + def test_entry_point_declared_group_is_independent_from_gui_heuristic(tmp_path: Path) -> None: (tmp_path / "pyproject.toml").write_text( """[project] diff --git a/tests/test_planning.py b/tests/test_planning.py index 035594b..72b19c8 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -135,6 +135,27 @@ def _write_custom_source_root_project(root: Path, *, declare_helper: bool = Fals (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") +def _write_external_packaging_root_project(root: Path, *, installed_only: bool = False) -> None: + shared = root.parent / f"{root.name}-shared" + (root / "src/app").mkdir(parents=True) + (shared / "helper").mkdir(parents=True) + (root / "src/app/__init__.py").write_text("", encoding="utf-8") + (root / "src/app/main.py").write_text( + "def main():\n from helper import value\n return value()\n", encoding="utf-8" + ) + (shared / "helper/__init__.py").write_text("def value(): return 1\n", encoding="utf-8") + target = "installed_app.main:main" if installed_only else "app.main:main" + (root / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='external-root-app'\nversion='1.0'\n" + f"[project.scripts]\nexternal-root='{target}'\n" + "[tool.setuptools.packages.find]\n" + f"where=['src', '../{shared.name}']\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + def test_import_promotion_uses_authoritative_custom_source_roots(tmp_path: Path) -> None: root = tmp_path / "custom-root" root.mkdir() @@ -175,6 +196,31 @@ def test_module_file_resolution_searches_all_safe_configured_roots(tmp_path: Pat )] == ["python/foo/bar/__init__.py"] +def test_external_packaging_root_blocks_both_source_and_package_mode_contracts( + tmp_path: Path, +) -> None: + source_root = tmp_path / "external-source" + _write_external_packaging_root_project(source_root) + source_assessment = assess_repository( + MaterializedRepository(root=source_root, source=str(source_root), source_kind="local") + ) + source_plan = create_deployment_plan(source_assessment, repository_root=source_root) + + assert source_plan.deployment_mode == "source" + assert "EXTERNAL_PACKAGING_ROOT_UNSUPPORTED" in source_plan.risk_gate.blocking_codes + assert "EXTERNAL_PACKAGING_ROOT_UNSUPPORTED" in source_plan.readiness.blocker_codes + + package_root = tmp_path / "external-package" + _write_external_packaging_root_project(package_root, installed_only=True) + package_assessment = assess_repository( + MaterializedRepository(root=package_root, source=str(package_root), source_kind="local") + ) + package_plan = create_deployment_plan(package_assessment, repository_root=package_root) + + assert package_plan.deployment_mode == "package" + assert "EXTERNAL_PACKAGING_ROOT_UNSUPPORTED" in package_plan.risk_gate.blocking_codes + + @pytest.mark.parametrize("backend", ["hatchling.build", "poetry.core.masonry.api"]) def test_unresolved_backend_src_entrypoint_preserves_source_mode( tmp_path: Path, backend: str From edae41bc9fb4bd7abb0c11e60c0bbef4dd20650f Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:28:33 -0400 Subject: [PATCH 31/59] Reject Windows-unsafe wheel paths and relocated metadata --- .../generation/artifacts.py | 85 ++++++++++++-- .../validation/static.py | 3 +- tests/test_generation.py | 108 ++++++++++++++++++ 3 files changed, 186 insertions(+), 10 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 6ab1785..e540de4 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -55,6 +55,26 @@ MAX_WHEEL_MEMBER_SIZE = 256 * 1024 * 1024 MAX_WHEEL_TOTAL_UNCOMPRESSED_SIZE = 512 * 1024 * 1024 +_WINDOWS_FORBIDDEN_COMPONENT_CHARACTERS = frozenset('<>:"|?*') +_WINDOWS_RESERVED_DEVICE_BASENAMES = frozenset( + { + "CON", + "PRN", + "AUX", + "NUL", + "CONIN$", + "CONOUT$", + *(f"COM{number}" for number in range(1, 10)), + *(f"LPT{number}" for number in range(1, 10)), + "COM¹", + "COM²", + "COM³", + "LPT¹", + "LPT²", + "LPT³", + } +) + def parse_artifact_argument(value: str) -> tuple[str, Path]: name, separator, raw_path = value.partition("=") @@ -63,6 +83,27 @@ def parse_artifact_argument(value: str) -> tuple[str, Path]: return canonicalize_name(name.strip()), Path(raw_path.strip()).expanduser() +def _windows_materializable_component(component: str) -> None: + """Reject components Windows cannot create through ordinary file APIs.""" + + if ( + not component + or component.endswith((".", " ")) + or any( + character in _WINDOWS_FORBIDDEN_COMPONENT_CHARACTERS + or character == "\x00" + or 1 <= ord(character) <= 31 + for character in component + ) + ): + raise PreparationError(f"Wheel contains a Windows-invalid path component: {component!r}") + # Windows reserves device basenames even when an ordinary-looking suffix + # follows (for example CON.py or NUL.tar.gz). + basename = component.split(".", 1)[0].upper() + if basename in _WINDOWS_RESERVED_DEVICE_BASENAMES: + raise PreparationError(f"Wheel contains a Windows-reserved path component: {component!r}") + + def _normalized_wheel_path(value: str) -> str: if ( not value @@ -75,6 +116,8 @@ def _normalized_wheel_path(value: str) -> str: path = PurePosixPath(value) if not path.parts or ".." in path.parts: raise PreparationError(f"Wheel contains an unsafe member: {value}") + for component in path.parts: + _windows_materializable_component(component) return path.as_posix() @@ -315,14 +358,15 @@ def _dist_info_members( return metadata_name, f"{dist_info}/WHEEL", f"{dist_info}/RECORD" -def installed_wheel_member_paths( +def installed_wheel_member_destinations( members: dict[str, zipfile.ZipInfo], wheel: Path -) -> set[str]: - """Return logical site-packages members after valid wheel relocation. +) -> dict[str, str]: + """Map every materialized site-packages destination after wheel relocation. - ``purelib`` members are relocated by installers into site-packages. Other - data schemes are deliberately not application import/package-data surface. - The caller has already validated the wheel's ``.data`` identity. + The authoritative root ``.dist-info`` tree participates in collision + accounting. ``purelib`` members are relocated by installers into that + same namespace; other data schemes remain outside this M6.1 application + surface. The caller has already validated the wheel's ``.data`` identity. """ installed: dict[str, str] = {} @@ -330,8 +374,6 @@ def installed_wheel_member_paths( if member.is_dir(): continue path = PurePosixPath(name) - if path.parts[0].endswith(".dist-info"): - continue destination: PurePosixPath | None = path if path.parts[0].endswith(".data"): if len(path.parts) < 3 or path.parts[1] != "purelib": @@ -345,6 +387,11 @@ def installed_wheel_member_paths( destination = None else: destination = PurePosixPath(*path.parts[2:]) + if any(part.endswith(".dist-info") for part in destination.parts): + raise PreparationError( + "Wheel .data/purelib content may not create an installed dist-info " + f"tree: {name}" + ) if destination is None or not destination.parts: continue normalized = destination.as_posix() @@ -355,7 +402,23 @@ def installed_wheel_member_paths( f"{previous}, {name}" ) installed[key] = normalized - return set(installed.values()) + return installed + + +def installed_wheel_member_paths( + members: dict[str, zipfile.ZipInfo], wheel: Path +) -> set[str]: + """Return the validated installed paths that form application surface. + + Metadata still contributes to installed-destination collision detection, + but never becomes Python/package-data application surface. + """ + + return { + destination + for destination in installed_wheel_member_destinations(members, wheel).values() + if not PurePosixPath(destination).parts[0].endswith(".dist-info") + } def _require_core_metadata(message, *, label: str, wheel: Path) -> tuple[str, str]: @@ -512,6 +575,10 @@ def validate_approved_wheel( raise PreparationError( f"Wheel is missing required WHEEL or RECORD metadata: {path.name}" ) + # Dependency wheels are installed by the same Windows runtime. + # Validate relocated site-packages destinations even though they + # have no first-party application-surface completeness contract. + installed_wheel_member_destinations(members, path) metadata = _metadata_message( bundle.read(members[metadata_name]), label="METADATA", wheel=path ) diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index ae20df9..5beab28 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -14,6 +14,7 @@ from pydantic import ValidationError +from python_deployment_builder.generation.acquisition import PreparationError from python_deployment_builder.generation.artifacts import installed_wheel_member_paths from python_deployment_builder.models import ( DeploymentManifest, @@ -382,7 +383,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR installed_wheel_member_paths(members, application_wheel) ) ) - except zipfile.BadZipFile: + except (zipfile.BadZipFile, PreparationError): entry_present = False entry_evidence = sorted(member_candidates) checks.append( diff --git a/tests/test_generation.py b/tests/test_generation.py index 736db65..2c28309 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -29,6 +29,8 @@ verify_uv_version, ) from python_deployment_builder.generation.artifacts import ( + _safe_wheel_members, + installed_wheel_member_destinations, validate_application_wheel, validate_approved_wheel, validate_artifact_set, @@ -1465,6 +1467,112 @@ def test_application_wheel_rejects_colliding_purelib_installed_member(tmp_path: validate_application_wheel(wheel, assessment, plan, repository_root=source) +@pytest.mark.parametrize( + "member", + [ + "app/CON.py", + "app/NUL.txt", + "app/AUX/config.json", + "app/COM1.py", + "app/LPT1/data.txt", + "app/CONIN$.txt", + "app/COM¹.txt", + "app/con.PY", + "app/Com1.txt", + "app/data.", + "app/data ", + "app/badname.txt", + "app/bad:name.txt", + 'app/bad"name.txt', + "app/bad|name.txt", + "app/bad?name.txt", + "app/bad*name.txt", + "app/bad\x01name.txt", + ], +) +def test_wheel_members_must_be_windows_materializable(tmp_path: Path, member: str) -> None: + wheel = tmp_path / "invalid-path.whl" + with zipfile.ZipFile(wheel, "w") as bundle: + bundle.writestr(member, b"content") + + with zipfile.ZipFile(wheel) as bundle, pytest.raises(PreparationError, match="Windows"): + _safe_wheel_members(bundle) + + +def test_wheel_windows_path_policy_preserves_unicode_and_existing_structure_checks( + tmp_path: Path, +) -> None: + valid = tmp_path / "unicode.whl" + with zipfile.ZipFile(valid, "w") as bundle: + bundle.writestr("app/données.json", b"{}") + bundle.writestr("app/日本語.txt", b"text") + with zipfile.ZipFile(valid) as bundle: + assert len(_safe_wheel_members(bundle)) == 2 + + collision = tmp_path / "ancestor.whl" + with zipfile.ZipFile(collision, "w") as bundle: + bundle.writestr("Demo", b"file") + bundle.writestr("demo/main.py", b"content") + with zipfile.ZipFile(collision) as bundle, pytest.raises( + PreparationError, match="ancestor collision" + ): + _safe_wheel_members(bundle) + + +def test_installed_destination_map_accounts_for_root_dist_info_and_purelib(tmp_path: Path) -> None: + wheel = tmp_path / "app-1.0-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as bundle: + bundle.writestr("app-1.0.dist-info/METADATA", b"metadata") + bundle.writestr("app-1.0.data/purelib/app/helper.py", b"VALUE = 1\n") + with zipfile.ZipFile(wheel) as bundle: + members = {item.filename: item for item in _safe_wheel_members(bundle)} + destinations = installed_wheel_member_destinations(members, wheel) + + assert set(destinations.values()) == { + "app-1.0.dist-info/METADATA", + "app/helper.py", + } + + +@pytest.mark.parametrize( + "member", + [ + "mapped_app-1.2.3.data/purelib/mapped_app-1.2.3.dist-info/METADATA", + "mapped_app-1.2.3.data/purelib/mapped_app-1.2.3.dist-info/WHEEL", + "mapped_app-1.2.3.data/purelib/mapped_app-1.2.3.dist-info/RECORD", + "mapped_app-1.2.3.data/purelib/other-1.0.dist-info/METADATA", + ], +) +def test_application_wheel_rejects_relocated_dist_info_tree(tmp_path: Path, member: str) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), additions={member: "untrusted metadata\n"} + ) + + with pytest.raises(PreparationError, match="may not create an installed dist-info"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + +def test_approved_dependency_wheel_rejects_relocated_dist_info_tree(tmp_path: Path) -> None: + plan = _plan("optional_map_app", ["map"]) + wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), + additions={ + "proxy_tools-0.1.0.data/purelib/proxy_tools-0.1.0.dist-info/METADATA": "bad\n" + }, + ) + + with pytest.raises(PreparationError, match="may not create an installed dist-info"): + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + @pytest.mark.parametrize( ("marker", "error"), [ From 40d81fefbf7175e6023e918d2d32eb6adc1180f7 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:12:28 -0400 Subject: [PATCH 32/59] Promote relative runtime imports and relocated path conflicts --- .../analysis/assessor.py | 19 ++++- .../analysis/inventory.py | 76 ++++++++++++++++--- .../generation/artifacts.py | 53 ++++++++----- .../planning/planner.py | 20 ++--- tests/test_analysis_scope.py | 72 ++++++++++++++++++ tests/test_generation.py | 45 +++++++++++ 6 files changed, 246 insertions(+), 39 deletions(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index 20ee6c5..8d3492b 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -256,7 +256,20 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen advisory_scope_imports = [ item for item in unusual_scope_imports if item not in generation_excluded_imports ] - if advisory_scope_imports: + promoted_non_runtime_scope = [ + item + for item in inventory.items + if item.role == RepositoryFileRole.APPLICATION_SOURCE + and any( + part.lower() in {"test", "tests", "doc", "docs", "example", "examples"} + for part in Path(item.path).parts + ) + and any( + "Application source imports local module" in evidence.detail + for evidence in item.evidence + ) + ] + if advisory_scope_imports or promoted_non_runtime_scope: risks.append( RiskFinding( code="APPLICATION_IMPORTS_NON_RUNTIME_SCOPE", @@ -272,7 +285,9 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen "Make the runtime dependency explicit or separate shared runtime code." ), evidence=[ - evidence for item in advisory_scope_imports for evidence in item.evidence + evidence + for item in [*advisory_scope_imports, *promoted_non_runtime_scope] + for evidence in item.evidence ], ) ) diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index f97c2f8..293359a 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -271,18 +271,76 @@ def inventory_repository(root: Path, source_roots: list[str]) -> InventoryResult return InventoryResult(items, application_files, summarize_inventory(items)) -def _imported_modules(tree: ast.AST) -> list[tuple[str, int]]: +def _source_package_contexts( + root: Path, source_path: Path, source_roots: list[str] +) -> set[str]: + """Return every safe package context that can contain ``source_path``. + + Source roots can overlap. Keep every valid interpretation so relative + imports conservatively promote all local candidates instead of selecting + an arbitrary root and potentially omitting runtime source. + """ + + resolved_root = root.resolve() + resolved_source = source_path.resolve() + contexts: set[str] = set() + for source_root in source_roots: + candidate_root = root / source_root + if candidate_root.is_symlink() or not candidate_root.is_dir(): + continue + try: + resolved_candidate_root = candidate_root.resolve() + resolved_candidate_root.relative_to(resolved_root) + relative = resolved_source.relative_to(resolved_candidate_root) + except ValueError: + continue + if relative.suffix != ".py": + continue + parts = list(relative.with_suffix("").parts) + if not parts: + continue + package_parts = parts if parts[-1] == "__init__" else parts[:-1] + if package_parts: + contexts.add(".".join(package_parts)) + return contexts + + +def _imported_modules( + tree: ast.AST, + source_path: Path, + root: Path, + source_roots: list[str], +) -> list[tuple[str, int]]: modules: list[tuple[str, int]] = [] for node in ast.walk(tree): if isinstance(node, ast.Import): modules.extend((alias.name, node.lineno) for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - modules.append((node.module, node.lineno)) - modules.extend( - (f"{node.module}.{alias.name}", node.lineno) - for alias in node.names - if alias.name != "*" - ) + elif isinstance(node, ast.ImportFrom): + if node.level == 0: + if not node.module: + continue + modules.append((node.module, node.lineno)) + modules.extend( + (f"{node.module}.{alias.name}", node.lineno) + for alias in node.names + if alias.name != "*" + ) + continue + for context in _source_package_contexts(root, source_path, source_roots): + context_parts = context.split(".") + parent_count = node.level - 1 + if parent_count >= len(context_parts): + # Python rejects imports above the top-level package. Do + # not fabricate a local module identity for that syntax. + continue + base = ".".join(context_parts[: len(context_parts) - parent_count]) + module = f"{base}.{node.module}" if node.module else base + modules.append((module, node.lineno)) + modules.extend( + (f"{module}.{alias.name}", node.lineno) + for alias in node.names + if alias.name != "*" + ) return modules @@ -336,7 +394,7 @@ def promote_imported_application_files( tree = ast.parse(source_path.read_text(encoding="utf-8-sig"), filename=relative_source) except (OSError, SyntaxError, UnicodeError): continue - for module, line in _imported_modules(tree): + for module, line in _imported_modules(tree, source_path, root, source_roots): for imported_path in _module_files(root, module, source_roots): relative = imported_path.relative_to(root).as_posix() item = by_path.get(relative) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index e540de4..0db2733 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -121,6 +121,35 @@ def _normalized_wheel_path(value: str) -> str: return path.as_posix() +def _validate_regular_file_path_collisions( + paths: list[tuple[str, str]], *, domain: str +) -> None: + """Reject Windows-equivalent file paths that cannot coexist on disk. + + ``paths`` contain materialized regular-file destinations and their archive + provenance. Archive paths and relocated ``purelib`` paths are different + domains, but a file can never also be a component ancestor in either one. + """ + + regular_paths: dict[str, str] = {} + for normalized, provenance in paths: + collision_key = normalized.casefold() + for index in range(1, len(PurePosixPath(normalized).parts)): + ancestor = "/".join(PurePosixPath(normalized).parts[:index]).casefold() + if previous := regular_paths.get(ancestor): + raise PreparationError( + f"Wheel contains a regular-file ancestor collision in {domain} paths: " + f"{previous}, {provenance}" + ) + for existing_key, existing_name in regular_paths.items(): + if existing_key.startswith(collision_key + "/"): + raise PreparationError( + f"Wheel contains a regular-file ancestor collision in {domain} paths: " + f"{existing_name}, {provenance}" + ) + regular_paths[collision_key] = provenance + + def _safe_wheel_members(bundle: zipfile.ZipFile) -> list[zipfile.ZipInfo]: members = bundle.infolist() if len(members) > MAX_WHEEL_MEMBERS: @@ -129,7 +158,7 @@ def _safe_wheel_members(bundle: zipfile.ZipFile) -> list[zipfile.ZipInfo]: if total_size > MAX_WHEEL_TOTAL_UNCOMPRESSED_SIZE: raise PreparationError("Wheel exceeds the maximum expanded archive size.") seen: dict[str, str] = {} - regular_paths: dict[str, str] = {} + regular_paths: list[tuple[str, str]] = [] for member in members: normalized = _normalized_wheel_path(member.filename) file_type = (member.external_attr >> 16) & 0o170000 @@ -148,23 +177,8 @@ def _safe_wheel_members(bundle: zipfile.ZipFile) -> list[zipfile.ZipInfo]: seen[collision_key] = member.filename if member.is_dir(): continue - parts = PurePosixPath(normalized).parts - for index in range(1, len(parts)): - ancestor = "/".join(parts[:index]).casefold() - if ancestor in regular_paths: - raise PreparationError( - "Wheel contains a regular-file ancestor collision: " - f"{regular_paths[ancestor]}, {member.filename}" - ) - for existing_key, existing_name in regular_paths.items(): - if collision_key.startswith(existing_key + "/") or existing_key.startswith( - collision_key + "/" - ): - raise PreparationError( - "Wheel contains a regular-file ancestor collision: " - f"{existing_name}, {member.filename}" - ) - regular_paths[collision_key] = member.filename + regular_paths.append((normalized.rstrip("/"), member.filename)) + _validate_regular_file_path_collisions(regular_paths, domain="archive") return members @@ -370,6 +384,7 @@ def installed_wheel_member_destinations( """ installed: dict[str, str] = {} + destinations: list[tuple[str, str]] = [] for name, member in members.items(): if member.is_dir(): continue @@ -402,6 +417,8 @@ def installed_wheel_member_destinations( f"{previous}, {name}" ) installed[key] = normalized + destinations.append((normalized, name)) + _validate_regular_file_path_collisions(destinations, domain="installed") return installed diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 726f454..7a43aab 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -158,6 +158,16 @@ def unresolved_surface_result() -> tuple[str, str, str, list[str]]: f"Python packaging-surface model, but {backend} is not modeled by M6.1." ], ) + if not surface_resolved and source_compatible: + return ( + "source", + "The project uses " + f"{backend}, whose installed Python packaging surface is not modeled by M6.1. " + "The authoritative entry point is source-import compatible, so source deployment " + "preserves the statically understood runtime surface.", + "SOURCE_COMPATIBLE", + [], + ) if source_constraints and source_compatible: return ( "source", @@ -213,16 +223,6 @@ def unresolved_surface_result() -> tuple[str, str, str, list[str]]: "SOURCE_COMPATIBLE", [], ) - if not surface_resolved: - return ( - "source", - "The project uses " - f"{backend}, whose installed Python packaging surface is not modeled by M6.1. " - "The authoritative entry point is source-import compatible, so source deployment " - "preserves the statically understood runtime surface.", - "SOURCE_COMPATIBLE", - [], - ) return ( "package", "The project has an install-oriented source layout without a source-only runtime " diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 9d13ca3..c8c6866 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -752,6 +752,78 @@ def test_analysis_roles_control_source_staging(tmp_path: Path) -> None: assert "historical/old.py" not in staged +def test_relative_imports_promote_test_scope_modules_and_stage_them(tmp_path: Path) -> None: + (tmp_path / "src/app/tests").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/main.py").write_text( + "from . import sibling\nfrom .tests import helper\nfrom .tests.helper import run\n\n" + "def main(): return sibling.value() + helper.value() + run()\n", + encoding="utf-8", + ) + (tmp_path / "src/app/sibling.py").write_text("def value(): return 1\n", encoding="utf-8") + (tmp_path / "src/app/tests/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/tests/helper.py").write_text( + "from . import nested\ndef value(): return nested.value()\ndef run(): return 1\n", + encoding="utf-8", + ) + (tmp_path / "src/app/tests/nested.py").write_text("def value(): return 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'relative-import-app'\nversion = '1.0'\n" + "[project.scripts]\nrelative-import-app = 'app.main:main'\n" + "[tool.setuptools]\npackages = ['app']\npackage-dir = {'' = 'src'}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + by_path = {item.path: item for item in assessment.file_inventory} + for path in ("src/app/tests/__init__.py", "src/app/tests/helper.py", "src/app/tests/nested.py"): + assert by_path[path].role == RepositoryFileRole.APPLICATION_SOURCE + assert any( + "Application source imports local module" in item.detail + for item in by_path[path].evidence + ) + assert "APPLICATION_IMPORTS_NON_RUNTIME_SCOPE" in {item.code for item in assessment.risks} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + + assert plan.deployment_mode == "source" + staged = _staging_files(tmp_path, assessment, plan, include=True) + assert { + "src/app/sibling.py", + "src/app/tests/__init__.py", + "src/app/tests/helper.py", + "src/app/tests/nested.py", + } <= staged.keys() + + +def test_parent_relative_import_uses_source_root_package_context(tmp_path: Path) -> None: + (tmp_path / "src/app/sub").mkdir(parents=True) + (tmp_path / "src/app/shared").mkdir() + for path in ("app/__init__.py", "app/sub/__init__.py", "app/shared/__init__.py"): + (tmp_path / "src" / path).write_text("", encoding="utf-8") + (tmp_path / "src/app/sub/main.py").write_text( + "from ..shared import helper\ndef main(): return helper.value()\n", encoding="utf-8" + ) + (tmp_path / "src/app/shared/helper.py").write_text("def value(): return 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'parent-relative-app'\nversion = '1.0'\n" + "[project.scripts]\nparent-relative-app = 'app.sub.main:main'\n" + "[tool.setuptools]\npackages = ['app', 'app.sub']\npackage-dir = {'' = 'src'}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + helper = next( + item for item in assessment.file_inventory if item.path == "src/app/shared/helper.py" + ) + + assert helper.role == RepositoryFileRole.APPLICATION_SOURCE + assert any("app.shared.helper" in item.detail for item in helper.evidence) + + @pytest.mark.parametrize( ("layout", "package_directory", "resource_path"), [ diff --git a/tests/test_generation.py b/tests/test_generation.py index 2c28309..7477e65 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1467,6 +1467,35 @@ def test_application_wheel_rejects_colliding_purelib_installed_member(tmp_path: validate_application_wheel(wheel, assessment, plan, repository_root=source) +@pytest.mark.parametrize( + "additions", + [ + { + "demo": "a regular file\n", + "mapped_app-1.2.3.data/purelib/demo/main.py": "def main(): return 0\n", + }, + { + "mapped_app-1.2.3.data/purelib/demo/main.py": "def main(): return 0\n", + "demo": "a regular file\n", + }, + ], +) +def test_application_wheel_rejects_post_relocation_file_ancestor_collision( + tmp_path: Path, additions: dict[str, str] +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel(_make_application_wheel(tmp_path), additions=additions) + + with pytest.raises(PreparationError, match="ancestor collision in installed paths"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + @pytest.mark.parametrize( "member", [ @@ -1573,6 +1602,22 @@ def test_approved_dependency_wheel_rejects_relocated_dist_info_tree(tmp_path: Pa validate_approved_wheel(f"proxy-tools={wheel}", plan) +def test_approved_dependency_wheel_rejects_post_relocation_file_ancestor_collision( + tmp_path: Path, +) -> None: + plan = _plan("optional_map_app", ["map"]) + wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), + additions={ + "demo": "a regular file\n", + "proxy_tools-0.1.0.data/purelib/demo/main.py": "def main(): return 0\n", + }, + ) + + with pytest.raises(PreparationError, match="ancestor collision in installed paths"): + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + @pytest.mark.parametrize( ("marker", "error"), [ From 2d4035000762e099f7c60d8f469af07612c7717e Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:52:38 -0400 Subject: [PATCH 33/59] Align setuptools defaults and package data resolution --- .../analysis/inventory.py | 5 +- .../analysis/metadata.py | 100 +++++++-- .../analysis/resources.py | 28 +-- tests/test_analysis_scope.py | 202 ++++++++++++++++++ 4 files changed, 299 insertions(+), 36 deletions(-) diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index 293359a..06e047c 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -299,7 +299,10 @@ def _source_package_contexts( parts = list(relative.with_suffix("").parts) if not parts: continue - package_parts = parts if parts[-1] == "__init__" else parts[:-1] + # A file's own stem never belongs to its package context. In + # particular, ``app/__init__.py`` runs in package ``app``, not the + # fictional package ``app.__init__``. + package_parts = parts[:-1] if package_parts: contexts.add(".".join(package_parts)) return contexts diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 5653a37..07e9672 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -71,29 +71,77 @@ def package_selection_present(self) -> bool: ) -# Setuptools' flat-layout auto-discovery deliberately avoids conventional -# development directories. Keep this bounded static subset explicit rather -# than treating every repository directory as application packaging surface. -_DEFAULT_FLAT_DISCOVERY_EXCLUDES = [ - "build", - "build.*", - "dist", - "dist.*", +# Verified against setuptools 79.0.1's FlatLayoutPackageFinder._EXCLUDE and +# DEFAULT_EXCLUDE. These defaults are specific to *automatic flat-layout* +# discovery; explicit ``packages.find`` remains a regular finder invocation. +_SETUPTOOLS_79_FLAT_PACKAGE_EXCLUDE_NAMES = ( + "ci", + "bin", + "debian", + "doc", "docs", - "docs.*", + "documentation", + "manpages", + "news", + "newsfragments", + "changelog", + "test", + "tests", + "unit_test", + "unit_tests", "example", - "example.*", "examples", - "examples.*", + "scripts", + "tools", + "util", + "utils", + "python", + "build", + "dist", + "venv", + "env", + "requirements", + "tasks", + "fabfile", + "site_scons", + "benchmark", + "benchmarks", + "exercise", + "exercises", + "htmlcov", + "[._]*", +) +_SETUPTOOLS_79_FLAT_PACKAGE_DEFAULT_EXCLUDES = tuple( + pattern + for name in _SETUPTOOLS_79_FLAT_PACKAGE_EXCLUDE_NAMES + for pattern in (name, f"{name}.*") +) + +# Verified against setuptools 79.0.1's FlatLayoutModuleFinder.DEFAULT_EXCLUDE. +# Unlike package defaults, these names are top-level module names only. +_SETUPTOOLS_79_FLAT_MODULE_DEFAULT_EXCLUDES = ( + "setup", + "conftest", "test", - "test.*", "tests", - "tests.*", - "venv", - "venv.*", - ".venv", - ".venv.*", -] + "example", + "examples", + "build", + "toxfile", + "noxfile", + "pavement", + "dodo", + "tasks", + "fabfile", + "[Ss][Cc]onstruct", + "conanfile", + "manage", + "benchmark", + "benchmarks", + "exercise", + "exercises", + "[._]*", +) # Verified against setuptools 79.0.1's PackageFinder and # PEP420PackageFinder. These are finder-level exclusions, applied before @@ -1227,7 +1275,9 @@ def safe_roots(values: list[str]) -> list[str]: ( [automatic_root], ["*"], - [] if automatic_root == "src" else _DEFAULT_FLAT_DISCOVERY_EXCLUDES, + [] + if automatic_root == "src" + else list(_SETUPTOOLS_79_FLAT_PACKAGE_DEFAULT_EXCLUDES), # Modern setuptools automatic discovery recognizes implicit # namespaces; explicit find configuration can still disable it. True, @@ -1261,12 +1311,18 @@ def safe_roots(values: list[str]) -> list[str]: ) and not automatic_setuptools_flat_surface_ambiguous: # Setuptools' default source-layout finder discovers top-level modules # as well as packages. Its flat-layout finder selects a package - # surface in preference to loose modules, so only a package-free flat - # layout contributes automatic py_modules. Explicit configuration - # never reaches this branch. + # surface in preference to loose modules: setuptools 79.0.1 implements + # ``_analyse_flat_packages() or _analyse_flat_modules()``. Therefore + # only a package-free flat layout contributes automatic py_modules. + # Explicit configuration never reaches this branch. discovered_modules = _discover_setuptools_py_modules( root, [automatic_setuptools_root], + excluded_modules=( + list(_SETUPTOOLS_79_FLAT_MODULE_DEFAULT_EXCLUDES) + if automatic_setuptools_root == "." + else None + ), ) if automatic_setuptools_root == "." and len(discovered_modules) > 1: # Mirroring the bounded flat package policy above prevents an diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index c576d3e..1a3c4e3 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -49,18 +49,15 @@ def _safe_package_data_pattern(pattern: str) -> bool: def _known_packages(project: PackagingAssessment) -> set[str]: - """Return only package identities established by static packaging metadata.""" - - return { - package - for package in [ - *project.packages, - *(name for name in project.package_directories if name), - *(name for name in project.package_data if name != "*"), - *(name for name in project.exclude_package_data if name != "*"), - ] - if package and package != "*" - } + """Return package identities selected by authoritative packaging metadata. + + ``package_data`` and ``exclude_package_data`` constrain files within a + selected package; they never select a package themselves. Physical + package-dir mappings likewise locate selected packages, but do not create + their identities. + """ + + return {package for package in project.packages if package and package != "*"} def _physical_package_roots( @@ -168,8 +165,13 @@ def resolve_package_data_members( resolved_root = root.resolve() resolved_members: list[ResolvedPackageDataMember] = [] seen: set[tuple[str, str, str, str]] = set() + selected_packages = _known_packages(project) for declared_package, patterns in project.package_data.items(): - packages = _known_packages(project) if declared_package == "*" else {declared_package} + packages = ( + selected_packages + if declared_package == "*" + else {declared_package} & selected_packages + ) for package in packages: for package_root in _physical_package_roots(root, project, package): exclusion_patterns = [ diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index c8c6866..ac1d21b 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -824,6 +824,44 @@ def test_parent_relative_import_uses_source_root_package_context(tmp_path: Path) assert any("app.shared.helper" in item.detail for item in helper.evidence) +def test_package_initializers_use_their_containing_package_as_relative_context( + tmp_path: Path, +) -> None: + (tmp_path / "src/app/tests").mkdir(parents=True) + (tmp_path / "src/app/sub").mkdir() + (tmp_path / "src/app/shared").mkdir() + (tmp_path / "src/app/__init__.py").write_text( + "from .tests import helper\n", encoding="utf-8" + ) + (tmp_path / "src/app/tests/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/tests/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "src/app/sub/__init__.py").write_text( + "from . import sibling\nfrom ..shared import helper\n", encoding="utf-8" + ) + (tmp_path / "src/app/sub/sibling.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "src/app/shared/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/shared/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'initializer-relative-app'\nversion = '1.0'\n" + "[tool.setuptools]\npackages = ['app', 'app.sub']\npackage-dir = {'' = 'src'}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + by_path = {item.path: item for item in assessment.file_inventory} + + for path in ("src/app/tests/__init__.py", "src/app/tests/helper.py"): + assert by_path[path].role == RepositoryFileRole.APPLICATION_SOURCE + assert any("app.tests" in item.detail for item in by_path[path].evidence) + sibling = by_path["src/app/sub/sibling.py"] + shared_helper = by_path["src/app/shared/helper.py"] + assert any("app.sub.sibling" in item.detail for item in sibling.evidence) + assert any("app.shared.helper" in item.detail for item in shared_helper.evidence) + assert "APPLICATION_IMPORTS_NON_RUNTIME_SCOPE" in {item.code for item in assessment.risks} + + @pytest.mark.parametrize( ("layout", "package_directory", "resource_path"), [ @@ -1040,6 +1078,85 @@ def test_setuptools_default_flat_discovery_excludes_development_directories(tmp_ assert "docs" not in project.packages +@pytest.mark.parametrize( + "reserved", + [ + "ci", + "bin", + "debian", + "doc", + "docs", + "manpages", + "news", + "newsfragments", + "changelog", + "test", + "tests", + "unit_test", + "example", + "examples", + "tools", + "scripts", + "util", + "utils", + "tasks", + "site_scons", + "benchmark", + "benchmarks", + "documentation", + "unit_tests", + "requirements", + "htmlcov", + "python", + "build", + "dist", + "venv", + "env", + "fabfile", + "exercise", + "exercises", + "_private", + ], +) +def test_setuptools_79_flat_package_defaults_exclude_reserved_names( + tmp_path: Path, reserved: str +) -> None: + (tmp_path / "app").mkdir() + (tmp_path / reserved / "internal").mkdir(parents=True) + for path in ( + tmp_path / "app/__init__.py", + tmp_path / reserved / "__init__.py", + tmp_path / reserved / "internal/__init__.py", + ): + path.write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools>=68']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'flat-defaults'\nversion = '1.0'\n", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == ["app"] + + +@pytest.mark.parametrize( + "package", ["app", "my_tools", "mytools", "toolbox", "utilities", "benchmarking"] +) +def test_setuptools_79_flat_package_defaults_do_not_exclude_ordinary_names( + tmp_path: Path, package: str +) -> None: + (tmp_path / package).mkdir() + (tmp_path / package / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools>=68']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'flat-ordinary'\nversion = '1.0'\n", + encoding="utf-8", + ) + + assert inspect_metadata(tmp_path).project.packages == [package] + + def test_setuptools_default_flat_single_module_defines_python_surface(tmp_path: Path) -> None: (tmp_path / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") (tmp_path / "pyproject.toml").write_text( @@ -1059,6 +1176,49 @@ def test_setuptools_default_flat_single_module_defines_python_surface(tmp_path: assert project.py_modules == ["helper"] +@pytest.mark.parametrize( + "reserved", + [ + "conftest", + "test", + "tests", + "example", + "examples", + "build", + "toxfile", + "noxfile", + "pavement", + "dodo", + "tasks", + "fabfile", + "SConstruct", + "conanfile", + "manage", + "benchmark", + "benchmarks", + "exercise", + "exercises", + "_private", + ], +) +def test_setuptools_79_flat_module_defaults_exclude_reserved_modules( + tmp_path: Path, reserved: str +) -> None: + (tmp_path / "setup.py").write_text("from setuptools import setup\nsetup()\n", encoding="utf-8") + (tmp_path / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / f"{reserved}.py").write_text("VALUE = 2\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools>=68']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'flat-modules'\nversion = '1.0'\n", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == [] + assert project.py_modules == ["helper"] + + def test_setuptools_default_flat_package_surface_omits_loose_module(tmp_path: Path) -> None: (tmp_path / "example_app").mkdir() (tmp_path / "example_app/__init__.py").write_text("", encoding="utf-8") @@ -1080,6 +1240,48 @@ def test_setuptools_default_flat_package_surface_omits_loose_module(tmp_path: Pa assert project.py_modules == [] +def test_package_data_does_not_create_an_unselected_package_identity(tmp_path: Path) -> None: + (tmp_path / "main.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "ghost").mkdir() + (tmp_path / "ghost/data.txt").write_text("not packaged\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools>=68']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'ghost-data'\nversion = '1.0'\n" + "[tool.setuptools]\npy-modules = ['main']\n" + "[tool.setuptools.package-data]\nghost = ['data.txt']\n", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + assert project.packages == [] + assert project.py_modules == ["main"] + assert resolve_package_data_members(tmp_path, project) == [] + + +def test_package_data_applies_only_to_selected_packages(tmp_path: Path) -> None: + for package in ("app", "ghost"): + (tmp_path / package / "data").mkdir(parents=True) + (tmp_path / package / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / package / "data/default.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools>=68']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'selected-data'\nversion = '1.0'\n" + "[tool.setuptools]\npackages = ['app']\n" + "[tool.setuptools.package-data]\napp = ['data/*.json']\nghost = ['data/*.json']\n" + "[tool.setuptools.exclude-package-data]\nghost = ['data/*.json']\n", + encoding="utf-8", + ) + + project = inspect_metadata(tmp_path).project + + resolved = resolve_package_data_members(tmp_path, project) + + assert [(item.source_path, item.installed_member_path) for item in resolved] == [ + ("app/data/default.json", "app/data/default.json") + ] + + def test_setuptools_default_flat_multi_package_surface_remains_unresolved(tmp_path: Path) -> None: for package in ("one", "two"): (tmp_path / package).mkdir() From de1a0c998f3c2b83e75e29e6ffd9c12644017597 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Mon, 7 Sep 2026 12:23:37 -0400 Subject: [PATCH 34/59] Guard analyzed metadata and flat discovery ambiguity --- .../analysis/assessor.py | 4 +- .../analysis/metadata.py | 32 +++ .../analysis/resources.py | 7 +- .../generation/generator.py | 31 ++- tests/test_analysis_scope.py | 11 +- tests/test_generation.py | 246 ++++++++++++++++++ 6 files changed, 320 insertions(+), 11 deletions(-) diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index 8d3492b..d73c19f 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -213,8 +213,8 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen severity=RiskSeverity.WARNING, status=FindingStatus.NEEDS_VALIDATION, description=( - "setup.py declares packaging-surface configuration that PDB cannot " - "statically resolve without executing target code." + "PDB cannot statically establish the authoritative setuptools packaging " + "surface for this project." ), recommendation=( "Use literal setuptools package configuration or retain source deployment; " diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 07e9672..491e9fd 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -540,6 +540,16 @@ def setuptools_packaging_roots_safe(root: Path) -> bool: return not inspect_metadata(root).setuptools_external_packaging_roots +def setuptools_packaging_surface_resolved(root: Path) -> bool: + """Whether a local setuptools project has a statically authoritative surface.""" + + metadata = inspect_metadata(root) + return ( + not metadata.setuptools_surface_unresolved + and not metadata.setuptools_external_packaging_roots + ) + + def _literal_module_attribute(root: Path, attribute: str) -> str | None: """Resolve a setuptools dynamic version attr only when it is a string literal.""" @@ -1305,6 +1315,17 @@ def safe_roots(values: list[str]) -> list[str]: # constraints remain conservative until metadata is explicit. discovered_packages = set() automatic_setuptools_flat_surface_ambiguous = True + setuptools_surface_unresolved = True + setuptools_surface_evidence.append( + _evidence( + root, + root_metadata_path, + "Automatic flat-layout package discovery found multiple top-level " + "packages; setuptools would refuse the ambiguous build until package " + "selection is explicit.", + _line_number(root_metadata_path, "build-backend"), + ) + ) packages = sorted({*packages, *discovered_packages}) if automatic_setuptools_root is not None and ( automatic_setuptools_root == "src" or not packages @@ -1329,6 +1350,17 @@ def safe_roots(values: list[str]) -> list[str]: # undeclared multi-module distribution from becoming a fabricated # wheel surface. discovered_modules = [] + setuptools_surface_unresolved = True + setuptools_surface_evidence.append( + _evidence( + root, + root_metadata_path, + "Automatic flat-layout module discovery found multiple top-level " + "modules; setuptools would refuse the ambiguous build until module " + "selection is explicit.", + _line_number(root_metadata_path, "build-backend"), + ) + ) py_modules = discovered_modules layout = ( "src" diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 1a3c4e3..6992504 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -334,13 +334,10 @@ def package_surface_resolved( # locally when planning or validating a wheel so a dynamic selector cannot # bypass the source-surface authority contract through a stale/manual plan. from python_deployment_builder.analysis.metadata import ( - setup_py_surface_resolved, - setuptools_packaging_roots_safe, + setuptools_packaging_surface_resolved, ) - return setup_py_surface_resolved(repository_root) and setuptools_packaging_roots_safe( - repository_root - ) + return setuptools_packaging_surface_resolved(repository_root) def _declared_package_data( diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 5788185..fd24df4 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -293,9 +293,36 @@ def _analysis_policy_paths(assessment) -> set[str]: } +def _analysis_metadata_paths(assessment) -> set[str]: + """Return parsed repository inputs that influence release planning. + + Packaging metadata and lockfiles are represented directly by the project + assessment. Python selection also records the exact files from which it + derived constraints (including ``.python-version`` and, when used, + documented version evidence). These inputs need Git provenance even when + role-aware staging intentionally does not copy them into a kit. + """ + + paths: set[str] = set() + for value in [ + *assessment.project.metadata_files, + *assessment.project.lockfiles, + *(evidence.file for evidence in assessment.python.evidence), + ]: + if not value: + continue + candidate = PurePosixPath(value.replace("\\", "/")) + if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts): + continue + paths.add(candidate.as_posix()) + return paths + + def _provenance_guard_paths(repository_root: Path, assessment, plan) -> set[str]: - return _selected_deployment_paths(repository_root, assessment, plan) | _analysis_policy_paths( - assessment + return ( + _selected_deployment_paths(repository_root, assessment, plan) + | _analysis_policy_paths(assessment) + | _analysis_metadata_paths(assessment) ) diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index ac1d21b..afbf707 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -11,6 +11,7 @@ from python_deployment_builder.analysis.metadata import inspect_metadata from python_deployment_builder.analysis.repository import MaterializedRepository from python_deployment_builder.analysis.resources import ( + package_surface_resolved, resolve_package_data_members, resolve_packaged_python_sources, ) @@ -1297,10 +1298,13 @@ def test_setuptools_default_flat_multi_package_surface_remains_unresolved(tmp_pa encoding="utf-8", ) - project = inspect_metadata(tmp_path).project + metadata = inspect_metadata(tmp_path) + project = metadata.project assert project.packages == [] assert project.py_modules == [] + assert metadata.setuptools_surface_unresolved + assert not package_surface_resolved(project, tmp_path) def test_setuptools_default_flat_multi_module_surface_remains_unresolved(tmp_path: Path) -> None: @@ -1317,10 +1321,13 @@ def test_setuptools_default_flat_multi_module_surface_remains_unresolved(tmp_pat encoding="utf-8", ) - project = inspect_metadata(tmp_path).project + metadata = inspect_metadata(tmp_path) + project = metadata.project assert project.packages == [] assert project.py_modules == [] + assert metadata.setuptools_surface_unresolved + assert not package_surface_resolved(project, tmp_path) @pytest.mark.parametrize( diff --git a/tests/test_generation.py b/tests/test_generation.py index 7477e65..2d05c17 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -37,8 +37,10 @@ ) from python_deployment_builder.generation.cmd import parse_certutil_sha256 from python_deployment_builder.generation.generator import ( + _analysis_metadata_paths, _git_tracked_paths, _planned_generated_paths, + _provenance_guard_paths, _render_owned_files, _selected_deployment_paths, _staging_files, @@ -4174,6 +4176,149 @@ def test_git_source_staging_blocks_dirty_tracked_project_metadata( assert relative in str(caught.value) +def _committed_metadata_provenance_fixture( + tmp_path: Path, source_kind: str +) -> tuple[Path, MaterializedRepository, str]: + """Build a Git fixture with a non-staged parsed input of each supported kind.""" + + source = tmp_path / f"dirty-{source_kind}" + source.mkdir() + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n", + encoding="utf-8", + ) + if source_kind == "setup.cfg": + relative = "setup.cfg" + (source / relative).write_text( + "[metadata]\nname = metadata-demo\nversion = 1.0\n" + "[options]\npackages =\n app\n" + "[options.entry_points]\nconsole_scripts =\n demo = app:main\n", + encoding="utf-8", + ) + elif source_kind == "setup.py": + relative = "setup.py" + (source / relative).write_text( + "from setuptools import setup\n" + "setup(name='metadata-demo', version='1.0', packages=['app'], " + "entry_points={'console_scripts': ['demo=app:main']})\n", + encoding="utf-8", + ) + else: + relative = ".python-version" + (source / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'metadata-demo'\nversion = '1.0'\n" + "[project.scripts]\ndemo = 'app:main'\n", + encoding="utf-8", + ) + (source / relative).write_text("3.11\n", encoding="utf-8") + (source / "app").mkdir() + (source / "app/__init__.py").write_text("", encoding="utf-8") + (source / "app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + return ( + source, + MaterializedRepository(root=source, source=str(source), source_kind="local"), + relative, + ) + + +@pytest.mark.parametrize("source_kind", ["setup.cfg", "setup.py", "python-version"]) +def test_git_source_staging_blocks_dirty_parsed_metadata_not_staged( + tmp_path: Path, source_kind: str +) -> None: + source, repository, relative = _committed_metadata_provenance_fixture( + tmp_path, source_kind + ) + path = source / relative + if source_kind in {"setup.cfg", "setup.py"}: + path.write_text( + path.read_text(encoding="utf-8").replace("1.0", "2.0"), encoding="utf-8" + ) + else: + path.write_text("3.12\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert relative in _analysis_metadata_paths(assessment) + assert relative in _provenance_guard_paths(source, assessment, plan) + if source_kind == "python-version": + assert assessment.python.python_version_file == "3.12" + else: + assert assessment.project.version == "2.0" + with pytest.raises(PreparationError, match=relative.replace(".", r"\.")): + _staging_files(source, assessment, plan, include=True) + + +@pytest.mark.parametrize("operation", ["deleted", "renamed"]) +def test_git_source_staging_uses_head_metadata_guard_for_removed_setup_cfg( + tmp_path: Path, operation: str +) -> None: + source, repository, relative = _committed_metadata_provenance_fixture(tmp_path, "setup.cfg") + path = source / relative + if operation == "deleted": + path.unlink() + else: + path.rename(source / "setup-moved.cfg") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match=r"setup\.cfg"): + _staging_files(source, assessment, plan, include=True) + + +@pytest.mark.parametrize("source_kind", ["setup.py", "python-version"]) +def test_git_source_staging_uses_head_metadata_guard_for_deleted_parsed_input( + tmp_path: Path, source_kind: str +) -> None: + source, repository, relative = _committed_metadata_provenance_fixture( + tmp_path, source_kind + ) + (source / relative).unlink() + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match=relative.replace(".", r"\.")): + _staging_files(source, assessment, plan, include=True) + + +def test_analysis_metadata_paths_include_parsed_requirements_and_python_evidence( + tmp_path: Path, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + (source / "requirements-runtime.txt").write_text("requests>=2\n", encoding="utf-8") + (source / ".python-version").write_text("3.12\n", encoding="utf-8") + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "metadata"], check=True) + assessment = assess_repository(repository) + + assert {"requirements-runtime.txt", ".python-version"} <= _analysis_metadata_paths( + assessment + ) + + +def test_git_source_staging_allows_unrelated_readme_without_python_evidence( + tmp_path: Path, +) -> None: + source, repository = _committed_source_fixture(tmp_path) + readme = source / "docs/readme.md" + readme.write_text("ordinary unrelated prose\n", encoding="utf-8") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert "docs/readme.md" not in _analysis_metadata_paths(assessment) + assert "app.py" in _staging_files(source, assessment, plan, include=True) + + def test_git_source_staging_allows_deleted_unrelated_documentation( tmp_path: Path, ) -> None: @@ -4820,6 +4965,107 @@ def test_dynamic_setup_selector_blocks_installed_only_entry_point(tmp_path: Path assert "PACKAGING_SURFACE_UNRESOLVED" in plan.readiness.blocker_codes +def _automatic_flat_ambiguity_project(root: Path, *, installed_only: bool = False) -> None: + for package in ("app_one", "app_two"): + (root / package).mkdir(parents=True) + (root / package / "__init__.py").write_text("", encoding="utf-8") + (root / "app_one/main.py").write_text("def main(): return 0\n", encoding="utf-8") + target = "installed_app.main:main" if installed_only else "app_one.main:main" + (root / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='flat-ambiguous'\nversion='1.0'\n" + f"[project.scripts]\ndemo='{target}'\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + +def test_automatic_flat_multi_package_surface_falls_back_to_source_and_blocks_wheel_bypass( + tmp_path: Path, +) -> None: + source = tmp_path / "flat-ambiguous-source" + _automatic_flat_ambiguity_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert "PACKAGING_SURFACE_UNRESOLVED" in [item.code for item in assessment.risks] + assert plan.deployment_mode == "source" + assert plan.deployment_mode_condition == "SOURCE_COMPATIBLE" + wheel = _make_application_wheel( + tmp_path, + name="flat-ambiguous", + version="1.0", + package="app_one", + target="app_one.main:main", + entry_group="console_scripts", + entry_name="demo", + ) + with pytest.raises(PreparationError, match="authoritative Python packaging-surface"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + +def test_automatic_flat_multi_package_surface_blocks_installed_only_entry_point( + tmp_path: Path, +) -> None: + source = tmp_path / "flat-ambiguous-installed" + _automatic_flat_ambiguity_project(source, installed_only=True) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert plan.deployment_mode == "package" + assert plan.deployment_mode_condition == "INSTALLED_PROJECT_REQUIRED" + assert "PACKAGING_SURFACE_UNRESOLVED" in plan.readiness.blocker_codes + + +def test_explicit_flat_multi_package_surface_is_authoritative(tmp_path: Path) -> None: + source = tmp_path / "flat-explicit" + _automatic_flat_ambiguity_project(source) + (source / "pyproject.toml").write_text( + (source / "pyproject.toml").read_text(encoding="utf-8") + + "[tool.setuptools]\npackages=['app_one', 'app_two']\n", + encoding="utf-8", + ) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + + assert "PACKAGING_SURFACE_UNRESOLVED" not in [item.code for item in assessment.risks] + assert assessment.project.packages == ["app_one", "app_two"] + + +@pytest.mark.parametrize( + ("target", "expected_mode", "expected_blocked"), + [ + ("main:main", "source", False), + ("installed_app:main", "package", True), + ], +) +def test_automatic_flat_multi_module_surface_falls_back_or_blocks_package_mode( + tmp_path: Path, target: str, expected_mode: str, expected_blocked: bool +) -> None: + source = tmp_path / "flat-modules" + source.mkdir() + for module in ("main", "helper"): + (source / f"{module}.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='flat-modules'\nversion='1.0'\n" + f"[project.scripts]\ndemo='{target}'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + + assert "PACKAGING_SURFACE_UNRESOLVED" in [item.code for item in assessment.risks] + assert plan.deployment_mode == expected_mode + assert ("PACKAGING_SURFACE_UNRESOLVED" in plan.readiness.blocker_codes) is expected_blocked + + @pytest.mark.parametrize( ("target", "mode"), [("app.main:main", "source"), ("installed_app.main:main", "package")], From 9f511709dbd38f84cece202cbfded9720869400f Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Mon, 7 Sep 2026 12:55:37 -0400 Subject: [PATCH 35/59] Strengthen lock artifact and dependency proofs --- .../generation/artifacts.py | 84 +++++- .../planning/lockfile.py | 25 +- tests/test_generation.py | 240 +++++++++++++++++- 3 files changed, 328 insertions(+), 21 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 0db2733..d946b23 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -35,7 +35,9 @@ RepositoryAssessment, ) from python_deployment_builder.planning.index import ( + TargetMarkerApplicability, TargetMarkerEnvironmentError, + target_marker_applicability, target_marker_applies, wheel_matches, ) @@ -563,6 +565,16 @@ def validate_approved_wheel( raise PreparationError( f"No developer-wheel requirement exists for {requested_name} in this deployment plan." ) + target_possible_versions = { + dependency.version + for dependency in (plan.lock_graph.dependencies if plan.lock_graph else []) + if canonicalize_name(dependency.name) == requested_name + } + if len(target_possible_versions) > 1: + raise PreparationError( + "Developer artifact substitution is ambiguous for target-possible locked versions: " + f"{requested_name} ({', '.join(sorted(target_possible_versions))})." + ) if len(versions) != 1: raise PreparationError( "Developer artifact substitution is ambiguous for target-possible locked versions: " @@ -638,6 +650,8 @@ def validate_approved_wheel( def validate_artifact_set( values: list[str], plan: DeploymentPlan ) -> list[tuple[ApprovedArtifact, Path]]: + for requirement in plan.lock_graph.artifact_requirements if plan.lock_graph else []: + validate_artifact_substitution_target(requirement.package, plan) validated = [validate_approved_wheel(value, plan) for value in values] names = [item[0].distribution_name for item in validated] if len(names) != len(set(names)): @@ -645,6 +659,22 @@ def validate_artifact_set( return validated +def validate_artifact_substitution_target(package: str, plan: DeploymentPlan) -> None: + """Reject a stale plan that would replace conditional versions by one wheel.""" + + canonical_name = canonicalize_name(package) + versions = { + dependency.version + for dependency in (plan.lock_graph.dependencies if plan.lock_graph else []) + if canonicalize_name(dependency.name) == canonical_name + } + if len(versions) > 1: + raise PreparationError( + "Developer artifact substitution is ambiguous for target-possible locked versions: " + f"{package} ({', '.join(sorted(versions))})." + ) + + def _application_requirement_applies(requirement: Requirement, plan: DeploymentPlan) -> bool: """Evaluate Core Metadata markers against the planned Windows target, not this host.""" @@ -670,18 +700,67 @@ def _application_requirement_applies(requirement: Requirement, plan: DeploymentP def _target_possible_dependencies(graph, canonical_name: str): - """Return all selected-target lock candidates for one distribution.""" + """Return direct selected-target candidates for one application dependency.""" return sorted( ( dependency for dependency in graph.dependencies - if canonicalize_name(dependency.name) == canonical_name + if dependency.direct and canonicalize_name(dependency.name) == canonical_name ), key=lambda dependency: (Version(dependency.version), dependency.version), ) +def _direct_dependency_presence_proven( + graph, plan: DeploymentPlan, application_name: str, canonical_name: str +) -> None: + """Require a definitely-applicable root lock edge for wheel metadata presence.""" + + selected_extras = {canonicalize_name(extra) for extra in graph.selected_extras} + matching = [] + unprovable: list[str] = [] + malformed: list[str] = [] + for edge in graph.edges: + if ( + canonicalize_name(edge.from_package) != application_name + or canonicalize_name(edge.to_package) != canonical_name + ): + continue + if edge.selected_extra and canonicalize_name(edge.selected_extra) not in selected_extras: + continue + matching.append(edge) + try: + applicability = target_marker_applicability( + edge.marker, + plan.runtime.python_version, + plan.runtime.architecture, + extra=edge.selected_extra or "", + ) + except TargetMarkerEnvironmentError as exc: + malformed.append(str(exc)) + continue + if applicability == TargetMarkerApplicability.APPLIES: + return + if applicability == TargetMarkerApplicability.UNPROVABLE: + unprovable.append(edge.marker or "") + if unprovable or malformed: + detail = "; ".join([*unprovable, *malformed]) + raise PreparationError( + "Application wheel Requires-Dist presence cannot be proven from a definitely " + f"applicable direct locked dependency edge for {canonical_name}: {detail}." + ) + if matching: + raise PreparationError( + "Application wheel Requires-Dist is absent from the selected locked environment: " + f"{canonical_name}; no direct locked dependency edge definitely applies." + ) + raise PreparationError( + "Application wheel Requires-Dist is absent from the selected locked environment: " + f"{canonical_name}; no direct locked dependency edge exists." + ) + + def _validate_dependency_extra_closure(requirement: Requirement, graph, candidates) -> None: """Prove a wheel dependency's requested extras are activated by the selected lock graph.""" @@ -784,6 +863,7 @@ def _validate_application_requires_dist( f"the selected locked environment: {requirement.name}." ) try: + _direct_dependency_presence_proven(graph, plan, application_name, name) candidates = _target_possible_dependencies(graph, name) except InvalidVersion as exc: raise PreparationError( diff --git a/src/python_deployment_builder/planning/lockfile.py b/src/python_deployment_builder/planning/lockfile.py index 40ed15a..2096c4b 100644 --- a/src/python_deployment_builder/planning/lockfile.py +++ b/src/python_deployment_builder/planning/lockfile.py @@ -263,23 +263,22 @@ def enqueue_edges( dependencies = sorted(locked.values(), key=lambda item: (not item.direct, item.name.lower())) findings: list[ArtifactPolicyFinding] = [] requirements: list[DeploymentArtifactRequirement] = [] - developer_artifact_versions: dict[str, set[str]] = {} + target_possible_versions: dict[str, set[str]] = {} + packages_needing_developer_substitution: set[str] = set() for dependency in dependencies: + canonical_name = canonicalize_name(dependency.name) + target_possible_versions.setdefault(canonical_name, set()).add(dependency.version) if ( dependency.artifact.policy == "developer_wheel_required" and dependency.artifact.source_distribution_available ): - developer_artifact_versions.setdefault(canonicalize_name(dependency.name), set()).add( - dependency.version - ) + packages_needing_developer_substitution.add(canonical_name) artifact_forks = { - package: versions - for package, versions in developer_artifact_versions.items() - if len(versions) > 1 + package: target_possible_versions[package] + for package in packages_needing_developer_substitution + if len(target_possible_versions[package]) > 1 } for dependency in dependencies: - if dependency.artifact.policy == "wheel_usable": - continue canonical_name = canonicalize_name(dependency.name) if canonical_name in artifact_forks: versions = ", ".join(sorted(artifact_forks[canonical_name])) @@ -293,13 +292,15 @@ def enqueue_edges( selected_extra=dependency.selected_extra, description=( "The selected target leaves multiple possible locked versions of " - f"{dependency.name} that require developer-supplied wheels ({versions}). " - "PDB cannot replace uv's conditional version selection with one " - "unconditional reviewed artifact." + f"{dependency.name} ({versions}), including a version that requires " + "a developer-supplied wheel. PDB cannot replace uv's conditional " + "version selection with one unconditional reviewed artifact." ), ) ) continue + if dependency.artifact.policy == "wheel_usable": + continue status = ( "developer_artifact_required" if dependency.artifact.source_distribution_available diff --git a/tests/test_generation.py b/tests/test_generation.py index 2d05c17..c8f18c5 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -328,6 +328,10 @@ def _application_plan_with_locked_dependencies( ) for name, version in dependencies ], + edges=[ + DependencyEdge(from_package="mapped-app", to_package=name) + for name, _version in dependencies + ], ) return configured @@ -339,8 +343,14 @@ def _application_plan_with_target_possible_dependencies( requested_extras: list[str] | None = None, available_extras: dict[str, list[str]] | None = None, edges=None, + include_default_direct_edges: bool = True, ): configured = plan.model_copy(deep=True) + direct_edges = ( + [DependencyEdge(from_package="mapped-app", to_package="foo") for _ in versions] + if include_default_direct_edges + else [] + ) configured.lock_graph = LockGraphAssessment( inspected=True, python_version=configured.runtime.python_version, @@ -360,7 +370,10 @@ def _application_plan_with_target_possible_dependencies( ) for version in versions ], - edges=edges or [], + edges=[ + *direct_edges, + *(edges or []), + ], ) return configured @@ -468,6 +481,30 @@ def _write_developer_artifact_fork_lock(root: Path, *, markers: bool = True) -> ) +def _write_mixed_artifact_policy_fork_lock(root: Path, second_policy: str) -> None: + second_artifact = { + "wheel": 'wheels = [{ url = "https://example.invalid/foo-2.0-py3-none-any.whl" }]\n', + "developer": 'sdist = { url = "https://example.invalid/foo-2.0.tar.gz" }\n', + "none": "", + }[second_policy] + (root / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = \">=3.12\"\n\n" + "[[package]]\nname = \"mapped-app\"\nversion = \"1.2.3\"\n" + "source = { virtual = \".\" }\n" + "dependencies = [\n" + " { name = \"foo\", version = \"1.0\", marker = \"python_full_version < '3.12.5'\" },\n" + " { name = \"foo\", version = \"2.0\", marker = \"python_full_version >= '3.12.5'\" },\n" + "]\n\n" + "[[package]]\nname = \"foo\"\nversion = \"1.0\"\n" + "source = { registry = \"https://pypi.org/simple\" }\n" + "sdist = { url = \"https://example.invalid/foo-1.0.tar.gz\" }\n\n" + "[[package]]\nname = \"foo\"\nversion = \"2.0\"\n" + "source = { registry = \"https://pypi.org/simple\" }\n" + + second_artifact, + encoding="utf-8", + ) + + def _plan_with_dependency_extra_lock(source: Path, *, selected_extras: list[str] | None = None): repository = MaterializedRepository(root=source, source=str(source), source_kind="local") assessment = assess_repository(repository) @@ -1125,12 +1162,13 @@ def test_application_wheel_requires_dist_proves_every_patch_marker_possible_lock ] configured = plan.model_copy(update={"lock_graph": graph}) - assert validate_application_wheel( - _make_application_wheel(tmp_path, requires_dist_values=["foo>=1"]), - assessment, - configured, - )[0] - with pytest.raises(PreparationError, match="every target-possible.*1.0"): + with pytest.raises(PreparationError, match="presence cannot be proven"): + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo>=1"]), + assessment, + configured, + ) + with pytest.raises(PreparationError, match="presence cannot be proven"): validate_application_wheel( _make_application_wheel(tmp_path, requires_dist_values=["foo>=2"]), assessment, @@ -1177,6 +1215,72 @@ def test_lock_inspection_blocks_multi_version_developer_artifact_fork( assert not output.exists() +@pytest.mark.parametrize("second_policy", ["wheel", "developer", "none"]) +def test_lock_inspection_blocks_mixed_policy_multi_version_artifact_fork( + tmp_path: Path, second_policy: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + _write_mixed_artifact_policy_fork_lock(source, second_policy) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + plan = create_deployment_plan(assess_repository(repository), repository_root=source) + + assert [(item.name, item.version) for item in plan.lock_graph.dependencies] == [ + ("foo", "1.0"), + ("foo", "2.0"), + ] + assert plan.lock_graph.artifact_requirements == [] + assert {item.code for item in plan.lock_graph.artifact_findings} == { + "MULTI_VERSION_ARTIFACT_FORK_UNSUPPORTED" + } + assert {item.version for item in plan.lock_graph.artifact_findings} == {"1.0", "2.0"} + for version in ("1.0", "2.0"): + wheel = _make_wheel(tmp_path, name="foo", version=version) + with pytest.raises(PreparationError, match="No developer-wheel requirement"): + validate_approved_wheel(f"foo={wheel}", plan) + + +def test_approved_wheel_rejects_stale_single_requirement_for_multi_version_graph( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + plan = create_deployment_plan( + assess_repository(repository), + repository_root=source, + ).model_copy(deep=True) + plan.lock_graph = LockGraphAssessment( + inspected=True, + python_version=plan.runtime.python_version, + architecture=plan.runtime.architecture, + dependencies=[ + LockedDependency( + name="foo", + version=version, + direct=True, + artifact=ArtifactAvailability( + compatible_wheel_available=version == "2.0", + source_distribution_available=version == "1.0", + policy="wheel_usable" if version == "2.0" else "developer_wheel_required", + ), + ) + for version in ("1.0", "2.0") + ], + artifact_requirements=[ + DeploymentArtifactRequirement( + package="foo", version="1.0", action="developer_wheel_required", reason="test" + ) + ], + ) + wheel = _make_wheel(tmp_path, name="foo", version="1.0") + + with pytest.raises(PreparationError, match="ambiguous.*1.0, 2.0"): + validate_artifact_set([f"foo={wheel}"], plan) + + def test_lock_inspection_keeps_unambiguous_developer_artifact_requirements( tmp_path: Path, ) -> None: @@ -1272,6 +1376,128 @@ def test_approved_wheel_rejects_programmatic_multi_version_requirement(tmp_path: validate_approved_wheel(f"foo={wheel}", ambiguous) +@pytest.mark.parametrize( + ("marker", "accepted"), + [ + (None, True), + ('python_version >= "3.12"', True), + ('python_version < "3.12"', False), + ('python_full_version < "3.12.5"', False), + ("python_version <", False), + ], +) +def test_application_wheel_requires_dist_needs_definitely_applicable_direct_edge( + tmp_path: Path, marker: str | None, accepted: bool +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_target_possible_dependencies( + create_deployment_plan(assessment, repository_root=source), + ["1.0"], + edges=[DependencyEdge(from_package="mapped-app", to_package="foo", marker=marker)], + include_default_direct_edges=False, + ) + wheel = _make_application_wheel(tmp_path, requires_dist_values=["foo>=1"]) + + if accepted: + assert validate_application_wheel(wheel, assessment, plan)[0] + else: + with pytest.raises(PreparationError, match="presence cannot be proven|absent"): + validate_application_wheel(wheel, assessment, plan) + + +def test_application_wheel_requires_dist_rejects_transitive_only_dependency( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source).model_copy(deep=True) + usable = ArtifactAvailability( + compatible_wheel_available=True, source_distribution_available=False, policy="wheel_usable" + ) + plan.lock_graph = LockGraphAssessment( + inspected=True, + python_version=plan.runtime.python_version, + architecture=plan.runtime.architecture, + dependencies=[ + LockedDependency(name="bar", version="1.0", direct=True, artifact=usable), + LockedDependency(name="foo", version="1.0", direct=False, artifact=usable), + ], + edges=[ + DependencyEdge(from_package="mapped-app", to_package="bar"), + DependencyEdge(from_package="bar", to_package="foo"), + ], + ) + + with pytest.raises(PreparationError, match="no direct locked dependency edge"): + validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo"]), assessment, plan + ) + + +def test_application_wheel_requires_dist_uses_direct_candidates_not_transitive_versions( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source).model_copy(deep=True) + usable = ArtifactAvailability( + compatible_wheel_available=True, source_distribution_available=False, policy="wheel_usable" + ) + plan.lock_graph = LockGraphAssessment( + inspected=True, + python_version=plan.runtime.python_version, + architecture=plan.runtime.architecture, + dependencies=[ + LockedDependency(name="foo", version="1.0", direct=True, artifact=usable), + LockedDependency(name="foo", version="2.0", direct=False, artifact=usable), + ], + edges=[DependencyEdge(from_package="mapped-app", to_package="foo")], + ) + + assert validate_application_wheel( + _make_application_wheel(tmp_path, requires_dist_values=["foo==1"]), assessment, plan + )[0] + + +def test_application_wheel_requires_dist_selected_extra_needs_direct_edge( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = _application_plan_with_target_possible_dependencies( + create_deployment_plan(assessment, repository_root=source), + ["1.0"], + edges=[DependencyEdge(from_package="mapped-app", to_package="foo", selected_extra="map")], + include_default_direct_edges=False, + ) + plan.lock_graph.selected_extras = ["map"] + + assert validate_application_wheel( + _make_application_wheel( + tmp_path, requires_dist_values=['foo; extra == "map"'] + ), + assessment, + plan, + )[0] + + def test_application_wheel_requires_dist_rejects_invalid_target_possible_version( tmp_path: Path, ) -> None: From ae119a458b8a0d988adaf458a47ccc7540f68ce8 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 09:28:39 -0400 Subject: [PATCH 36/59] Validate staged wheel trust and install schemes --- .../generation/artifacts.py | 84 +++-- .../generation/structural.py | 35 +- .../validation/static.py | 47 ++- tests/test_generation.py | 318 ++++++++++++++++++ 4 files changed, 459 insertions(+), 25 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index d946b23..589e495 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -381,8 +381,9 @@ def installed_wheel_member_destinations( The authoritative root ``.dist-info`` tree participates in collision accounting. ``purelib`` members are relocated by installers into that - same namespace; other data schemes remain outside this M6.1 application - surface. The caller has already validated the wheel's ``.data`` identity. + same namespace. All other ``.data`` installation schemes are active wheel + payloads outside M6.1's bounded pure-Python installation model and are + rejected. The caller has already validated the wheel's ``.data`` identity. """ installed: dict[str, str] = {} @@ -393,22 +394,23 @@ def installed_wheel_member_destinations( path = PurePosixPath(name) destination: PurePosixPath | None = path if path.parts[0].endswith(".data"): - if len(path.parts) < 3 or path.parts[1] != "purelib": - # scripts/headers/data have no site-packages application - # representation; platlib is outside the pure-Python contract. - if len(path.parts) >= 2 and path.parts[1] == "platlib": - raise PreparationError( - "Application wheel uses .data/platlib, which is not accepted by the " - f"pure-Python package-mode contract: {wheel.name}" - ) - destination = None - else: - destination = PurePosixPath(*path.parts[2:]) - if any(part.endswith(".dist-info") for part in destination.parts): - raise PreparationError( - "Wheel .data/purelib content may not create an installed dist-info " - f"tree: {name}" - ) + if len(path.parts) < 3: + raise PreparationError( + "Wheel uses a malformed .data installation layout: " + f"{name}" + ) + scheme = path.parts[1] + if scheme != "purelib": + raise PreparationError( + "Wheel uses unsupported .data installation scheme " + f"{scheme!r}; M6.1 accepts only purelib: {wheel.name}" + ) + destination = PurePosixPath(*path.parts[2:]) + if any(part.endswith(".dist-info") for part in destination.parts): + raise PreparationError( + "Wheel .data/purelib content may not create an installed dist-info " + f"tree: {name}" + ) if destination is None or not destination.parts: continue normalized = destination.as_posix() @@ -440,6 +442,40 @@ def installed_wheel_member_paths( } +def validate_wheel_installation_layout(path: Path) -> None: + """Validate archive, identity, RECORD, and install-destination safety. + + This structural subset is reusable by static-kit validation, where the + original source assessment/lock plan is unavailable but staged artifacts + must still be safe to materialize into the managed environment. + """ + + try: + with zipfile.ZipFile(path) as bundle: + members = _member_map(_safe_wheel_members(bundle)) + if bundle.testzip() is not None: + raise PreparationError(f"Wheel archive entries are corrupt: {path.name}") + _metadata_name, wheel_name, record_name = _dist_info_members(members, path) + if wheel_name not in members or record_name not in members: + raise PreparationError( + f"Wheel is missing required WHEEL or RECORD metadata: {path.name}" + ) + installed_wheel_member_destinations(members, path) + _validate_record(bundle, members, record_name, path) + nested_wheels = [ + name + for name, member in members.items() + if not member.is_dir() and PurePosixPath(name).suffix.lower() == ".whl" + ] + if nested_wheels: + raise PreparationError( + "NESTED_WHEEL_UNSUPPORTED: Wheel contains opaque nested wheel members: " + + ", ".join(sorted(nested_wheels)) + ) + except zipfile.BadZipFile as exc: + raise PreparationError(f"Malformed wheel archive: {path.name}") from exc + + def _require_core_metadata(message, *, label: str, wheel: Path) -> tuple[str, str]: metadata_versions = message.get_all("Metadata-Version", []) names = message.get_all("Name", []) @@ -504,7 +540,7 @@ def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> No ) -def _validate_application_security( +def _validate_wheel_security( bundle: zipfile.ZipFile, members: dict[str, zipfile.ZipInfo], *, @@ -515,6 +551,11 @@ def _validate_application_security( member_path = PurePosixPath(name) if member.is_dir(): continue + if member_path.suffix.lower() == ".whl": + raise PreparationError( + "NESTED_WHEEL_UNSUPPORTED: Wheel contains an opaque nested wheel member: " + f"{name}" + ) if member_path.suffix.lower() == ".ps1" or is_secret_filename(member_path.name): failures.append(name) continue @@ -533,7 +574,7 @@ def _validate_application_security( failures.append(name) if failures: raise PreparationError( - "Application wheel content violates deployment security policy: " + "Wheel content violates deployment security policy: " + ", ".join(sorted(failures)) ) @@ -615,6 +656,7 @@ def validate_approved_wheel( bundle.read(members[wheel_name]), label="WHEEL", wheel=path ) _validate_record(bundle, members, record_name, path) + _validate_wheel_security(bundle, members) except zipfile.BadZipFile as exc: raise PreparationError(f"Malformed wheel archive: {path.name}") from exc @@ -1043,7 +1085,7 @@ def validate_application_wheel( for item in plan.configuration if item.secret and (value := os.environ.get(item.name)) is not None ) - _validate_application_security( + _validate_wheel_security( bundle, members, configured_secret_values=configured_secret_values, diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index c1756b1..bc74447 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -33,6 +33,23 @@ def _check(condition: bool, code: str, description: str) -> RiskFinding: ) +def trusted_artifact_wheel_paths(manifest: DeploymentManifest) -> set[str]: + """Return exact manifest-owned wheel paths with dedicated validation. + + A wheel hash/index proves identity only. The sole wheels exempt from + ordinary staged-file scanning are artifacts already validated through the + application/dependency wheel validators and named by this manifest. + """ + + paths = { + f"deployment/wheels/{artifact.filename}" + for artifact in manifest.approved_artifacts + } + if manifest.application_artifact is not None: + paths.add(f"deployment/application/{manifest.application_artifact.filename}") + return paths + + def validate_rendered_files( files: dict[str, bytes], manifest: DeploymentManifest, @@ -115,11 +132,16 @@ def validate_rendered_files( program_files_hits: list[str] = [] secret_hits: list[str] = [] undecodable_text: list[str] = [] - # Every intentionally staged file is release content. Wheels remain - # opaque here because their member-level validator owns their security scan. + unvalidated_wheels: list[str] = [] + trusted_wheels = trusted_artifact_wheel_paths(manifest) + # Every intentionally staged file is release content. Only exact + # manifest-owned artifacts may remain opaque because their dedicated + # member-level validators own their security scans. for relative, content in files.items(): path = PurePosixPath(relative) if path.suffix.lower() == ".whl": + if relative not in trusted_wheels: + unvalidated_wheels.append(relative) continue if is_secret_filename(path.name): secret_hits.append(relative) @@ -185,6 +207,15 @@ def validate_rendered_files( "NO_PROGRAM_FILES_WRITES", f"Program Files write targets: {program_files_hits or 'none'}", ), + _check( + not unvalidated_wheels, + "NO_UNVALIDATED_STAGED_WHEELS", + "All staged wheels are exact manifest-declared artifacts with dedicated " + "wheel validation." + if not unvalidated_wheels + else "Staged wheels have not passed dedicated artifact validation: " + f"{sorted(unvalidated_wheels)}", + ), _check( not undecodable_text, "TEXT_SECURITY_DECODABLE", diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 5beab28..95f1b08 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -15,7 +15,11 @@ from pydantic import ValidationError from python_deployment_builder.generation.acquisition import PreparationError -from python_deployment_builder.generation.artifacts import installed_wheel_member_paths +from python_deployment_builder.generation.artifacts import ( + installed_wheel_member_paths, + validate_wheel_installation_layout, +) +from python_deployment_builder.generation.structural import trusted_artifact_wheel_paths from python_deployment_builder.models import ( DeploymentManifest, ManualValidationItem, @@ -290,6 +294,21 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=unexpected_paths, ) ) + trusted_wheels = trusted_artifact_wheel_paths(manifest) + unvalidated_wheels = sorted( + path + for path in actual_paths + if PurePosixPath(path).suffix.lower() == ".whl" and path not in trusted_wheels + ) + checks.append( + _check( + "NO_UNVALIDATED_STAGED_WHEELS", + not unvalidated_wheels, + "Every staged wheel is an exact manifest-declared artifact.", + "A staged wheel is not an exact manifest-declared artifact.", + evidence=unvalidated_wheels, + ) + ) metadata_hash_failures = [] for name, expected in ( @@ -338,6 +357,24 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=artifact_failures, ) ) + wheel_layout_failures: list[str] = [] + for relative in sorted(trusted_wheels): + path = _safe_kit_path(root, relative) + if path is None or not path.is_file(): + continue + try: + validate_wheel_installation_layout(path) + except PreparationError as exc: + wheel_layout_failures.append(f"{relative}: {exc}") + checks.append( + _check( + "WHEEL_INSTALLATION_LAYOUT", + not wheel_layout_failures, + "Manifest-declared wheels have safe archive and installation layouts.", + "A manifest-declared wheel has an unsafe archive or installation layout.", + evidence=wheel_layout_failures, + ) + ) missing_source_roots = [ item @@ -478,9 +515,15 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR *(root / "deployment").rglob("*"), } for path in security_paths: - if path is None or not path.is_file() or path.suffix.lower() == ".whl": + if path is None or not path.is_file(): continue relative = str(path.relative_to(root)) + if path.suffix.lower() == ".whl": + if relative in trusted_wheels: + continue + # The dedicated unvalidated-wheel check above owns this opaque + # member; do not claim an ordinary text scan proved it safe. + continue content = path.read_bytes() if not is_textual_content(Path(relative), content): continue diff --git a/tests/test_generation.py b/tests/test_generation.py index c8f18c5..618818d 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -1830,6 +1830,82 @@ def test_approved_dependency_wheel_rejects_relocated_dist_info_tree(tmp_path: Pa validate_approved_wheel(f"proxy-tools={wheel}", plan) +@pytest.mark.parametrize("scheme", ["platlib", "scripts", "headers", "data", "unknown"]) +def test_application_wheel_rejects_unsupported_data_installation_schemes( + tmp_path: Path, scheme: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={f"mapped_app-1.2.3.data/{scheme}/payload.txt": "payload\n"}, + ) + + with pytest.raises(PreparationError, match="unsupported .data installation scheme"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + +def test_application_wheel_rejects_malformed_data_installation_layout(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"mapped_app-1.2.3.data/payload.txt": "payload\n"}, + ) + + with pytest.raises(PreparationError, match="malformed .data installation layout"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + +def test_approved_dependency_wheel_rejects_unsupported_data_installation_scheme( + tmp_path: Path, +) -> None: + plan = _plan("optional_map_app", ["map"]) + wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), + additions={"proxy_tools-0.1.0.data/data/Lib/site-packages/proxy_tools/x.py": "x = 1\n"}, + ) + + with pytest.raises(PreparationError, match="unsupported .data installation scheme"): + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + +@pytest.mark.parametrize("validator", ["application", "approved"]) +def test_wheel_validators_reject_opaque_nested_wheels(tmp_path: Path, validator: str) -> None: + nested = _make_wheel(tmp_path, name="vendor", version="1.0") + if validator == "application": + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/vendor.WHL": nested.read_bytes()}, + ) + with pytest.raises(PreparationError, match="NESTED_WHEEL_UNSUPPORTED"): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + else: + plan = _plan("optional_map_app", ["map"]) + wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"proxy_tools/vendor.whl": nested.read_bytes()} + ) + with pytest.raises(PreparationError, match="NESTED_WHEEL_UNSUPPORTED"): + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + def test_approved_dependency_wheel_rejects_post_relocation_file_ancestor_collision( tmp_path: Path, ) -> None: @@ -3364,6 +3440,124 @@ def test_package_mode_release_is_deterministic_and_survives_extraction( assert "first-party application artifact is mapped-app==1.2.3" in smoke +def _add_indexed_file(kit: Path, relative: str, content: bytes) -> None: + target = kit / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + index_path = kit / "deployment/generated-files.json" + index = json.loads(index_path.read_text(encoding="utf-8")) + index["files"].append( + {"path": relative, "sha256": hashlib.sha256(content).hexdigest()} + ) + index_path.write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") + + +def test_static_validation_rejects_indexed_unvalidated_staged_wheel( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit(_repository("prepared_gui"), kit, bootstrap_mode="online_cmd") + _add_indexed_file(kit, "code/resources/vendor.whl", _make_wheel(tmp_path).read_bytes()) + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "NO_UNVALIDATED_STAGED_WHEELS" and item.status.value == "FAIL" + for item in report.static_checks + ) + + +def test_static_validation_rejects_unsupported_application_wheel_data_scheme( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + wheel = _make_application_wheel(tmp_path) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + application_wheel=wheel, + bootstrap_mode="online_cmd", + ) + staged = kit / "deployment/application" / wheel.name + _rewrite_application_wheel( + staged, + additions={"mapped_app-1.2.3.data/data/payload.dat": b"payload"}, + ) + + report = validate_static_kit(kit) + + assert any( + item.code == "WHEEL_INSTALLATION_LAYOUT" and item.status.value == "FAIL" + for item in report.static_checks + ) + + +def test_static_validation_rejects_unsupported_approved_artifact_data_scheme( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + staged = kit / "deployment/wheels" / artifact.name + _rewrite_application_wheel( + staged, + additions={"proxy_tools-0.1.0.data/scripts/tool.exe": b"payload"}, + ) + + report = validate_static_kit(kit) + + assert any( + item.code == "WHEEL_INSTALLATION_LAYOUT" and item.status.value == "FAIL" + for item in report.static_checks + ) + + def test_package_regeneration_removes_unchanged_obsolete_application_wheel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -4970,6 +5164,88 @@ def test_templates_are_thin_and_forbid_prohibited_shells(tmp_path: Path) -> None assert any(item.code == "NO_FORBIDDEN_SHELL" for item in checks) +@pytest.mark.parametrize( + "relative", + [ + "app/resources/vendor.whl", + "deployment/wheels/extra.whl", + "deployment/application/extra.WHL", + ], +) +def test_rendered_files_reject_wheels_not_exactly_declared_by_manifest( + tmp_path: Path, relative: str +) -> None: + plan = _plan() + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + owned, manifest = _render_owned_files( + plan, + FIXTURES / "prepared_gui", + bootstrap_mode="bundled_uv", + system_certs=False, + approved=[], + bundled_uv=fake_uv, + ) + owned["deployment/generated-files.json"] = b"{}" + files = { + "pyproject.toml": (FIXTURES / "prepared_gui" / "pyproject.toml").read_bytes(), + "uv.lock": (FIXTURES / "prepared_gui" / "uv.lock").read_bytes(), + relative: _make_wheel(tmp_path).read_bytes(), + **owned, + } + + with pytest.raises(PreparationError, match="NO_UNVALIDATED_STAGED_WHEELS"): + validate_rendered_files(files, manifest, generated_paths=set(owned), secret_values=[]) + + +def test_rendered_files_allows_exact_manifest_declared_artifact_wheels(tmp_path: Path) -> None: + plan = _plan() + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + approved_wheel = _make_wheel(tmp_path) + approved = ApprovedArtifact( + distribution_name="proxy-tools", + version="0.1.0", + filename=approved_wheel.name, + sha256=hashlib.sha256(approved_wheel.read_bytes()).hexdigest(), + ) + application_wheel = _make_application_wheel(tmp_path) + owned, manifest = _render_owned_files( + plan, + FIXTURES / "prepared_gui", + bootstrap_mode="bundled_uv", + system_certs=False, + approved=[(approved, approved_wheel)], + bundled_uv=fake_uv, + ) + owned["deployment/generated-files.json"] = b"{}" + manifest = manifest.model_copy( + update={ + "application_artifact": ApplicationArtifact( + distribution_name="mapped-app", + version="1.2.3", + filename=application_wheel.name, + sha256=hashlib.sha256(application_wheel.read_bytes()).hexdigest(), + entry_point_name="mapped-app", + entry_point_target="installed_app.main:main", + ) + } + ) + files = { + "pyproject.toml": (FIXTURES / "prepared_gui" / "pyproject.toml").read_bytes(), + "uv.lock": (FIXTURES / "prepared_gui" / "uv.lock").read_bytes(), + f"deployment/application/{application_wheel.name}": application_wheel.read_bytes(), + **owned, + } + + checks = validate_rendered_files(files, manifest, generated_paths=set(owned), secret_values=[]) + + assert any( + item.code == "NO_UNVALIDATED_STAGED_WHEELS" and item.severity.value == "info" + for item in checks + ) + + def test_generation_structural_validation_scans_shared_textual_configuration_formats( tmp_path: Path, ) -> None: @@ -5112,6 +5388,48 @@ def test_authoritative_source_package_data_non_utf8_fails_before_output_write( assert not (tmp_path / "kit").exists() +def test_authoritative_source_package_data_wheel_fails_before_output_write( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + (source / "app/resources").mkdir(parents=True) + (source / "app/__init__.py").write_text("", encoding="utf-8") + (source / "app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + vendor = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"vendor/.env": "credential=sk-abcdefghijklmnop\n"} + ) + shutil.copy2(vendor, source / "app/resources/vendor.whl") + (source / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'vendor-wheel-app'\nversion = '1.0'\ndependencies = []\n" + "[project.scripts]\nvendor = 'app.main:main'\n" + "[tool.setuptools]\npackages = ['app']\n" + "[tool.setuptools.package-data]\napp = ['resources/*.whl']\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + + with pytest.raises(PreparationError, match="NO_UNVALIDATED_STAGED_WHEELS"): + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + bootstrap_mode="online_cmd", + ) + assert not (tmp_path / "kit").exists() + + def test_application_wheel_known_text_with_non_utf8_content_fails_closed(tmp_path: Path) -> None: source = tmp_path / "source" source.mkdir() From ae115ac1c7165b8715969c599a6899245ba222bb Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:05:26 -0400 Subject: [PATCH 37/59] Validate local wheel security and collisions --- .../generation/artifacts.py | 88 +++++- .../generation/generator.py | 13 + .../validation/static.py | 43 ++- tests/test_generation.py | 291 ++++++++++++++++++ 4 files changed, 421 insertions(+), 14 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 589e495..50e3b92 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -13,6 +13,7 @@ import re import stat import zipfile +from collections.abc import Iterable from email.parser import BytesParser from email.policy import default from pathlib import Path, PurePosixPath @@ -136,6 +137,11 @@ def _validate_regular_file_path_collisions( regular_paths: dict[str, str] = {} for normalized, provenance in paths: collision_key = normalized.casefold() + if previous := regular_paths.get(collision_key): + raise PreparationError( + f"Wheel contains colliding regular files in {domain} paths: " + f"{previous}, {provenance}" + ) for index in range(1, len(PurePosixPath(normalized).parts)): ancestor = "/".join(PurePosixPath(normalized).parts[:index]).casefold() if previous := regular_paths.get(ancestor): @@ -386,6 +392,15 @@ def installed_wheel_member_destinations( rejected. The caller has already validated the wheel's ``.data`` identity. """ + destinations = installed_wheel_file_destinations(members, wheel) + return {destination.casefold(): destination for destination, _name in destinations} + + +def installed_wheel_file_destinations( + members: dict[str, zipfile.ZipInfo], wheel: Path +) -> list[tuple[str, str]]: + """Return validated materialized destinations with archive-member provenance.""" + installed: dict[str, str] = {} destinations: list[tuple[str, str]] = [] for name, member in members.items(): @@ -423,7 +438,7 @@ def installed_wheel_member_destinations( installed[key] = normalized destinations.append((normalized, name)) _validate_regular_file_path_collisions(destinations, domain="installed") - return installed + return destinations def installed_wheel_member_paths( @@ -442,6 +457,22 @@ def installed_wheel_member_paths( } +def configured_secret_values(secret_names: Iterable[str]) -> tuple[str, ...]: + """Return current secret values without serializing or reporting them.""" + + values: list[str] = [] + seen: set[str] = set() + for name in secret_names: + if (value := os.environ.get(name)) is not None and value not in seen: + values.append(value) + seen.add(value) + return tuple(values) + + +def _plan_configured_secret_values(plan: DeploymentPlan) -> tuple[str, ...]: + return configured_secret_values(item.name for item in plan.configuration if item.secret) + + def validate_wheel_installation_layout(path: Path) -> None: """Validate archive, identity, RECORD, and install-destination safety. @@ -476,6 +507,48 @@ def validate_wheel_installation_layout(path: Path) -> None: raise PreparationError(f"Malformed wheel archive: {path.name}") from exc +def validate_wheel_static_safety( + path: Path, *, configured_secret_values: tuple[str, ...] = () +) -> None: + """Validate a manifest-owned wheel without source assessment or lock-plan state.""" + + try: + with zipfile.ZipFile(path) as bundle: + members = _member_map(_safe_wheel_members(bundle)) + if bundle.testzip() is not None: + raise PreparationError(f"Wheel archive entries are corrupt: {path.name}") + _metadata_name, wheel_name, record_name = _dist_info_members(members, path) + if wheel_name not in members or record_name not in members: + raise PreparationError( + f"Wheel is missing required WHEEL or RECORD metadata: {path.name}" + ) + installed_wheel_member_destinations(members, path) + _validate_record(bundle, members, record_name, path) + _validate_wheel_security( + bundle, members, configured_secret_values=configured_secret_values + ) + except zipfile.BadZipFile as exc: + raise PreparationError(f"Malformed wheel archive: {path.name}") from exc + + +def validate_combined_wheel_installation_paths(paths: Iterable[Path]) -> None: + """Require all locally supplied wheel payloads to coexist on Windows.""" + + destinations: list[tuple[str, str]] = [] + for path in paths: + validate_wheel_installation_layout(path) + try: + with zipfile.ZipFile(path) as bundle: + members = _member_map(_safe_wheel_members(bundle)) + destinations.extend( + (destination, f"{path.name}: {member}") + for destination, member in installed_wheel_file_destinations(members, path) + ) + except zipfile.BadZipFile as exc: # pragma: no cover - guarded above + raise PreparationError(f"Malformed wheel archive: {path.name}") from exc + _validate_regular_file_path_collisions(destinations, domain="combined installed") + + def _require_core_metadata(message, *, label: str, wheel: Path) -> tuple[str, str]: metadata_versions = message.get_all("Metadata-Version", []) names = message.get_all("Name", []) @@ -633,6 +706,7 @@ def validate_approved_wheel( f"on Windows {plan.runtime.architecture}." ) + secret_values = _plan_configured_secret_values(plan) try: with zipfile.ZipFile(path) as bundle: members = _member_map(_safe_wheel_members(bundle)) @@ -656,7 +730,9 @@ def validate_approved_wheel( bundle.read(members[wheel_name]), label="WHEEL", wheel=path ) _validate_record(bundle, members, record_name, path) - _validate_wheel_security(bundle, members) + _validate_wheel_security( + bundle, members, configured_secret_values=secret_values + ) except zipfile.BadZipFile as exc: raise PreparationError(f"Malformed wheel archive: {path.name}") from exc @@ -1080,15 +1156,11 @@ def validate_application_wheel( "Application wheel contains unexpected native binaries: " + ", ".join(native_members) ) - configured_secret_values = tuple( - value - for item in plan.configuration - if item.secret and (value := os.environ.get(item.name)) is not None - ) + secret_values = _plan_configured_secret_values(plan) _validate_wheel_security( bundle, members, - configured_secret_values=configured_secret_values, + configured_secret_values=secret_values, ) parser = configparser.ConfigParser(interpolation=None) diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index fd24df4..aa2c359 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -28,6 +28,7 @@ from python_deployment_builder.generation.artifacts import ( validate_application_wheel, validate_artifact_set, + validate_combined_wheel_installation_paths, ) from python_deployment_builder.generation.manifest import ( build_deployment_manifest, @@ -965,6 +966,12 @@ def generate_deployment_kit( else None ) approved = validate_artifact_set(artifact_values, plan) + validate_combined_wheel_installation_paths( + [ + *(path for _artifact, path in approved), + *([application_artifact[1]] if application_artifact is not None else []), + ] + ) requirements = { (canonicalize_name(item.package), item.version) for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) @@ -1057,6 +1064,12 @@ def generate_deployment_kit( if application_wheel is not None else None ) + validate_combined_wheel_installation_paths( + [ + *(path for _artifact, path in approved), + *([application_artifact[1]] if application_artifact is not None else []), + ] + ) requirements = { (canonicalize_name(item.package), item.version) for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 95f1b08..37eefc1 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -16,8 +16,11 @@ from python_deployment_builder.generation.acquisition import PreparationError from python_deployment_builder.generation.artifacts import ( + configured_secret_values, installed_wheel_member_paths, + validate_combined_wheel_installation_paths, validate_wheel_installation_layout, + validate_wheel_static_safety, ) from python_deployment_builder.generation.structural import trusted_artifact_wheel_paths from python_deployment_builder.models import ( @@ -295,6 +298,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR ) ) trusted_wheels = trusted_artifact_wheel_paths(manifest) + secret_values = configured_secret_values(manifest.configuration_secret_names) unvalidated_wheels = sorted( path for path in actual_paths @@ -358,12 +362,14 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR ) ) wheel_layout_failures: list[str] = [] + safe_trusted_wheel_paths: list[Path] = [] for relative in sorted(trusted_wheels): path = _safe_kit_path(root, relative) if path is None or not path.is_file(): continue try: validate_wheel_installation_layout(path) + safe_trusted_wheel_paths.append(path) except PreparationError as exc: wheel_layout_failures.append(f"{relative}: {exc}") checks.append( @@ -375,6 +381,36 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=wheel_layout_failures, ) ) + wheel_security_failures: list[str] = [] + for path in safe_trusted_wheel_paths: + try: + validate_wheel_static_safety(path, configured_secret_values=secret_values) + except PreparationError as exc: + wheel_security_failures.append(f"{path.relative_to(root)}: {exc}") + checks.append( + _check( + "WHEEL_SECURITY", + not wheel_security_failures, + "Manifest-declared wheels pass member security validation.", + "A manifest-declared wheel violates member security validation.", + evidence=wheel_security_failures, + ) + ) + combined_wheel_failures: list[str] = [] + if not wheel_layout_failures: + try: + validate_combined_wheel_installation_paths(safe_trusted_wheel_paths) + except PreparationError as exc: + combined_wheel_failures.append(str(exc)) + checks.append( + _check( + "WHEEL_INSTALLATION_COLLISIONS", + not combined_wheel_failures, + "Manifest-declared wheels have no combined installed-path collisions.", + "Manifest-declared wheels have colliding installed destinations.", + evidence=combined_wheel_failures, + ) + ) missing_source_roots = [ item @@ -504,11 +540,6 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR program_files: list[str] = [] obvious_secrets: list[str] = [] undecodable_text: list[str] = [] - configured_secret_values = [ - value - for name in manifest.configuration_secret_names - if (value := os.environ.get(name)) is not None - ] security_paths = { *(_safe_kit_path(root, relative) for relative in indexed_paths), *root.glob("*.bat"), @@ -535,7 +566,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR if text is None: continue findings = text_security_findings( - text, configured_secret_values=configured_secret_values + text, configured_secret_values=secret_values ) if "forbidden_shell" in findings: lowered = text.lower() diff --git a/tests/test_generation.py b/tests/test_generation.py index 618818d..7ae6417 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -34,6 +34,7 @@ validate_application_wheel, validate_approved_wheel, validate_artifact_set, + validate_combined_wheel_installation_paths, ) from python_deployment_builder.generation.cmd import parse_certutil_sha256 from python_deployment_builder.generation.generator import ( @@ -59,6 +60,7 @@ ApprovedArtifact, ArtifactAvailability, BootstrapArtifact, + ConfigurationPlan, DependencyEdge, DeploymentArtifactRequirement, LockedDependency, @@ -1906,6 +1908,102 @@ def test_wheel_validators_reject_opaque_nested_wheels(tmp_path: Path, validator: validate_approved_wheel(f"proxy-tools={wheel}", plan) +@pytest.mark.parametrize( + ("application_member", "approved_member", "error"), + [ + ("app/main.py", "app/main.py", "colliding regular files"), + ("App/Main.py", "app/main.py", "colliding regular files"), + ("demo", "demo/helper.py", "ancestor collision"), + ("demo/helper.py", "demo", "ancestor collision"), + ], +) +def test_application_and_approved_wheels_must_have_combined_installation_paths( + tmp_path: Path, application_member: str, approved_member: str, error: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + application_plan = create_deployment_plan(assessment, repository_root=source) + application_wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), additions={application_member: "application\n"} + ) + _application, application_path = validate_application_wheel( + application_wheel, assessment, application_plan, repository_root=source + ) + approved_plan = _plan("optional_map_app", ["map"]) + approved_wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={approved_member: "dependency\n"} + ) + _approved, approved_path = validate_approved_wheel( + f"proxy-tools={approved_wheel}", approved_plan + ) + + with pytest.raises(PreparationError, match=error): + validate_combined_wheel_installation_paths([application_path, approved_path]) + + +def test_approved_wheels_must_not_collide_but_may_share_directories(tmp_path: Path) -> None: + plan = _plan("optional_map_app", ["map"]) + assert plan.lock_graph is not None + plan.lock_graph.artifact_requirements.append( + DeploymentArtifactRequirement( + package="helper-dep", version="1.0", action="developer_wheel_required", reason="test" + ) + ) + first = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"namespace/a.py": "first\n", "app/shared.py": "first\n"} + ) + second = _rewrite_application_wheel( + _make_wheel(tmp_path, name="helper-dep", version="1.0"), + additions={"namespace/b.py": "second\n", "app/shared.py": "second\n"}, + ) + _first, first_path = validate_approved_wheel(f"proxy-tools={first}", plan) + _second, second_path = validate_approved_wheel(f"helper-dep={second}", plan) + + with pytest.raises(PreparationError, match="colliding regular files"): + validate_combined_wheel_installation_paths([first_path, second_path]) + + disjoint = _rewrite_application_wheel( + second, + removals={"app/shared.py"}, + additions={"namespace/b.py": "second\n"}, + ) + _second, disjoint_path = validate_approved_wheel(f"helper-dep={disjoint}", plan) + validate_combined_wheel_installation_paths([first_path, disjoint_path]) + + ancestor_first = _rewrite_application_wheel( + first, + removals={"app/shared.py"}, + additions={"Demo": "a regular file\n"}, + ) + ancestor_second = _rewrite_application_wheel( + disjoint, + additions={"demo/helper.py": "dependency\n"}, + ) + _first, ancestor_first_path = validate_approved_wheel(f"proxy-tools={ancestor_first}", plan) + _second, ancestor_second_path = validate_approved_wheel( + f"helper-dep={ancestor_second}", plan + ) + with pytest.raises(PreparationError, match="ancestor collision"): + validate_combined_wheel_installation_paths([ancestor_first_path, ancestor_second_path]) + + +def test_purelib_relocation_participates_in_cross_wheel_collision_checks(tmp_path: Path) -> None: + application = _rewrite_application_wheel( + _make_application_wheel(tmp_path), additions={"shared.py": "application\n"} + ) + dependency = _rewrite_application_wheel( + _make_wheel(tmp_path), + additions={"proxy_tools-0.1.0.data/purelib/shared.py": "dependency\n"}, + ) + + with pytest.raises(PreparationError, match="colliding regular files"): + validate_combined_wheel_installation_paths([application, dependency]) + + def test_approved_dependency_wheel_rejects_post_relocation_file_ancestor_collision( tmp_path: Path, ) -> None: @@ -3186,6 +3284,85 @@ def test_application_wheel_rejects_configured_secret_in_metadata( assert secret not in str(caught.value) +def _secret_configuration(name: str) -> ConfigurationPlan: + return ConfigurationPlan( + name=name, + secret=True, + supply_strategy="environment", + rationale="test configured-secret policy", + ) + + +def test_approved_wheel_rejects_current_configured_secret_value( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + secret = "correct-horse-demo-token-937451" + monkeypatch.setenv("DEMO_API_TOKEN", secret) + plan = _plan("optional_map_app", ["map"]) + plan.configuration = [_secret_configuration("DEMO_API_TOKEN")] + wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"helper/config/settings.txt": f"token={secret}\n"} + ) + + with pytest.raises(PreparationError, match="security policy") as caught: + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + assert secret not in str(caught.value) + + +def test_wheel_configured_secret_policy_is_shared_and_deduplicated( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + secret = "correct-horse-demo-token-937451" + monkeypatch.setenv("DEMO_API_TOKEN", secret) + monkeypatch.setenv("SECOND_DEMO_TOKEN", secret) + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + application_plan = create_deployment_plan(assessment, repository_root=source) + application_plan.configuration = [ + _secret_configuration("DEMO_API_TOKEN"), + _secret_configuration("SECOND_DEMO_TOKEN"), + _secret_configuration("UNSET_DEMO_TOKEN"), + ] + application_wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/settings.txt": f"token={secret}\n"}, + ) + approved_plan = _plan("optional_map_app", ["map"]) + approved_plan.configuration = application_plan.configuration + approved_wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"helper/settings.txt": f"token={secret}\n"} + ) + + with pytest.raises(PreparationError, match="security policy"): + validate_application_wheel( + application_wheel, assessment, application_plan, repository_root=source + ) + with pytest.raises(PreparationError, match="security policy"): + validate_approved_wheel(f"proxy-tools={approved_wheel}", approved_plan) + + +def test_approved_wheel_security_allows_unconfigured_text_and_rejects_obvious_secret( + tmp_path: Path, +) -> None: + plan = _plan("optional_map_app", ["map"]) + benign = _rewrite_application_wheel( + _make_wheel(tmp_path), + additions={"helper/settings.txt": "token=correct-horse-demo-token-937451\n"}, + ) + assert validate_approved_wheel(f"proxy-tools={benign}", plan)[0].filename == benign.name + + obvious = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"helper/settings.txt": "token=sk-abcdefghijklmnop\n"} + ) + with pytest.raises(PreparationError, match="security policy"): + validate_approved_wheel(f"proxy-tools={obvious}", plan) + + @pytest.mark.parametrize( "filename", [ @@ -3452,6 +3629,18 @@ def _add_indexed_file(kit: Path, relative: str, content: bytes) -> None: index_path.write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") +def _update_indexed_hashes(kit: Path, *relatives: str) -> None: + index_path = kit / "deployment/generated-files.json" + index = json.loads(index_path.read_text(encoding="utf-8")) + requested = set(relatives) + for item in index["files"]: + if item["path"] in requested: + item["sha256"] = hashlib.sha256((kit / item["path"]).read_bytes()).hexdigest() + requested.remove(item["path"]) + assert not requested + index_path.write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") + + def test_static_validation_rejects_indexed_unvalidated_staged_wheel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -3558,6 +3747,108 @@ def test_static_validation_rejects_unsupported_approved_artifact_data_scheme( ) +def test_static_validation_rescans_trusted_wheel_configured_secrets( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + secret = "correct-horse-demo-token-937451" + monkeypatch.setenv("DEMO_API_TOKEN", secret) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + staged_relative = f"deployment/wheels/{artifact.name}" + staged = kit / staged_relative + _rewrite_application_wheel(staged, additions={"helper/settings.txt": f"token={secret}\n"}) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["configuration_secret_names"] = ["DEMO_API_TOKEN"] + manifest["approved_artifacts"][0]["sha256"] = hashlib.sha256(staged.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, staged_relative, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert any( + item.code == "WHEEL_SECURITY" and item.status.value == "FAIL" + for item in report.static_checks + ) + assert secret not in json.dumps(report.model_dump(mode="json")) + + +def test_static_validation_rejects_combined_trusted_wheel_collisions( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + proxy_relative = f"deployment/wheels/{artifact.name}" + proxy = kit / proxy_relative + _rewrite_application_wheel(proxy, additions={"app/shared.py": "proxy\n"}) + helper = _rewrite_application_wheel( + _make_wheel(tmp_path, name="helper-dep", version="1.0"), + additions={"app/shared.py": "helper\n"}, + ) + helper_relative = f"deployment/wheels/{helper.name}" + _add_indexed_file(kit, helper_relative, helper.read_bytes()) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["approved_artifacts"][0]["sha256"] = hashlib.sha256(proxy.read_bytes()).hexdigest() + manifest["approved_artifacts"].append( + { + "distribution_name": "helper-dep", + "version": "1.0", + "filename": helper.name, + "sha256": hashlib.sha256(helper.read_bytes()).hexdigest(), + "wheel_tags": ["py3-none-any"], + "requirement_action": "developer_wheel_required", + } + ) + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, proxy_relative, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert any( + item.code == "WHEEL_INSTALLATION_COLLISIONS" and item.status.value == "FAIL" + for item in report.static_checks + ) + + def test_package_regeneration_removes_unchanged_obsolete_application_wheel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From ec499f739f0aa611d1d5a7f5726be17324aca35d Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:25:23 -0400 Subject: [PATCH 38/59] Harden artifact and resource validation --- .../analysis/resources.py | 187 ++++++++++++++- .../generation/artifacts.py | 74 ++++-- .../generation/structural.py | 77 +++++-- .../validation/static.py | 36 ++- tests/test_analysis_scope.py | 99 ++++++++ tests/test_generation.py | 214 +++++++++++++++++- 6 files changed, 646 insertions(+), 41 deletions(-) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 6992504..bc74115 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -504,6 +504,164 @@ def bind_pattern(target: ast.AST, values: list[ast.AST]) -> None: return assignments, returns +def _resource_import_bindings(tree: ast.AST) -> tuple[set[str], set[str]]: + """Return proven ``importlib.resources`` module and ``files`` bindings.""" + + modules: set[str] = set() + files: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "importlib.resources": + modules.add(alias.asname or alias.name) + elif isinstance(node, ast.ImportFrom): + if node.module == "importlib": + for alias in node.names: + if alias.name == "resources": + modules.add(alias.asname or alias.name) + elif node.module == "importlib.resources": + for alias in node.names: + if alias.name == "files": + files.add(alias.asname or alias.name) + return modules, files + + +def _resource_package_roots( + root: Path, + package: str, + source_roots: list[str], + project: PackagingAssessment | None, +) -> list[Path]: + """Resolve a literal package anchor only through safe in-repository roots.""" + + if not package or not all(part.isidentifier() for part in package.split(".")): + return [] + candidates = _physical_package_roots(root, project, package) if project else [] + relative = Path(*package.split(".")) + candidates.extend(root / source_root / relative for source_root in source_roots) + candidates.append(root / relative) + resolved_root = root.resolve() + safe: list[Path] = [] + for candidate in candidates: + if candidate.is_symlink() or not candidate.is_dir(): + continue + try: + candidate.resolve().relative_to(resolved_root) + except ValueError: + continue + if candidate not in safe: + safe.append(candidate) + return safe + + +def _is_resource_files_call( + node: ast.AST, module_bindings: set[str], files_bindings: set[str] +) -> bool: + if not isinstance(node, ast.Call): + return False + name = _qualified_name(node.func) + return name in files_bindings or any(name == f"{binding}.files" for binding in module_bindings) + + +def _safe_resource_member(values: list[str]) -> bool: + return all( + value + and not PurePosixPath(value.replace("\\", "/")).is_absolute() + and not PureWindowsPath(value).is_absolute() + and ".." not in PurePosixPath(value.replace("\\", "/")).parts + for value in values + ) + + +def _importlib_resource_path_values( + node: ast.AST, + *, + root: Path, + source_roots: list[str], + project: PackagingAssessment | None, + assignments: dict[str, ast.AST], + returns: dict[str, ast.AST], + module_bindings: set[str], + files_bindings: set[str], +) -> list[str] | None: + """Resolve a bounded ``importlib.resources.files`` path expression statically.""" + + if _is_resource_files_call(node, module_bindings, files_bindings): + if not isinstance(node, ast.Call) or len(node.args) != 1: + return [] + package_values = _path_values( + node.args[0], + root=root, + source_path=root, + assignments=assignments, + returns=returns, + ) + if len(package_values) != 1: + return [] + return [ + package_root.relative_to(root.resolve()).as_posix() + for package_root in _resource_package_roots( + root, package_values[0], source_roots, project + ) + ] + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "joinpath" + ): + base = _importlib_resource_path_values( + node.func.value, + root=root, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + module_bindings=module_bindings, + files_bindings=files_bindings, + ) + if base is None: + return None + parts = [ + _path_values( + argument, + root=root, + source_path=root, + assignments=assignments, + returns=returns, + ) + for argument in node.args + ] + if not base or any(not part or not _safe_resource_member(part) for part in parts): + return [] + for part in parts: + base = _combine_paths(base, part) + return base + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + base = _importlib_resource_path_values( + node.left, + root=root, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + module_bindings=module_bindings, + files_bindings=files_bindings, + ) + if base is None: + return None + parts = _path_values( + node.right, + root=root, + source_path=root, + assignments=assignments, + returns=returns, + ) + if not base or not parts or not _safe_resource_member(parts): + return [] + return _combine_paths(base, parts) + return None + + def _combine_paths(left: list[str], right: list[str]) -> list[str]: return [(Path(first) / second).as_posix() for first in left for second in right] @@ -780,6 +938,7 @@ def _literal_evidence( root: Path, source_roots: list[str], application_files: list[Path] | None, + project: PackagingAssessment | None, ) -> tuple[dict[str, list[Evidence]], dict[str, set[str]], set[str]]: found: dict[str, list[Evidence]] = defaultdict(list) access: dict[str, set[str]] = defaultdict(set) @@ -804,16 +963,31 @@ def _literal_evidence( continue lines = source.splitlines() assignments, returns = _bindings(tree) + module_bindings, files_bindings = _resource_import_bindings(tree) for node in ast.walk(tree): if not isinstance(node, ast.Call): continue for expression, mode in _path_uses(node): - values = _path_values( + resource_values = _importlib_resource_path_values( expression, root=root, - source_path=path, + source_roots=source_roots, + project=project, assignments=assignments, returns=returns, + module_bindings=module_bindings, + files_bindings=files_bindings, + ) + values = ( + resource_values + if resource_values is not None + else _path_values( + expression, + root=root, + source_path=path, + assignments=assignments, + returns=returns, + ) ) for value in values: if not _looks_like_resource_literal(value): @@ -828,7 +1002,10 @@ def _literal_evidence( file=relative, line=node.lineno, detail=( - f"Static {mode} path use through " + f"Static {mode} resource use through " + "importlib.resources.files() resolves here." + if resource_values is not None + else f"Static {mode} path use through " f"{_qualified_name(node.func)} resolves here." ), excerpt=lines[node.lineno - 1].strip(), @@ -875,7 +1052,9 @@ def inspect_resources( application_files: list[Path] | None = None, project: PackagingAssessment | None = None, ) -> tuple[list[ResourceRequirement], list[ConfigurationRequirement]]: - literals, access_modes, unresolved = _literal_evidence(root, source_roots, application_files) + literals, access_modes, unresolved = _literal_evidence( + root, source_roots, application_files, project + ) declared_package_data = _declared_package_data(root, project) resources: list[ResourceRequirement] = [] for relative, references in sorted(literals.items()): diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 50e3b92..27f4975 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -879,24 +879,69 @@ def _direct_dependency_presence_proven( ) +def _direct_dependency_extras_proven( + requirement: Requirement, + graph, + plan: DeploymentPlan, + application_name: str, +) -> None: + """Require wheel dependency extras on definitely-applicable root lock edges. + + ``LockedDependency`` merges evidence from target-possible paths for planning, + so its aggregate requested extras cannot prove one application requirement is + always activated. Only root edges which definitely apply may provide that + proof. + """ + + requested = {canonicalize_name(extra) for extra in requirement.extras} + if not requested: + return + dependency_name = canonicalize_name(requirement.name) + selected_extras = {canonicalize_name(extra) for extra in graph.selected_extras} + guaranteed: set[str] = set() + unprovable: list[str] = [] + malformed: list[str] = [] + for edge in graph.edges: + if ( + canonicalize_name(edge.from_package) != application_name + or canonicalize_name(edge.to_package) != dependency_name + ): + continue + if edge.selected_extra and canonicalize_name(edge.selected_extra) not in selected_extras: + continue + try: + applicability = target_marker_applicability( + edge.marker, + plan.runtime.python_version, + plan.runtime.architecture, + extra=edge.selected_extra or "", + ) + except TargetMarkerEnvironmentError as exc: + malformed.append(str(exc)) + continue + if applicability == TargetMarkerApplicability.APPLIES: + guaranteed.update( + canonicalize_name(extra) for extra in edge.requested_dependency_extras + ) + elif applicability == TargetMarkerApplicability.UNPROVABLE: + unprovable.append(edge.marker or "") + missing = requested - guaranteed + if not missing: + return + detail = "; ".join([*sorted(set(unprovable)), *sorted(set(malformed))]) + suffix = f" Unprovable direct-edge markers: {detail}." if detail else "" + raise PreparationError( + "Application wheel Requires-Dist dependency extra activation cannot be proven from " + "definitely applicable direct locked dependency edges: " + f"{requirement.name}[{','.join(sorted(missing))}].{suffix}" + ) + + def _validate_dependency_extra_closure(requirement: Requirement, graph, candidates) -> None: - """Prove a wheel dependency's requested extras are activated by the selected lock graph.""" + """Prove every target-possible version declares the requested extras and closure.""" requested = {canonicalize_name(extra) for extra in requirement.extras} name = canonicalize_name(requirement.name) - without_activation = [ - dependency.version - for dependency in candidates - if not requested - <= {canonicalize_name(extra) for extra in dependency.requested_dependency_extras} - ] - if without_activation: - raise PreparationError( - "Application wheel Requires-Dist dependency extra is not activated for every " - "target-possible locked version: " - f"{requirement.name}[{','.join(sorted(requested))}]. Missing activation: " - + ", ".join(sorted(set(without_activation))) - ) without_declaration = [ dependency.version for dependency in candidates @@ -1008,6 +1053,7 @@ def _validate_application_requires_dist( f"incompatible possible versions: {', '.join(incompatible)}." ) if requirement.extras: + _direct_dependency_extras_proven(requirement, graph, plan, application_name) _validate_dependency_extra_closure(requirement, graph, candidates) diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index bc74447..26845cc 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -5,7 +5,7 @@ import hashlib import json import re -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePosixPath, PureWindowsPath from python_deployment_builder.generation.acquisition import PreparationError from python_deployment_builder.models import ( @@ -33,6 +33,26 @@ def _check(condition: bool, code: str, description: str) -> RiskFinding: ) +def manifest_artifact_wheel_path(directory: str, filename: str) -> str | None: + """Return one canonical kit-relative artifact path, or reject an unsafe filename.""" + + posix = PurePosixPath(filename) + windows = PureWindowsPath(filename) + if ( + not filename + or filename in {".", ".."} + or "/" in filename + or "\\" in filename + or posix.is_absolute() + or windows.is_absolute() + or windows.drive + or posix.name != filename + or not filename.lower().endswith(".whl") + ): + return None + return f"deployment/{directory}/{filename}" + + def trusted_artifact_wheel_paths(manifest: DeploymentManifest) -> set[str]: """Return exact manifest-owned wheel paths with dedicated validation. @@ -42,11 +62,16 @@ def trusted_artifact_wheel_paths(manifest: DeploymentManifest) -> set[str]: """ paths = { - f"deployment/wheels/{artifact.filename}" + path for artifact in manifest.approved_artifacts + if (path := manifest_artifact_wheel_path("wheels", artifact.filename)) is not None } - if manifest.application_artifact is not None: - paths.add(f"deployment/application/{manifest.application_artifact.filename}") + if manifest.application_artifact is not None and ( + path := manifest_artifact_wheel_path( + "application", manifest.application_artifact.filename + ) + ): + paths.add(path) return paths @@ -58,6 +83,36 @@ def validate_rendered_files( secret_values: list[str] | None = None, ) -> list[RiskFinding]: checks: list[RiskFinding] = [] + application_path = ( + manifest_artifact_wheel_path("application", manifest.application_artifact.filename) + if manifest.application_artifact is not None + else None + ) + approved_paths = [ + manifest_artifact_wheel_path("wheels", artifact.filename) + for artifact in manifest.approved_artifacts + ] + unsafe_artifacts = [ + *( + [f"application: {manifest.application_artifact.filename}"] + if manifest.application_artifact is not None and application_path is None + else [] + ), + *( + f"approved: {artifact.filename}" + for artifact, path in zip(manifest.approved_artifacts, approved_paths, strict=True) + if path is None + ), + ] + checks.append( + _check( + not unsafe_artifacts, + "MANIFEST_ARTIFACT_FILENAMES", + "Manifest artifact filenames are safe wheel basenames." + if not unsafe_artifacts + else f"Unsafe manifest artifact filenames: {unsafe_artifacts}", + ) + ) missing = sorted(set(manifest.referenced_files) - set(files)) checks.append( _check(not missing, "MANIFEST_REFERENCES", f"Missing referenced files: {missing or 'none'}") @@ -75,11 +130,8 @@ def validate_rendered_files( ) ) application_hash_ok = manifest.application_artifact is None or ( - ( - data := files.get( - f"deployment/application/{manifest.application_artifact.filename}" - ) - ) + application_path is not None + and (data := files.get(application_path)) is not None and hashlib.sha256(data).hexdigest() == manifest.application_artifact.sha256 ) @@ -91,11 +143,10 @@ def validate_rendered_files( ) ) artifact_hashes_ok = all( - ( - data := files.get(f"deployment/wheels/{artifact.filename}") - ) is not None + path is not None + and (data := files.get(path)) is not None and hashlib.sha256(data).hexdigest() == artifact.sha256 - for artifact in manifest.approved_artifacts + for artifact, path in zip(manifest.approved_artifacts, approved_paths, strict=True) ) checks.append( _check( diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 37eefc1..32c4f10 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -22,7 +22,10 @@ validate_wheel_installation_layout, validate_wheel_static_safety, ) -from python_deployment_builder.generation.structural import trusted_artifact_wheel_paths +from python_deployment_builder.generation.structural import ( + manifest_artifact_wheel_path, + trusted_artifact_wheel_paths, +) from python_deployment_builder.models import ( DeploymentManifest, ManualValidationItem, @@ -85,6 +88,13 @@ def _safe_kit_path(root: Path, relative: str) -> Path | None: return candidate +def _safe_manifest_artifact_path(root: Path, directory: str, filename: str) -> Path | None: + """Return a contained manifest-owned wheel path without touching unsafe names.""" + + relative = manifest_artifact_wheel_path(directory, filename) + return _safe_kit_path(root, relative) if relative is not None else None + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -155,14 +165,20 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR if manifest.application_artifact is None: application_artifact_failures.append("manifest application artifact is missing") else: - application_wheel = ( - root / "deployment" / "application" / manifest.application_artifact.filename + application_wheel = _safe_manifest_artifact_path( + root, "application", manifest.application_artifact.filename ) if ( - not application_wheel.is_file() + application_wheel is None + or not application_wheel.is_file() or _sha256(application_wheel) != manifest.application_artifact.sha256 ): - application_artifact_failures.append(manifest.application_artifact.filename) + application_artifact_failures.append( + f"unsafe application artifact filename: " + f"{manifest.application_artifact.filename}" + if application_wheel is None + else manifest.application_artifact.filename + ) elif manifest.application_artifact is not None: application_artifact_failures.append( "source mode unexpectedly declares an application wheel" @@ -349,9 +365,13 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR artifact_failures: list[str] = [] for artifact in manifest.approved_artifacts: - path = root / "deployment" / "wheels" / artifact.filename - if not path.is_file() or _sha256(path) != artifact.sha256: - artifact_failures.append(artifact.filename) + path = _safe_manifest_artifact_path(root, "wheels", artifact.filename) + if path is None or not path.is_file() or _sha256(path) != artifact.sha256: + artifact_failures.append( + f"unsafe approved artifact filename: {artifact.filename}" + if path is None + else artifact.filename + ) checks.append( _check( "APPROVED_ARTIFACT_HASHES", diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index afbf707..405e905 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -929,6 +929,105 @@ def test_authoritative_setuptools_package_data_is_promoted_and_staged( assert assess_repository(_repository(tmp_path)).repository.fingerprint != original +@pytest.mark.parametrize( + ("imports", "files_call"), + [ + ("import importlib.resources", "importlib.resources.files('app')"), + ("import importlib.resources as ir", "ir.files('app')"), + ("from importlib import resources", "resources.files('app')"), + ("from importlib import resources as ir", "ir.files('app')"), + ("from importlib.resources import files", "files('app')"), + ("from importlib.resources import files as resource_files", "resource_files('app')"), + ], +) +def test_importlib_resources_files_promotes_concrete_source_resource( + tmp_path: Path, imports: str, files_call: str +) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{"default": true}\n', encoding="utf-8") + (package / "main.py").write_text( + f"{imports}\n\ndef main():\n" + f" return {files_call}.joinpath('defaults.json').read_text(encoding='utf-8')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'resource-app'\nversion = '1.0'\n" + "[project.scripts]\nresource-app = 'app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + resource = next(item for item in assessment.resources if item.path == "src/app/defaults.json") + inventory = next(item for item in assessment.file_inventory if item.path == resource.path) + staged = _staging_files(tmp_path, assessment, plan, include=True) + + assert resource.packaging_status == "repository_adjacent" + assert inventory.role == RepositoryFileRole.RUNTIME_RESOURCE + assert "importlib.resources.files" in resource.evidence[-1].detail + assert resource.path in staged + + +def test_importlib_resources_rejects_unproven_or_escaping_resource_paths(tmp_path: Path) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{}\n', encoding="utf-8") + (package / "main.py").write_text( + "from pathlib import Path\n" + "def files(name):\n return Path(name)\n" + "def main(name='defaults.json'):\n" + " files('app').joinpath('defaults.json').read_text()\n" + " from importlib.resources import files as resource_files\n" + " return resource_files('app').joinpath('../secret.txt').read_text()\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'unproven-resource-app'\nversion = '1.0'\n" + "[project.scripts]\nunproven-resource-app = 'app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + staged = _staging_files(tmp_path, assessment, plan, include=True) + + assert all(item.path != "src/app/defaults.json" for item in assessment.resources) + escaped = next(item for item in assessment.resources if "secret.txt" in item.path) + assert escaped.status == FindingStatus.NEEDS_VALIDATION + assert all("secret.txt" not in path for path in staged) + + +def test_importlib_resources_uses_package_dir_parent_mapping(tmp_path: Path) -> None: + package = tmp_path / "lib/sub" + package.mkdir(parents=True) + (tmp_path / "lib/__init__.py").write_text("", encoding="utf-8") + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{}\n', encoding="utf-8") + (tmp_path / "lib/main.py").write_text( + "from importlib.resources import files\n" + "def main():\n return files('app.sub').joinpath('defaults.json').read_bytes()\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'mapped-resource-app'\nversion = '1.0'\n" + "[project.scripts]\nmapped-resource-app = 'app.main:main'\n" + "[tool.setuptools]\npackages = ['app', 'app.sub']\npackage-dir = {app = 'lib'}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + resource = next(item for item in assessment.resources if item.path == "lib/sub/defaults.json") + assert resource.packaging_status == "repository_adjacent" + + def test_wildcard_setuptools_package_data_uses_known_physical_package_mapping( tmp_path: Path, ) -> None: diff --git a/tests/test_generation.py b/tests/test_generation.py index 7ae6417..505fd4d 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -72,6 +72,7 @@ from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.planning.planner import create_deployment_plan from python_deployment_builder.security_policy import is_textual_content +from python_deployment_builder.validation import static as static_validation from python_deployment_builder.validation.static import validate_static_kit FIXTURES = Path(__file__).parent / "fixtures" @@ -349,7 +350,14 @@ def _application_plan_with_target_possible_dependencies( ): configured = plan.model_copy(deep=True) direct_edges = ( - [DependencyEdge(from_package="mapped-app", to_package="foo") for _ in versions] + [ + DependencyEdge( + from_package="mapped-app", + to_package="foo", + requested_dependency_extras=requested_extras or [], + ) + for _ in versions + ] if include_default_direct_edges else [] ) @@ -459,6 +467,56 @@ def _write_dependency_extra_lock( ) +def _write_path_merged_dependency_extra_lock( + root: Path, + *, + direct_extras: list[str] | None = None, + direct_marker: str | None = None, + selected_map_extra: bool = False, +) -> None: + """Write plain/direct and transitive foo[bar] paths sharing one locked version.""" + + direct = ", ".join(f'"{extra}"' for extra in direct_extras or []) + marker = ( + f', marker = "{direct_marker.replace(chr(34), chr(92) + chr(34))}"' + if direct_marker + else "" + ) + optional = ( + '\n[package.optional-dependencies]\nmap = [{ name = "foo", extra = ["bar"] }]\n' + if selected_map_extra + else "" + ) + if selected_map_extra: + root_dependencies = '{ name = "helper" }' + elif direct_marker and direct_extras: + root_dependencies = ( + '{ name = "foo" }, ' + f'{{ name = "foo", extra = [{direct}]{marker} }}, {{ name = "helper" }}' + ) + else: + root_dependencies = f'{{ name = "foo", extra = [{direct}]{marker} }}, {{ name = "helper" }}' + (root / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = \">=3.12\"\n\n" + "[[package]]\nname = \"mapped-app\"\nversion = \"1.2.3\"\n" + "source = { virtual = \".\" }\n" + f"dependencies = [{root_dependencies}]\n" + + optional + + "\n[[package]]\nname = \"foo\"\nversion = \"1.0\"\n" + "source = { registry = \"https://pypi.org/simple\" }\n" + "wheels = [{ url = \"https://example.invalid/foo-1.0-py3-none-any.whl\" }]\n" + "\n[package.optional-dependencies]\nbar = [{ name = \"bar-helper\" }]\n\n" + "[[package]]\nname = \"helper\"\nversion = \"1.0\"\n" + "source = { registry = \"https://pypi.org/simple\" }\n" + "dependencies = [{ name = \"foo\", extra = [\"bar\"] }]\n" + "wheels = [{ url = \"https://example.invalid/helper-1.0-py3-none-any.whl\" }]\n\n" + "[[package]]\nname = \"bar-helper\"\nversion = \"1.0\"\n" + "source = { registry = \"https://pypi.org/simple\" }\n" + "wheels = [{ url = \"https://example.invalid/bar_helper-1.0-py3-none-any.whl\" }]\n", + encoding="utf-8", + ) + + def _write_developer_artifact_fork_lock(root: Path, *, markers: bool = True) -> None: """Write target-possible source-only foo branches for artifact-fork tests.""" first_marker, second_marker = ( @@ -1545,7 +1603,7 @@ def test_application_wheel_dependency_extra_requires_every_candidate_activation_ requested_extras=[], available_extras={"1.0": ["bar"], "2.0": ["bar"]}, ) - with pytest.raises(PreparationError, match="not activated.*1.0.*2.0"): + with pytest.raises(PreparationError, match="extra activation cannot be proven"): validate_application_wheel(wheel, assessment, missing_activation) missing_declaration = _application_plan_with_target_possible_dependencies( @@ -2212,6 +2270,53 @@ def test_application_wheel_dependency_requested_extra_variants_and_markers( ) +def test_application_dependency_extra_activation_uses_definite_direct_root_edges( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + wheel = _make_application_wheel(tmp_path, requires_dist_values=["foo[bar]>=1"]) + + _write_path_merged_dependency_extra_lock(source) + assessment, transitive_only = _plan_with_dependency_extra_lock(source) + merged = next(item for item in transitive_only.lock_graph.dependencies if item.name == "foo") + assert merged.requested_dependency_extras == ["bar"] + with pytest.raises(PreparationError, match="extra activation cannot be proven"): + validate_application_wheel(wheel, assessment, transitive_only) + + _write_path_merged_dependency_extra_lock(source, direct_extras=["bar"]) + assessment, direct = _plan_with_dependency_extra_lock(source) + validate_application_wheel(wheel, assessment, direct) + + _write_path_merged_dependency_extra_lock( + source, + direct_extras=["bar"], + direct_marker='python_full_version < "3.12.5"', + ) + assessment, unprovable = _plan_with_dependency_extra_lock(source) + with pytest.raises(PreparationError, match="extra activation cannot be proven"): + validate_application_wheel(wheel, assessment, unprovable) + + +def test_application_dependency_extra_activation_honors_selected_application_extra( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + _write_path_merged_dependency_extra_lock(source, selected_map_extra=True) + wheel = _make_application_wheel( + tmp_path, requires_dist_values=['foo[bar]>=1; extra == "map"'] + ) + + assessment, selected = _plan_with_dependency_extra_lock(source, selected_extras=["map"]) + validate_application_wheel(wheel, assessment, selected) + + assessment, unselected = _plan_with_dependency_extra_lock(source) + validate_application_wheel(wheel, assessment, unselected) + + def test_package_prepare_lock_reassesses_before_requires_dist_validation( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -3669,6 +3774,111 @@ def test_static_validation_rejects_indexed_unvalidated_staged_wheel( ) +@pytest.mark.parametrize( + "filename", + [ + "../sample_app-1.0-py3-none-any.whl", + "../../../../outside/sample_app-1.0-py3-none-any.whl", + "/outside/sample_app-1.0-py3-none-any.whl", + "C:\\outside\\sample_app-1.0-py3-none-any.whl", + "\\\\server\\share\\sample_app-1.0-py3-none-any.whl", + "nested/sample_app-1.0-py3-none-any.whl", + ], +) +def test_static_validation_rejects_unsafe_application_artifact_filenames_without_opening_them( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, filename: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + outside = tmp_path / "outside" + outside.mkdir() + external = _make_application_wheel(outside) + kit = tmp_path / "kit" + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + application_wheel=_make_application_wheel(tmp_path), + bootstrap_mode="online_cmd", + ) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["application_artifact"]["filename"] = filename + manifest["application_artifact"]["sha256"] = hashlib.sha256(external.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + original_sha256 = static_validation._sha256 + + def checked_sha256(path: Path) -> str: + assert path != external + return original_sha256(path) + + monkeypatch.setattr(static_validation, "_sha256", checked_sha256) + + report = validate_static_kit(kit) + + artifact_hash = next( + item for item in report.static_checks if item.code == "APPLICATION_ARTIFACT_HASH" + ) + assert artifact_hash.status.value == "FAIL" + assert any("unsafe application artifact filename" in item for item in artifact_hash.evidence) + assert next( + item for item in report.static_checks if item.code == "ENTRY_POINT_STRUCTURE" + ).status.value == "FAIL" + + +def test_static_validation_rejects_unsafe_approved_artifact_filename( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["approved_artifacts"][0]["filename"] = "../proxy_tools-0.1.0-py3-none-any.whl" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + approved_hashes = next( + item for item in report.static_checks if item.code == "APPROVED_ARTIFACT_HASHES" + ) + assert approved_hashes.status.value == "FAIL" + assert any("unsafe approved artifact filename" in item for item in approved_hashes.evidence) + + def test_static_validation_rejects_unsupported_application_wheel_data_scheme( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 2d5c040338db571f0b96f6b5d79a237e06ee0ba7 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:37:40 -0400 Subject: [PATCH 39/59] Resolve implicit importlib resources anchors --- .../analysis/resources.py | 88 ++++++++++++-- tests/test_analysis_scope.py | 113 ++++++++++++++++++ 2 files changed, 188 insertions(+), 13 deletions(-) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index bc74115..74da466 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -573,10 +573,56 @@ def _safe_resource_member(values: list[str]) -> bool: ) +def _implicit_resource_root(root: Path, source_path: Path) -> list[str]: + """Return the safe caller-adjacent container for ``files()``. + + Python 3.12 resolves an omitted ``importlib.resources.files`` anchor from + the caller module. Resource analysis already visits only application + source files, but still validates that the particular caller is a regular + in-repository Python file before using its physical parent as an anchor. + """ + + source = _safe_python_source(source_path, root) + if source is None: + return [] + try: + return [source.parent.relative_to(root.resolve()).as_posix()] + except ValueError: + return [] + + +def _resource_package_anchor_values( + node: ast.AST, + *, + root: Path, + source_path: Path, + source_roots: list[str], + project: PackagingAssessment | None, + assignments: dict[str, ast.AST], + returns: dict[str, ast.AST], +) -> list[str]: + package_values = _path_values( + node, + root=root, + source_path=source_path, + assignments=assignments, + returns=returns, + ) + if len(package_values) != 1: + return [] + return [ + package_root.relative_to(root.resolve()).as_posix() + for package_root in _resource_package_roots( + root, package_values[0], source_roots, project + ) + ] + + def _importlib_resource_path_values( node: ast.AST, *, root: Path, + source_path: Path, source_roots: list[str], project: PackagingAssessment | None, assignments: dict[str, ast.AST], @@ -587,23 +633,36 @@ def _importlib_resource_path_values( """Resolve a bounded ``importlib.resources.files`` path expression statically.""" if _is_resource_files_call(node, module_bindings, files_bindings): - if not isinstance(node, ast.Call) or len(node.args) != 1: + if not isinstance(node, ast.Call): + return [] + if not node.args: + # ``anchor=`` is the Python 3.12 spelling. Deliberately leave + # deprecated ``package=`` unresolved rather than treating either + # keyword form as the zero-argument implicit caller anchor. + if not node.keywords: + return _implicit_resource_root(root, source_path) + if len(node.keywords) == 1 and node.keywords[0].arg == "anchor": + return _resource_package_anchor_values( + node.keywords[0].value, + root=root, + source_path=source_path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + ) + return [] + if len(node.args) != 1 or node.keywords: return [] - package_values = _path_values( + return _resource_package_anchor_values( node.args[0], root=root, - source_path=root, + source_path=source_path, + source_roots=source_roots, + project=project, assignments=assignments, returns=returns, ) - if len(package_values) != 1: - return [] - return [ - package_root.relative_to(root.resolve()).as_posix() - for package_root in _resource_package_roots( - root, package_values[0], source_roots, project - ) - ] if ( isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) @@ -612,6 +671,7 @@ def _importlib_resource_path_values( base = _importlib_resource_path_values( node.func.value, root=root, + source_path=source_path, source_roots=source_roots, project=project, assignments=assignments, @@ -625,7 +685,7 @@ def _importlib_resource_path_values( _path_values( argument, root=root, - source_path=root, + source_path=source_path, assignments=assignments, returns=returns, ) @@ -640,6 +700,7 @@ def _importlib_resource_path_values( base = _importlib_resource_path_values( node.left, root=root, + source_path=source_path, source_roots=source_roots, project=project, assignments=assignments, @@ -652,7 +713,7 @@ def _importlib_resource_path_values( parts = _path_values( node.right, root=root, - source_path=root, + source_path=source_path, assignments=assignments, returns=returns, ) @@ -971,6 +1032,7 @@ def _literal_evidence( resource_values = _importlib_resource_path_values( expression, root=root, + source_path=path, source_roots=source_roots, project=project, assignments=assignments, diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 405e905..954bacc 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -971,6 +971,119 @@ def test_importlib_resources_files_promotes_concrete_source_resource( assert resource.path in staged +@pytest.mark.parametrize( + ("imports", "files_call"), + [ + ("import importlib.resources", "importlib.resources.files()"), + ("import importlib.resources as ir", "ir.files()"), + ("from importlib import resources", "resources.files()"), + ("from importlib import resources as ir", "ir.files()"), + ("from importlib.resources import files", "files()"), + ("from importlib.resources import files as resource_files", "resource_files()"), + ], +) +def test_importlib_resources_implicit_anchor_promotes_caller_resource( + tmp_path: Path, imports: str, files_call: str +) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{"default": true}\n', encoding="utf-8") + (package / "main.py").write_text( + f"{imports}\n\ndef main():\n" + f" return {files_call}.joinpath('defaults.json').read_text(encoding='utf-8')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'implicit-resource-app'\nversion = '1.0'\n" + "[project.scripts]\nimplicit-resource-app = 'app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + resource = next(item for item in assessment.resources if item.path == "src/app/defaults.json") + staged = _staging_files(tmp_path, assessment, plan, include=True) + + assert resource.packaging_status == "repository_adjacent" + assert resource.path in staged + + +@pytest.mark.parametrize( + ("source", "entry_point", "resource"), + [ + ("src/app/__init__.py", "app:main", "src/app/defaults.json"), + ("src/app/sub/__init__.py", "app.sub:main", "src/app/sub/defaults.json"), + ("src/app/sub/module.py", "app.sub.module:main", "src/app/sub/defaults.json"), + ("src/main.py", "main:main", "src/defaults.json"), + ], +) +def test_importlib_resources_implicit_anchor_uses_source_parent( + tmp_path: Path, source: str, entry_point: str, resource: str +) -> None: + source_path = tmp_path / source + source_path.parent.mkdir(parents=True) + for parent in source_path.parents: + if parent == tmp_path / "src": + break + init = parent / "__init__.py" + if not init.exists() and parent != source_path.parent: + init.write_text("", encoding="utf-8") + (tmp_path / resource).write_text('{}\n', encoding="utf-8") + source_path.write_text( + "from importlib.resources import files\n\n" + "def main():\n return files().joinpath('defaults.json').read_bytes()\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'caller-resource-app'\nversion = '1.0'\n" + f"[project.scripts]\ncaller-resource-app = '{entry_point}'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + assert next(item for item in assessment.resources if item.path == resource).path == resource + + +def test_importlib_resources_implicit_anchor_honors_source_root_and_keywords( + tmp_path: Path, +) -> None: + package = tmp_path / "lib/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{}\n', encoding="utf-8") + (package / "main.py").write_text( + "from importlib.resources import files\n\n" + "def main():\n" + " files().joinpath('defaults.json').read_text()\n" + " files(anchor='app').joinpath('defaults.json').read_text()\n" + " return files(package='app').joinpath('defaults.json').read_text()\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires = ['setuptools']\nbuild-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'lib-implicit-resource-app'\nversion = '1.0'\n" + "[project.scripts]\nlib-implicit-resource-app = 'app.main:main'\n" + "[tool.setuptools]\npackages = ['app']\npackage-dir = {'' = 'lib'}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + resource = next(item for item in assessment.resources if item.path == "lib/app/defaults.json") + assert len( + [ + evidence + for evidence in resource.evidence + if "importlib.resources.files" in evidence.detail + ] + ) == 2 + + def test_importlib_resources_rejects_unproven_or_escaping_resource_paths(tmp_path: Path) -> None: package = tmp_path / "src/app" package.mkdir(parents=True) From df9dc480f8be3970ead4039f0219b484a5246949 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:04:55 -0400 Subject: [PATCH 40/59] Harden static package and wheel analysis --- .../analysis/inventory.py | 7 ++ .../analysis/metadata.py | 69 ++++++++---- .../generation/artifacts.py | 65 ++++++++++++ .../validation/static.py | 53 ++++++++++ tests/test_analysis_scope.py | 63 +++++++++++ tests/test_generation.py | 100 ++++++++++++++++++ tests/test_metadata.py | 53 ++++++++++ tests/test_planning.py | 2 +- 8 files changed, 392 insertions(+), 20 deletions(-) diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index 06e047c..4fc4291 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -373,6 +373,13 @@ def _module_files(root: Path, module: str, source_roots: list[str]) -> list[Path candidate_root / relative / "__init__.py", ) ) + # Importing a dotted local module executes every existing regular + # package initializer on its path. Preserve those files as application + # source without fabricating namespace-package initializers. + candidates.extend( + candidate_root / Path(*relative.parts[:index]) / "__init__.py" + for index in range(1, len(relative.parts)) + ) return sorted({path for path in candidates if path.is_file()}) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 491e9fd..9148931 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -344,17 +344,37 @@ def _package_data_mapping( ) -> dict[str, list[str]]: """Read the supported literal setuptools package-data mapping shape.""" - if not isinstance(value, dict): + declarations = _literal_package_data_mapping(value) + if declarations is None: return {} return { - ("*" if empty_key_is_wildcard and package == "" else package): [ - pattern for pattern in patterns if isinstance(pattern, str) - ] - for package, patterns in value.items() - if isinstance(package, str) and isinstance(patterns, list) + "*" if empty_key_is_wildcard and package == "" else package: patterns + for package, patterns in declarations.items() } +def _literal_string_sequence(value: Any) -> list[str] | None: + """Return a fully literal setuptools string sequence without coercion.""" + + if isinstance(value, (list, tuple)) and all(isinstance(item, str) for item in value): + return list(value) + return None + + +def _literal_package_data_mapping(value: Any) -> dict[str, list[str]] | None: + """Return a fully literal package-data mapping or mark it unresolved.""" + + if not isinstance(value, dict): + return None + declarations: dict[str, list[str]] = {} + for package, patterns in value.items(): + sequence = _literal_string_sequence(patterns) + if not isinstance(package, str) or sequence is None: + return None + declarations[package] = sequence + return declarations + + def _string_list(value: Any, *, default: list[str] | None = None) -> list[str]: """Return the supported TOML/list-or-string metadata shape without coercion.""" @@ -518,6 +538,18 @@ def inspect_setup_call(path: Path) -> SetupCallInspection: except (ValueError, TypeError): if keyword.arg in _SETUP_SURFACE_FIELDS: unresolved.add(keyword.arg) + else: + if keyword.arg in {"packages", "py_modules"} and ( + _literal_string_sequence(values[keyword.arg]) is None + ): + # A malformed literal selection is no more authoritative + # than a dynamic one. Do not retain a string subset and + # silently claim a complete setuptools surface. + unresolved.add(keyword.arg) + elif keyword.arg in {"package_data", "exclude_package_data"} and ( + _literal_package_data_mapping(values[keyword.arg]) is None + ): + unresolved.add(keyword.arg) return SetupCallInspection( literal_values=values, present_keywords=frozenset(present), @@ -1024,7 +1056,9 @@ def inspect_metadata(root: Path) -> MetadataResult: setuptools_package_selection_configured or setup_inspection.package_selection_present ) - setuptools_surface_unresolved = setup_inspection.surface_unresolved + setuptools_surface_unresolved = ( + setuptools_surface_unresolved or setup_inspection.surface_unresolved + ) if setuptools_surface_unresolved: unresolved = sorted( setup_inspection.unresolved_keywords & _SETUP_SURFACE_FIELDS @@ -1044,22 +1078,19 @@ def inspect_metadata(root: Path) -> MetadataResult: _line_number(setup_py_path, "setup("), ) ) - literal_packages = setup_values.get("packages") - if isinstance(literal_packages, list) and not packages: - packages = [package for package in literal_packages if isinstance(package, str)] - if not py_modules and isinstance(setup_values.get("py_modules"), list): - py_modules = [ - module for module in setup_values["py_modules"] if isinstance(module, str) - ] + literal_packages = _literal_string_sequence(setup_values.get("packages")) + if literal_packages is not None and not packages: + packages = literal_packages + literal_py_modules = _literal_string_sequence(setup_values.get("py_modules")) + if literal_py_modules is not None and not py_modules: + py_modules = literal_py_modules if distribution_name is None and isinstance(setup_values.get("name"), str): distribution_name = setup_values["name"] if project_version is None and isinstance(setup_values.get("version"), str): project_version = setup_values["version"] if requires_python is None and isinstance(setup_values.get("python_requires"), str): requires_python = setup_values["python_requires"] - for specification in setup_values.get("install_requires", []): - if not isinstance(specification, str): - continue + for specification in _literal_string_sequence(setup_values.get("install_requires")) or []: parsed = _dependency( specification, "runtime", @@ -1075,8 +1106,8 @@ def inspect_metadata(root: Path) -> MetadataResult: setup_entry_points = setup_values.get("entry_points", {}) if isinstance(setup_entry_points, dict): for group in ("console_scripts", "gui_scripts"): - values = setup_entry_points.get(group, []) - if not isinstance(values, list): + values = _literal_string_sequence(setup_entry_points.get(group)) + if values is None: continue for specification in values: if not isinstance(specification, str) or "=" not in specification: diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 27f4975..19f6a14 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -14,6 +14,7 @@ import stat import zipfile from collections.abc import Iterable +from dataclasses import dataclass from email.parser import BytesParser from email.policy import default from pathlib import Path, PurePosixPath @@ -58,6 +59,16 @@ MAX_WHEEL_MEMBER_SIZE = 256 * 1024 * 1024 MAX_WHEEL_TOTAL_UNCOMPRESSED_SIZE = 512 * 1024 * 1024 + +@dataclass(frozen=True) +class WheelStaticMetadata: + """Installer-facing wheel identity proven without source or lock-plan state.""" + + distribution_name: str + version: Version + filename_tags: frozenset[str] + declared_tags: frozenset[str] + _WINDOWS_FORBIDDEN_COMPONENT_CHARACTERS = frozenset('<>:"|?*') _WINDOWS_RESERVED_DEVICE_BASENAMES = frozenset( { @@ -586,6 +597,60 @@ def _require_wheel_metadata(message, *, wheel: Path) -> set[str]: return tags +def validate_wheel_metadata_semantics(path: Path) -> WheelStaticMetadata: + """Validate filename, Core METADATA, and WHEEL semantics without execution. + + This deliberately does not prove target compatibility, source-surface + completeness, or lock dependencies. It is the reusable installer-facing + semantic subset available to both generated-kit static validation and the + generation-time artifact validators. + """ + + try: + filename_name, filename_version, _build, filename_tags = parse_wheel_filename(path.name) + except ValueError as exc: + raise PreparationError(f"Malformed wheel filename: {path.name}") from exc + filename_distribution = canonicalize_name(str(filename_name)) + try: + with zipfile.ZipFile(path) as bundle: + members = _member_map(_safe_wheel_members(bundle)) + metadata_name, wheel_name, _record_name = _dist_info_members(members, path) + if wheel_name not in members: + raise PreparationError( + f"Wheel is missing required WHEEL metadata: {path.name}" + ) + metadata = _metadata_message( + bundle.read(members[metadata_name]), label="METADATA", wheel=path + ) + wheel_metadata = _metadata_message( + bundle.read(members[wheel_name]), label="WHEEL", wheel=path + ) + except zipfile.BadZipFile as exc: + raise PreparationError(f"Malformed wheel archive: {path.name}") from exc + + metadata_name_value, metadata_version_value = _require_core_metadata( + metadata, label="METADATA", wheel=path + ) + try: + metadata_version = Version(metadata_version_value) + except InvalidVersion as exc: + raise PreparationError(f"Malformed METADATA in wheel: {path.name}") from exc + if canonicalize_name(metadata_name_value) != filename_distribution: + raise PreparationError(f"Wheel METADATA name does not match its filename: {path.name}") + if metadata_version != filename_version: + raise PreparationError(f"Wheel METADATA version does not match its filename: {path.name}") + declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) + filename_tag_values = {str(item) for item in filename_tags} + if not filename_tag_values <= declared_tags: + raise PreparationError(f"Wheel tag metadata does not match its filename: {path.name}") + return WheelStaticMetadata( + distribution_name=filename_distribution, + version=filename_version, + filename_tags=frozenset(filename_tag_values), + declared_tags=frozenset(declared_tags), + ) + + def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> None: """Require a precision-safe Requires-Python proof for a minor-only runtime.""" diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 32c4f10..39c577b 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -12,6 +12,8 @@ from datetime import UTC, datetime from pathlib import Path, PurePosixPath +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version from pydantic import ValidationError from python_deployment_builder.generation.acquisition import PreparationError @@ -20,6 +22,7 @@ installed_wheel_member_paths, validate_combined_wheel_installation_paths, validate_wheel_installation_layout, + validate_wheel_metadata_semantics, validate_wheel_static_safety, ) from python_deployment_builder.generation.structural import ( @@ -401,6 +404,56 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=wheel_layout_failures, ) ) + expected_wheel_identities: dict[str, tuple[str, str]] = {} + if manifest.application_artifact is not None and ( + relative := manifest_artifact_wheel_path( + "application", manifest.application_artifact.filename + ) + ): + expected_wheel_identities[relative] = ( + manifest.application_artifact.distribution_name, + manifest.application_artifact.version, + ) + for artifact in manifest.approved_artifacts: + if relative := manifest_artifact_wheel_path("wheels", artifact.filename): + expected_wheel_identities[relative] = ( + artifact.distribution_name, + artifact.version, + ) + wheel_metadata_failures: list[str] = [] + for path in safe_trusted_wheel_paths: + relative = path.relative_to(root).as_posix() + try: + metadata = validate_wheel_metadata_semantics(path) + expected = expected_wheel_identities.get(relative) + if expected is None: + raise PreparationError("Wheel is not an exact manifest-owned artifact.") + expected_name, expected_version = expected + try: + expected_version_value = Version(expected_version) + except InvalidVersion as exc: + raise PreparationError( + f"Manifest wheel version is invalid: {path.name}" + ) from exc + if canonicalize_name(expected_name) != metadata.distribution_name: + raise PreparationError( + f"Wheel METADATA name does not match manifest artifact: {path.name}" + ) + if expected_version_value != metadata.version: + raise PreparationError( + f"Wheel METADATA version does not match manifest artifact: {path.name}" + ) + except PreparationError as exc: + wheel_metadata_failures.append(f"{relative}: {exc}") + checks.append( + _check( + "WHEEL_METADATA_SEMANTICS", + not wheel_metadata_failures, + "Manifest-declared wheels have valid metadata matching their filenames and manifests.", + "A manifest-declared wheel has invalid or mismatched installer metadata.", + evidence=wheel_metadata_failures, + ) + ) wheel_security_failures: list[str] = [] for path in safe_trusted_wheel_paths: try: diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 954bacc..5c4208c 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -1115,6 +1115,69 @@ def test_importlib_resources_rejects_unproven_or_escaping_resource_paths(tmp_pat assert all("secret.txt" not in path for path in staged) +def test_dotted_import_promotion_includes_and_scans_regular_package_initializers( + tmp_path: Path, +) -> None: + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/docs").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/main.py").write_text( + "import docs.helper\n\ndef main(): return docs.helper.VALUE\n", encoding="utf-8" + ) + (tmp_path / "src/docs/__init__.py").write_text( + "REGISTERED = True\nimport docs.config\n", encoding="utf-8" + ) + (tmp_path / "src/docs/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "src/docs/config.py").write_text("VALUE = 2\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='dotted-import-app'\nversion='1.0'\n" + "[project.scripts]\ndotted-import-app='app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + staged = _staging_files(tmp_path, assessment, plan, include=True) + inventory = {item.path: item for item in assessment.file_inventory} + + for path in ("src/docs/__init__.py", "src/docs/helper.py", "src/docs/config.py"): + assert inventory[path].role == RepositoryFileRole.APPLICATION_SOURCE + assert path in staged + assert any( + "docs.helper" in evidence.detail + for evidence in inventory["src/docs/__init__.py"].evidence + ) + + +def test_dotted_import_promotion_preserves_existing_ancestor_initializers(tmp_path: Path) -> None: + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/pkg/sub").mkdir(parents=True) + for relative in ( + "src/app/__init__.py", + "src/pkg/__init__.py", + "src/pkg/sub/__init__.py", + ): + (tmp_path / relative).write_text("", encoding="utf-8") + (tmp_path / "src/app/main.py").write_text( + "import pkg.sub.helper\n\ndef main(): return pkg.sub.helper.VALUE\n", + encoding="utf-8", + ) + (tmp_path / "src/pkg/sub/helper.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='nested-import-app'\nversion='1.0'\n" + "[project.scripts]\nnested-import-app='app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + inventory = {item.path: item for item in assessment.file_inventory} + + for path in ("src/pkg/__init__.py", "src/pkg/sub/__init__.py", "src/pkg/sub/helper.py"): + assert inventory[path].role == RepositoryFileRole.APPLICATION_SOURCE + + def test_importlib_resources_uses_package_dir_parent_mapping(tmp_path: Path) -> None: package = tmp_path / "lib/sub" package.mkdir(parents=True) diff --git a/tests/test_generation.py b/tests/test_generation.py index 505fd4d..ff35740 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -3957,6 +3957,106 @@ def test_static_validation_rejects_unsupported_approved_artifact_data_scheme( ) +@pytest.mark.parametrize( + ("member", "content"), + [ + ( + "mapped_app-1.2.3.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: maybe\nTag: py3-none-any\n", + ), + ( + "mapped_app-1.2.3.dist-info/WHEEL", + "Wheel-Version: 2.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ), + ("mapped_app-1.2.3.dist-info/METADATA", "Metadata-Version: 2.1\nVersion: 1.2.3\n\n"), + ( + "mapped_app-1.2.3.dist-info/METADATA", + "Metadata-Version: 2.1\nName: mapped-app\nVersion: invalid\n\n", + ), + ], +) +def test_static_validation_rejects_semantically_invalid_application_wheel_metadata( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, member: str, content: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + wheel = _make_application_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + application_wheel=wheel, + bootstrap_mode="online_cmd", + ) + relative = f"deployment/application/{wheel.name}" + staged = kit / relative + _rewrite_application_wheel(staged, replacements={member: content}) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["application_artifact"]["sha256"] = hashlib.sha256(staged.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, relative, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert any( + item.code == "WHEEL_METADATA_SEMANTICS" and item.status.value == "FAIL" + for item in report.static_checks + ) + + +def test_static_validation_requires_wheel_metadata_to_match_manifest_artifacts( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["approved_artifacts"][0]["distribution_name"] = "other-package" + manifest["approved_artifacts"][0]["version"] = "9.9" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert any( + item.code == "WHEEL_METADATA_SEMANTICS" and item.status.value == "FAIL" + for item in report.static_checks + ) + + def test_static_validation_rescans_trusted_wheel_configured_secrets( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/test_metadata.py b/tests/test_metadata.py index a9b6702..2240bb8 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -3,6 +3,8 @@ import zipfile from pathlib import Path +import pytest + from python_deployment_builder.analysis.metadata import ( inspect_metadata, inspect_setup_call, @@ -116,6 +118,57 @@ def test_setup_call_inspection_distinguishes_absent_literal_and_unresolved_surfa assert dynamic_result.surface_unresolved +def test_setup_py_tuple_package_and_module_sequences_are_authoritative(tmp_path: Path) -> None: + (tmp_path / "lib/app/data").mkdir(parents=True) + (tmp_path / "lib/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "lib/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (tmp_path / "lib/app/data/default.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "lib/helper.py").write_text("def main(): return 0\n", encoding="utf-8") + (tmp_path / "setup.py").write_text( + "from setuptools import setup\n" + "setup(name='tuple-demo', version='1.0', packages=('app',), " + "py_modules=('helper',), package_dir={'': 'lib'}, " + "package_data={'app': ('data/*.json',)}, install_requires=('requests>=2',))\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.project.packages == ["app"] + assert result.project.py_modules == ["helper"] + assert result.project.source_roots == ["lib"] + assert not result.setuptools_surface_unresolved + assert [item.distribution_name for item in result.dependencies] == ["requests"] + package_data_paths = [ + item.source_path for item in resolve_package_data_members(tmp_path, result.project) + ] + assert package_data_paths == [ + "lib/app/data/default.json" + ] + + +@pytest.mark.parametrize( + "field, value", + [ + ("packages", "('app', 1)"), + ("py_modules", "('helper', 1)"), + ], +) +def test_setup_py_malformed_literal_selection_remains_unresolved( + tmp_path: Path, field: str, value: str +) -> None: + (tmp_path / "setup.py").write_text( + f"from setuptools import setup\nsetup(name='bad', version='1', {field}={value})\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.setuptools_surface_unresolved + assert result.project.packages == [] + assert result.project.py_modules == [] + + def test_dynamic_setup_package_selector_does_not_trigger_automatic_discovery( tmp_path: Path, ) -> None: diff --git a/tests/test_planning.py b/tests/test_planning.py index 72b19c8..4d1da99 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -193,7 +193,7 @@ def test_module_file_resolution_searches_all_safe_configured_roots(tmp_path: Pat )] == ["lib/helper.py", "python/helper.py"] assert [path.relative_to(tmp_path).as_posix() for path in _module_files( tmp_path, "foo.bar", ["lib", "python"] - )] == ["python/foo/bar/__init__.py"] + )] == ["lib/foo/__init__.py", "python/foo/bar/__init__.py"] def test_external_packaging_root_blocks_both_source_and_package_mode_contracts( From 04a20f12013dfa0efcca2afe1a9b3e830f9238bd Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:17:41 -0400 Subject: [PATCH 41/59] Fix entry point and resource analysis contracts --- .../analysis/metadata.py | 27 +++- .../analysis/resources.py | 142 +++++++++++++++--- .../templates/windows_uv/launch.py | 23 ++- tests/test_analysis_scope.py | 68 +++++++++ tests/test_generation.py | 46 ++++++ tests/test_launch_contract.py | 41 ++++- tests/test_metadata.py | 4 +- 7 files changed, 326 insertions(+), 25 deletions(-) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 9148931..1e1fee3 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -41,6 +41,14 @@ class MetadataResult: setuptools_external_packaging_root_evidence: list[Evidence] = field(default_factory=list) +@dataclass(frozen=True) +class LiteralModuleAttribute: + """A literal dynamic setuptools value and the source file that supplied it.""" + + value: str + source_path: str + + _SETUP_SURFACE_FIELDS = frozenset( {"packages", "py_modules", "package_dir", "package_data", "exclude_package_data"} ) @@ -582,7 +590,7 @@ def setuptools_packaging_surface_resolved(root: Path) -> bool: ) -def _literal_module_attribute(root: Path, attribute: str) -> str | None: +def _literal_module_attribute(root: Path, attribute: str) -> LiteralModuleAttribute | None: """Resolve a setuptools dynamic version attr only when it is a string literal.""" try: @@ -629,7 +637,13 @@ def _literal_module_attribute(root: Path, attribute: str) -> str | None: return None resolved_values.append(value) if len(resolved_values) == 1: - return resolved_values[0] + try: + source_path = path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + return None + return LiteralModuleAttribute( + value=resolved_values[0], source_path=source_path + ) if resolved_values: return None return None @@ -708,7 +722,14 @@ def inspect_metadata(root: Path) -> MetadataResult: version_rule = dynamic.get("version") version_attr = version_rule.get("attr") if isinstance(version_rule, dict) else None if isinstance(version_attr, str): - project_version = _literal_module_attribute(root, version_attr) + resolved_version = _literal_module_attribute(root, version_attr) + if resolved_version is not None: + project_version = resolved_version.value + # metadata_files is also the model-derived provenance input list. + # A literal dynamic-version module is parsed to establish the + # authoritative project version even when package mode does not + # stage the source file. + metadata_files.append(resolved_version.source_path) requires_python = ( project.get("requires-python") if isinstance(project.get("requires-python"), str) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 74da466..bfba216 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -504,11 +504,19 @@ def bind_pattern(target: ast.AST, values: list[ast.AST]) -> None: return assignments, returns -def _resource_import_bindings(tree: ast.AST) -> tuple[set[str], set[str]]: - """Return proven ``importlib.resources`` module and ``files`` bindings.""" +_LEGACY_IMPORTLIB_RESOURCE_READS = frozenset( + {"read_text", "read_binary", "open_text", "open_binary"} +) + + +def _resource_import_bindings( + tree: ast.AST, +) -> tuple[set[str], set[str], dict[str, str]]: + """Return proven importlib-resources module, ``files``, and read bindings.""" modules: set[str] = set() files: set[str] = set() + reads: dict[str, str] = {} for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: @@ -523,7 +531,9 @@ def _resource_import_bindings(tree: ast.AST) -> tuple[set[str], set[str]]: for alias in node.names: if alias.name == "files": files.add(alias.asname or alias.name) - return modules, files + elif alias.name in _LEGACY_IMPORTLIB_RESOURCE_READS: + reads[alias.asname or alias.name] = alias.name + return modules, files, reads def _resource_package_roots( @@ -563,6 +573,25 @@ def _is_resource_files_call( return name in files_bindings or any(name == f"{binding}.files" for binding in module_bindings) +def _legacy_resource_function_name( + node: ast.AST, + module_bindings: set[str], + read_bindings: dict[str, str], +) -> str | None: + """Return a proven legacy importlib.resources functional read name.""" + + if not isinstance(node, ast.Call): + return None + name = _qualified_name(node.func) + if name in read_bindings: + return read_bindings[name] + for binding in module_bindings: + for function in _LEGACY_IMPORTLIB_RESOURCE_READS: + if name == f"{binding}.{function}": + return function + return None + + def _safe_resource_member(values: list[str]) -> bool: return all( value @@ -618,6 +647,56 @@ def _resource_package_anchor_values( ] +def _legacy_importlib_resource_path_values( + node: ast.Call, + *, + root: Path, + source_path: Path, + source_roots: list[str], + project: PackagingAssessment | None, + assignments: dict[str, ast.AST], + returns: dict[str, ast.AST], + module_bindings: set[str], + read_bindings: dict[str, str], +) -> tuple[str, list[str]] | None: + """Resolve the bounded two-argument legacy resource read API statically.""" + + function = _legacy_resource_function_name(node, module_bindings, read_bindings) + if function is None: + return None + # Python 3.12 legacy resource calls address one direct member of a + # package. Keep dynamic, nested, and traversal-like members unresolved. + if len(node.args) != 2: + return function, [] + allowed_keywords = {"encoding", "errors"} if function.endswith("text") else set() + if any(keyword.arg not in allowed_keywords for keyword in node.keywords): + return function, [] + package_roots = _resource_package_anchor_values( + node.args[0], + root=root, + source_path=source_path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + ) + members = _path_values( + node.args[1], + root=root, + source_path=source_path, + assignments=assignments, + returns=returns, + ) + if ( + not package_roots + or len(members) != 1 + or not _safe_resource_member(members) + or len(PurePosixPath(members[0].replace("\\", "/")).parts) != 1 + ): + return function, [] + return function, _combine_paths(package_roots, members) + + def _importlib_resource_path_values( node: ast.AST, *, @@ -1024,22 +1103,51 @@ def _literal_evidence( continue lines = source.splitlines() assignments, returns = _bindings(tree) - module_bindings, files_bindings = _resource_import_bindings(tree) + module_bindings, files_bindings, read_bindings = _resource_import_bindings(tree) for node in ast.walk(tree): if not isinstance(node, ast.Call): continue - for expression, mode in _path_uses(node): - resource_values = _importlib_resource_path_values( - expression, - root=root, - source_path=path, - source_roots=source_roots, - project=project, - assignments=assignments, - returns=returns, - module_bindings=module_bindings, - files_bindings=files_bindings, + legacy_resource = _legacy_importlib_resource_path_values( + node, + root=root, + source_path=path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + module_bindings=module_bindings, + read_bindings=read_bindings, + ) + uses: list[tuple[ast.AST, str, list[str] | None, str]] = [] + if legacy_resource is not None: + function, values = legacy_resource + # Detect these before generic ``receiver.read_text()`` handling; + # their receiver is an importlib module, not a filesystem path. + uses.append( + ( + node, + "read", + values, + f"importlib.resources.{function}()", + ) ) + else: + for expression, mode in _path_uses(node): + resource_values = _importlib_resource_path_values( + expression, + root=root, + source_path=path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + module_bindings=module_bindings, + files_bindings=files_bindings, + ) + uses.append( + (expression, mode, resource_values, "importlib.resources.files()") + ) + for expression, mode, resource_values, resource_api in uses: values = ( resource_values if resource_values is not None @@ -1052,7 +1160,7 @@ def _literal_evidence( ) ) for value in values: - if not _looks_like_resource_literal(value): + if resource_values is None and not _looks_like_resource_literal(value): continue resolved_any = False for resolved in _resolve_literal(root, path, value): @@ -1065,7 +1173,7 @@ def _literal_evidence( line=node.lineno, detail=( f"Static {mode} resource use through " - "importlib.resources.files() resolves here." + f"{resource_api} resolves here." if resource_values is not None else f"Static {mode} path use through " f"{_qualified_name(node.func)} resolves here." diff --git a/src/python_deployment_builder/templates/windows_uv/launch.py b/src/python_deployment_builder/templates/windows_uv/launch.py index ec1ceb6..5598192 100644 --- a/src/python_deployment_builder/templates/windows_uv/launch.py +++ b/src/python_deployment_builder/templates/windows_uv/launch.py @@ -27,6 +27,25 @@ def configure_source_paths(manifest: dict, project_root: Path) -> None: sys.path.insert(0, str(path.resolve())) +def resolve_entry_point_object(module: object, qualified_name: str) -> object: + """Resolve a PyPA entry-point object reference without evaluating code.""" + + components = qualified_name.split(".") + if not qualified_name or any(not component.isidentifier() for component in components): + raise DeploymentRuntimeError( + f"Invalid entry-point object reference: {qualified_name!r}" + ) + target = module + for component in components: + try: + target = getattr(target, component) + except AttributeError as exc: + raise DeploymentRuntimeError( + f"Entry-point callable is unavailable: {qualified_name}" + ) from exc + return target + + def check_entry_point(manifest: dict, project_root: Path) -> None: configure_source_paths(manifest, project_root) if importlib.util.find_spec(manifest["entry_point_module"]) is None: @@ -34,7 +53,7 @@ def check_entry_point(manifest: dict, project_root: Path) -> None: f"Entry-point module is not importable: {manifest['entry_point_module']}" ) module = importlib.import_module(manifest["entry_point_module"]) - target = getattr(module, manifest["entry_point_callable"], None) + target = resolve_entry_point_object(module, manifest["entry_point_callable"]) if not callable(target): raise DeploymentRuntimeError( "Entry-point callable is unavailable: " @@ -63,7 +82,7 @@ def invoke(manifest: dict, project_root: Path) -> int: if manifest.get("project_write_probe_required"): probe_project_write(project_root, manifest["application_display_name"]) module = importlib.import_module(manifest["entry_point_module"]) - target = getattr(module, manifest["entry_point_callable"]) + target = resolve_entry_point_object(module, manifest["entry_point_callable"]) original_argv = sys.argv[:] try: # Helper arguments are private deployment details. Application arguments are diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 5c4208c..7b2d523 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -971,6 +971,74 @@ def test_importlib_resources_files_promotes_concrete_source_resource( assert resource.path in staged +@pytest.mark.parametrize( + ("imports", "resource_call"), + [ + ("import importlib.resources", "importlib.resources.read_text('app', 'defaults.json')"), + ("import importlib.resources as ir", "ir.read_binary('app', 'defaults.json')"), + ("from importlib import resources", "resources.read_text('app', 'defaults.json')"), + ("from importlib import resources as ir", "ir.open_binary('app', 'defaults.json')"), + ("from importlib.resources import read_text", "read_text('app', 'defaults.json')"), + ( + "from importlib.resources import read_binary as resource_read_binary", + "resource_read_binary('app', 'defaults.json')", + ), + ("from importlib.resources import open_text", "open_text('app', 'defaults.json')"), + ], +) +def test_legacy_importlib_resources_reads_promote_concrete_source_resource( + tmp_path: Path, imports: str, resource_call: str +) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{"default": true}\n', encoding="utf-8") + (package / "main.py").write_text( + f"{imports}\n\ndef main():\n return {resource_call}\n", encoding="utf-8" + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'legacy-resource-app'\nversion = '1.0'\n" + "[project.scripts]\nlegacy-resource-app = 'app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + resource = next(item for item in assessment.resources if item.path == "src/app/defaults.json") + staged = _staging_files(tmp_path, assessment, plan, include=True) + + assert resource.packaging_status == "repository_adjacent" + assert resource.path in staged + assert "importlib.resources." in resource.evidence[-1].detail + + +def test_legacy_importlib_resources_rejects_dynamic_and_escaping_members(tmp_path: Path) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{}\n', encoding="utf-8") + (tmp_path / "src/secret.json").write_text('{}\n', encoding="utf-8") + (package / "main.py").write_text( + "from importlib.resources import read_text\n" + "def main(name='defaults.json'):\n" + " read_text('app', name)\n" + " return read_text('app', '../secret.json')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'unsafe-legacy-resource-app'\nversion = '1.0'\n" + "[project.scripts]\nunsafe-legacy-resource-app = 'app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + assert all(item.path != "src/app/defaults.json" for item in assessment.resources) + assert any(item.status == FindingStatus.NEEDS_VALIDATION for item in assessment.resources) + + @pytest.mark.parametrize( ("imports", "files_call"), [ diff --git a/tests/test_generation.py b/tests/test_generation.py index ff35740..13c5225 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -5327,6 +5327,52 @@ def test_analysis_metadata_paths_include_parsed_requirements_and_python_evidence ) +@pytest.mark.parametrize("operation", ["modified", "deleted", "renamed"]) +def test_git_source_staging_guards_literal_dynamic_version_module( + tmp_path: Path, operation: str +) -> None: + source = tmp_path / "dynamic-version-source" + source.mkdir() + (source / "src/app").mkdir(parents=True) + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools']\nbuild-backend='setuptools.build_meta'\n" + "[project]\nname='dynamic-version-source'\ndynamic=['version']\n" + "[project.scripts]\ndynamic-version-source='app:main'\n" + "[tool.setuptools.dynamic]\nversion={attr='app.__version__'}\n", + encoding="utf-8", + ) + (source / "src/app/__init__.py").write_text( + "__version__ = '1.0'\ndef main(): return 0\n", encoding="utf-8" + ) + (source / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], check=True + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + version_module = source / "src/app/__init__.py" + if operation == "modified": + version_module.write_text( + "__version__ = '2.0'\ndef main(): return 0\n", encoding="utf-8" + ) + elif operation == "deleted": + version_module.unlink() + else: + version_module.rename(source / "src/app/version.py") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + # The resolved source is a metadata/provenance input, even where a package + # deployment intentionally does not stage application source. + if operation == "modified": + assert "src/app/__init__.py" in _provenance_guard_paths(source, assessment, plan) + with pytest.raises(PreparationError, match=r"src/app/__init__\.py"): + _staging_files(source, assessment, plan, include=True) + + def test_git_source_staging_allows_unrelated_readme_without_python_evidence( tmp_path: Path, ) -> None: diff --git a/tests/test_launch_contract.py b/tests/test_launch_contract.py index d6fd3d8..65b960c 100644 --- a/tests/test_launch_contract.py +++ b/tests/test_launch_contract.py @@ -13,7 +13,9 @@ from python_deployment_builder.generation.templates import TEMPLATE_ROOT -def _write_launch_fixture(tmp_path: Path, target_source: str) -> tuple[Path, Path, Path]: +def _write_launch_fixture( + tmp_path: Path, target_source: str, *, callable_name: str = "main" +) -> tuple[Path, Path, Path]: project_root = tmp_path / "application" project_root.mkdir() (project_root / "synthetic_target.py").write_text(dedent(target_source), encoding="utf-8") @@ -33,7 +35,7 @@ def _write_launch_fixture(tmp_path: Path, target_source: str) -> tuple[Path, Pat "entry_point_name": "synthetic-entry", "entry_point_kind": "gui", "entry_point_module": "synthetic_target", - "entry_point_callable": "main", + "entry_point_callable": callable_name, "deployment_mode": "source", "source_roots": ["."], "project_write_probe_required": False, @@ -179,6 +181,41 @@ def main(): assert not _failure_logs(local_app_data) +def test_generated_launch_resolves_qualified_entry_point_object_in_source_and_package_modes( + tmp_path: Path, +) -> None: + target_source = """ + from pathlib import Path + + class Runner: + @staticmethod + def main(): + Path("qualified-target-called.txt").write_text("yes", encoding="utf-8") + """ + project_root, launcher, local_app_data = _write_launch_fixture( + tmp_path, target_source, callable_name="Runner.main" + ) + + source_result = _run_launch(project_root, launcher, local_app_data) + + assert source_result.returncode == 0, source_result.stderr + assert (project_root / "qualified-target-called.txt").is_file() + (project_root / "qualified-target-called.txt").unlink() + (launcher.parent / "synthetic_target.py").write_text( + dedent(target_source), encoding="utf-8" + ) + manifest_path = project_root / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["deployment_mode"] = "package" + manifest["source_roots"] = [] + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + package_result = _run_launch(project_root, launcher, local_app_data) + + assert package_result.returncode == 0, package_result.stderr + assert (project_root / "qualified-target-called.txt").is_file() + + @pytest.mark.parametrize( ("statement", "expected_code", "failure_expected"), [ diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 2240bb8..a51468a 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -551,7 +551,9 @@ def test_literal_dynamic_version_attr_is_resolved_without_import(tmp_path: Path) f'open({str(marker)!r}, "w").write("executed")\n__version__ = "1.0"\n', encoding="utf-8", ) - assert inspect_metadata(tmp_path).project.version == "1.0" + metadata = inspect_metadata(tmp_path) + assert metadata.project.version == "1.0" + assert "version_module.py" in metadata.project.metadata_files assert not marker.exists() From 6b128b01988cc98446ee2534f16383b7eb792a44 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:56:22 -0400 Subject: [PATCH 42/59] Validate wheel dependency and target contracts --- .../analysis/metadata.py | 103 ++++-- .../generation/artifacts.py | 293 ++++++++++++++++-- .../validation/static.py | 22 ++ tests/test_generation.py | 78 +++++ tests/test_metadata.py | 44 +++ 5 files changed, 500 insertions(+), 40 deletions(-) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 1e1fee3..27114db 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -590,7 +590,13 @@ def setuptools_packaging_surface_resolved(root: Path) -> bool: ) -def _literal_module_attribute(root: Path, attribute: str) -> LiteralModuleAttribute | None: +def _literal_module_attribute( + root: Path, + attribute: str, + *, + package_directories: dict[str, str] | None = None, + source_roots: list[str] | None = None, +) -> LiteralModuleAttribute | None: """Resolve a setuptools dynamic version attr only when it is a string literal.""" try: @@ -599,20 +605,49 @@ def _literal_module_attribute(root: Path, attribute: str) -> LiteralModuleAttrib return None if not all(part.isidentifier() for part in module_name.split(".")): return None - relative = Path(*module_name.split(".")) - candidates = [ - root / relative.with_suffix(".py"), - root / relative / "__init__.py", - root / "src" / relative.with_suffix(".py"), - root / "src" / relative / "__init__.py", + module_parts = module_name.split(".") + package_directories = package_directories or {} + source_roots = source_roots or [] + candidate_bases: list[Path] = [] + named_mappings = [ + name + for name in package_directories + if name and (module_name == name or module_name.startswith(f"{name}.")) ] + if named_mappings: + mapping = max(named_mappings, key=lambda name: len(name.split("."))) + inspection = inspect_setuptools_packaging_root(root, package_directories[mapping]) + if inspection.status != "UNSAFE" and inspection.normalized_root is not None: + remainder = module_parts[len(mapping.split(".")) :] + candidate_bases.append(root / inspection.normalized_root / Path(*remainder)) + elif "" in package_directories: + inspection = inspect_setuptools_packaging_root(root, package_directories[""]) + if inspection.status != "UNSAFE" and inspection.normalized_root is not None: + candidate_bases.append(root / inspection.normalized_root / Path(*module_parts)) + else: + for source_root in [*source_roots, "src", "."]: + inspection = inspect_setuptools_packaging_root(root, source_root) + if inspection.status != "UNSAFE" and inspection.normalized_root is not None: + candidate_bases.append(root / inspection.normalized_root / Path(*module_parts)) + candidates: list[Path] = [] + for base in candidate_bases: + candidates.extend((base.with_suffix(".py"), base / "__init__.py")) + seen_candidates: set[Path] = set() + resolutions: list[LiteralModuleAttribute] = [] for path in candidates: - if not path.is_file(): + if path in seen_candidates: + continue + seen_candidates.add(path) + if path.is_symlink() or not path.is_file(): + continue + try: + path.resolve().relative_to(root.resolve()) + except ValueError: continue try: tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path)) except (OSError, SyntaxError, UnicodeError): - return None + continue resolved_values: list[str] = [] for node in tree.body: value_node: ast.expr | None = None @@ -632,21 +667,23 @@ def _literal_module_attribute(root: Path, attribute: str) -> LiteralModuleAttrib try: value = ast.literal_eval(value_node) except (ValueError, TypeError): - return None + resolved_values = [] + break if not isinstance(value, str) or not value.strip(): - return None + resolved_values = [] + break resolved_values.append(value) if len(resolved_values) == 1: try: source_path = path.resolve().relative_to(root.resolve()).as_posix() except ValueError: return None - return LiteralModuleAttribute( - value=resolved_values[0], source_path=source_path + resolutions.append( + LiteralModuleAttribute(value=resolved_values[0], source_path=source_path) ) - if resolved_values: - return None - return None + # Multiple configured roots must not let declaration order choose the + # authoritative version module. One literal source is the bounded model. + return resolutions[0] if len(resolutions) == 1 else None def _documented_python_versions(root: Path) -> tuple[list[str], list[Evidence]]: @@ -713,16 +750,42 @@ def inspect_metadata(root: Path) -> MetadataResult: project_version = ( project.get("version") if isinstance(project.get("version"), str) else None ) + tool = document.get("tool") if isinstance(document.get("tool"), dict) else {} + setuptools = tool.get("setuptools") if isinstance(tool.get("setuptools"), dict) else {} + dynamic_package_directories = ( + { + name: path + for name, path in setuptools.get("package-dir", {}).items() + if isinstance(name, str) and isinstance(path, str) + } + if isinstance(setuptools.get("package-dir"), dict) + else {} + ) + dynamic_package_find = ( + setuptools.get("packages", {}).get("find", {}) + if isinstance(setuptools.get("packages"), dict) + else {} + ) + dynamic_source_roots = ( + _string_list(dynamic_package_find.get("where"), default=["."]) + if isinstance(dynamic_package_find, dict) + else [] + ) + if not dynamic_source_roots and isinstance(dynamic_package_directories.get(""), str): + dynamic_source_roots = [dynamic_package_directories[""]] if project_version is None and "version" in project.get("dynamic", []): - tool = document.get("tool") if isinstance(document.get("tool"), dict) else {} - setuptools = tool.get("setuptools") if isinstance(tool.get("setuptools"), dict) else {} dynamic = ( setuptools.get("dynamic") if isinstance(setuptools.get("dynamic"), dict) else {} ) version_rule = dynamic.get("version") version_attr = version_rule.get("attr") if isinstance(version_rule, dict) else None if isinstance(version_attr, str): - resolved_version = _literal_module_attribute(root, version_attr) + resolved_version = _literal_module_attribute( + root, + version_attr, + package_directories=dynamic_package_directories, + source_roots=dynamic_source_roots, + ) if resolved_version is not None: project_version = resolved_version.value # metadata_files is also the model-derived provenance input list. @@ -792,7 +855,6 @@ def inspect_metadata(root: Path) -> MetadataResult: for name, target in values.items() if isinstance(name, str) and isinstance(target, str) ) - tool = document.get("tool") if isinstance(document.get("tool"), dict) else {} uv = tool.get("uv") if isinstance(tool.get("uv"), dict) else {} uv_workspace = isinstance(uv.get("workspace"), dict) uv_sources = uv.get("sources") if isinstance(uv.get("sources"), dict) else {} @@ -859,7 +921,6 @@ def inspect_metadata(root: Path) -> MetadataResult: # The supported string form maps to Poetry's standard # console-script entry-point behavior. entry_points.append(_entry_point(root, pyproject_path, name, target, "scripts")) - setuptools = tool.get("setuptools") if isinstance(tool.get("setuptools"), dict) else {} if build_backend is None and setuptools: # A project that supplies setuptools' own pyproject configuration # but omits [build-system] follows the conventional setuptools diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 19f6a14..4ffe135 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -68,6 +68,7 @@ class WheelStaticMetadata: version: Version filename_tags: frozenset[str] declared_tags: frozenset[str] + requires_python: str | None _WINDOWS_FORBIDDEN_COMPONENT_CHARACTERS = frozenset('<>:"|?*') _WINDOWS_RESERVED_DEVICE_BASENAMES = frozenset( @@ -643,25 +644,29 @@ def validate_wheel_metadata_semantics(path: Path) -> WheelStaticMetadata: filename_tag_values = {str(item) for item in filename_tags} if not filename_tag_values <= declared_tags: raise PreparationError(f"Wheel tag metadata does not match its filename: {path.name}") + requires_python_values = metadata.get_all("Requires-Python", []) + if len(requires_python_values) > 1 or ( + requires_python_values and not requires_python_values[0].strip() + ): + raise PreparationError(f"Malformed Requires-Python metadata in wheel: {path.name}") return WheelStaticMetadata( distribution_name=filename_distribution, version=filename_version, filename_tags=frozenset(filename_tag_values), declared_tags=frozenset(declared_tags), + requires_python=(requires_python_values[0].strip() if requires_python_values else None), ) -def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> None: +def _validate_requires_python_constraint( + constraint: str | None, python_version: str, wheel: Path +) -> None: """Require a precision-safe Requires-Python proof for a minor-only runtime.""" - values = metadata.get_all("Requires-Python", []) - if not values: + if constraint is None: return - if len(values) != 1 or not values[0].strip(): - raise PreparationError(f"Malformed Requires-Python metadata in wheel: {wheel.name}") - constraint = values[0].strip() try: - compatibility = minor_python_compatibility(plan.runtime.python_version, constraint) + compatibility = minor_python_compatibility(python_version, constraint) except (InvalidSpecifier, ValueError) as exc: raise PreparationError( f"Malformed Requires-Python metadata in wheel: {wheel.name}" @@ -669,13 +674,39 @@ def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> No if compatibility == MinorPythonCompatibility.INCOMPATIBLE: raise PreparationError( f"Wheel Requires-Python {constraint!r} is incompatible with selected Python " - f"{plan.runtime.python_version}." + f"{python_version}." ) if compatibility == MinorPythonCompatibility.UNPROVABLE: raise PreparationError( f"Wheel Requires-Python {constraint!r} cannot be proven for minor-only selected " - f"Python {plan.runtime.python_version}." + f"Python {python_version}." + ) + + +def _validate_requires_python(metadata, plan: DeploymentPlan, wheel: Path) -> None: + values = metadata.get_all("Requires-Python", []) + if len(values) > 1 or (values and not values[0].strip()): + raise PreparationError(f"Malformed Requires-Python metadata in wheel: {wheel.name}") + _validate_requires_python_constraint( + values[0].strip() if values else None, plan.runtime.python_version, wheel + ) + + +def validate_wheel_target_compatibility( + path: Path, + *, + python_version: str, + architecture: str, + requires_python: str | None = None, +) -> None: + """Prove a wheel is installable for PDB's planned Windows target.""" + + if not wheel_matches(path.name, python_version, architecture): + raise PreparationError( + f"Wheel {path.name} is incompatible with CPython {python_version} " + f"on Windows {architecture}." ) + _validate_requires_python_constraint(requires_python, python_version, path) def _validate_wheel_security( @@ -765,11 +796,11 @@ def validate_approved_wheel( f"Artifact version mismatch for {requested_name}: expected {requirement.version}, " f"received {filename_version}." ) - if not wheel_matches(path.name, plan.runtime.python_version, plan.runtime.architecture): - raise PreparationError( - f"Wheel {path.name} is incompatible with CPython {plan.runtime.python_version} " - f"on Windows {plan.runtime.architecture}." - ) + validate_wheel_target_compatibility( + path, + python_version=plan.runtime.python_version, + architecture=plan.runtime.architecture, + ) secret_values = _plan_configured_secret_values(plan) try: @@ -814,6 +845,7 @@ def validate_approved_wheel( f"received {metadata_version}." ) _validate_requires_python(metadata, plan, path) + _validate_approved_requires_dist(metadata, plan, requested_name, filename_version) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: @@ -858,6 +890,229 @@ def validate_artifact_substitution_target(package: str, plan: DeploymentPlan) -> ) +def _approved_package_activated_extras(graph, plan: DeploymentPlan, package_name: str) -> set[str]: + """Return extras definitely requested of an approved package by incoming edges.""" + + activated: set[str] = set() + for edge in graph.edges: + if canonicalize_name(edge.to_package) != package_name: + continue + try: + applicability = target_marker_applicability( + edge.marker, + plan.runtime.python_version, + plan.runtime.architecture, + extra=edge.selected_extra or "", + ) + except TargetMarkerEnvironmentError: + continue + if applicability == TargetMarkerApplicability.APPLIES: + activated.update(canonicalize_name(extra) for extra in edge.requested_dependency_extras) + return activated + + +def _approved_requirement_applies( + requirement: Requirement, plan: DeploymentPlan, activated_extras: set[str] +) -> bool: + """Strictly evaluate an approved wheel's own Core Metadata marker.""" + + marker = str(requirement.marker) if requirement.marker else None + if marker is None: + return True + results: list[TargetMarkerApplicability] = [] + for extra in ["", *sorted(activated_extras)]: + try: + results.append( + target_marker_applicability( + marker, plan.runtime.python_version, plan.runtime.architecture, extra=extra + ) + ) + except TargetMarkerEnvironmentError as exc: + raise PreparationError( + "Approved wheel Requires-Dist marker cannot be proven for the selected " + f"target: {requirement}. {exc}" + ) from exc + if TargetMarkerApplicability.UNPROVABLE in results: + raise PreparationError( + "Approved wheel Requires-Dist marker cannot be proven for the selected target: " + f"{requirement}." + ) + return TargetMarkerApplicability.APPLIES in results + + +def _parent_dependency_presence_proven( + graph, + plan: DeploymentPlan, + parent_name: str, + dependency_name: str, + activated_parent_extras: set[str], +) -> None: + """Require a definitely-applicable lock edge from an approved wheel's parent.""" + + matching = False + unprovable: list[str] = [] + for edge in graph.edges: + if ( + canonicalize_name(edge.from_package) != parent_name + or canonicalize_name(edge.to_package) != dependency_name + ): + continue + if ( + edge.selected_extra + and canonicalize_name(edge.selected_extra) not in activated_parent_extras + ): + continue + matching = True + try: + applicability = target_marker_applicability( + edge.marker, + plan.runtime.python_version, + plan.runtime.architecture, + extra=edge.selected_extra or "", + ) + except TargetMarkerEnvironmentError as exc: + unprovable.append(str(exc)) + continue + if applicability == TargetMarkerApplicability.APPLIES: + return + if applicability == TargetMarkerApplicability.UNPROVABLE: + unprovable.append(edge.marker or "") + if unprovable: + raise PreparationError( + "Approved wheel Requires-Dist presence cannot be proven from a definitely " + f"applicable locked dependency edge for {dependency_name}: {'; '.join(unprovable)}." + ) + if matching: + raise PreparationError( + "Approved wheel Requires-Dist is absent from the selected locked environment: " + f"{dependency_name}; no parent dependency edge definitely applies." + ) + raise PreparationError( + "Approved wheel Requires-Dist is absent from the selected locked environment: " + f"{dependency_name}; no {parent_name} dependency edge exists." + ) + + +def _parent_dependency_extras_proven( + requirement: Requirement, + graph, + plan: DeploymentPlan, + parent_name: str, + activated_parent_extras: set[str], +) -> None: + requested = {canonicalize_name(extra) for extra in requirement.extras} + if not requested: + return + guaranteed: set[str] = set() + for edge in graph.edges: + if ( + canonicalize_name(edge.from_package) != parent_name + or canonicalize_name(edge.to_package) != canonicalize_name(requirement.name) + ): + continue + if ( + edge.selected_extra + and canonicalize_name(edge.selected_extra) not in activated_parent_extras + ): + continue + try: + applicability = target_marker_applicability( + edge.marker, + plan.runtime.python_version, + plan.runtime.architecture, + extra=edge.selected_extra or "", + ) + except TargetMarkerEnvironmentError: + continue + if applicability == TargetMarkerApplicability.APPLIES: + guaranteed.update( + canonicalize_name(extra) for extra in edge.requested_dependency_extras + ) + missing = requested - guaranteed + if missing: + raise PreparationError( + "Approved wheel Requires-Dist dependency extra activation cannot be proven from " + "definitely applicable parent locked dependency edges: " + f"{requirement.name}[{','.join(sorted(missing))}]." + ) + + +def _validate_approved_requires_dist( + metadata, plan: DeploymentPlan, approved_name: str, approved_version: Version +) -> None: + """Prove an approved ``--no-deps`` wheel fits its selected lock environment.""" + + raw_requirements = metadata.get_all("Requires-Dist", []) + if not raw_requirements: + return + graph = plan.lock_graph + if graph is None or not graph.inspected: + raise PreparationError( + "Approved wheel Requires-Dist validation requires an inspected selected uv.lock " + "dependency graph." + ) + activated_extras = _approved_package_activated_extras(graph, plan, approved_name) + for raw in raw_requirements: + try: + requirement = Requirement(raw) + except InvalidRequirement as exc: + raise PreparationError( + f"Approved wheel has malformed Requires-Dist metadata: {raw!r}." + ) from exc + if not _approved_requirement_applies(requirement, plan, activated_extras): + continue + name = canonicalize_name(requirement.name) + if name == approved_name: + if ( + requirement.url + or requirement.extras + or approved_version not in requirement.specifier + ): + raise PreparationError( + "Approved wheel self Requires-Dist is not compatible with its artifact " + f"version: {requirement}." + ) + continue + if requirement.url: + raise PreparationError( + "Approved wheel Requires-Dist direct references are not provable against the " + f"selected locked environment: {requirement.name}." + ) + _parent_dependency_presence_proven( + graph, plan, approved_name, name, activated_extras + ) + candidates = sorted( + ( + dependency + for dependency in graph.dependencies + if canonicalize_name(dependency.name) == name + ), + key=lambda dependency: (Version(dependency.version), dependency.version), + ) + if not candidates: + raise PreparationError( + "Approved wheel Requires-Dist is absent from the selected locked environment: " + f"{requirement.name}." + ) + incompatible = [ + candidate.version + for candidate in candidates + if Version(candidate.version) not in requirement.specifier + ] + if incompatible: + raise PreparationError( + "Approved wheel Requires-Dist cannot be proven for every target-possible locked " + f"version: {requirement}. Incompatible possible versions: " + + ", ".join(sorted(set(incompatible))) + + "." + ) + if requirement.extras: + _parent_dependency_extras_proven( + requirement, graph, plan, approved_name, activated_extras + ) + _validate_dependency_extra_closure(requirement, graph, candidates) + + def _application_requirement_applies(requirement: Requirement, plan: DeploymentPlan) -> bool: """Evaluate Core Metadata markers against the planned Windows target, not this host.""" @@ -1173,11 +1428,11 @@ def validate_application_wheel( f"Application wheel version mismatch: expected {expected_version}, received " f"{filename_version}." ) - if not wheel_matches(path.name, plan.runtime.python_version, plan.runtime.architecture): - raise PreparationError( - f"Application wheel {path.name} is incompatible with CPython " - f"{plan.runtime.python_version} on Windows {plan.runtime.architecture}." - ) + validate_wheel_target_compatibility( + path, + python_version=plan.runtime.python_version, + architecture=plan.runtime.architecture, + ) entry_point = plan.entry_point if entry_point is None: diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 39c577b..313e1ca 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -24,6 +24,7 @@ validate_wheel_installation_layout, validate_wheel_metadata_semantics, validate_wheel_static_safety, + validate_wheel_target_compatibility, ) from python_deployment_builder.generation.structural import ( manifest_artifact_wheel_path, @@ -454,6 +455,27 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=wheel_metadata_failures, ) ) + wheel_target_failures: list[str] = [] + for path in safe_trusted_wheel_paths: + try: + metadata = validate_wheel_metadata_semantics(path) + validate_wheel_target_compatibility( + path, + python_version=manifest.python_version, + architecture=manifest.architecture, + requires_python=metadata.requires_python, + ) + except PreparationError as exc: + wheel_target_failures.append(f"{path.relative_to(root)}: {exc}") + checks.append( + _check( + "WHEEL_TARGET_COMPATIBILITY", + not wheel_target_failures, + "Manifest-declared wheels are compatible with the planned Windows target.", + "A manifest-declared wheel is incompatible with the planned Windows target.", + evidence=wheel_target_failures, + ) + ) wheel_security_failures: list[str] = [] for path in safe_trusted_wheel_paths: try: diff --git a/tests/test_generation.py b/tests/test_generation.py index 13c5225..886b459 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -35,6 +35,7 @@ validate_approved_wheel, validate_artifact_set, validate_combined_wheel_installation_paths, + validate_wheel_target_compatibility, ) from python_deployment_builder.generation.cmd import parse_certutil_sha256 from python_deployment_builder.generation.generator import ( @@ -100,6 +101,7 @@ def _make_wheel( dist_info: str | None = None, requires_python: str | None = None, requires_python_values: list[str] | None = None, + requires_dist_values: list[str] | None = None, wheel_version: str = "1.0", ) -> Path: normalized = name.replace("-", "_") @@ -115,6 +117,7 @@ def _make_wheel( or ([requires_python] if requires_python else []) ) ) + + "".join(f"Requires-Dist: {value}\n" for value in requires_dist_values or []) + "\n" ), f"{dist_info}/WHEEL": ( @@ -1002,6 +1005,81 @@ def test_approved_wheel_uses_shared_dist_info_identity_validation(tmp_path: Path assert validate_approved_wheel(f"proxy-tools={historical}", plan)[0].filename == historical.name +@pytest.mark.parametrize( + ("requires_dist", "helper_version", "edge", "error"), + [ + (["helper>=1"], "1.0", True, None), + (["helper>=2"], "1.0", True, "every target-possible"), + (["helper>=1"], "1.0", False, "no proxy-tools dependency edge"), + ], +) +def test_approved_wheel_requires_dist_uses_approved_parent_lock_edges( + tmp_path: Path, + requires_dist: list[str], + helper_version: str, + edge: bool, + error: str | None, +) -> None: + plan = _plan("optional_map_app", ["map"]).model_copy(deep=True) + assert plan.lock_graph is not None + plan.lock_graph.dependencies.append( + LockedDependency( + name="helper", + version=helper_version, + direct=False, + artifact=ArtifactAvailability( + compatible_wheel_available=True, + source_distribution_available=False, + policy="wheel_usable", + ), + ) + ) + if edge: + plan.lock_graph.edges.append( + DependencyEdge(from_package="proxy-tools", to_package="helper") + ) + wheel = _make_wheel(tmp_path, requires_dist_values=requires_dist) + + if error is None: + assert validate_approved_wheel(f"proxy-tools={wheel}", plan)[0].filename == wheel.name + else: + with pytest.raises(PreparationError, match=error): + validate_approved_wheel(f"proxy-tools={wheel}", plan) + + +@pytest.mark.parametrize( + ("filename", "requires_python", "error"), + [ + ("proxy_tools-0.1.0-py3-none-any.whl", None, None), + ("proxy_tools-0.1.0-cp311-cp311-win_amd64.whl", None, "incompatible"), + ("proxy_tools-0.1.0-py3-none-any.whl", ">=3.13", "Requires-Python"), + ], +) +def test_wheel_target_compatibility_uses_manifest_precision_rules( + tmp_path: Path, filename: str, requires_python: str | None, error: str | None +) -> None: + wheel = _make_wheel(tmp_path, requires_python=requires_python) + target = tmp_path / filename + if target != wheel: + target.write_bytes(wheel.read_bytes()) + + if error is None: + validate_wheel_target_compatibility( + target, + python_version="3.12", + architecture="x86_64", + requires_python=requires_python, + ) + else: + with pytest.raises(PreparationError, match=error): + validate_wheel_target_compatibility( + target, + python_version="3.12", + architecture="x86_64", + requires_python=requires_python, + ) + + @pytest.mark.parametrize( ("requires_dist", "locked", "error"), [ diff --git a/tests/test_metadata.py b/tests/test_metadata.py index a51468a..364b8c5 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -568,3 +568,47 @@ def test_nonliteral_dynamic_version_attr_remains_unresolved(tmp_path: Path) -> N encoding="utf-8", ) assert inspect_metadata(tmp_path).project.version is None + + +@pytest.mark.parametrize( + ("configuration", "relative"), + [ + ("[tool.setuptools]\npackage-dir = {'' = 'lib'}\n", "lib/app/__init__.py"), + ("[tool.setuptools.packages.find]\nwhere = ['python_src']\n", "python_src/app/__init__.py"), + ("[tool.setuptools]\npackage-dir = {app = 'lib'}\n", "lib/__init__.py"), + ], +) +def test_literal_dynamic_version_attr_uses_safe_setuptools_package_roots( + tmp_path: Path, configuration: str, relative: str +) -> None: + source = tmp_path / relative + source.parent.mkdir(parents=True) + source.write_text("__version__ = '1.2.3'\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'dynamic-root-demo'\ndynamic = ['version']\n" + + configuration + + "[tool.setuptools.dynamic]\nversion = {attr = 'app.__version__'}\n", + encoding="utf-8", + ) + + metadata = inspect_metadata(tmp_path) + + assert metadata.project.version == "1.2.3" + assert relative in metadata.project.metadata_files + + +def test_literal_dynamic_version_attr_uses_parent_package_dir_mapping(tmp_path: Path) -> None: + source = tmp_path / "lib/sub/__init__.py" + source.parent.mkdir(parents=True) + source.write_text("__version__ = '1.2.3'\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'dynamic-parent-root-demo'\ndynamic = ['version']\n" + "[tool.setuptools]\npackage-dir = {app = 'lib'}\n" + "[tool.setuptools.dynamic]\nversion = {attr = 'app.sub.__version__'}\n", + encoding="utf-8", + ) + + metadata = inspect_metadata(tmp_path) + + assert metadata.project.version == "1.2.3" + assert "lib/sub/__init__.py" in metadata.project.metadata_files From 8fd875031c4c493291300202d8f9c8440f1c30ef Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:01:06 -0400 Subject: [PATCH 43/59] Validate staged wheel dependencies semantically --- .../generation/artifacts.py | 46 +++- .../validation/static.py | 110 ++++++++++ tests/test_generation.py | 202 +++++++++++++++++- 3 files changed, 346 insertions(+), 12 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 4ffe135..e589b91 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -69,6 +69,7 @@ class WheelStaticMetadata: filename_tags: frozenset[str] declared_tags: frozenset[str] requires_python: str | None + requires_dist: tuple[str, ...] _WINDOWS_FORBIDDEN_COMPONENT_CHARACTERS = frozenset('<>:"|?*') _WINDOWS_RESERVED_DEVICE_BASENAMES = frozenset( @@ -655,6 +656,7 @@ def validate_wheel_metadata_semantics(path: Path) -> WheelStaticMetadata: filename_tags=frozenset(filename_tag_values), declared_tags=frozenset(declared_tags), requires_python=(requires_python_values[0].strip() if requires_python_values else None), + requires_dist=tuple(metadata.get_all("Requires-Dist", [])), ) @@ -791,7 +793,14 @@ def validate_approved_wheel( f"{requested_name} ({', '.join(sorted(versions))})." ) requirement = requirements[0] - if str(filename_version) != requirement.version: + try: + requirement_version = Version(requirement.version) + except InvalidVersion as exc: + raise PreparationError( + f"Approved artifact requirement version is invalid for {requested_name}: " + f"{requirement.version!r}." + ) from exc + if filename_version != requirement_version: raise PreparationError( f"Artifact version mismatch for {requested_name}: expected {requirement.version}, " f"received {filename_version}." @@ -839,13 +848,21 @@ def validate_approved_wheel( raise PreparationError( f"Wheel metadata name mismatch: expected {requested_name}, received {metadata_name}." ) - if metadata_version != requirement.version: + try: + metadata_version_value = Version(metadata_version) + except InvalidVersion as exc: + raise PreparationError( + f"Wheel metadata version is invalid for {requested_name}: {metadata_version!r}." + ) from exc + if metadata_version_value != requirement_version: raise PreparationError( f"Wheel metadata version mismatch: expected {requirement.version}, " f"received {metadata_version}." ) _validate_requires_python(metadata, plan, path) - _validate_approved_requires_dist(metadata, plan, requested_name, filename_version) + validate_approved_requires_dist( + metadata.get_all("Requires-Dist", []), plan, requested_name, filename_version + ) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} if not declared_tags or not filename_tag_values <= declared_tags: @@ -1037,12 +1054,14 @@ def _parent_dependency_extras_proven( ) -def _validate_approved_requires_dist( - metadata, plan: DeploymentPlan, approved_name: str, approved_version: Version +def validate_approved_requires_dist( + raw_requirements: tuple[str, ...] | list[str], + plan: DeploymentPlan, + approved_name: str, + approved_version: Version, ) -> None: """Prove an approved ``--no-deps`` wheel fits its selected lock environment.""" - raw_requirements = metadata.get_all("Requires-Dist", []) if not raw_requirements: return graph = plan.lock_graph @@ -1298,12 +1317,14 @@ def _validate_dependency_extra_closure(requirement: Requirement, graph, candidat pending.append(child) -def _validate_application_requires_dist( - metadata, plan: DeploymentPlan, application_name: str, application_version: Version +def validate_application_requires_dist( + raw_requirements: tuple[str, ...] | list[str], + plan: DeploymentPlan, + application_name: str, + application_version: Version, ) -> None: """Prove every applicable first-party wheel requirement is in the selected lock graph.""" - raw_requirements = metadata.get_all("Requires-Dist", []) if not raw_requirements: return graph = plan.lock_graph @@ -1482,8 +1503,11 @@ def validate_application_wheel( raise PreparationError("Application wheel METADATA version is wrong.") _validate_requires_python(metadata, plan, path) if validate_locked_dependencies: - _validate_application_requires_dist( - metadata, plan, expected_name, expected_version_value + validate_application_requires_dist( + metadata.get_all("Requires-Dist", []), + plan, + expected_name, + expected_version_value, ) declared_tags = _require_wheel_metadata(wheel_metadata, wheel=path) filename_tag_values = {str(item) for item in filename_tags} diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 313e1ca..082016b 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -8,9 +8,11 @@ import platform import re import socket +import tomllib import zipfile from datetime import UTC, datetime from pathlib import Path, PurePosixPath +from types import SimpleNamespace from packaging.utils import canonicalize_name from packaging.version import InvalidVersion, Version @@ -20,6 +22,8 @@ from python_deployment_builder.generation.artifacts import ( configured_secret_values, installed_wheel_member_paths, + validate_application_requires_dist, + validate_approved_requires_dist, validate_combined_wheel_installation_paths, validate_wheel_installation_layout, validate_wheel_metadata_semantics, @@ -39,6 +43,7 @@ ValidationHost, ValidationReport, ) +from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.security_policy import ( FORBIDDEN_SHELL, TextContentEncodingError, @@ -117,6 +122,51 @@ def _load_manifest(root: Path) -> DeploymentManifest: raise KitValidationError(f"Deployment manifest is invalid: {exc}") from exc +def _static_lock_root_name(root: Path, manifest: DeploymentManifest) -> str | None: + """Find the staged lock root without re-assessing source packaging metadata.""" + + if manifest.application_artifact is not None: + return manifest.application_artifact.distribution_name + try: + with (root / "pyproject.toml").open("rb") as handle: + document = tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError): + return None + project = document.get("project") + name = project.get("name") if isinstance(project, dict) else None + return name if isinstance(name, str) and name.strip() else None + + +def _static_lock_plan(root: Path, manifest: DeploymentManifest): + """Build the transient staged-lock proof context used by wheel validators.""" + + application_name = _static_lock_root_name(root, manifest) + if application_name is None: + raise PreparationError( + "Trusted wheel Requires-Dist validation cannot identify the staged lock root." + ) + graph = inspect_uv_lock( + root, + application_name, + manifest.python_version, + manifest.architecture, + manifest.selected_extras, + ) + if not graph.inspected: + detail = "; ".join(graph.limitations) or "uv.lock could not be inspected." + raise PreparationError( + "Trusted wheel Requires-Dist validation requires an inspected staged uv.lock: " + f"{detail}" + ) + return SimpleNamespace( + runtime=SimpleNamespace( + python_version=manifest.python_version, + architecture=manifest.architecture, + ), + lock_graph=graph, + ) + + def _manual_gui_checks(manifest: DeploymentManifest) -> list[ManualValidationItem]: checks = [ ManualValidationItem( @@ -422,6 +472,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR artifact.version, ) wheel_metadata_failures: list[str] = [] + wheel_metadata_by_path = {} for path in safe_trusted_wheel_paths: relative = path.relative_to(root).as_posix() try: @@ -444,6 +495,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR raise PreparationError( f"Wheel METADATA version does not match manifest artifact: {path.name}" ) + wheel_metadata_by_path[path] = metadata except PreparationError as exc: wheel_metadata_failures.append(f"{relative}: {exc}") checks.append( @@ -476,6 +528,64 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=wheel_target_failures, ) ) + wheel_dependency_failures: list[str] = [] + dependency_wheels = [ + path + for path in safe_trusted_wheel_paths + if path in wheel_metadata_by_path and wheel_metadata_by_path[path].requires_dist + ] + if dependency_wheels: + try: + static_plan = _static_lock_plan(root, manifest) + except (PreparationError, InvalidVersion, ValueError) as exc: + wheel_dependency_failures.append(str(exc)) + else: + application_relative = ( + manifest_artifact_wheel_path( + "application", manifest.application_artifact.filename + ) + if manifest.application_artifact is not None + else None + ) + approved_by_relative = { + relative: artifact + for artifact in manifest.approved_artifacts + if (relative := manifest_artifact_wheel_path("wheels", artifact.filename)) + } + for path in dependency_wheels: + relative = path.relative_to(root).as_posix() + try: + metadata = wheel_metadata_by_path[path] + if relative == application_relative: + if manifest.application_artifact is None: + raise PreparationError("Application wheel is not manifest-owned.") + validate_application_requires_dist( + metadata.requires_dist, + static_plan, + canonicalize_name(manifest.application_artifact.distribution_name), + metadata.version, + ) + continue + artifact = approved_by_relative.get(relative) + if artifact is None: + raise PreparationError("Wheel is not an exact manifest-owned artifact.") + validate_approved_requires_dist( + metadata.requires_dist, + static_plan, + canonicalize_name(artifact.distribution_name), + metadata.version, + ) + except (PreparationError, InvalidVersion, ValueError) as exc: + wheel_dependency_failures.append(f"{relative}: {exc}") + checks.append( + _check( + "WHEEL_DEPENDENCY_COMPATIBILITY", + not wheel_dependency_failures, + "Manifest-declared wheel dependencies are proven against the staged uv.lock.", + "A manifest-declared wheel dependency is not proven by the staged uv.lock.", + evidence=wheel_dependency_failures, + ) + ) wheel_security_failures: list[str] = [] for path in safe_trusted_wheel_paths: try: diff --git a/tests/test_generation.py b/tests/test_generation.py index 886b459..ca2acb8 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -102,6 +102,7 @@ def _make_wheel( requires_python: str | None = None, requires_python_values: list[str] | None = None, requires_dist_values: list[str] | None = None, + metadata_version: str | None = None, wheel_version: str = "1.0", ) -> Path: normalized = name.replace("-", "_") @@ -109,7 +110,8 @@ def _make_wheel( dist_info = dist_info or f"{normalized}-{version}.dist-info" files = { f"{dist_info}/METADATA": ( - f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + f"Metadata-Version: 2.1\nName: {name}\nVersion: " + f"{metadata_version if metadata_version is not None else version}\n" + "".join( f"Requires-Python: {value}\n" for value in ( @@ -3824,6 +3826,157 @@ def _update_indexed_hashes(kit: Path, *relatives: str) -> None: index_path.write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") +def _refresh_manifest_wheel_hash(kit: Path, relative: str, *, approved: bool = False) -> None: + """Keep a deliberately re-authored static-kit test internally hash-consistent.""" + + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + sha256 = hashlib.sha256((kit / relative).read_bytes()).hexdigest() + if approved: + manifest["approved_artifacts"][0]["sha256"] = sha256 + else: + manifest["application_artifact"]["sha256"] = sha256 + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, relative, "deployment/manifest.json") + + +def test_static_validation_proves_application_wheel_requires_dist_against_staged_lock( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "uv.lock").write_text( + """version = 1 +revision = 3 +[[package]] +name = "mapped-app" +version = "1.2.3" +source = { virtual = "." } +dependencies = [{ name = "helper", version = "1.0" }] +[[package]] +name = "helper" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +wheels = [{ url = "https://example.invalid/helper-1.0-py3-none-any.whl" }] +""", + encoding="utf-8", + ) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + wheel = _make_application_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + application_wheel=wheel, + bootstrap_mode="online_cmd", + ) + relative = f"deployment/application/{wheel.name}" + _rewrite_application_wheel( + kit / relative, + replacements={ + "mapped_app-1.2.3.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: mapped-app\nVersion: 1.2.3\n" + "Requires-Dist: helper>=2\n\n" + ) + }, + ) + _refresh_manifest_wheel_hash(kit, relative) + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "WHEEL_DEPENDENCY_COMPATIBILITY" and item.status.value == "FAIL" + for item in report.static_checks + ) + + +def test_static_validation_proves_approved_wheel_requires_dist_against_staged_lock( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + shutil.copytree(FIXTURES / "optional_map_app", source) + lock_path = source / "uv.lock" + lock_path.write_text( + lock_path.read_text(encoding="utf-8") + + """ +[[package]] +name = "helper" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +wheels = [{ url = "https://example.invalid/helper-1.0-py3-none-any.whl" }] +""", + encoding="utf-8", + ) + # Add the approved package's own selected lock edge, rather than letting an + # unrelated helper occurrence satisfy the later static proof. + proxy_package = ( + 'name = "proxy-tools"\nversion = "0.1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\nsdist' + ) + proxy_with_helper = ( + 'name = "proxy-tools"\nversion = "0.1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'dependencies = [{ name = "helper", version = "1.0" }]\nsdist' + ) + lock_path.write_text( + lock_path.read_text(encoding="utf-8").replace(proxy_package, proxy_with_helper), + encoding="utf-8", + ) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + relative = f"deployment/wheels/{artifact.name}" + _rewrite_application_wheel( + kit / relative, + replacements={ + "proxy_tools-0.1.0.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: proxy-tools\nVersion: 0.1.0\n" + "Requires-Dist: helper>=2\n\n" + ) + }, + ) + _refresh_manifest_wheel_hash(kit, relative, approved=True) + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "WHEEL_DEPENDENCY_COMPATIBILITY" and item.status.value == "FAIL" + for item in report.static_checks + ) + + def test_static_validation_rejects_indexed_unvalidated_staged_wheel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -6497,6 +6650,53 @@ def test_approved_wheel_rejects_wrong_name_version_and_metadata(tmp_path: Path) validate_approved_wheel(f"proxy-tools={wrong}", plan) +def test_approved_wheel_compares_core_metadata_version_with_pep440_semantics( + tmp_path: Path, +) -> None: + plan = _plan("optional_map_app", ["map"]).model_copy(deep=True) + assert plan.lock_graph is not None + plan.lock_graph.artifact_requirements = [ + DeploymentArtifactRequirement( + package="proxy-tools", + version="1.0.0", + action="developer_wheel_required", + reason="test", + ) + ] + wheel = _make_wheel(tmp_path, version="1.0.0", metadata_version="1.0") + + approved, _ = validate_approved_wheel(f"proxy-tools={wheel}", plan) + + # Retain the locked spelling in the manifest while comparing identity with Version. + assert approved.version == "1.0.0" + + +@pytest.mark.parametrize("metadata_version", ["not-a-version", ""]) +def test_approved_wheel_rejects_invalid_core_metadata_version_cleanly( + tmp_path: Path, metadata_version: str +) -> None: + wheel = _make_wheel(tmp_path, metadata_version=metadata_version) + + with pytest.raises(PreparationError, match="metadata version is invalid|Malformed METADATA"): + validate_approved_wheel(f"proxy-tools={wheel}", _plan("optional_map_app", ["map"])) + + +def test_approved_wheel_rejects_invalid_requirement_version_cleanly(tmp_path: Path) -> None: + plan = _plan("optional_map_app", ["map"]).model_copy(deep=True) + assert plan.lock_graph is not None + plan.lock_graph.artifact_requirements = [ + DeploymentArtifactRequirement( + package="proxy-tools", + version="not-a-version", + action="developer_wheel_required", + reason="test", + ) + ] + + with pytest.raises(PreparationError, match="requirement version is invalid"): + validate_approved_wheel(f"proxy-tools={_make_wheel(tmp_path)}", plan) + + @pytest.mark.parametrize( ("requires_python", "accepted"), [ From a44b0793ae08ca3502298a5c607c48de5cc76b4d Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:04:17 -0400 Subject: [PATCH 44/59] Close wheel policy and runtime resource gaps --- .../analysis/resources.py | 100 +++++++- .../generation/artifacts.py | 89 ++++--- .../generation/generator.py | 34 +++ .../planning/planner.py | 34 +++ .../validation/static.py | 16 ++ tests/test_analysis_scope.py | 185 ++++++++++++++ tests/test_generation.py | 235 +++++++++++++++++- tests/test_planning.py | 72 ++++++ 8 files changed, 726 insertions(+), 39 deletions(-) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index bfba216..324a848 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -511,17 +511,21 @@ def bind_pattern(target: ast.AST, values: list[ast.AST]) -> None: def _resource_import_bindings( tree: ast.AST, -) -> tuple[set[str], set[str], dict[str, str]]: - """Return proven importlib-resources module, ``files``, and read bindings.""" +) -> tuple[set[str], set[str], dict[str, str], set[str], set[str]]: + """Return proven importlib.resources and pkgutil resource bindings.""" modules: set[str] = set() files: set[str] = set() reads: dict[str, str] = {} + pkgutil_modules: set[str] = set() + pkgutil_get_data: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: if alias.name == "importlib.resources": modules.add(alias.asname or alias.name) + elif alias.name == "pkgutil": + pkgutil_modules.add(alias.asname or alias.name) elif isinstance(node, ast.ImportFrom): if node.module == "importlib": for alias in node.names: @@ -533,7 +537,11 @@ def _resource_import_bindings( files.add(alias.asname or alias.name) elif alias.name in _LEGACY_IMPORTLIB_RESOURCE_READS: reads[alias.asname or alias.name] = alias.name - return modules, files, reads + elif node.module == "pkgutil": + for alias in node.names: + if alias.name == "get_data": + pkgutil_get_data.add(alias.asname or alias.name) + return modules, files, reads, pkgutil_modules, pkgutil_get_data def _resource_package_roots( @@ -541,6 +549,8 @@ def _resource_package_roots( package: str, source_roots: list[str], project: PackagingAssessment | None, + *, + require_initializer: bool = False, ) -> list[Path]: """Resolve a literal package anchor only through safe in-repository roots.""" @@ -553,7 +563,11 @@ def _resource_package_roots( resolved_root = root.resolve() safe: list[Path] = [] for candidate in candidates: - if candidate.is_symlink() or not candidate.is_dir(): + if ( + candidate.is_symlink() + or not candidate.is_dir() + or (require_initializer and not (candidate / "__init__.py").is_file()) + ): continue try: candidate.resolve().relative_to(resolved_root) @@ -629,6 +643,7 @@ def _resource_package_anchor_values( project: PackagingAssessment | None, assignments: dict[str, ast.AST], returns: dict[str, ast.AST], + require_initializer: bool = False, ) -> list[str]: package_values = _path_values( node, @@ -642,11 +657,63 @@ def _resource_package_anchor_values( return [ package_root.relative_to(root.resolve()).as_posix() for package_root in _resource_package_roots( - root, package_values[0], source_roots, project + root, + package_values[0], + source_roots, + project, + require_initializer=require_initializer, ) ] +def _pkgutil_resource_path_values( + node: ast.Call, + *, + root: Path, + source_path: Path, + source_roots: list[str], + project: PackagingAssessment | None, + assignments: dict[str, ast.AST], + returns: dict[str, ast.AST], + module_bindings: set[str], + get_data_bindings: set[str], +) -> list[str] | None: + """Resolve a proven filesystem-package ``pkgutil.get_data`` read.""" + + name = _qualified_name(node.func) + if name not in get_data_bindings and not any( + name == f"{binding}.get_data" for binding in module_bindings + ): + return None + if len(node.args) != 2 or node.keywords: + return [] + package_roots = _resource_package_anchor_values( + node.args[0], + root=root, + source_path=source_path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + require_initializer=True, + ) + members = _path_values( + node.args[1], + root=root, + source_path=source_path, + assignments=assignments, + returns=returns, + ) + if ( + not package_roots + or len(members) != 1 + or "\\" in members[0] + or not _safe_resource_member(members) + ): + return [] + return _combine_paths(package_roots, members) + + def _legacy_importlib_resource_path_values( node: ast.Call, *, @@ -1103,10 +1170,27 @@ def _literal_evidence( continue lines = source.splitlines() assignments, returns = _bindings(tree) - module_bindings, files_bindings, read_bindings = _resource_import_bindings(tree) + ( + module_bindings, + files_bindings, + read_bindings, + pkgutil_modules, + pkgutil_get_data, + ) = _resource_import_bindings(tree) for node in ast.walk(tree): if not isinstance(node, ast.Call): continue + pkgutil_resource = _pkgutil_resource_path_values( + node, + root=root, + source_path=path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + module_bindings=pkgutil_modules, + get_data_bindings=pkgutil_get_data, + ) legacy_resource = _legacy_importlib_resource_path_values( node, root=root, @@ -1119,7 +1203,9 @@ def _literal_evidence( read_bindings=read_bindings, ) uses: list[tuple[ast.AST, str, list[str] | None, str]] = [] - if legacy_resource is not None: + if pkgutil_resource is not None: + uses.append((node, "read", pkgutil_resource, "pkgutil.get_data()")) + elif legacy_resource is not None: function, values = legacy_resource # Detect these before generic ``receiver.read_text()`` handling; # their receiver is an importlib module, not a filesystem path. diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index e589b91..ed83800 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -750,6 +750,63 @@ def _validate_wheel_security( ) +def _validate_application_wheel_content_policy( + members: dict[str, zipfile.ZipInfo], wheel_metadata +) -> None: + """Enforce the bounded first-party pure-Python application policy.""" + + if wheel_metadata.get("Root-Is-Purelib", "").strip().lower() != "true": + raise PreparationError( + "Application wheels containing platform/native installation content require " + "explicit future project evidence and are not accepted by package mode." + ) + cache_members = [ + name + for name in members + if name.lower().endswith((".pyc", ".pyo")) + or any( + re.fullmatch(r"__pycache__(?:\s*\(\d+\))?", part, re.IGNORECASE) + for part in PurePosixPath(name).parts + ) + ] + native_members = [ + name + for name in members + if PurePosixPath(name).suffix.lower() + in {".dll", ".pyd", ".so", ".dylib", ".exe", ".lib"} + ] + if cache_members: + raise PreparationError( + "Application wheel contains Python runtime cache files: " + + ", ".join(cache_members) + ) + if native_members: + raise PreparationError( + "Application wheel contains unexpected native binaries: " + + ", ".join(native_members) + ) + + +def validate_application_wheel_content_policy(path: Path) -> None: + """Validate first-party content rules without source or plan state.""" + + try: + with zipfile.ZipFile(path) as bundle: + members = _member_map(_safe_wheel_members(bundle)) + _metadata_name, wheel_name, _record_name = _dist_info_members(members, path) + if wheel_name not in members: + raise PreparationError( + f"Wheel is missing required WHEEL metadata: {path.name}" + ) + wheel_metadata = _metadata_message( + bundle.read(members[wheel_name]), label="WHEEL", wheel=path + ) + _require_wheel_metadata(wheel_metadata, wheel=path) + _validate_application_wheel_content_policy(members, wheel_metadata) + except zipfile.BadZipFile as exc: + raise PreparationError(f"Malformed application wheel: {path.name}") from exc + + def validate_approved_wheel( value: str, plan: DeploymentPlan, @@ -1515,37 +1572,7 @@ def validate_application_wheel( raise PreparationError( f"Application wheel tag metadata does not match its filename: {path.name}" ) - if wheel_metadata.get("Root-Is-Purelib", "").strip().lower() != "true": - raise PreparationError( - "Application wheels containing platform/native installation content require " - "explicit future project evidence and are not accepted by package mode." - ) - - cache_members = [ - name - for name in names - if name.lower().endswith((".pyc", ".pyo")) - or any( - re.fullmatch(r"__pycache__(?:\s*\(\d+\))?", part, re.IGNORECASE) - for part in PurePosixPath(name).parts - ) - ] - native_members = [ - name - for name in names - if PurePosixPath(name).suffix.lower() - in {".dll", ".pyd", ".so", ".dylib", ".exe", ".lib"} - ] - if cache_members: - raise PreparationError( - "Application wheel contains Python runtime cache files: " - + ", ".join(cache_members) - ) - if native_members: - raise PreparationError( - "Application wheel contains unexpected native binaries: " - + ", ".join(native_members) - ) + _validate_application_wheel_content_policy(members, wheel_metadata) secret_values = _plan_configured_secret_values(plan) _validate_wheel_security( bundle, diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index aa2c359..55caed4 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -879,6 +879,40 @@ def generate_deployment_kit( f"Deployment planning is blocked: {workspace_code}. " "M6.1 standalone deployment does not preserve or install uv workspace members." ) + runtime_sync_blockers = { + code + for code in plan.risk_gate.blocking_codes + if code == "RUNTIME_SYNC_METADATA_UNSUPPORTED" + } + if runtime_sync_blockers: + runtime_sync_code = next(iter(runtime_sync_blockers)) + if dry_run: + preview = _preview( + plan, + output_root, + dry_run=True, + bootstrap_mode=bootstrap_mode, + system_certs=system_certs, + prepare_lock=prepare_lock, + approved=[], + application_artifact=None, + staging_source_paths=[], + ) + preview.developer_actions.insert( + 0, + f"Stop: {runtime_sync_code} prevents immutable dependency synchronization.", + ) + return GenerationResult( + output_directory=str(output_root), + dry_run=True, + generated=False, + preview=preview, + ) + raise PreparationError( + f"Deployment planning is blocked: {runtime_sync_code}. The pinned uv 0.12.5 " + "lock workflow does not consume backend-only setup.cfg/setup.py dependency " + "metadata, and PDB will not execute project metadata on the end-user system." + ) # An escaping setuptools root is outside both the source/provenance # boundary and this kit's standalone staging model. Stop before lock # preparation, application-wheel work, artifact work, or output writes. diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 7a43aab..32f12b9 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -530,6 +530,26 @@ def create_deployment_plan( applicable_dependencies = selected_dependencies( assessment, selected_extras, python_version, architecture ) + backend_only_dependencies = sorted( + dependency.distribution_name + for dependency in applicable_dependencies + if any( + evidence.file in {"setup.cfg", "setup.py"} + for evidence in dependency.evidence + ) + and not any(evidence.file == "pyproject.toml" for evidence in dependency.evidence) + ) + if backend_only_dependencies: + # Exact uv 0.12.5 evidence shows both ``uv lock`` and end-user sync + # ignore setup.cfg/setup.py dependency declarations. Copying those + # backend files cannot make the prepared lock authoritative, and + # executing project metadata on the end-user system is outside M6.1. + mode_blockers.append( + "RUNTIME_SYNC_METADATA_UNSUPPORTED: uv 0.12.5 lock/sync does not consume " + "setup.cfg or setup.py dependency declarations, so the immutable deployment lock " + "cannot represent selected dependencies: " + + ", ".join(backend_only_dependencies) + ) extras = build_extra_plans(assessment, selected_extras, python_version, architecture) configuration = [ ConfigurationPlan( @@ -617,6 +637,20 @@ def create_deployment_plan( ), ] gate = _risk_gate(assessment) + if backend_only_dependencies: + gate = gate.model_copy( + update={ + "outcome": "block", + "blocking_codes": sorted( + {*gate.blocking_codes, "RUNTIME_SYNC_METADATA_UNSUPPORTED"} + ), + "rationale": ( + gate.rationale + + " Backend-only dependency metadata is not representable by the pinned " + "uv lock workflow." + ), + } + ) fingerprint = hashlib.sha256( json.dumps(sorted(selected_extras), separators=(",", ":")).encode() ).hexdigest() diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 082016b..fac7588 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -23,6 +23,7 @@ configured_secret_values, installed_wheel_member_paths, validate_application_requires_dist, + validate_application_wheel_content_policy, validate_approved_requires_dist, validate_combined_wheel_installation_paths, validate_wheel_installation_layout, @@ -528,6 +529,21 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=wheel_target_failures, ) ) + application_content_failures: list[str] = [] + if application_wheel is not None and application_wheel in safe_trusted_wheel_paths: + try: + validate_application_wheel_content_policy(application_wheel) + except PreparationError as exc: + application_content_failures.append(str(exc)) + checks.append( + _check( + "APPLICATION_WHEEL_CONTENT_POLICY", + not application_content_failures, + "The first-party application wheel satisfies the pure-Python content policy.", + "The first-party application wheel violates the pure-Python content policy.", + evidence=application_content_failures, + ) + ) wheel_dependency_failures: list[str] = [] dependency_wheels = [ path diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 7b2d523..60581a7 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -1039,6 +1039,191 @@ def test_legacy_importlib_resources_rejects_dynamic_and_escaping_members(tmp_pat assert any(item.status == FindingStatus.NEEDS_VALIDATION for item in assessment.resources) +@pytest.mark.parametrize( + ("imports", "resource_call"), + [ + ("import pkgutil", "pkgutil.get_data('app', 'defaults.json')"), + ("import pkgutil as pu", "pu.get_data('app', 'defaults.json')"), + ("from pkgutil import get_data", "get_data('app', 'defaults.json')"), + ( + "from pkgutil import get_data as resource_data", + "resource_data('app', 'defaults.json')", + ), + ], +) +def test_pkgutil_get_data_promotes_concrete_package_resource( + tmp_path: Path, imports: str, resource_call: str +) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text('{"default": true}\n', encoding="utf-8") + (package / "main.py").write_text( + f"{imports}\n\ndef main():\n return {resource_call}\n", encoding="utf-8" + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname = 'pkgutil-app'\nversion = '1.0'\n" + "[project.scripts]\npkgutil-app = 'app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + resource = next(item for item in assessment.resources if item.path == "src/app/defaults.json") + + assert resource.packaging_status == "repository_adjacent" + assert "pkgutil.get_data" in resource.evidence[-1].detail + assert resource.path in _staging_files(tmp_path, assessment, plan, include=True) + + +def test_pkgutil_get_data_supports_nested_members_and_package_dir_mapping( + tmp_path: Path, +) -> None: + package = tmp_path / "lib/app" + resource_path = package / "templates/defaults.json" + resource_path.parent.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + resource_path.write_text("{}\n", encoding="utf-8") + (package / "main.py").write_text( + "import pkgutil\ndef main():\n" + " return pkgutil.get_data('app', 'templates/defaults.json')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='pkgutil-mapped'\nversion='1.0'\n" + "[project.scripts]\npkgutil-mapped='app.main:main'\n" + "[tool.setuptools]\npackage-dir={\"\"='lib'}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + assert any(item.path == "lib/app/templates/defaults.json" for item in assessment.resources) + + +@pytest.mark.parametrize( + ("anchor", "entry_point", "package_directories", "physical_package", "resource"), + [ + ("app", "app.main:main", "{app='code'}", "code", "code/defaults.json"), + ( + "app.sub", + "app.sub.main:main", + "{app='lib'}", + "lib/sub", + "lib/sub/defaults.json", + ), + ], +) +def test_pkgutil_get_data_uses_exact_and_parent_package_dir_mappings( + tmp_path: Path, + anchor: str, + entry_point: str, + package_directories: str, + physical_package: str, + resource: str, +) -> None: + package = tmp_path / physical_package + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text( + "from pkgutil import get_data\n" + f"def main(): return get_data('{anchor}', 'defaults.json')\n", + encoding="utf-8", + ) + (package / "defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='pkgutil-exact-mapped'\nversion='1.0'\n" + f"[project.scripts]\npkgutil-exact-mapped='{entry_point}'\n" + "[tool.setuptools]\n" + f"package-dir={package_directories}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + assert any(item.path == resource for item in assessment.resources) + + +def test_pkgutil_get_data_keeps_declared_package_data_package_backed( + tmp_path: Path, +) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text( + "import pkgutil\ndef main(): return pkgutil.get_data('app', 'defaults.json')\n", + encoding="utf-8", + ) + (package / "defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='pkgutil-package-data'\nversion='1.0'\n" + "[project.scripts]\npkgutil-package-data='app.main:main'\n" + "[tool.setuptools]\npackages=['app']\npackage-dir={\"\"='src'}\n" + "[tool.setuptools.package-data]\napp=['defaults.json']\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + resource = next(item for item in assessment.resources if item.path == "src/app/defaults.json") + + assert resource.packaging_status == "packaged" + + +def test_unrelated_get_data_function_does_not_receive_pkgutil_semantics( + tmp_path: Path, +) -> None: + package = tmp_path / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "defaults.json").write_text("{}\n", encoding="utf-8") + (package / "main.py").write_text( + "def get_data(package, resource): return None\n" + "def main(): return get_data('app', 'defaults.json')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='unrelated-get-data'\nversion='1.0'\n" + "[project.scripts]\nunrelated-get-data='app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + assert all(item.path != "src/app/defaults.json" for item in assessment.resources) + + +def test_pkgutil_get_data_rejects_dynamic_unsafe_and_namespace_only_resources( + tmp_path: Path, +) -> None: + namespace = tmp_path / "src/ns" + namespace.mkdir(parents=True) + (namespace / "defaults.json").write_text("{}\n", encoding="utf-8") + (namespace / "main.py").write_text( + "from pkgutil import get_data\n" + "def main(name='defaults.json'):\n" + " get_data('ns', name)\n" + " get_data('ns', '../defaults.json')\n" + " return get_data('ns', 'C:\\\\outside.json')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='pkgutil-unsafe'\nversion='1.0'\n" + "[project.scripts]\npkgutil-unsafe='ns.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + assert all(item.path != "src/ns/defaults.json" for item in assessment.resources) + assert any(item.status == FindingStatus.NEEDS_VALIDATION for item in assessment.resources) + + @pytest.mark.parametrize( ("imports", "files_call"), [ diff --git a/tests/test_generation.py b/tests/test_generation.py index ca2acb8..5d13eda 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -32,6 +32,7 @@ _safe_wheel_members, installed_wheel_member_destinations, validate_application_wheel, + validate_application_wheel_content_policy, validate_approved_wheel, validate_artifact_set, validate_combined_wheel_installation_paths, @@ -68,7 +69,7 @@ LockGraphAssessment, ) from python_deployment_builder.packaging.archive import safe_extract_zip -from python_deployment_builder.packaging.packager import package_deployment_kit +from python_deployment_builder.packaging.packager import PackageError, package_deployment_kit from python_deployment_builder.planning.index import target_marker_applies from python_deployment_builder.planning.lockfile import inspect_uv_lock from python_deployment_builder.planning.planner import create_deployment_plan @@ -3840,6 +3841,238 @@ def _refresh_manifest_wheel_hash(kit: Path, relative: str, *, approved: bool = F _update_indexed_hashes(kit, relative, "deployment/manifest.json") +@pytest.mark.parametrize( + "member", + [ + "native.pyd", + "native.dll", + "tool.exe", + "native.so", + "native.dylib", + "native.lib", + "main.pyc", + "main.pyo", + ], +) +def test_shared_application_wheel_content_policy_rejects_prohibited_members( + tmp_path: Path, member: str +) -> None: + wheel = _make_application_wheel(tmp_path) + _rewrite_application_wheel(wheel, additions={f"installed_app/{member}": b"synthetic"}) + + with pytest.raises(PreparationError, match="native binaries|runtime cache"): + validate_application_wheel_content_policy(wheel) + + +@pytest.mark.parametrize("case", ["not-purelib", "native", "cache"]) +def test_static_validation_enforces_application_wheel_content_policy( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, case: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + wheel = _make_application_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + application_wheel=wheel, + bootstrap_mode="online_cmd", + ) + relative = f"deployment/application/{wheel.name}" + staged = kit / relative + if case == "not-purelib": + _rewrite_application_wheel( + staged, + replacements={ + "mapped_app-1.2.3.dist-info/WHEEL": ( + "Wheel-Version: 1.0\nRoot-Is-Purelib: false\nTag: py3-none-any\n" + ) + }, + ) + elif case == "native": + _rewrite_application_wheel(staged, additions={"installed_app/native.pyd": b""}) + else: + _rewrite_application_wheel( + staged, + additions={"installed_app/__PYcache__/main.cpython-312.pyc": b""}, + ) + _refresh_manifest_wheel_hash(kit, relative) + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "APPLICATION_WHEEL_CONTENT_POLICY" and item.status.value == "FAIL" + for item in report.static_checks + ) + if case == "native": + with pytest.raises(PackageError, match="APPLICATION_WHEEL_CONTENT_POLICY"): + package_deployment_kit(kit, output_directory=tmp_path / "release") + + +def test_static_application_policy_does_not_restrict_reviewed_dependency_wheels( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"proxy_tools/native.pyd": b"synthetic"} + ) + kit = tmp_path / "kit" + + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + report = validate_static_kit(kit) + + assert report.final_state.value == "STATIC_VALID" + assert next( + item + for item in report.static_checks + if item.code == "APPLICATION_WHEEL_CONTENT_POLICY" + ).status.value == "PASS" + + +@pytest.mark.parametrize( + ("metadata_kind", "expected_mode"), + [("setup.cfg", "package"), ("setup.py", "source")], +) +def test_backend_only_dependencies_block_generation_before_lock_or_output( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + metadata_kind: str, + expected_mode: str, +) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1']\n" + "build-backend='setuptools.build_meta'\n", + encoding="utf-8", + ) + if metadata_kind == "setup.cfg": + package = source / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / metadata_kind).write_text( + "[metadata]\nname=backend-only\nversion=1.0\n" + "[options]\npackages=find:\npackage_dir=\n =src\n" + "install_requires=colorama==0.4.6\n" + "[options.packages.find]\nwhere=src\n" + "[options.entry_points]\nconsole_scripts=\n" + " backend-only=app.main:main\n", + encoding="utf-8", + ) + else: + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / metadata_kind).write_text( + "from setuptools import setup\n" + "setup(name='backend-only', version='1.0', py_modules=['app'], " + "install_requires=['colorama==0.4.6'], " + "entry_points={'console_scripts': ['backend-only=app:main']})\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = '>=3.12'\n", encoding="utf-8" + ) + repository = MaterializedRepository( + root=source, source=str(source), source_kind="local" + ) + output = tmp_path / "kit" + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail("runtime-sync blocker must run before uv acquisition"), + ) + + plan = create_deployment_plan( + assess_repository(repository), repository_root=source + ) + preview = generate_deployment_kit(repository, output, dry_run=True).preview + + assert plan.deployment_mode == expected_mode + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" in plan.risk_gate.blocking_codes + assert any( + "RUNTIME_SYNC_METADATA_UNSUPPORTED" in action + for action in preview.developer_actions + ) + assert not output.exists() + with pytest.raises(PreparationError, match="RUNTIME_SYNC_METADATA_UNSUPPORTED"): + generate_deployment_kit(repository, output, prepare_lock=True) + assert not output.exists() + + +def test_pkgutil_selected_resource_receives_normal_release_security_scan( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + package = source / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "main.py").write_text( + "import pkgutil\ndef main(): return pkgutil.get_data('app', 'defaults.txt')\n", + encoding="utf-8", + ) + (package / "defaults.txt").write_text( + "API_KEY = 'sk-abcdefghijklmnop'\n", encoding="utf-8" + ) + (source / "pyproject.toml").write_text( + "[project]\nname='pkgutil-secure'\nversion='1.0'\n" + "[project.scripts]\npkgutil-secure='app.main:main'\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = '>=3.12'\n", encoding="utf-8" + ) + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + + with pytest.raises(PreparationError, match="NO_SECRET_VALUES"): + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + ) + + assert not (tmp_path / "kit").exists() + + def test_static_validation_proves_application_wheel_requires_dist_against_staged_lock( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/test_planning.py b/tests/test_planning.py index 4d1da99..beb6fd4 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -1008,3 +1008,75 @@ def test_platform_specific_findings_are_filtered_for_windows() -> None: assert treatments[0].decision == "ignored_for_windows" assert treatments[1].decision == "applicable" + + +@pytest.mark.parametrize("metadata_kind", ["setup.cfg", "setup.py"]) +def test_backend_only_dependencies_block_immutable_uv_sync_contract( + tmp_path: Path, metadata_kind: str +) -> None: + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1']\n" + "build-backend='setuptools.build_meta'\n", + encoding="utf-8", + ) + (tmp_path / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + if metadata_kind == "setup.cfg": + (tmp_path / metadata_kind).write_text( + "[metadata]\nname=backend-only\nversion=1.0\n" + "[options]\npy_modules=app\ninstall_requires=colorama==0.4.6\n" + "[options.entry_points]\nconsole_scripts=\n backend-only=app:main\n", + encoding="utf-8", + ) + else: + (tmp_path / metadata_kind).write_text( + "from setuptools import setup\n" + "setup(name='backend-only', version='1.0', py_modules=['app'], " + "install_requires=['colorama==0.4.6'], " + "entry_points={'console_scripts': ['backend-only=app:main']})\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = '>=3.12'\n", encoding="utf-8" + ) + repository = MaterializedRepository( + root=tmp_path, source=str(tmp_path), source_kind="local" + ) + + plan = create_deployment_plan( + assess_repository(repository), repository_root=tmp_path + ) + + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" in plan.risk_gate.blocking_codes + assert "--locked" in plan.runtime.sync_command.arguments + assert "--frozen" not in plan.runtime.sync_command.arguments + assert any("colorama" in item for item in plan.readiness.blockers) + + +def test_backend_metadata_without_dependencies_needs_no_runtime_staging( + tmp_path: Path, +) -> None: + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1']\n" + "build-backend='setuptools.build_meta'\n", + encoding="utf-8", + ) + (tmp_path / "setup.cfg").write_text( + "[metadata]\nname=metadata-only\nversion=1.0\n" + "[options]\npy_modules=app\n" + "[options.entry_points]\nconsole_scripts=\n metadata-only=app:main\n", + encoding="utf-8", + ) + (tmp_path / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + (tmp_path / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = '>=3.12'\n", encoding="utf-8" + ) + repository = MaterializedRepository( + root=tmp_path, source=str(tmp_path), source_kind="local" + ) + + plan = create_deployment_plan( + assess_repository(repository), repository_root=tmp_path + ) + + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" not in plan.risk_gate.blocking_codes + assert plan.runtime.sync_command.arguments[-1] == "--no-install-project" From b9a419d1acff12ea70ef66260d1920d336a40371 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:18:33 -0400 Subject: [PATCH 45/59] Handle entry-point extras and short secrets --- src/python_deployment_builder/entry_points.py | 82 ++++++ .../generation/artifacts.py | 39 ++- .../generation/generator.py | 67 ++++- .../planning/planner.py | 62 ++++- .../validation/static.py | 16 +- tests/test_entry_points.py | 68 +++++ tests/test_generation.py | 258 ++++++++++++++++++ tests/test_planning.py | 62 +++++ 8 files changed, 639 insertions(+), 15 deletions(-) create mode 100644 src/python_deployment_builder/entry_points.py create mode 100644 tests/test_entry_points.py diff --git a/src/python_deployment_builder/entry_points.py b/src/python_deployment_builder/entry_points.py new file mode 100644 index 0000000..063d85a --- /dev/null +++ b/src/python_deployment_builder/entry_points.py @@ -0,0 +1,82 @@ +"""Static parsing for PyPA entry-point object references.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from packaging.utils import InvalidName, canonicalize_name + + +class EntryPointTargetError(ValueError): + """Raised when an entry-point value is not a bounded object reference.""" + + +@dataclass(frozen=True) +class ParsedEntryPointTarget: + """Normalized semantics of an entry-point value without importing it.""" + + module: str + attributes: tuple[str, ...] + extras: tuple[str, ...] + + @property + def callable_name(self) -> str: + return ".".join(self.attributes) + + +_EXTRAS_SUFFIX = re.compile(r"^(?P.*?)\s*\[\s*(?P[^\[\]]*)\s*\]\s*$") + + +def _identifier_path(value: str, *, label: str, allow_empty: bool = False) -> tuple[str, ...]: + value = value.strip() + if not value: + if allow_empty: + return () + raise EntryPointTargetError(f"Entry-point {label} is empty.") + components = tuple(value.split(".")) + if any(not component.isidentifier() for component in components): + raise EntryPointTargetError( + f"Entry-point {label} is not a dotted Python identifier: {value!r}" + ) + return components + + +def _canonical_extras(value: str) -> tuple[str, ...]: + raw = [item.strip() for item in value.split(",")] + if not raw or any(not item for item in raw): + raise EntryPointTargetError("Entry-point extras contain an empty name.") + try: + return tuple(sorted({canonicalize_name(item, validate=True) for item in raw})) + except InvalidName as exc: + raise EntryPointTargetError("Entry-point extras contain an invalid name.") from exc + + +def parse_entry_point_target(value: str) -> ParsedEntryPointTarget: + """Parse ``module[:attributes] [extras]`` using the PyPA bounded grammar.""" + + target = value.strip() + extras: tuple[str, ...] = () + if "[" in target or "]" in target: + matched = _EXTRAS_SUFFIX.fullmatch(target) + if matched is None: + raise EntryPointTargetError("Entry-point extras use malformed bracket syntax.") + target = matched.group("object").strip() + extras = _canonical_extras(matched.group("extras")) + if target.count(":") > 1: + raise EntryPointTargetError("Entry-point object reference contains multiple colons.") + if ":" in target: + module_text, attributes_text = target.split(":", 1) + attributes = _identifier_path(attributes_text, label="attribute path") + else: + module_text = target + attributes = () + module = ".".join(_identifier_path(module_text, label="module")) + return ParsedEntryPointTarget(module=module, attributes=attributes, extras=extras) + + +__all__ = [ + "EntryPointTargetError", + "ParsedEntryPointTarget", + "parse_entry_point_target", +] diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index ed83800..e47a8d4 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -29,6 +29,10 @@ resolve_package_data_members, resolve_packaged_python_sources, ) +from python_deployment_builder.entry_points import ( + EntryPointTargetError, + parse_entry_point_target, +) from python_deployment_builder.generation.acquisition import PreparationError, sha256_file from python_deployment_builder.models import ( ApplicationArtifact, @@ -58,6 +62,7 @@ MAX_WHEEL_MEMBERS = 10_000 MAX_WHEEL_MEMBER_SIZE = 256 * 1024 * 1024 MAX_WHEEL_TOTAL_UNCOMPRESSED_SIZE = 512 * 1024 * 1024 +MIN_CONFIGURED_SECRET_SCAN_LENGTH = 8 @dataclass(frozen=True) @@ -471,14 +476,34 @@ def installed_wheel_member_paths( def configured_secret_values(secret_names: Iterable[str]) -> tuple[str, ...]: - """Return current secret values without serializing or reporting them.""" + """Return scannable current secrets or fail closed without exposing values. + + Exact substring scanning below eight characters is too noisy to prove + absence safely. Empty environment values remain unset-equivalent, while a + non-empty short configured secret makes release security unprovable. + """ values: list[str] = [] seen: set[str] = set() + unscannable_names: list[str] = [] for name in secret_names: - if (value := os.environ.get(name)) is not None and value not in seen: + value = os.environ.get(name) + if not value: + continue + if len(value) < MIN_CONFIGURED_SECRET_SCAN_LENGTH: + unscannable_names.append(name) + continue + if value not in seen: values.append(value) seen.add(value) + if unscannable_names: + raise PreparationError( + "SHORT_CONFIGURED_SECRET_UNSCANNABLE: Configured secret environment variable(s) " + + ", ".join(sorted(set(unscannable_names))) + + " have non-empty values shorter than the minimum reliable content-scan " + "length. Unset them during release validation if not needed, or use longer " + "credentials." + ) return tuple(values) @@ -1588,7 +1613,15 @@ def validate_application_wheel( entry_point.name, fallback="", ).strip() - if installed_target != entry_point.target: + try: + expected_entry_point = parse_entry_point_target(entry_point.target) + installed_entry_point = parse_entry_point_target(installed_target) + except EntryPointTargetError as exc: + raise PreparationError( + "Application wheel entry point disagrees with authoritative metadata: " + "the installed target is missing or invalid." + ) from exc + if installed_entry_point != expected_entry_point: raise PreparationError( "Application wheel entry point disagrees with authoritative metadata: " f"expected {entry_point.name} = {entry_point.target}, received " diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index 55caed4..ce86c4a 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -4,7 +4,6 @@ import hashlib import json -import os import re import subprocess import tempfile @@ -26,6 +25,7 @@ sha256_file, ) from python_deployment_builder.generation.artifacts import ( + configured_secret_values, validate_application_wheel, validate_artifact_set, validate_combined_wheel_installation_paths, @@ -804,6 +804,61 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) + try: + secret_values = configured_secret_values( + item.name for item in plan.configuration if item.secret + ) + except PreparationError as exc: + if not dry_run: + raise + preview = _preview( + plan, + output_root, + dry_run=True, + bootstrap_mode=bootstrap_mode, + system_certs=system_certs, + prepare_lock=prepare_lock, + approved=[], + application_artifact=None, + staging_source_paths=[], + ) + preview.developer_actions.insert(0, f"Stop: {exc}") + return GenerationResult( + output_directory=str(output_root), + dry_run=True, + generated=False, + preview=preview, + ) + entrypoint_extra_codes = { + "ENTRYPOINT_EXTRA_NOT_SELECTED", + "ENTRYPOINT_EXTRA_UNDECLARED", + } & set(plan.risk_gate.blocking_codes) + if entrypoint_extra_codes: + code = sorted(entrypoint_extra_codes)[0] + if dry_run: + preview = _preview( + plan, + output_root, + dry_run=True, + bootstrap_mode=bootstrap_mode, + system_certs=system_certs, + prepare_lock=prepare_lock, + approved=[], + application_artifact=None, + staging_source_paths=[], + ) + preview.developer_actions.insert( + 0, f"Stop: {code} prevents entry-point dependency readiness." + ) + return GenerationResult( + output_directory=str(output_root), + dry_run=True, + generated=False, + preview=preview, + ) + raise PreparationError( + "Deployment planning is blocked: " + "; ".join(plan.readiness.blockers) + ) # A skip-worktree index bit means the filesystem PDB assessed may omit a # tracked part of HEAD. Block both deployment modes before staging, lock # preparation, artifact work, or output mutation rather than claiming the @@ -1090,6 +1145,9 @@ def generate_deployment_kit( selected_extras=selected_extras, repository_root=repository_root, ) + secret_values = configured_secret_values( + item.name for item in plan.configuration if item.secret + ) approved = validate_artifact_set(artifact_values, plan) application_artifact = ( validate_application_wheel( @@ -1159,16 +1217,11 @@ def generate_deployment_kit( } else: files_for_validation = files - secret_values = [ - value - for name in manifest.configuration_secret_names - if (value := os.environ.get(name)) - ] structural_checks = validate_rendered_files( files_for_validation, manifest, generated_paths=set(owned), - secret_values=secret_values, + secret_values=list(secret_values), ) previous = _load_previous_index(output_root) _create, _replace, final_collisions = _classify_output( diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 32f12b9..8048d96 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -17,6 +17,10 @@ resolve_packaged_python_sources, ) from python_deployment_builder.backends.uv_managed import UvManagedBackend +from python_deployment_builder.entry_points import ( + EntryPointTargetError, + parse_entry_point_target, +) from python_deployment_builder.models import ( ConfigurationPlan, DependencyAssessment, @@ -237,20 +241,49 @@ def _entrypoint(assessment: RepositoryAssessment) -> EntrypointPlan | None: if not entries: return None chosen = next((item for item in entries if item.kind == "gui"), entries[0]) - if ":" not in chosen.target: + try: + parsed = parse_entry_point_target(chosen.target) + except EntryPointTargetError as exc: + raise ValueError(f"Entry point target is invalid: {chosen.target}") from exc + if not parsed.attributes: raise ValueError(f"Entry point target is not module:callable: {chosen.target}") - module, callable_name = chosen.target.split(":", 1) return EntrypointPlan( name=chosen.name, target=chosen.target, kind=chosen.kind, declared_group=chosen.declared_group, - module=module, - callable=callable_name, + module=parsed.module, + callable=parsed.callable_name, alternatives=[item.name for item in entries if item.name != chosen.name], ) +def _entrypoint_extra_blockers( + assessment: RepositoryAssessment, + entry_point: EntrypointPlan | None, + selected_extra_names: set[str], +) -> list[str]: + if entry_point is None: + return [] + parsed = parse_entry_point_target(entry_point.target) + available = { + canonicalize_name(item) for item in assessment.project.optional_dependency_groups + } + undeclared = sorted(set(parsed.extras) - available) + missing = sorted((set(parsed.extras) & available) - selected_extra_names) + blockers = [ + "ENTRYPOINT_EXTRA_UNDECLARED: the selected entry point declares optional extra " + f"'{item}', but application metadata does not declare it." + for item in undeclared + ] + blockers.extend( + "ENTRYPOINT_EXTRA_NOT_SELECTED: the selected entry point declares required extra " + f"'{item}'. Regenerate with that application extra selected." + for item in missing + ) + return blockers + + def _risk_gate(assessment: RepositoryAssessment) -> RiskGate: blocking = [item.code for item in assessment.risks if item.severity == RiskSeverity.BLOCKING] warnings = [item.code for item in assessment.risks if item.severity == RiskSeverity.WARNING] @@ -506,6 +539,10 @@ def create_deployment_plan( mode, mode_rationale, mode_condition, mode_blockers = _deployment_mode( assessment, entry_point, repository_root ) + entrypoint_extra_blockers = _entrypoint_extra_blockers( + assessment, entry_point, selected_extra_names + ) + mode_blockers.extend(entrypoint_extra_blockers) runtime = UvManagedBackend().build_plan( app_id, python_version, @@ -637,6 +674,23 @@ def create_deployment_plan( ), ] gate = _risk_gate(assessment) + if entrypoint_extra_blockers: + entrypoint_extra_codes = { + item.split(":", 1)[0] for item in entrypoint_extra_blockers + } + gate = gate.model_copy( + update={ + "outcome": "block", + "blocking_codes": sorted( + {*gate.blocking_codes, *entrypoint_extra_codes} + ), + "rationale": ( + gate.rationale + + " Entry-point extras must already be declared and selected in the " + "immutable deployment graph." + ), + } + ) if backend_only_dependencies: gate = gate.model_copy( update={ diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index fac7588..29b164a 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -369,7 +369,21 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR ) ) trusted_wheels = trusted_artifact_wheel_paths(manifest) - secret_values = configured_secret_values(manifest.configuration_secret_names) + secret_scanability_failures: list[str] = [] + try: + secret_values = configured_secret_values(manifest.configuration_secret_names) + except PreparationError as exc: + secret_values = () + secret_scanability_failures.append(str(exc)) + checks.append( + _check( + "CONFIGURED_SECRET_SCANABILITY", + not secret_scanability_failures, + "Current configured secret values can be scanned reliably when present.", + "A current configured secret value is too short for reliable content scanning.", + evidence=secret_scanability_failures, + ) + ) unvalidated_wheels = sorted( path for path in actual_paths diff --git a/tests/test_entry_points.py b/tests/test_entry_points.py new file mode 100644 index 0000000..0589c84 --- /dev/null +++ b/tests/test_entry_points.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import pytest + +from python_deployment_builder.entry_points import ( + EntryPointTargetError, + parse_entry_point_target, +) + + +@pytest.mark.parametrize( + ("value", "module", "attributes", "extras"), + [ + ("app.main:main", "app.main", ("main",), ()), + ("app.main:main[feature]", "app.main", ("main",), ("feature",)), + ("app.main:main [feature]", "app.main", ("main",), ("feature",)), + ( + " app.cli : Runner.main [ gui, Map_Feature ] ", + "app.cli", + ("Runner", "main"), + ("gui", "map-feature"), + ), + ( + "app.cli:Factory.handlers.start [Feature.One]", + "app.cli", + ("Factory", "handlers", "start"), + ("feature-one",), + ), + ], +) +def test_parse_entry_point_target_separates_object_reference_and_extras( + value: str, + module: str, + attributes: tuple[str, ...], + extras: tuple[str, ...], +) -> None: + parsed = parse_entry_point_target(value) + + assert parsed.module == module + assert parsed.attributes == attributes + assert parsed.extras == extras + + +def test_entry_point_extra_order_and_spelling_are_semantically_equal() -> None: + source = parse_entry_point_target("app.main : main [gui, Feature_One]") + wheel = parse_entry_point_target("app.main:main[feature-one,gui]") + + assert source == wheel + + +@pytest.mark.parametrize( + "value", + [ + "app..main:main", + "app.main:", + "app.main:Runner..main", + "app.main:Runner-main", + "app.main:Runner()[0]", + "app.main:main[feature", + "app.main:main feature]", + "app.main:main[]", + "app.main:main[feature,,map]", + "app.main:main[bad extra]", + ], +) +def test_parse_entry_point_target_rejects_malformed_values(value: str) -> None: + with pytest.raises(EntryPointTargetError): + parse_entry_point_target(value) diff --git a/tests/test_generation.py b/tests/test_generation.py index 5d13eda..32dd8b0 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -30,6 +30,7 @@ ) from python_deployment_builder.generation.artifacts import ( _safe_wheel_members, + configured_secret_values, installed_wheel_member_destinations, validate_application_wheel, validate_application_wheel_content_policy, @@ -830,6 +831,112 @@ def test_gui_scripts_wheel_must_match_gui_declared_group(tmp_path: Path) -> None validate_application_wheel(wrong_group, assessment, plan) +def test_application_wheel_compares_entry_point_target_semantics( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source, target="installed_app.main : main [ Feature_One, map ]") + pyproject = source / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + "[project.gui-scripts]", + "[project.optional-dependencies]\nFeature_One=[]\nmap=[]\n" + "[project.gui-scripts]", + ), + encoding="utf-8", + ) + repository = MaterializedRepository( + root=source, source=str(source), source_kind="local" + ) + assessment = assess_repository(repository) + plan = create_deployment_plan( + assessment, selected_extras=["feature-one", "map"], repository_root=source + ) + + assert plan.entry_point is not None + assert (plan.entry_point.module, plan.entry_point.callable) == ( + "installed_app.main", + "main", + ) + valid = _make_application_wheel( + tmp_path, target="installed_app.main:main[map,feature.one]" + ) + validate_application_wheel(valid, assessment, plan, repository_root=source) + + valid.unlink() + wrong_extras = _make_application_wheel( + tmp_path, target="installed_app.main:main[feature-one]" + ) + with pytest.raises(PreparationError, match="entry point disagrees"): + validate_application_wheel( + wrong_extras, assessment, plan, repository_root=source + ) + + +def test_qualified_entry_point_with_extra_keeps_clean_runtime_callable( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source, target="installed_app.main:Factory.handlers.start [feature]") + pyproject = source / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + "[project.gui-scripts]", + "[project.optional-dependencies]\nfeature=[]\n[project.gui-scripts]", + ), + encoding="utf-8", + ) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + + plan = create_deployment_plan( + assessment, selected_extras=["feature"], repository_root=source + ) + + assert plan.entry_point is not None + assert plan.entry_point.callable == "Factory.handlers.start" + + +def test_unselected_entry_point_extra_blocks_generation_and_dry_run( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "pyproject.toml").write_text( + "[project]\nname='entry-extra-app'\nversion='1.0'\n" + "[project.optional-dependencies]\nfeature=[]\n" + "[project.scripts]\nentry-extra-app='app:main [feature]'\n" + "[tool.setuptools]\npy-modules=['app']\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = '>=3.12'\n", encoding="utf-8" + ) + repository = MaterializedRepository( + root=source, source=str(source), source_kind="local" + ) + output = tmp_path / "kit" + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail("entry-point extra blocker must precede uv"), + ) + + preview = generate_deployment_kit(repository, output, dry_run=True).preview + + assert any( + "ENTRYPOINT_EXTRA_NOT_SELECTED" in action + for action in preview.developer_actions + ) + assert not output.exists() + with pytest.raises(PreparationError, match="ENTRYPOINT_EXTRA_NOT_SELECTED"): + generate_deployment_kit(repository, output) + assert not output.exists() + + def test_poetry_string_script_uses_console_scripts_for_wheel_validation(tmp_path: Path) -> None: source = tmp_path / "source" source.mkdir() @@ -3532,6 +3639,74 @@ def test_wheel_configured_secret_policy_is_shared_and_deduplicated( validate_approved_wheel(f"proxy-tools={approved_wheel}", approved_plan) +@pytest.mark.parametrize( + ("secret", "name"), + [("1234567", "SHORT_TOKEN"), ("482731", "DEMO_PIN"), ("¤", "ONE_CHAR_SECRET")], +) +def test_nonempty_short_configured_secrets_fail_scanability_without_value_leak( + monkeypatch: pytest.MonkeyPatch, secret: str, name: str +) -> None: + monkeypatch.setenv(name, secret) + + with pytest.raises( + PreparationError, match=f"SHORT_CONFIGURED_SECRET_UNSCANNABLE.*{name}" + ) as caught: + configured_secret_values([name]) + + assert secret not in str(caught.value) + + +def test_empty_unset_and_scannable_configured_secret_values( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("EMPTY_SECRET", "") + monkeypatch.delenv("UNSET_SECRET", raising=False) + monkeypatch.setenv("EIGHT_CHAR_SECRET", "12345678") + monkeypatch.setenv("WHITESPACE_SECRET", " 123456 ") + + assert configured_secret_values( + ["EMPTY_SECRET", "UNSET_SECRET", "EIGHT_CHAR_SECRET", "WHITESPACE_SECRET"] + ) == ("12345678", " 123456 ") + + +def test_non_identifier_configured_secret_name_remains_supported( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEMO-TOKEN", "12345678") + + assert configured_secret_values(["DEMO-TOKEN"]) == ("12345678",) + + +def test_application_and_approved_wheels_reject_unscannable_configured_secret( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + secret = "482731" + monkeypatch.setenv("DEMO_PIN", secret) + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + application_plan = create_deployment_plan(assessment, repository_root=source) + application_plan.configuration = [_secret_configuration("DEMO_PIN")] + approved_plan = _plan("optional_map_app", ["map"]) + approved_plan.configuration = [_secret_configuration("DEMO_PIN")] + + with pytest.raises(PreparationError, match="SHORT_CONFIGURED_SECRET_UNSCANNABLE") as app: + validate_application_wheel( + _make_application_wheel(tmp_path), + assessment, + application_plan, + repository_root=source, + ) + with pytest.raises(PreparationError, match="SHORT_CONFIGURED_SECRET_UNSCANNABLE") as dep: + validate_approved_wheel(f"proxy-tools={_make_wheel(tmp_path)}", approved_plan) + + assert secret not in str(app.value) + assert secret not in str(dep.value) + + def test_approved_wheel_security_allows_unconfigured_text_and_rejects_obvious_secret( tmp_path: Path, ) -> None: @@ -4566,6 +4741,89 @@ def test_static_validation_rescans_trusted_wheel_configured_secrets( assert secret not in json.dumps(report.model_dump(mode="json")) +def test_static_validation_reports_unscannable_secret_without_value_leak( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit(_repository("prepared_gui"), kit, bootstrap_mode="online_cmd") + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["configuration_secret_names"] = ["DEMO_PIN"] + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + secret = "482731" + monkeypatch.setenv("DEMO_PIN", secret) + + report = validate_static_kit(kit) + + assert any( + item.code == "CONFIGURED_SECRET_SCANABILITY" and item.status.value == "FAIL" + for item in report.static_checks + ) + assert secret not in json.dumps(report.model_dump(mode="json")) + + +@pytest.mark.parametrize("deployment_mode", ["source", "package"]) +def test_generation_and_dry_run_block_unscannable_secret_before_writes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, deployment_mode: str +) -> None: + source = tmp_path / deployment_mode + source.mkdir() + if deployment_mode == "package": + _write_mapped_project(source) + (source / "code/main.py").write_text( + "import os\nPIN = os.environ['DEMO_API_TOKEN']\ndef main(): return 0\n", + encoding="utf-8", + ) + else: + (source / "app.py").write_text( + "import os\nPIN = os.environ['DEMO_API_TOKEN']\ndef main(): return 0\n", + encoding="utf-8", + ) + (source / "pyproject.toml").write_text( + "[project]\nname='short-secret-source'\nversion='1.0'\n" + "[project.scripts]\nshort-secret-source='app:main'\n" + "[tool.setuptools]\npy-modules=['app']\n", + encoding="utf-8", + ) + (source / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = '>=3.12'\n", + encoding="utf-8", + ) + secret = "482731" + monkeypatch.setenv("DEMO_API_TOKEN", secret) + repository = MaterializedRepository( + root=source, source=str(source), source_kind="local" + ) + output = tmp_path / f"{deployment_mode}-kit" + + dry_run = generate_deployment_kit(repository, output, dry_run=True) + + assert not dry_run.generated + assert any( + "SHORT_CONFIGURED_SECRET_UNSCANNABLE" in action + for action in dry_run.preview.developer_actions + ) + assert secret not in json.dumps(dry_run.model_dump(mode="json")) + assert not output.exists() + with pytest.raises(PreparationError, match="SHORT_CONFIGURED_SECRET_UNSCANNABLE") as caught: + generate_deployment_kit(repository, output) + assert secret not in str(caught.value) + assert not output.exists() + + def test_static_validation_rejects_combined_trusted_wheel_collisions( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/test_planning.py b/tests/test_planning.py index beb6fd4..b185e8b 100644 --- a/tests/test_planning.py +++ b/tests/test_planning.py @@ -1080,3 +1080,65 @@ def test_backend_metadata_without_dependencies_needs_no_runtime_staging( assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" not in plan.risk_gate.blocking_codes assert plan.runtime.sync_command.arguments[-1] == "--no-install-project" + + +def _entry_point_extra_project(tmp_path: Path, *, declare_extra: bool = True): + (tmp_path / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + optional = ( + "[project.optional-dependencies]\nFeature_One=[]\n" if declare_extra else "" + ) + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools>=79.0.1']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='entry-extra-app'\nversion='1.0'\n" + + optional + + "[project.scripts]\n" + "entry-extra-app='app : main [ feature-one ]'\n" + "[tool.setuptools]\npy-modules=['app']\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text( + "version = 1\nrevision = 3\nrequires-python = '>=3.12'\n", encoding="utf-8" + ) + repository = MaterializedRepository( + root=tmp_path, source=str(tmp_path), source_kind="local" + ) + return assess_repository(repository) + + +def test_selected_declared_entry_point_extra_is_parsed_for_runtime( + tmp_path: Path, +) -> None: + plan = create_deployment_plan( + _entry_point_extra_project(tmp_path), + selected_extras=["Feature.One"], + repository_root=tmp_path, + ) + + assert plan.entry_point is not None + assert plan.entry_point.target == "app : main [ feature-one ]" + assert plan.entry_point.module == "app" + assert plan.entry_point.callable == "main" + assert plan.runtime.selected_extras == ["Feature_One"] + assert "ENTRYPOINT_EXTRA_NOT_SELECTED" not in plan.readiness.blocker_codes + + +def test_unselected_entry_point_extra_blocks_planning(tmp_path: Path) -> None: + plan = create_deployment_plan( + _entry_point_extra_project(tmp_path), repository_root=tmp_path + ) + + assert "ENTRYPOINT_EXTRA_NOT_SELECTED" in plan.readiness.blocker_codes + assert "ENTRYPOINT_EXTRA_NOT_SELECTED" in plan.risk_gate.blocking_codes + assert any("feature-one" in blocker for blocker in plan.readiness.blockers) + + +def test_undeclared_entry_point_extra_blocks_planning(tmp_path: Path) -> None: + plan = create_deployment_plan( + _entry_point_extra_project(tmp_path, declare_extra=False), + selected_extras=[], + repository_root=tmp_path, + ) + + assert "ENTRYPOINT_EXTRA_UNDECLARED" in plan.readiness.blocker_codes + assert "ENTRYPOINT_EXTRA_UNDECLARED" in plan.risk_gate.blocking_codes From 4c64e771ef178ae9d51f31d5e9b5813062818666 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:20:49 -0400 Subject: [PATCH 46/59] Validate approved artifacts and source setup --- .../generation/artifacts.py | 88 +++++++--- .../templates/windows_uv/manage.py | 24 ++- .../validation/static.py | 97 ++++++++++- tests/test_generation.py | 135 +++++++++++++++ tests/test_validation.py | 156 ++++++++++++++++++ 5 files changed, 462 insertions(+), 38 deletions(-) diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index e47a8d4..ba1291f 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -37,6 +37,7 @@ from python_deployment_builder.models import ( ApplicationArtifact, ApprovedArtifact, + DeploymentArtifactRequirement, DeploymentPlan, RepositoryAssessment, ) @@ -832,6 +833,64 @@ def validate_application_wheel_content_policy(path: Path) -> None: raise PreparationError(f"Malformed application wheel: {path.name}") from exc +def validate_approved_artifact_lock_identity( + distribution_name: str, + version: str | Version, + plan: DeploymentPlan, +) -> DeploymentArtifactRequirement: + """Prove one approved artifact is the unambiguous staged-lock substitution.""" + + requested_name = canonicalize_name(distribution_name) + requirements = [ + item + for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) + if canonicalize_name(item.package) == requested_name + ] + if not requirements: + raise PreparationError( + f"No developer-wheel requirement exists for {requested_name} in this deployment plan." + ) + + try: + supplied_version = version if isinstance(version, Version) else Version(version) + except InvalidVersion as exc: + raise PreparationError( + f"Approved artifact version is invalid for {requested_name}: {version!r}." + ) from exc + try: + requirement_versions = {Version(item.version) for item in requirements} + except InvalidVersion as exc: + raise PreparationError( + f"Approved artifact requirement version is invalid for {requested_name}." + ) from exc + try: + target_possible_versions = { + Version(dependency.version) + for dependency in (plan.lock_graph.dependencies if plan.lock_graph else []) + if canonicalize_name(dependency.name) == requested_name + } + except InvalidVersion as exc: + raise PreparationError( + f"Target-possible locked version is invalid for {requested_name}." + ) from exc + + if len(target_possible_versions) > 1 or len(requirement_versions) != 1: + versions = sorted( + str(item) for item in target_possible_versions or requirement_versions + ) + raise PreparationError( + "Developer artifact substitution is ambiguous for target-possible locked versions: " + f"{requested_name} ({', '.join(versions)})." + ) + required_version = next(iter(requirement_versions)) + if supplied_version != required_version: + raise PreparationError( + f"Artifact version mismatch for {requested_name}: expected " + f"{requirements[0].version}, received {supplied_version}." + ) + return requirements[0] + + def validate_approved_wheel( value: str, plan: DeploymentPlan, @@ -849,32 +908,9 @@ def validate_approved_wheel( f"Artifact name mismatch: option requested {requested_name}, filename contains " f"{filename_name}." ) - requirements = [ - item - for item in (plan.lock_graph.artifact_requirements if plan.lock_graph else []) - if canonicalize_name(item.package) == requested_name - ] - versions = {item.version for item in requirements} - if not requirements: - raise PreparationError( - f"No developer-wheel requirement exists for {requested_name} in this deployment plan." - ) - target_possible_versions = { - dependency.version - for dependency in (plan.lock_graph.dependencies if plan.lock_graph else []) - if canonicalize_name(dependency.name) == requested_name - } - if len(target_possible_versions) > 1: - raise PreparationError( - "Developer artifact substitution is ambiguous for target-possible locked versions: " - f"{requested_name} ({', '.join(sorted(target_possible_versions))})." - ) - if len(versions) != 1: - raise PreparationError( - "Developer artifact substitution is ambiguous for target-possible locked versions: " - f"{requested_name} ({', '.join(sorted(versions))})." - ) - requirement = requirements[0] + requirement = validate_approved_artifact_lock_identity( + requested_name, filename_version, plan + ) try: requirement_version = Version(requirement.version) except InvalidVersion as exc: diff --git a/src/python_deployment_builder/templates/windows_uv/manage.py b/src/python_deployment_builder/templates/windows_uv/manage.py index cdd690b..640e1b2 100644 --- a/src/python_deployment_builder/templates/windows_uv/manage.py +++ b/src/python_deployment_builder/templates/windows_uv/manage.py @@ -166,12 +166,26 @@ def _promote_environment( environment=runtime_env, logger=logger, ) - run_logged( - [str(uv_executable), "pip", "check", "--python", str(python)], - cwd=project_root, - environment=runtime_env, - logger=logger, + deployment_mode = manifest.get("deployment_mode") or ( + "package" if application_artifact else "source" ) + if deployment_mode == "package": + logger.info("Checking installed package metadata compatibility.") + run_logged( + [str(uv_executable), "pip", "check", "--python", str(python)], + cwd=project_root, + environment=runtime_env, + logger=logger, + ) + else: + # The source-mode root is intentionally absent from installed + # distribution metadata. Generic pip check would misreport valid + # dependency back-edges to that source-provided root as missing. + logger.info( + "Skipping uv pip check because the source-mode root project is " + "intentionally not installed; dependency compatibility was validated " + "against the locked deployment plan." + ) launch_check = [ str(python), diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 29b164a..fa62d6d 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -24,6 +24,7 @@ installed_wheel_member_paths, validate_application_requires_dist, validate_application_wheel_content_policy, + validate_approved_artifact_lock_identity, validate_approved_requires_dist, validate_combined_wheel_installation_paths, validate_wheel_installation_layout, @@ -139,12 +140,12 @@ def _static_lock_root_name(root: Path, manifest: DeploymentManifest) -> str | No def _static_lock_plan(root: Path, manifest: DeploymentManifest): - """Build the transient staged-lock proof context used by wheel validators.""" + """Build the transient staged-lock proof context used by artifact validators.""" application_name = _static_lock_root_name(root, manifest) if application_name is None: raise PreparationError( - "Trusted wheel Requires-Dist validation cannot identify the staged lock root." + "Staged-lock artifact validation cannot identify the root application." ) graph = inspect_uv_lock( root, @@ -156,7 +157,7 @@ def _static_lock_plan(root: Path, manifest: DeploymentManifest): if not graph.inspected: detail = "; ".join(graph.limitations) or "uv.lock could not be inspected." raise PreparationError( - "Trusted wheel Requires-Dist validation requires an inspected staged uv.lock: " + "Staged-lock artifact validation requires an inspected uv.lock: " f"{detail}" ) return SimpleNamespace( @@ -558,6 +559,90 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=application_content_failures, ) ) + static_plan = None + static_lock_failures: list[str] = [] + try: + static_plan = _static_lock_plan(root, manifest) + except (PreparationError, InvalidVersion, ValueError) as exc: + static_lock_failures.append(str(exc)) + + approved_identity_failures = list(static_lock_failures) + if static_plan is not None: + approved_identities: list[tuple[str, Version]] = [] + for artifact in manifest.approved_artifacts: + try: + artifact_identity = ( + canonicalize_name(artifact.distribution_name), + Version(artifact.version), + ) + validate_approved_artifact_lock_identity( + artifact.distribution_name, artifact.version, static_plan + ) + except (PreparationError, InvalidVersion, ValueError) as exc: + approved_identity_failures.append( + f"{artifact.distribution_name}=={artifact.version}: {exc}" + ) + else: + if artifact_identity in approved_identities: + approved_identity_failures.append( + "Manifest repeats an approved artifact lock identity: " + f"{artifact.distribution_name}=={artifact.version}." + ) + approved_identities.append(artifact_identity) + for requirement in static_plan.lock_graph.artifact_requirements: + try: + requirement_version = Version(requirement.version) + except InvalidVersion: + approved_identity_failures.append( + f"Invalid staged-lock artifact requirement version: " + f"{requirement.package}=={requirement.version}" + ) + continue + matching = [] + for artifact in manifest.approved_artifacts: + try: + artifact_version = Version(artifact.version) + except InvalidVersion: + continue + if ( + canonicalize_name(artifact.distribution_name) + == canonicalize_name(requirement.package) + and artifact_version == requirement_version + ): + matching.append(artifact) + if len(matching) != 1: + approved_identity_failures.append( + "Staged-lock developer artifact requirement does not have exactly one " + f"manifest-approved wheel: {requirement.package}=={requirement.version}." + ) + suppressed_packages: list[str] = [] + for index, argument in enumerate(manifest.sync_arguments): + if argument != "--no-install-package": + continue + if index + 1 >= len(manifest.sync_arguments): + approved_identity_failures.append( + "Runtime sync arguments end with --no-install-package without a distribution." + ) + continue + suppressed_packages.append(canonicalize_name(manifest.sync_arguments[index + 1])) + approved_packages = [ + canonicalize_name(artifact.distribution_name) + for artifact in manifest.approved_artifacts + ] + if sorted(suppressed_packages) != sorted(approved_packages): + approved_identity_failures.append( + "Runtime --no-install-package arguments do not match manifest-approved artifacts." + ) + checks.append( + _check( + "APPROVED_ARTIFACT_LOCK_IDENTITY", + not approved_identity_failures, + "Manifest-approved artifacts exactly match staged-lock developer requirements.", + "Manifest-approved artifacts and staged-lock developer requirements disagree.", + evidence=approved_identity_failures, + ) + ) + wheel_dependency_failures: list[str] = [] dependency_wheels = [ path @@ -565,10 +650,8 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR if path in wheel_metadata_by_path and wheel_metadata_by_path[path].requires_dist ] if dependency_wheels: - try: - static_plan = _static_lock_plan(root, manifest) - except (PreparationError, InvalidVersion, ValueError) as exc: - wheel_dependency_failures.append(str(exc)) + if static_plan is None: + wheel_dependency_failures.extend(static_lock_failures) else: application_relative = ( manifest_artifact_wheel_path( diff --git a/tests/test_generation.py b/tests/test_generation.py index 32dd8b0..b7eb905 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -4385,6 +4385,141 @@ def test_static_validation_proves_approved_wheel_requires_dist_against_staged_lo ) +def test_static_validation_rejects_dependency_free_approved_wheel_absent_from_lock( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit(_repository("prepared_gui"), kit, bootstrap_mode="online_cmd") + intruder = _make_wheel(tmp_path, name="intruder", version="1.0") + relative = f"deployment/wheels/{intruder.name}" + _add_indexed_file(kit, relative, intruder.read_bytes()) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["approved_artifacts"].append( + { + "distribution_name": "intruder", + "version": "1.0", + "filename": intruder.name, + "sha256": hashlib.sha256(intruder.read_bytes()).hexdigest(), + "wheel_tags": ["py3-none-any"], + "requirement_action": "developer_wheel_required", + } + ) + manifest["sync_arguments"].extend(["--no-install-package", "intruder"]) + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "APPROVED_ARTIFACT_LOCK_IDENTITY" + and item.status.value == "FAIL" + for item in report.static_checks + ) + + +def test_static_validation_requires_every_locked_artifact_substitution_in_manifest( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + relative = f"deployment/wheels/{artifact.name}" + (kit / relative).unlink() + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["approved_artifacts"] = [] + manifest["referenced_files"] = [ + item for item in manifest["referenced_files"] if item != relative + ] + manifest["sync_arguments"] = manifest["sync_arguments"][:-2] + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + index_path = kit / "deployment/generated-files.json" + index = json.loads(index_path.read_text(encoding="utf-8")) + index["files"] = [item for item in index["files"] if item["path"] != relative] + index_path.write_text(json.dumps(index, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "APPROVED_ARTIFACT_LOCK_IDENTITY" + and item.status.value == "FAIL" + for item in report.static_checks + ) + + +def test_static_approved_artifact_lock_identity_uses_pep440_versions( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["approved_artifacts"][0]["version"] = "0.1" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert report.final_state.value == "STATIC_VALID", [ + (item.code, item.evidence) + for item in report.static_checks + if item.status.value == "FAIL" + ] + + def test_static_validation_rejects_indexed_unvalidated_staged_wheel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/test_validation.py b/tests/test_validation.py index 8adf3a1..cffce04 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -437,6 +437,7 @@ def test_package_runtime_installs_artifacts_in_order_and_states_success_last( events: list[str] = [] manifest = { "application_id": "sample", + "deployment_mode": "package", "runtime_paths": { "application_root": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample", "environment_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\env", @@ -485,6 +486,161 @@ def record(command, **kwargs): ] +def test_source_runtime_skips_pip_check_for_intentionally_uninstalled_root( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import importlib.util + import sys + + template_root = ( + Path(__file__).parents[1] + / "src" + / "python_deployment_builder" + / "templates" + / "windows_uv" + ) + monkeypatch.syspath_prepend(str(template_root)) + spec = importlib.util.spec_from_file_location("source_manage", template_root / "manage.py") + assert spec and spec.loader + manage = importlib.util.module_from_spec(spec) + sys.modules["source_manage"] = manage + spec.loader.exec_module(manage) + + local = tmp_path / "LocalAppData" + monkeypatch.setenv("LOCALAPPDATA", str(local)) + project = tmp_path / "project" + project.mkdir() + deployment = tmp_path / "deployment" + (deployment / "wheels").mkdir(parents=True) + plugin = deployment / "wheels/plugin.whl" + plugin.write_bytes(b"plugin requiring source-only app") + monkeypatch.setattr(manage, "deployment_directory", lambda: deployment) + environment = local / "PythonDeploymentBuilder/apps/sample/env" + events: list[str] = [] + manifest = { + "application_id": "sample", + "deployment_mode": "source", + "runtime_paths": { + "application_root": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample", + "environment_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\env", + "logs_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\logs", + "state_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\state", + }, + "runtime_environment": {"PYTHONPATH": r"%PROJECT_ROOT%"}, + "sync_arguments": [ + "sync", + "--locked", + "--no-build", + "--no-install-project", + ], + "approved_artifacts": [{"filename": plugin.name}], + "application_artifact": None, + } + + def record(command, **kwargs): + if command[1] == "sync": + events.append("locked-sync") + (environment / "Scripts").mkdir(parents=True) + (environment / "Scripts/python.exe").write_bytes(b"python") + (environment / "Scripts/pythonw.exe").write_bytes(b"pythonw") + elif str(plugin) in command: + events.append("dependency-artifact") + elif command[1:3] == ["pip", "check"]: + events.append("pip-check") + else: + events.append("entry-point-check") + + monkeypatch.setattr(manage, "run_logged", record) + monkeypatch.setattr(manage, "write_state", lambda *args: events.append("state-success")) + + manage._promote_environment( + manifest, project, tmp_path / "uv.exe", manage.logging.getLogger("test-source") + ) + + assert events == [ + "locked-sync", + "dependency-artifact", + "entry-point-check", + "state-success", + ] + + +def test_package_runtime_pip_check_failure_restores_previous_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import importlib.util + import sys + + template_root = ( + Path(__file__).parents[1] + / "src" + / "python_deployment_builder" + / "templates" + / "windows_uv" + ) + monkeypatch.syspath_prepend(str(template_root)) + spec = importlib.util.spec_from_file_location( + "pip_check_failure_manage", template_root / "manage.py" + ) + assert spec and spec.loader + manage = importlib.util.module_from_spec(spec) + sys.modules["pip_check_failure_manage"] = manage + spec.loader.exec_module(manage) + + local = tmp_path / "LocalAppData" + monkeypatch.setenv("LOCALAPPDATA", str(local)) + project = tmp_path / "project" + project.mkdir() + deployment = tmp_path / "deployment" + (deployment / "application").mkdir(parents=True) + application = deployment / "application/application.whl" + application.write_bytes(b"application") + monkeypatch.setattr(manage, "deployment_directory", lambda: deployment) + app_root = local / "PythonDeploymentBuilder/apps/sample" + environment = app_root / "env" + (environment / "Scripts").mkdir(parents=True) + old_python = environment / "Scripts/python.exe" + old_python.write_bytes(b"known-good") + (environment / "Scripts/pythonw.exe").write_bytes(b"known-good") + state_writes: list[str] = [] + manifest = { + "application_id": "sample", + "deployment_mode": "package", + "runtime_paths": { + "application_root": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample", + "environment_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\env", + "logs_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\logs", + "state_path": r"%LOCALAPPDATA%\PythonDeploymentBuilder\apps\sample\state", + }, + "runtime_environment": {}, + "sync_arguments": ["sync", "--locked", "--no-build", "--no-install-project"], + "approved_artifacts": [], + "application_artifact": {"filename": application.name}, + } + + def fail_check(command, **kwargs): + if command[1] == "sync": + (environment / "Scripts").mkdir(parents=True) + (environment / "Scripts/python.exe").write_bytes(b"candidate") + (environment / "Scripts/pythonw.exe").write_bytes(b"candidate") + return + if command[1:3] == ["pip", "check"]: + raise manage.DeploymentRuntimeError("controlled pip check failure") + + monkeypatch.setattr(manage, "run_logged", fail_check) + monkeypatch.setattr(manage, "write_state", lambda *args: state_writes.append("written")) + + with pytest.raises(manage.DeploymentRuntimeError, match="pip check failure"): + manage._promote_environment( + manifest, project, tmp_path / "uv.exe", manage.logging.getLogger("test-pip-check") + ) + + assert old_python.read_bytes() == b"known-good" + assert not state_writes + assert not (app_root / "env.previous").exists() + assert not (app_root / "env.failed").exists() + + def test_package_runtime_application_install_failure_restores_previous_environment( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 8c6a27936cd79a21f2802152ec03236b7007c22f Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:47:11 -0400 Subject: [PATCH 47/59] Validate the complete uv sync contract --- .../backends/uv_managed.py | 45 +++- .../generation/manifest.py | 9 +- .../validation/static.py | 43 ++-- tests/test_generation.py | 225 +++++++++++++++++- 4 files changed, 278 insertions(+), 44 deletions(-) diff --git a/src/python_deployment_builder/backends/uv_managed.py b/src/python_deployment_builder/backends/uv_managed.py index 152af87..ae5499c 100644 --- a/src/python_deployment_builder/backends/uv_managed.py +++ b/src/python_deployment_builder/backends/uv_managed.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Sequence + from python_deployment_builder.models import ( BootstrapArtifact, PlannedCommand, @@ -23,6 +25,32 @@ } +def uv_sync_arguments( + *, + python_version: str, + selected_extras: Sequence[str], + approved_artifact_names: Sequence[str] = (), +) -> list[str]: + """Return the exact immutable uv sync command accepted by the M6.1 runtime.""" + + arguments = [ + "sync", + "--locked", + "--no-build", + "--managed-python", + "--python", + python_version, + ] + if "dev" not in selected_extras: + arguments.append("--no-dev") + for extra in selected_extras: + arguments.extend(["--extra", extra]) + arguments.append("--no-install-project") + for distribution_name in approved_artifact_names: + arguments.extend(["--no-install-package", distribution_name]) + return arguments + + class UvManagedBackend: """Plan exact LocalAppData paths and commands for the first runtime backend.""" @@ -85,19 +113,10 @@ def build_plan( purpose="Install the selected managed CPython without PATH or registry integration.", ) selected_extras = selected_extras or [] - sync_arguments = [ - "sync", - "--locked", - "--no-build", - "--managed-python", - "--python", - python_version, - ] - if "dev" not in selected_extras: - sync_arguments.append("--no-dev") - for extra in selected_extras: - sync_arguments.extend(["--extra", extra]) - sync_arguments.append("--no-install-project") + sync_arguments = uv_sync_arguments( + python_version=python_version, + selected_extras=selected_extras, + ) sync = PlannedCommand( executable=paths.uv_executable, arguments=sync_arguments, diff --git a/src/python_deployment_builder/generation/manifest.py b/src/python_deployment_builder/generation/manifest.py index d3de986..36b8a48 100644 --- a/src/python_deployment_builder/generation/manifest.py +++ b/src/python_deployment_builder/generation/manifest.py @@ -8,6 +8,7 @@ from pathlib import Path from python_deployment_builder import __version__ +from python_deployment_builder.backends.uv_managed import uv_sync_arguments from python_deployment_builder.generation.acquisition import PreparationError, sha256_file from python_deployment_builder.models import ( ApplicationArtifact, @@ -58,9 +59,11 @@ def build_deployment_manifest( if plan.deployment_mode == "source" and not source_roots: raise PreparationError("Source deployment plan does not provide a runtime source root.") - sync_arguments = list(plan.runtime.sync_command.arguments) - for artifact in approved_artifacts: - sync_arguments.extend(["--no-install-package", artifact.distribution_name]) + sync_arguments = uv_sync_arguments( + python_version=plan.runtime.python_version, + selected_extras=plan.runtime.selected_extras, + approved_artifact_names=[item.distribution_name for item in approved_artifacts], + ) timestamp = generated_at or datetime.now(UTC) fingerprint_payload: dict[str, object] = { "schema_version": plan.schema_version, diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index fa62d6d..59c05d4 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -18,6 +18,7 @@ from packaging.version import InvalidVersion, Version from pydantic import ValidationError +from python_deployment_builder.backends.uv_managed import uv_sync_arguments from python_deployment_builder.generation.acquisition import PreparationError from python_deployment_builder.generation.artifacts import ( configured_secret_values, @@ -615,24 +616,6 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR "Staged-lock developer artifact requirement does not have exactly one " f"manifest-approved wheel: {requirement.package}=={requirement.version}." ) - suppressed_packages: list[str] = [] - for index, argument in enumerate(manifest.sync_arguments): - if argument != "--no-install-package": - continue - if index + 1 >= len(manifest.sync_arguments): - approved_identity_failures.append( - "Runtime sync arguments end with --no-install-package without a distribution." - ) - continue - suppressed_packages.append(canonicalize_name(manifest.sync_arguments[index + 1])) - approved_packages = [ - canonicalize_name(artifact.distribution_name) - for artifact in manifest.approved_artifacts - ] - if sorted(suppressed_packages) != sorted(approved_packages): - approved_identity_failures.append( - "Runtime --no-install-package arguments do not match manifest-approved artifacts." - ) checks.append( _check( "APPROVED_ARTIFACT_LOCK_IDENTITY", @@ -788,17 +771,23 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR ) ) - sync_extras: list[str] = [] - for index_arg, value in enumerate(manifest.sync_arguments[:-1]): - if value == "--extra": - sync_extras.append(manifest.sync_arguments[index_arg + 1]) + expected_sync_arguments = uv_sync_arguments( + python_version=manifest.python_version, + selected_extras=manifest.selected_extras, + approved_artifact_names=[ + artifact.distribution_name for artifact in manifest.approved_artifacts + ], + ) checks.append( _check( - "SELECTED_EXTRAS", - sorted(sync_extras) == sorted(manifest.selected_extras), - "Selected extras exactly match the locked sync command.", - "Selected extras and locked sync arguments differ.", - evidence=[f"manifest={manifest.selected_extras}", f"sync={sync_extras}"], + "SYNC_ARGUMENTS_CONTRACT", + manifest.sync_arguments == expected_sync_arguments, + "Runtime sync arguments exactly match the immutable uv-managed contract.", + "Runtime sync arguments differ from the immutable uv-managed contract.", + evidence=[ + f"expected={expected_sync_arguments}", + f"actual={manifest.sync_arguments}", + ], ) ) diff --git a/tests/test_generation.py b/tests/test_generation.py index b7eb905..a9edc30 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -21,6 +21,7 @@ git_skip_worktree_paths, ) from python_deployment_builder.analysis.resources import resolve_package_data_members +from python_deployment_builder.backends.uv_managed import uv_sync_arguments from python_deployment_builder.cli import build_parser, main from python_deployment_builder.generation.acquisition import ( PreparationError, @@ -3908,6 +3909,11 @@ def manifest(plan, application): ) original = manifest(package_plan, artifact) + assert original.sync_arguments == uv_sync_arguments( + python_version=package_plan.runtime.python_version, + selected_extras=package_plan.runtime.selected_extras, + ) + assert "--no-install-package" not in original.sync_arguments renamed_wheel = tmp_path / "mapped_app-1.2.3-1-py3-none-any.whl" renamed_wheel.write_bytes(wheel.read_bytes()) renamed_artifact, _ = validate_application_wheel( @@ -4520,6 +4526,191 @@ def test_static_approved_artifact_lock_identity_uses_pep440_versions( ] +@pytest.mark.parametrize( + "mutation", + [ + "missing-sync", + "missing-locked", + "missing-no-build", + "missing-managed-python", + "missing-python", + "missing-no-install-project", + "missing-no-dev", + "wrong-python", + "injected-upgrade", + "duplicate-locked", + "unexpected-positional", + "reordered-critical-options", + "extra-unselected-extra", + ], +) +def test_static_validation_rejects_noncanonical_sync_arguments( + mutation: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit(_repository("prepared_gui"), kit, bootstrap_mode="online_cmd") + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + arguments = manifest["sync_arguments"] + if mutation == "missing-sync": + arguments.remove("sync") + elif mutation == "missing-locked": + arguments.remove("--locked") + elif mutation == "missing-no-build": + arguments.remove("--no-build") + elif mutation == "missing-managed-python": + arguments.remove("--managed-python") + elif mutation == "missing-python": + index = arguments.index("--python") + del arguments[index : index + 2] + elif mutation == "missing-no-install-project": + arguments.remove("--no-install-project") + elif mutation == "missing-no-dev": + arguments.remove("--no-dev") + elif mutation == "wrong-python": + arguments[arguments.index("--python") + 1] = "3.11" + elif mutation == "injected-upgrade": + arguments.append("--upgrade") + elif mutation == "duplicate-locked": + arguments.insert(arguments.index("--locked"), "--locked") + elif mutation == "unexpected-positional": + arguments.append("unexpected-project") + elif mutation == "reordered-critical-options": + locked = arguments.index("--locked") + no_build = arguments.index("--no-build") + arguments[locked], arguments[no_build] = arguments[no_build], arguments[locked] + elif mutation == "extra-unselected-extra": + index = arguments.index("--no-install-project") + arguments[index:index] = ["--extra", "rogue"] + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "SYNC_ARGUMENTS_CONTRACT" and item.status.value == "FAIL" + for item in report.static_checks + ) + + +@pytest.mark.parametrize( + "mutation", + ["unexpected-no-dev", "missing-selected-extra", "duplicate-selected-extra"], +) +def test_static_sync_contract_enforces_selected_dev_exactly( + mutation: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["dev"], + bootstrap_mode="online_cmd", + ) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + arguments = manifest["sync_arguments"] + extra = arguments.index("--extra") + if mutation == "unexpected-no-dev": + arguments.insert(extra, "--no-dev") + elif mutation == "missing-selected-extra": + del arguments[extra : extra + 2] + elif mutation == "duplicate-selected-extra": + arguments[extra:extra] = ["--extra", "dev"] + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "SYNC_ARGUMENTS_CONTRACT" and item.status.value == "FAIL" + for item in report.static_checks + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "missing-approved-suppression", + "extra-approved-suppression", + "duplicate-approved-suppression", + "wrong-approved-suppression", + ], +) +def test_static_sync_contract_enforces_approved_suppression_exactly( + mutation: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + artifact = _make_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={artifact}"], + bootstrap_mode="online_cmd", + ) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + arguments = manifest["sync_arguments"] + suppression = arguments.index("--no-install-package") + if mutation == "missing-approved-suppression": + del arguments[suppression : suppression + 2] + elif mutation == "extra-approved-suppression": + arguments.extend(["--no-install-package", "intruder"]) + elif mutation == "duplicate-approved-suppression": + arguments.extend(["--no-install-package", "proxy-tools"]) + elif mutation == "wrong-approved-suppression": + arguments[suppression + 1] = "intruder" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _update_indexed_hashes(kit, "deployment/manifest.json") + + report = validate_static_kit(kit) + + assert report.final_state.value == "FAILED" + assert any( + item.code == "SYNC_ARGUMENTS_CONTRACT" and item.status.value == "FAIL" + for item in report.static_checks + ) + + def test_static_validation_rejects_indexed_unvalidated_staged_wheel( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -6596,6 +6787,10 @@ def test_source_mode_staged_secret_fails_before_writes( def test_manifest_renders_flat_source_system_certs_and_selected_extra(tmp_path: Path) -> None: plan = _plan("optional_map_app", ["map"]) + assert plan.runtime.sync_command.arguments == uv_sync_arguments( + python_version=plan.runtime.python_version, + selected_extras=["map"], + ) wheel = _make_wheel(tmp_path) approved, _path = validate_approved_wheel(f"proxy-tools={wheel}", plan) manifest = build_deployment_manifest( @@ -6611,10 +6806,38 @@ def test_manifest_renders_flat_source_system_certs_and_selected_extra(tmp_path: assert manifest.source_roots == ["."] assert manifest.system_certs assert manifest.selected_extras == ["map"] - assert manifest.sync_arguments[-2:] == ["--no-install-package", "proxy-tools"] + assert manifest.sync_arguments == uv_sync_arguments( + python_version=plan.runtime.python_version, + selected_extras=["map"], + approved_artifact_names=["proxy-tools"], + ) assert manifest.approved_artifacts[0].sha256 +def test_uv_sync_arguments_preserve_selected_extra_and_artifact_order() -> None: + assert uv_sync_arguments( + python_version="3.12", + selected_extras=["dev", "map"], + approved_artifact_names=["reviewed-b", "reviewed-a"], + ) == [ + "sync", + "--locked", + "--no-build", + "--managed-python", + "--python", + "3.12", + "--extra", + "dev", + "--extra", + "map", + "--no-install-project", + "--no-install-package", + "reviewed-b", + "--no-install-package", + "reviewed-a", + ] + + def test_src_manifest_uses_src_root() -> None: plan = _plan("target_app") lock = FIXTURES / "target_app" / "uv.lock" From 9e5c7e30e3585794f6a75a0bb99ca6a2bf252408 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:33:07 -0400 Subject: [PATCH 48/59] Guard unmodeled setuptools package data --- .../analysis/metadata.py | 166 +++++++++- .../generation/generator.py | 15 + tests/test_generation.py | 163 +++++++++ tests/test_metadata.py | 308 ++++++++++++++++++ 4 files changed, 646 insertions(+), 6 deletions(-) diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 27114db..9b26cba 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -50,7 +50,14 @@ class LiteralModuleAttribute: _SETUP_SURFACE_FIELDS = frozenset( - {"packages", "py_modules", "package_dir", "package_data", "exclude_package_data"} + { + "packages", + "py_modules", + "package_dir", + "package_data", + "exclude_package_data", + "include_package_data", + } ) @@ -733,6 +740,12 @@ def inspect_metadata(root: Path) -> MetadataResult: setuptools_surface_evidence: list[Evidence] = [] setuptools_external_packaging_roots: list[str] = [] setuptools_external_packaging_root_evidence: list[Evidence] = [] + pyproject_controls_include_package_data = False + pyproject_include_package_data: bool | None = None + pyproject_include_package_data_invalid = False + setup_cfg_include_package_data: bool | None = False + setup_py_include_package_data: bool | None = False + setuptools_file_finder_requirements: list[str] = [] uv_workspace = False uv_workspace_source = False uv_workspace_evidence: list[Evidence] = [] @@ -746,6 +759,15 @@ def inspect_metadata(root: Path) -> MetadataResult: build_system = ( document.get("build-system") if isinstance(document.get("build-system"), dict) else {} ) + for specification in build_system.get("requires", []): + if not isinstance(specification, str): + continue + try: + build_requirement = Requirement(specification) + except InvalidRequirement: + continue + if canonicalize_name(build_requirement.name) == "setuptools-scm": + setuptools_file_finder_requirements.append(specification) distribution_name = project.get("name") if isinstance(project.get("name"), str) else None project_version = ( project.get("version") if isinstance(project.get("version"), str) else None @@ -803,6 +825,22 @@ def inspect_metadata(root: Path) -> MetadataResult: if isinstance(build_system.get("build-backend"), str) else None ) + pyproject_controls_include_package_data = bool( + ( + isinstance(build_backend, str) + and build_backend.startswith("setuptools.") + and isinstance(document.get("project"), dict) + ) + or setuptools + ) + if pyproject_controls_include_package_data: + configured_include_package_data = setuptools.get( + "include-package-data", True + ) + if isinstance(configured_include_package_data, bool): + pyproject_include_package_data = configured_include_package_data + else: + pyproject_include_package_data_invalid = True if requires_python: python_evidence.append( _evidence( @@ -1009,6 +1047,13 @@ def inspect_metadata(root: Path) -> MetadataResult: package_data_parser = configparser.ConfigParser() package_data_parser.optionxform = str package_data_parser.read(setup_cfg_path, encoding="utf-8") + if parser.has_option("options", "include_package_data"): + try: + setup_cfg_include_package_data = parser.getboolean( + "options", "include_package_data" + ) + except ValueError: + setup_cfg_include_package_data = None if distribution_name is None: distribution_name = parser.get("metadata", "name", fallback=None) project_version = parser.get("metadata", "version", fallback=None) @@ -1134,17 +1179,37 @@ def inspect_metadata(root: Path) -> MetadataResult: metadata_files.append("setup.py") setup_inspection = inspect_setup_call(setup_py_path) setup_values = setup_inspection.literal_values + if setup_inspection.has_kwargs_expansion: + setup_py_include_package_data = None + elif "include_package_data" in setup_inspection.present_keywords: + configured_include_package_data = setup_values.get("include_package_data") + setup_py_include_package_data = ( + configured_include_package_data + if isinstance(configured_include_package_data, bool) + else None + ) setuptools_package_selection_configured = ( setuptools_package_selection_configured or setup_inspection.package_selection_present ) + setup_unresolved_fields = set( + setup_inspection.unresolved_keywords & _SETUP_SURFACE_FIELDS + ) + if pyproject_controls_include_package_data: + # Setuptools 79.0.1's pyproject configuration is authoritative for + # include-package-data. A literal legacy setup() value does not + # override an explicit pyproject value. + setup_unresolved_fields.discard("include_package_data") + setup_surface_unresolved = bool( + setup_inspection.parse_failed + or setup_inspection.has_kwargs_expansion + or setup_unresolved_fields + ) setuptools_surface_unresolved = ( - setuptools_surface_unresolved or setup_inspection.surface_unresolved + setuptools_surface_unresolved or setup_surface_unresolved ) - if setuptools_surface_unresolved: - unresolved = sorted( - setup_inspection.unresolved_keywords & _SETUP_SURFACE_FIELDS - ) + if setup_surface_unresolved: + unresolved = sorted(setup_unresolved_fields) detail = ( "setup() expands **kwargs, so modeled packaging-surface fields cannot be " "statically established." @@ -1251,6 +1316,95 @@ def inspect_metadata(root: Path) -> MetadataResult: ), ) + manifest_path = root / "MANIFEST.in" + modeled_setuptools = bool( + isinstance(build_backend, str) and build_backend.startswith("setuptools.") + ) or (build_backend is None and (setup_cfg_path.is_file() or setup_py_path.is_file())) + effective_include_package_data: bool | None = False + include_package_data_evidence_path: Path | None = None + if modeled_setuptools: + if pyproject_controls_include_package_data: + effective_include_package_data = ( + None + if pyproject_include_package_data_invalid + else pyproject_include_package_data + ) + include_package_data_evidence_path = pyproject_path + else: + legacy_values = [ + value + for path, value in ( + (setup_cfg_path, setup_cfg_include_package_data), + (setup_py_path, setup_py_include_package_data), + ) + if path.is_file() + ] + if any(value is True for value in legacy_values): + # Setuptools 79.0.1 keeps the mechanism enabled when either + # legacy configuration source explicitly enables it. + effective_include_package_data = True + elif any(value is None for value in legacy_values): + effective_include_package_data = None + else: + effective_include_package_data = False + include_package_data_evidence_path = next( + (path for path in (setup_py_path, setup_cfg_path) if path.is_file()), + None, + ) + if modeled_setuptools and effective_include_package_data is None: + setuptools_surface_unresolved = True + evidence_path = include_package_data_evidence_path or pyproject_path + setuptools_surface_evidence.append( + _evidence( + root, + evidence_path, + "Setuptools include_package_data is present but cannot be resolved to a " + "literal boolean without executing project configuration.", + _line_number(evidence_path, "include_package_data"), + ) + ) + if ( + modeled_setuptools + and manifest_path.is_file() + and effective_include_package_data is not False + ): + # MANIFEST.in is authoritative build metadata only while setuptools' + # file-list package-data mechanism can affect the wheel. It remains a + # provenance input, never an inferred runtime resource or staged file. + metadata_files.append("MANIFEST.in") + setuptools_surface_unresolved = True + state = "True" if effective_include_package_data is True else "unresolved" + setuptools_surface_evidence.append( + _evidence( + root, + manifest_path, + "MANIFEST.in may contribute package data through effective " + f"include_package_data={state}; M6.1 does not interpret setuptools " + "manifest/file-list semantics.", + 1, + ) + ) + if ( + modeled_setuptools + and effective_include_package_data is not False + and setuptools_file_finder_requirements + ): + # setuptools-scm registers a setuptools file-finder hook that can add + # version-controlled package files without explicit package_data. + # M6.1 identifies only this reproduced standard plugin; arbitrary build + # requirements are not guessed to be file finders. + setuptools_surface_unresolved = True + setuptools_surface_evidence.append( + _evidence( + root, + pyproject_path, + "Declared setuptools-scm build requirement may contribute package data " + "through the active setuptools file-finder mechanism; M6.1 does not " + "interpret plugin-provided file lists.", + _line_number(pyproject_path, "setuptools-scm"), + ) + ) + requirements = _requirements_files(root) for path in requirements: metadata_files.append(path.relative_to(root).as_posix()) diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index ce86c4a..c808787 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -392,6 +392,21 @@ def _tracked_deployment_paths( ) tracked.add("uv.lock") if assessment.repository.revision is not None: + # Selected inputs retain their more specific staging diagnostic below. + # This check closes the separate gap for analyzed metadata (for example + # an active MANIFEST.in or dynamic-version module) that intentionally + # is not copied into the runtime kit. + metadata_for_tracking = _analysis_metadata_paths(assessment) - selected + if missing_lock_is_previewed or created_lock is not None: + metadata_for_tracking.discard("uv.lock") + untracked_metadata = sorted(metadata_for_tracking - tracked) + if untracked_metadata: + raise PreparationError( + "Analyzed metadata inputs must be tracked for Git release generation " + f"at recorded source revision {assessment.repository.revision}: " + + ", ".join(untracked_metadata) + + ". Commit or remove those metadata inputs before release generation." + ) selected_for_tracking = selected - ({"uv.lock"} if missing_lock_is_previewed else set()) untracked_selected = sorted(selected_for_tracking - tracked) if untracked_selected: diff --git a/tests/test_generation.py b/tests/test_generation.py index a9edc30..9251d09 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -3000,6 +3000,169 @@ def test_application_wheel_requires_every_concrete_declared_package_data_member( assert artifact.filename == complete.name +def test_application_wheel_rejects_unmodeled_manifest_package_data_surface( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + (source / "src/app").mkdir(parents=True) + (source / "src/app/__init__.py").write_text("", encoding="utf-8") + (source / "src/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "src/app/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "MANIFEST.in").write_text( + "include src/app/defaults.json\n", encoding="utf-8" + ) + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1','wheel']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='manifest-demo'\nversion='1.0.0'\n" + "[project.scripts]\nmanifest-demo='app.main:main'\n" + "[tool.setuptools.packages.find]\nwhere=['src']\n", + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + assert "PACKAGING_SURFACE_UNRESOLVED" in [item.code for item in assessment.risks] + assert plan.deployment_mode == "source" + assert plan.deployment_mode_condition == "SOURCE_COMPATIBLE" + assert "MANIFEST.in" in _analysis_metadata_paths(assessment) + assert "MANIFEST.in" not in _selected_deployment_paths(source, assessment, plan) + package_plan = plan.model_copy(deep=True) + package_plan.deployment_mode = "package" + incomplete = _make_application_wheel( + tmp_path, + name="manifest-demo", + version="1.0.0", + package="app", + target="app.main:main", + entry_group="console_scripts", + entry_name="manifest-demo", + ) + + with pytest.raises(PreparationError, match="authoritative Python packaging-surface"): + validate_application_wheel( + incomplete, + assessment, + package_plan, + repository_root=source, + ) + + +def test_installed_only_manifest_surface_blocks_package_readiness(tmp_path: Path) -> None: + source = tmp_path / "source" + (source / "src/app").mkdir(parents=True) + (source / "src/app/__init__.py").write_text("", encoding="utf-8") + (source / "src/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (source / "src/app/defaults.json").write_text("{}\n", encoding="utf-8") + (source / "MANIFEST.in").write_text( + "include src/app/defaults.json\n", encoding="utf-8" + ) + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1','wheel']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='manifest-demo'\nversion='1.0.0'\n" + "[project.scripts]\nmanifest-demo='installed_app.main:main'\n" + "[tool.setuptools]\npackages=['installed_app']\n" + "package-dir={'installed_app'='src/app'}\n", + encoding="utf-8", + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + assert plan.deployment_mode == "package" + assert plan.deployment_mode_condition == "INSTALLED_PROJECT_REQUIRED" + assert "PACKAGING_SURFACE_UNRESOLVED" in plan.readiness.blocker_codes + + +def _write_git_manifest_surface_project(root: Path, *, manifest: bool = True) -> None: + (root / "src/app").mkdir(parents=True) + (root / "src/app/__init__.py").write_text("", encoding="utf-8") + (root / "src/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "src/app/defaults.json").write_text("{}\n", encoding="utf-8") + (root / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1','wheel']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='manifest-demo'\nversion='1.0.0'\n" + "[project.scripts]\nmanifest-demo='app.main:main'\n" + "[tool.setuptools.packages.find]\nwhere=['src']\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.11'\n" + "[[package]]\nname='manifest-demo'\nversion='1.0.0'\nsource={virtual='.'}\n", + encoding="utf-8", + ) + if manifest: + (root / "MANIFEST.in").write_text( + "include src/app/defaults.json\n", encoding="utf-8" + ) + + +@pytest.mark.parametrize("operation", ["modified", "deleted", "renamed"]) +def test_git_provenance_guards_active_manifest_input( + tmp_path: Path, operation: str +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_git_manifest_surface_project(source) + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + manifest = source / "MANIFEST.in" + if operation == "modified": + manifest.write_text("recursive-include src/app *.json\n", encoding="utf-8") + elif operation == "deleted": + manifest.unlink() + else: + manifest.rename(source / "MANIFEST-renamed.in") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises( + PreparationError, match="Tracked deployment inputs differ.*MANIFEST.in" + ): + _staging_files(source, assessment, plan, include=True) + + +def test_untracked_active_manifest_blocks_generation_and_dry_run(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _write_git_manifest_surface_project(source, manifest=False) + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "pdb@example.invalid"], + check=True, + ) + subprocess.run( + ["git", "-C", str(source), "config", "user.name", "PDB Test"], check=True + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + (source / "MANIFEST.in").write_text( + "include src/app/defaults.json\n", encoding="utf-8" + ) + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=source) + + with pytest.raises(PreparationError, match="Analyzed metadata inputs.*MANIFEST.in"): + _staging_files(source, assessment, plan, include=True) + output = tmp_path / "kit" + with pytest.raises(PreparationError, match="Analyzed metadata inputs.*MANIFEST.in"): + generate_deployment_kit(repository, output, dry_run=True) + assert not output.exists() + + def test_application_wheel_and_source_staging_honor_excluded_package_data( tmp_path: Path, ) -> None: diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 364b8c5..374025e 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -16,6 +16,48 @@ FIXTURES = Path(__file__).parent / "fixtures" +def _write_manifest_pyproject_project( + root: Path, + *, + include_package_data: bool | None = None, + package_data: bool = False, + exclude_package_data: bool = False, +) -> None: + (root / "src/app").mkdir(parents=True) + (root / "src/app/__init__.py").write_text("", encoding="utf-8") + (root / "src/app/main.py").write_text("def main(): return 0\n", encoding="utf-8") + (root / "src/app/defaults.json").write_text("{}\n", encoding="utf-8") + (root / "MANIFEST.in").write_text( + "include src/app/defaults.json\n", encoding="utf-8" + ) + include_setting = ( + "" + if include_package_data is None + else "[tool.setuptools]\n" + f"include-package-data = {str(include_package_data).lower()}\n" + ) + package_data_setting = ( + "[tool.setuptools.package-data]\napp=['defaults.json']\n" + if package_data + else "" + ) + exclude_setting = ( + "[tool.setuptools.exclude-package-data]\napp=['*.secret']\n" + if exclude_package_data + else "" + ) + (root / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1','wheel']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='manifest-demo'\nversion='1.0.0'\n" + "[project.scripts]\nmanifest-demo='app.main:main'\n" + f"{include_setting}" + "[tool.setuptools.packages.find]\nwhere=['src']\n" + f"{package_data_setting}{exclude_setting}", + encoding="utf-8", + ) + + def test_pyproject_parsing_and_entry_points() -> None: result = inspect_metadata(FIXTURES / "simple_cli") @@ -169,6 +211,272 @@ def test_setup_py_malformed_literal_selection_remains_unresolved( assert result.project.py_modules == [] +@pytest.mark.parametrize("setting", [None, True]) +def test_active_pyproject_manifest_package_data_surface_is_unresolved( + tmp_path: Path, setting: bool | None +) -> None: + _write_manifest_pyproject_project(tmp_path, include_package_data=setting) + + result = inspect_metadata(tmp_path) + + assert result.project.package_data == {} + assert result.setuptools_surface_unresolved + assert "MANIFEST.in" in result.project.metadata_files + assert any( + evidence.file == "MANIFEST.in" + and "include_package_data=True" in evidence.detail + for evidence in result.setuptools_surface_evidence + ) + + +def test_pyproject_manifest_is_inactive_when_include_package_data_is_false( + tmp_path: Path, +) -> None: + _write_manifest_pyproject_project(tmp_path, include_package_data=False) + + result = inspect_metadata(tmp_path) + + assert not result.setuptools_surface_unresolved + assert "MANIFEST.in" not in result.project.metadata_files + assert result.project.packages == ["app"] + + +@pytest.mark.parametrize( + ("setting", "expected_unresolved"), + [(None, False), (True, True), (False, False)], +) +def test_setup_cfg_manifest_uses_legacy_include_package_data_default( + tmp_path: Path, setting: bool | None, expected_unresolved: bool +) -> None: + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "MANIFEST.in").write_text( + "include src/app/defaults.json\n", encoding="utf-8" + ) + configured = ( + "" + if setting is None + else f"include_package_data = {str(setting).lower()}\n" + ) + (tmp_path / "setup.cfg").write_text( + "[metadata]\nname=manifest-demo\nversion=1.0.0\n" + "[options]\npackages=find:\npackage_dir=\n = src\n" + f"{configured}" + "[options.packages.find]\nwhere=src\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.setuptools_surface_unresolved is expected_unresolved + assert ("MANIFEST.in" in result.project.metadata_files) is expected_unresolved + + +@pytest.mark.parametrize( + ("setting", "expected_unresolved"), + [("True", True), ("False", False), ("SOME_VALUE", True)], +) +def test_setup_py_manifest_requires_literal_include_package_data( + tmp_path: Path, setting: str, expected_unresolved: bool +) -> None: + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "MANIFEST.in").write_text( + "include src/app/defaults.json\n", encoding="utf-8" + ) + (tmp_path / "setup.py").write_text( + "from setuptools import setup\n" + "setup(name='manifest-demo', version='1.0.0', package_dir={'': 'src'}, " + f"packages=['app'], include_package_data={setting})\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.setuptools_surface_unresolved is expected_unresolved + assert ("MANIFEST.in" in result.project.metadata_files) is expected_unresolved + + +def test_invalid_setup_cfg_include_package_data_is_controlled_unresolved( + tmp_path: Path, +) -> None: + (tmp_path / "setup.cfg").write_text( + "[metadata]\nname=manifest-demo\nversion=1.0\n" + "[options]\ninclude_package_data=perhaps\n", + encoding="utf-8", + ) + (tmp_path / "MANIFEST.in").write_text("include app/data.txt\n", encoding="utf-8") + + result = inspect_metadata(tmp_path) + + assert result.setuptools_surface_unresolved + assert "MANIFEST.in" in result.project.metadata_files + + +def test_manifest_does_not_create_setuptools_finding_for_other_backend(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['hatchling']\nbuild-backend='hatchling.build'\n" + "[project]\nname='other-backend'\nversion='1.0'\n", + encoding="utf-8", + ) + (tmp_path / "MANIFEST.in").write_text("include data.txt\n", encoding="utf-8") + + result = inspect_metadata(tmp_path) + + assert not result.setuptools_surface_unresolved + assert "MANIFEST.in" not in result.project.metadata_files + + +def test_known_setuptools_scm_file_finder_keeps_active_surface_unresolved( + tmp_path: Path, +) -> None: + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "src/app/scm-data.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\n" + "requires=['setuptools==79.0.1','wheel','setuptools-scm>=8']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='scm-finder-demo'\nversion='1.0.0'\n" + "[tool.setuptools.packages.find]\nwhere=['src']\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert result.setuptools_surface_unresolved + assert any( + "setuptools-scm" in evidence.detail + for evidence in result.setuptools_surface_evidence + ) + + +def test_include_package_data_false_disables_known_file_finder_surface( + tmp_path: Path, +) -> None: + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\n" + "requires=['setuptools==79.0.1','wheel','setuptools-scm>=8']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='scm-finder-demo'\nversion='1.0.0'\n" + "[tool.setuptools]\ninclude-package-data=false\n" + "[tool.setuptools.packages.find]\nwhere=['src']\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert not result.setuptools_surface_unresolved + + +def test_unrecognized_build_requirement_is_not_guessed_to_be_a_file_finder( + tmp_path: Path, +) -> None: + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1','custom-build-plugin']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='custom-plugin-demo'\nversion='1.0.0'\n" + "[tool.setuptools.packages.find]\nwhere=['src']\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert not result.setuptools_surface_unresolved + + +def test_pyproject_include_false_remains_authoritative_over_setup_py_true( + tmp_path: Path, +) -> None: + _write_manifest_pyproject_project(tmp_path, include_package_data=False) + (tmp_path / "setup.py").write_text( + "from setuptools import setup\nsetup(include_package_data=True)\n", + encoding="utf-8", + ) + + result = inspect_metadata(tmp_path) + + assert not result.setuptools_surface_unresolved + assert "MANIFEST.in" not in result.project.metadata_files + + +def test_explicit_package_data_remains_modeled_when_manifest_is_inactive( + tmp_path: Path, +) -> None: + _write_manifest_pyproject_project( + tmp_path, include_package_data=False, package_data=True + ) + + result = inspect_metadata(tmp_path) + members = resolve_package_data_members(tmp_path, result.project) + + assert not result.setuptools_surface_unresolved + assert [(item.source_path, item.installed_member_path) for item in members] == [ + ("src/app/defaults.json", "app/defaults.json") + ] + + +def test_explicit_include_and_exclude_do_not_resolve_active_manifest_surface( + tmp_path: Path, +) -> None: + _write_manifest_pyproject_project( + tmp_path, + include_package_data=True, + package_data=True, + exclude_package_data=True, + ) + + result = inspect_metadata(tmp_path) + + assert result.project.package_data == {"app": ["defaults.json"]} + assert result.project.exclude_package_data == {"app": ["*.secret"]} + assert result.setuptools_surface_unresolved + + +@pytest.mark.parametrize( + ("setting", "expected_included"), + [(None, True), (True, True), (False, False)], +) +def test_setuptools_79_pyproject_manifest_wheel_behavior( + tmp_path: Path, setting: bool | None, expected_included: bool +) -> None: + import setuptools + + if setuptools.__version__ != "79.0.1": + pytest.skip("Exact setuptools 79.0.1 behavioral evidence requires that version.") + _write_manifest_pyproject_project(tmp_path, include_package_data=setting) + dist = tmp_path / "dist" + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "wheel", + ".", + "--no-build-isolation", + "--no-deps", + "--wheel-dir", + str(dist), + ], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + wheel = next(dist.glob("manifest_demo-1.0.0-*.whl")) + + with zipfile.ZipFile(wheel) as bundle: + included = "app/defaults.json" in bundle.namelist() + + assert included is expected_included + + def test_dynamic_setup_package_selector_does_not_trigger_automatic_discovery( tmp_path: Path, ) -> None: From a94c1b69f7934a7a0f89f725c17f135ee3296cea Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:20:26 -0400 Subject: [PATCH 49/59] Fix keyword AST runtime analysis --- .../analysis/ast_utils.py | 27 +++ .../analysis/inventory.py | 29 +++ .../analysis/resources.py | 11 +- .../analysis/runtime_assumptions.py | 15 +- tests/test_analysis_scope.py | 169 ++++++++++++++++++ tests/test_generation.py | 39 ++++ tests/test_runtime_assumptions.py | 43 +++++ 7 files changed, 323 insertions(+), 10 deletions(-) create mode 100644 src/python_deployment_builder/analysis/ast_utils.py diff --git a/src/python_deployment_builder/analysis/ast_utils.py b/src/python_deployment_builder/analysis/ast_utils.py new file mode 100644 index 0000000..46e95e7 --- /dev/null +++ b/src/python_deployment_builder/analysis/ast_utils.py @@ -0,0 +1,27 @@ +"""Small bounded helpers for non-executing AST analysis.""" + +from __future__ import annotations + +import ast + + +def call_argument( + node: ast.Call, + *, + position: int, + keyword: str, +) -> ast.AST | None: + """Return one API argument without attempting general signature binding. + + The supported APIs use an equivalent positional-or-keyword form for this + argument. Positional syntax deliberately wins if a syntactically valid + call supplies both forms: that call is runtime-invalid, and this bounded + static analysis must neither invent a second value nor evaluate it. + """ + + if len(node.args) > position: + return node.args[position] + return next( + (item.value for item in node.keywords if item.arg == keyword), + None, + ) diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index 4fc4291..a54a725 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -315,6 +315,17 @@ def _imported_modules( source_roots: list[str], ) -> list[tuple[str, int]]: modules: list[tuple[str, int]] = [] + importlib_modules: set[str] = set() + import_module_functions: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "importlib": + importlib_modules.add(alias.asname or "importlib") + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module == "importlib": + for alias in node.names: + if alias.name == "import_module": + import_module_functions.add(alias.asname or alias.name) for node in ast.walk(tree): if isinstance(node, ast.Import): modules.extend((alias.name, node.lineno) for alias in node.names) @@ -344,6 +355,24 @@ def _imported_modules( for alias in node.names if alias.name != "*" ) + elif isinstance(node, ast.Call): + target: ast.AST | None = None + function = node.func + direct_import = isinstance(function, ast.Name) and ( + function.id == "__import__" or function.id in import_module_functions + ) + module_import = ( + isinstance(function, ast.Attribute) + and function.attr == "import_module" + and isinstance(function.value, ast.Name) + and function.value.id in importlib_modules + ) + if direct_import or module_import: + target = node.args[0] if node.args else None + if isinstance(target, ast.Constant) and isinstance(target.value, str): + module = target.value + if module and all(part.isidentifier() for part in module.split(".")): + modules.append((module, node.lineno)) return modules diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 324a848..d873b05 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from pathlib import Path, PurePosixPath, PureWindowsPath +from python_deployment_builder.analysis.ast_utils import call_argument from python_deployment_builder.analysis.imports import EXCLUDED_DIRECTORIES from python_deployment_builder.models import ( ConfigurationRequirement, @@ -1091,8 +1092,9 @@ def _open_access(name: str, node: ast.Call) -> str: def _path_uses(node: ast.Call) -> list[tuple[ast.AST, str]]: name = _qualified_name(node.func) method = name.split(".")[-1].lower() - if name == "open" and node.args: - return [(node.args[0], _open_access(name, node))] + if name == "open": + path = call_argument(node, position=0, keyword="file") + return [(path, _open_access(name, node))] if path is not None else [] if method == "open" and isinstance(node.func, ast.Attribute): library_open = _qualified_name(node.func.value).split(".")[0].lower() in { "fitz", @@ -1126,8 +1128,9 @@ def _path_uses(node: ast.Call) -> list[tuple[ast.AST, str]]: ] if method in {"iterdir", "glob", "rglob"} and isinstance(node.func, ast.Attribute): return [(node.func.value, "read")] - if name in {"os.listdir", "os.scandir"} and node.args: - return [(node.args[0], "read")] + if name in {"os.listdir", "os.scandir"}: + path = call_argument(node, position=0, keyword="path") + return [(path, "read")] if path is not None else [] return [] diff --git a/src/python_deployment_builder/analysis/runtime_assumptions.py b/src/python_deployment_builder/analysis/runtime_assumptions.py index dd9ebe3..c15d9e6 100644 --- a/src/python_deployment_builder/analysis/runtime_assumptions.py +++ b/src/python_deployment_builder/analysis/runtime_assumptions.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from pathlib import Path +from python_deployment_builder.analysis.ast_utils import call_argument from python_deployment_builder.analysis.imports import EXCLUDED_DIRECTORIES from python_deployment_builder.models import ( ConfigurationRequirement, @@ -298,8 +299,10 @@ def visit_Subscript(self, node: ast.Subscript) -> None: # noqa: N802 def visit_Call(self, node: ast.Call) -> None: # noqa: N802 name = _qualified_name(node.func) - if name in {"os.getenv", "os.environ.get"} and node.args: - variable = _literal_string(node.args[0]) + if name in {"os.getenv", "os.environ.get"}: + variable = _literal_string( + call_argument(node, position=0, keyword="key") + ) if variable: self.config[variable].append(self._evidence(node, f"Read through {name}.")) if name in {"Path.cwd", "pathlib.Path.cwd", "os.getcwd"}: @@ -316,7 +319,7 @@ def visit_Call(self, node: ast.Call) -> None: # noqa: N802 "Popen", "run", }: - command = _command_name(node.args[0]) if node.args else None + command = _command_name(call_argument(node, position=0, keyword="args")) self._runtime( "external_executable", command or "dynamic subprocess command", @@ -331,7 +334,7 @@ def visit_Call(self, node: ast.Call) -> None: # noqa: N802 }: self._runtime("external_launcher", name, node, f"Called {name}.") if name in {"ctypes.CDLL", "ctypes.WinDLL", "ctypes.OleDLL"}: - library = _literal_string(node.args[0]) if node.args else None + library = _literal_string(call_argument(node, position=0, keyword="name")) self._runtime("native_runtime", library or "dynamic DLL", node, f"Called {name}.") method = name.split(".")[-1] if method in WRITE_METHODS: @@ -348,8 +351,8 @@ def visit_Call(self, node: ast.Call) -> None: # noqa: N802 ): self.generic_visit(node) return - if name == "open" and node.args: - value = node.args[0] + if name == "open": + value = call_argument(node, position=0, keyword="file") or node elif name.startswith("shutil.") and len(node.args) >= 2: value = node.args[1] else: diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 60581a7..1a32df5 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -1431,6 +1431,175 @@ def test_dotted_import_promotion_preserves_existing_ancestor_initializers(tmp_pa assert inventory[path].role == RepositoryFileRole.APPLICATION_SOURCE +@pytest.mark.parametrize( + "source", + [ + "import importlib\nplugin = importlib.import_module('app.examples.plugin')\n", + "import importlib as il\nplugin = il.import_module('app.examples.plugin')\n", + "from importlib import import_module\nplugin = import_module('app.examples.plugin')\n", + "from importlib import import_module as load_module\n" + "plugin = load_module('app.examples.plugin')\n", + "plugin = __import__('app.examples.plugin')\n", + ], +) +def test_literal_dynamic_import_promotes_excluded_module_and_initializers( + tmp_path: Path, source: str +) -> None: + package = tmp_path / "src/app/examples" + package.mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "plugin.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "src/app/main.py").write_text(source, encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='dynamic-import-app'\nversion='1.0'\n" + "[project.scripts]\ndynamic-import-app='app.main:main'\n" + "[tool.setuptools]\npackage-dir={''='src'}\npackages=['app']\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + staged = _staging_files( + tmp_path, + assessment, + create_deployment_plan(assessment, repository_root=tmp_path), + include=True, + ) + inventory = {item.path: item for item in assessment.file_inventory} + + for path in ( + "src/app/__init__.py", + "src/app/examples/__init__.py", + "src/app/examples/plugin.py", + ): + assert inventory[path].role == RepositoryFileRole.APPLICATION_SOURCE + assert path in staged + + +@pytest.mark.parametrize( + "source", + [ + "def import_module(name): return None\nimport_module('app.examples.plugin')\n", + "import importlib\nimportlib.import_module(module_name)\n", + "import importlib\nimportlib.import_module(f'app.{name}')\n", + "import importlib\nimportlib.import_module('.plugin', package='app')\n", + "import importlib\nimportlib.import_module('app-plugin')\n", + ], +) +def test_dynamic_import_requires_proven_binding_and_absolute_literal_module_name( + tmp_path: Path, source: str +) -> None: + package = tmp_path / "src/app/examples" + package.mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("", encoding="utf-8") + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "plugin.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "src/app/main.py").write_text(source, encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='unresolved-dynamic-import-app'\nversion='1.0'\n" + "[project.scripts]\nunresolved-dynamic-import-app='app.main:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + inventory = {item.path: item for item in assessment.file_inventory} + + assert inventory["src/app/examples/plugin.py"].role == RepositoryFileRole.EXAMPLE_OR_SNIPPET + + +def test_literal_dynamic_import_uses_custom_package_directory_source_root(tmp_path: Path) -> None: + package = tmp_path / "lib/app/docs" + package.mkdir(parents=True) + (tmp_path / "lib/app/__init__.py").write_text("", encoding="utf-8") + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "plugin.py").write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / "lib/app/main.py").write_text( + "import importlib\nplugin = importlib.import_module('app.docs.plugin')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='lib-dynamic-import-app'\nversion='1.0'\n" + "[project.scripts]\nlib-dynamic-import-app='app.main:main'\n" + "[tool.setuptools]\npackages=['app']\npackage-dir={''='lib'}\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + inventory = {item.path: item for item in assessment.file_inventory} + + assert inventory["lib/app/docs/plugin.py"].role == RepositoryFileRole.APPLICATION_SOURCE + + +@pytest.mark.parametrize( + ("call", "expected_access"), + [ + ("open('defaults.json')", "read"), + ("open(file='defaults.json', mode='r', encoding='utf-8')", "read"), + ("open(file='defaults.json', mode='rb')", "read"), + ("open(file='defaults.json', mode='w')", "write"), + ("open(file='defaults.json', mode='a')", "write"), + ("open(file='defaults.json', mode='x')", "write"), + ("open(file='defaults.json', mode='r+')", "read_write"), + ("open(file='defaults.json', mode='w+')", "read_write"), + ], +) +def test_builtin_open_file_keyword_promotes_read_resources_only( + tmp_path: Path, call: str, expected_access: str +) -> None: + (tmp_path / "defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "app.py").write_text(f"def main():\n return {call}\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='open-keyword-app'\nversion='1.0'\n" + "[project.scripts]\nopen-keyword-app='app:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + resources = [item for item in assessment.resources if item.path == "defaults.json"] + + if expected_access == "read": + assert len(resources) == 1 + assert resources[0].access_mode == "read" + assert "defaults.json" in _staging_files( + tmp_path, + assessment, + create_deployment_plan(assessment, repository_root=tmp_path), + include=True, + ) + else: + assert resources == [] + + +def test_builtin_open_file_keyword_keeps_static_variables_and_ignores_object_methods( + tmp_path: Path, +) -> None: + (tmp_path / "defaults.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "app.py").write_text( + "def main(obj, path):\n" + " filename = 'defaults.json'\n" + " open(file=filename)\n" + " open(file=path)\n" + " obj.open(file='defaults.json')\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='open-variable-app'\nversion='1.0'\n" + "[project.scripts]\nopen-variable-app='app:main'\n", + encoding="utf-8", + ) + (tmp_path / "uv.lock").write_text("version = 1\nrevision = 3\n", encoding="utf-8") + + assessment = assess_repository(_repository(tmp_path)) + + resource = next(item for item in assessment.resources if item.path == "defaults.json") + assert resource.access_mode == "read" + assert [item.path for item in assessment.resources] == ["defaults.json"] + + def test_importlib_resources_uses_package_dir_parent_mapping(tmp_path: Path) -> None: package = tmp_path / "lib/sub" package.mkdir(parents=True) diff --git a/tests/test_generation.py b/tests/test_generation.py index 9251d09..62c4968 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -3710,6 +3710,45 @@ def test_application_wheel_rejects_configured_secret_value( validate_application_wheel(wheel, assessment, plan) +def test_keyword_environment_secret_reaches_application_and_approved_wheel_scans( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + (source / "code/main.py").write_text( + "import os\nAPI_TOKEN = os.getenv(key='DB_PASSWORD')\ndef main(): return 0\n", + encoding="utf-8", + ) + secret = "keyword-form-secret-that-must-not-ship" + monkeypatch.setenv("DB_PASSWORD", secret) + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + application_plan = create_deployment_plan(assessment, repository_root=source) + application_wheel = _rewrite_application_wheel( + _make_application_wheel(tmp_path), + additions={"installed_app/config.py": f"TOKEN = {secret!r}"}, + ) + approved_plan = _plan("optional_map_app", ["map"]) + approved_plan.configuration = application_plan.configuration + approved_wheel = _rewrite_application_wheel( + _make_wheel(tmp_path), additions={"helper/settings.txt": f"token={secret}\n"} + ) + + assert [item.name for item in application_plan.configuration if item.secret] == [ + "DB_PASSWORD" + ] + assert secret not in assessment.model_dump_json() + assert secret not in application_plan.model_dump_json() + with pytest.raises(PreparationError, match="security policy"): + validate_application_wheel( + application_wheel, assessment, application_plan, repository_root=source + ) + with pytest.raises(PreparationError, match="security policy"): + validate_approved_wheel(f"proxy-tools={approved_wheel}", approved_plan) + + def test_application_wheel_rejects_configured_secret_in_metadata( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/test_runtime_assumptions.py b/tests/test_runtime_assumptions.py index afe3c2b..5cc6c87 100644 --- a/tests/test_runtime_assumptions.py +++ b/tests/test_runtime_assumptions.py @@ -1,5 +1,7 @@ from pathlib import Path +import pytest + from python_deployment_builder.analysis.runtime_assumptions import scan_runtime_assumptions FIXTURES = Path(__file__).parent / "fixtures" @@ -19,3 +21,44 @@ def test_environment_gui_paths_and_writes_are_detected() -> None: def test_external_executable_is_detected() -> None: result = scan_runtime_assumptions(FIXTURES / "external_executable", ["."]) assert any(item.name == "convert.exe" for item in result.runtime_requirements) + + +@pytest.mark.parametrize( + ("call", "name"), + [ + ("os.getenv('X')", "X"), + ("os.getenv(key='X')", "X"), + ("os.getenv('X', 'fallback')", "X"), + ("os.getenv(key='X', default='fallback')", "X"), + ("os.environ.get('X')", "X"), + ("os.environ.get(key='X')", "X"), + ("os.environ.get('X', 'fallback')", "X"), + ("os.environ.get(key='X', default='fallback')", "X"), + ("os.environ['X']", "X"), + ("os.environ.get(key='API-KEY')", "API-KEY"), + ], +) +def test_environment_reads_accept_literal_positional_and_keyword_keys( + tmp_path: Path, call: str, name: str +) -> None: + (tmp_path / "app.py").write_text(f"import os\nVALUE = {call}\n", encoding="utf-8") + + result = scan_runtime_assumptions(tmp_path, ["."]) + + requirement = next(item for item in result.configuration_requirements if item.name == name) + assert requirement.secret is False + + +def test_environment_keyword_key_stays_literal_only_and_positional_wins_duplicates( + tmp_path: Path, +) -> None: + (tmp_path / "app.py").write_text( + "import os\nname = 'DYNAMIC'\n" + "first = os.getenv(key=name)\n" + "second = os.getenv('POSITIONAL', key='KEYWORD')\n", + encoding="utf-8", + ) + + result = scan_runtime_assumptions(tmp_path, ["."]) + + assert {item.name for item in result.configuration_requirements} == {"POSITIONAL"} From bc682d75b5d77580d5e76b7f17c3a375843368fe Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 09:57:01 -0400 Subject: [PATCH 50/59] Fix proven API bindings and staged lock root identity --- .../pr9-alias-keyword-lock-root-2026-09-11.md | 123 +++++ scripts/verify_legacy_lock_root.py | 112 ++++ .../analysis/inventory.py | 3 +- .../analysis/resources.py | 42 +- .../analysis/runtime_assumptions.py | 36 +- .../generation/generator.py | 13 +- .../planning/lockfile.py | 34 ++ .../planning/planner.py | 33 +- .../validation/static.py | 23 +- tests/test_generation.py | 16 +- tests/test_review_binding_and_lock_root.py | 490 ++++++++++++++++++ 11 files changed, 900 insertions(+), 25 deletions(-) create mode 100644 docs/acceptance/pr9-alias-keyword-lock-root-2026-09-11.md create mode 100644 scripts/verify_legacy_lock_root.py create mode 100644 tests/test_review_binding_and_lock_root.py diff --git a/docs/acceptance/pr9-alias-keyword-lock-root-2026-09-11.md b/docs/acceptance/pr9-alias-keyword-lock-root-2026-09-11.md new file mode 100644 index 0000000..7ecbb29 --- /dev/null +++ b/docs/acceptance/pr9-alias-keyword-lock-root-2026-09-11.md @@ -0,0 +1,123 @@ +# PR #9: proven bindings, keyword reads, and legacy lock roots + +Starting head: `a94c1b69f7934a7a0f89f725c17f135ee3296cea`. +Review submitted: `2026-09-11T12:28:22Z`. +Accounting before the next review: 111 findings, 108 unresolved inline threads, +three review-level-only findings, and one NOT_APPLICABLE inline finding. +This correction does not resolve threads, dismiss reviews, or authorize merge. + +## Fresh findings and reproduction + +| Finding | Comment | Correction | +| --- | --- | --- | +| P1: Track aliased environment APIs when collecting secrets | 3989107557 | Precompute explicit stdlib import bindings before visiting runtime reads. | +| P2: Preserve the root identity for legacy source manifests | 3989107568 | Cross-check structural lock identity; block actual rootless legacy projects before generation. | +| P2: Read the dynamic-import name keyword | 3989107573 | Use the existing `call_argument` for `name`. | +| P2: Resolve keyword-form pkgutil resource reads | 3989107579 | Use the same helper for `package` and `resource`. | + +The initial regression run, before production edits, produced **55 failures and +22 passes**. Aliased `os.getenv`, imported/aliased `getenv`, and imported/aliased +`environ.get` calls were present in the AST but retained alias spellings and +produced no configuration requirements. Thus their names did not reach manifest +secret lists or configured-value scanning. Keyword dynamic imports did not stage +their excluded-scope targets; keyword pkgutil reads did not stage their resources. +The legacy source kit was generated but failed static lock identity validation. + +## Bounded AST contract + +Explicit absolute imports provide file-level binding evidence, independently of +the textual order of imports and function declarations. Wildcard imports, +relative imports, and unrelated user functions are not proof of stdlib origin. +This is not runtime import execution or a general lexical name-resolution engine. + +The environment matrix covers canonical and aliased os modules, directly imported +and aliased getenv functions, and directly imported/aliased environ objects, +including get/subscript reads, positional/keyword keys, defaults, dynamic keys, +and non-identifier environment names. Aliased secrets reach assessment, planning, +manifest name lists, configured-value lookup, staged resource scans, and +application/approved-wheel scans. Synthetic-value assertions cover serialization +and exception/log output; short configured secrets retain their fail-closed rule. + +Dynamic imports use the existing proven importlib/builtin bindings, literal +absolute dotted-name policy, and local-module resolver. Keyword names follow the +same source-root, initializer, containment, staging, and deployment-mode paths as +positional names. Dynamic, f-string, relative, and malformed names remain unresolved. + +Pkgutil accepts positional, mixed, and fully keyword-bound arguments for the +existing proven aliases. Positional values win duplicate bindings, as in the +shared helper. Extra arguments, unrelated keywords, dynamic values, unsafe paths, +and namespace-only packages remain unresolved. A keyword argument followed by a +bare positional resource is invalid Python syntax, not another supported form. +Nested safe resources and declared package-data behavior retain existing rules. + +The one bounded binding audit also corrected aliases of already-supported runtime +APIs in os, subprocess, ctypes, shutil, and webbrowser; resource directory reads +through os.listdir/scandir aliases; and relative imports incorrectly supplying +stdlib resource binding evidence. Focused tests accompany these changes. No new +API family, general argument binder, or generic name-resolution engine was added. + +## Actual uv 0.12.5 evidence and identity decision + +The explicit developer script `scripts/verify_legacy_lock_root.py` was run against +`uv 0.12.5 (210d1f678 2026-08-14 x86_64-pc-windows-msvc)` with `uv lock --python 3.12`. +It creates disposable fixtures outside the repository. Applications are not run. + +| Fixture | Actual root representation | +| --- | --- | +| Ordinary PEP 621 source project | `source = { virtual = "." }` | +| PEP 621 package-mode project with setuptools/src layout | `source = { editable = "." }` | +| PEP 621 source project with selected `feature` extra | `source = { virtual = "." }`; metadata lists the extra | +| Build-system-only pyproject + literal setup.py, zero dependencies | **No package records** | +| Build-system-only pyproject + setup.cfg, zero dependencies | **No package records** | + +Both actual legacy locks contain only `version = 1`, `revision = 3`, and +`requires-python = ">=3.12"`. Assessment identifies `legacy-demo` and source mode; +the backend-dependency blocker does not apply. Before this correction, generation +succeeded but `APPROVED_ARTIFACT_LOCK_IDENTITY` failed because no root identity +could be proved. Afterward, `LEGACY_LOCK_ROOT_UNIDENTIFIABLE` blocks planning and +generation, including an explicit dry-run action, before creating the kit. +The remedy is standardized `[project]` metadata and a regenerated lock. + +A new TOML-only helper returns the exact name of one unambiguous virtual/editable +`.` root. Static validation cross-checks canonical distribution names from the +application artifact, standardized project metadata, and structural lock root; +contradictions fail closed. There is no application-ID/directory-name inference, +source reassessment, or metadata execution. Root-bearing older source manifests, +including synthetic legacy compatibility fixtures, remain valid. Such fixtures +are not claimed to be the output of the actual rootless uv legacy workflow. + +Merely persisting the legacy name would not supply the absent root graph, so the +allowed early planning blocker is used instead of a new manifest field. Analysis +schema **1.4** and planning schema **1.3** remain unchanged. Static lock proof is +still unconditional: an empty approved-artifact list does not bypass reverse +artifact completeness. + +## Read-only acceptance checks + +| Repository | Unchanged SHA | Result | +| --- | --- | --- | +| SimpleGeorefGUI | `f484570d89fb1f9e9170fac915475358dfc1234e` | package; ENTRYPOINT_REQUIRES_PACKAGE_MODE; BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python / 77 data members | +| Geo Map Explanation Extractor | `5e7b321d0aeb9ba1d586bfc548c79793d84c6033` | source / SOURCE_COMPATIBLE | +| TN Coordinate Converter | `e1e7a1588c37a99c2d02efaf3eef3d04636f12f0` | source / SOURCE_COMPATIBLE; existing reviewed kit STATIC_VALID | + +Before/after Git working-tree status also matched. Configuration-name lists did +not change: SGG has APPDATA, LOCALAPPDATA, SIMPLE_GEOREF_GUI_DATA_DIR, XDG_DATA_HOME; +Geo has LOCALAPPDATA and OPENAI_API_KEY; TN has none. No environment values are +included in this report. TN's existing kit retains the approved +`proxy_tools-0.1.0-py3-none-any.whl` artifact and `proxy-tools==0.1.0` identity. + +## Verification + +Focused final regression run: **127 passed, 430 deselected** (new regression file, +keyword wheel-secret propagation, and existing backend-only metadata cases). +The single complete suite passed: **998 passed, 3 skipped in 259.25 seconds** +(previous baseline: 874 passed, 3 skipped). `ruff check .` and +`git diff --check` passed. The final diff contains only the four corrections, +the bounded audit, regression tests, and this reproduction/acceptance evidence. + +The complete suite retains coverage of the preceding keyword AST correction, +MANIFEST/include-package-data and setuptools-scm guards, exact sync contract, +approved-artifact identity/reverse completeness, mode-aware pip checks, entry +points, short secrets, LOCALAPPDATA ordering, wheel/dependency policy, resource +and inventory rules, package surfaces, wheel security/integrity/collisions, +provenance, prepare-lock/dry-run/reporting/rollback, and deterministic packaging. diff --git a/scripts/verify_legacy_lock_root.py b/scripts/verify_legacy_lock_root.py new file mode 100644 index 0000000..c63730a --- /dev/null +++ b/scripts/verify_legacy_lock_root.py @@ -0,0 +1,112 @@ +"""Explicit developer acceptance: observe uv 0.12.5 roots without running application code.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import tomllib +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.generator import generate_deployment_kit +from python_deployment_builder.planning.planner import create_deployment_plan +from python_deployment_builder.validation.static import validate_static_kit + + +def main() -> None: + uv = Path(sys.argv[1]).resolve() + version = subprocess.check_output([str(uv), "--version"], text=True).strip() + assert version.startswith("uv 0.12.5 "), version + workspace = Path(tempfile.mkdtemp(prefix="pdb-legacy-root-")) + print(version, workspace, flush=True) + environment = os.environ.copy() + environment.update(UV_SYSTEM_CERTS="true", UV_NO_ENV_FILE="1", UV_NO_PROGRESS="1") + for kind in ("setup.py", "setup.cfg", "pep621", "package", "extra"): + root = workspace / kind + root.mkdir() + build = ( + "[build-system]\nrequires=['setuptools==79.0.1']\n" + "build-backend='setuptools.build_meta'\n" + ) + (root / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + if kind == "setup.py": + metadata = ( + "from setuptools import setup\n" + "setup(name='legacy-demo', version='1.0.0', py_modules=['app'], " + "entry_points={'console_scripts':['legacy-demo=app:main']})\n" + ) + elif kind == "setup.cfg": + metadata = ( + "[metadata]\nname=legacy-demo\nversion=1.0.0\n" + "[options]\npy_modules=app\n[options.entry_points]\n" + "console_scripts=\n legacy-demo=app:main\n" + ) + else: + metadata = "" + if metadata: + (root / kind).write_text(metadata, encoding="utf-8") + pyproject = build + else: + pyproject = ( + "[project]\nname='legacy-demo'\nversion='1.0.0'\n" + "requires-python='>=3.12'\n" + "[project.scripts]\nlegacy-demo='app:main'\n" + ) + if kind == "package": + (root / "src/app").mkdir(parents=True) + (root / "src/app/__init__.py").write_text("def main(): return 0\n") + (root / "app.py").unlink() + pyproject = build + pyproject + "[tool.setuptools]\npackage-dir={''='src'}\n" + if kind == "extra": + pyproject += "[project.optional-dependencies]\nfeature=[]\n" + (root / "pyproject.toml").write_text(pyproject, encoding="utf-8") + subprocess.run( + [str(uv), "lock", "--python", "3.12"], + cwd=root, + env=environment, + check=True, + capture_output=True, + text=True, + ) + lock = tomllib.loads((root / "uv.lock").read_text(encoding="utf-8")) + repo = MaterializedRepository(root=root, source=str(root), source_kind="local") + assessment = assess_repository(repo) + plan = create_deployment_plan( + assessment, repository_root=root, selected_extras=["feature"] if kind == "extra" else [] + ) + result = { + "kind": kind, + "packages": lock.get("package"), + "distribution": assessment.project.distribution_name, + "mode": plan.deployment_mode, + "condition": plan.deployment_mode_condition, + "blocking_codes": plan.risk_gate.blocking_codes, + } + if metadata: + kit = workspace / (kind + "-kit") + try: + generate_deployment_kit(repo, kit, bootstrap_mode="online_cmd", system_certs=True) + except PreparationError as exc: + assert "LEGACY_LOCK_ROOT_UNIDENTIFIABLE" in str(exc) + assert not kit.exists() + result["generation_blocker"] = str(exc) + else: + report = validate_static_kit(kit) + result["static"] = report.final_state.value + result["failures"] = [ + item.model_dump(mode="json") + for item in report.static_checks + if item.status.value == "FAIL" + ] + print(json.dumps(result), flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index a54a725..40e4b6e 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -12,6 +12,7 @@ from packaging.requirements import InvalidRequirement, Requirement from pathspec import PathSpec +from python_deployment_builder.analysis.ast_utils import call_argument from python_deployment_builder.models import ( AnalysisScopeSummary, DependencyAssessment, @@ -368,7 +369,7 @@ def _imported_modules( and function.value.id in importlib_modules ) if direct_import or module_import: - target = node.args[0] if node.args else None + target = call_argument(node, position=0, keyword="name") if isinstance(target, ast.Constant) and isinstance(target.value, str): module = target.value if module and all(part.isidentifier() for part in module.split(".")): diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index d873b05..e42a53c 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -527,7 +527,7 @@ def _resource_import_bindings( modules.add(alias.asname or alias.name) elif alias.name == "pkgutil": pkgutil_modules.add(alias.asname or alias.name) - elif isinstance(node, ast.ImportFrom): + elif isinstance(node, ast.ImportFrom) and node.level == 0: if node.module == "importlib": for alias in node.names: if alias.name == "resources": @@ -686,10 +686,16 @@ def _pkgutil_resource_path_values( name == f"{binding}.get_data" for binding in module_bindings ): return None - if len(node.args) != 2 or node.keywords: + if len(node.args) > 2 or any( + keyword.arg not in {"package", "resource"} for keyword in node.keywords + ): + return [] + package = call_argument(node, position=0, keyword="package") + resource = call_argument(node, position=1, keyword="resource") + if package is None or resource is None: return [] package_roots = _resource_package_anchor_values( - node.args[0], + package, root=root, source_path=source_path, source_roots=source_roots, @@ -699,7 +705,7 @@ def _pkgutil_resource_path_values( require_initializer=True, ) members = _path_values( - node.args[1], + resource, root=root, source_path=source_path, assignments=assignments, @@ -1089,7 +1095,28 @@ def _open_access(name: str, node: ast.Call) -> str: return "write" if writes else "read" -def _path_uses(node: ast.Call) -> list[tuple[ast.AST, str]]: +def _directory_read_bindings(tree: ast.AST) -> set[str]: + """Collect explicit os.listdir/scandir spellings for the existing reader.""" + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "os": + names.update( + f"{alias.asname or 'os'}.{method}" for method in ("listdir", "scandir") + ) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module == "os": + names.update( + alias.asname or alias.name + for alias in node.names + if alias.name in {"listdir", "scandir"} + ) + return names + + +def _path_uses( + node: ast.Call, directory_reads: set[str] | None = None +) -> list[tuple[ast.AST, str]]: name = _qualified_name(node.func) method = name.split(".")[-1].lower() if name == "open": @@ -1128,7 +1155,7 @@ def _path_uses(node: ast.Call) -> list[tuple[ast.AST, str]]: ] if method in {"iterdir", "glob", "rglob"} and isinstance(node.func, ast.Attribute): return [(node.func.value, "read")] - if name in {"os.listdir", "os.scandir"}: + if name in {"os.listdir", "os.scandir"} | (directory_reads or set()): path = call_argument(node, position=0, keyword="path") return [(path, "read")] if path is not None else [] return [] @@ -1173,6 +1200,7 @@ def _literal_evidence( continue lines = source.splitlines() assignments, returns = _bindings(tree) + directory_reads = _directory_read_bindings(tree) ( module_bindings, files_bindings, @@ -1221,7 +1249,7 @@ def _literal_evidence( ) ) else: - for expression, mode in _path_uses(node): + for expression, mode in _path_uses(node, directory_reads): resource_values = _importlib_resource_path_values( expression, root=root, diff --git a/src/python_deployment_builder/analysis/runtime_assumptions.py b/src/python_deployment_builder/analysis/runtime_assumptions.py index c15d9e6..cc0148b 100644 --- a/src/python_deployment_builder/analysis/runtime_assumptions.py +++ b/src/python_deployment_builder/analysis/runtime_assumptions.py @@ -61,6 +61,27 @@ def _literal_string(node: ast.AST | None) -> str | None: return None +def _runtime_import_bindings(tree: ast.AST) -> dict[str, str]: + """Collect explicit imports for the already-supported stdlib runtime APIs. + + File-level evidence is intentionally independent of declaration order. This + is not execution or lexical name resolution; wildcard/relative imports + provide no proof. Conflicting import identities are left unresolved. + """ + supported = {"os", "subprocess", "ctypes", "shutil", "webbrowser"} + candidates: dict[str, set[str]] = defaultdict(set) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name in supported: + candidates[alias.asname or alias.name].add(alias.name) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module in supported: + for alias in node.names: + if alias.name != "*": + candidates[alias.asname or alias.name].add(f"{node.module}.{alias.name}") + return {name: next(iter(values)) for name, values in candidates.items() if len(values) == 1} + + def _command_name(node: ast.AST | None) -> str | None: if isinstance(node, (ast.List, ast.Tuple)) and node.elts: return _literal_string(node.elts[0]) @@ -147,6 +168,7 @@ def __init__( relative: str, source_lines: list[str], function_returns: dict[str, str] | None = None, + import_bindings: dict[str, str] | None = None, ) -> None: self.relative = relative self.lines = source_lines @@ -155,8 +177,15 @@ def __init__( self.writes: dict[tuple[str, str], list[Evidence]] = defaultdict(list) self.assignments: dict[str, str] = {} self.function_returns = function_returns or {} + self.import_bindings = import_bindings or {} self.lexical_scopes: list[tuple[str, str | None]] = [("module", None)] + def _api_name(self, node: ast.AST) -> str: + name = _qualified_name(node) + first, separator, rest = name.partition(".") + binding = self.import_bindings.get(first) + return binding + separator + rest if binding is not None else "" + def _call_summary_identity(self, function: ast.expr) -> str | None: if isinstance(function, ast.Name): return function.id @@ -291,15 +320,15 @@ def visit_Constant(self, node: ast.Constant) -> None: # noqa: N802 self._runtime("permissions", "Program Files path", node, value) def visit_Subscript(self, node: ast.Subscript) -> None: # noqa: N802 - if _qualified_name(node.value) == "os.environ": + if self._api_name(node.value) == "os.environ": name = _literal_string(node.slice) if name: self.config[name].append(self._evidence(node, "Read from os.environ.")) self.generic_visit(node) def visit_Call(self, node: ast.Call) -> None: # noqa: N802 - name = _qualified_name(node.func) - if name in {"os.getenv", "os.environ.get"}: + name = self._api_name(node.func) or _qualified_name(node.func) + if self._api_name(node.func) in {"os.getenv", "os.environ.get"}: variable = _literal_string( call_argument(node, position=0, keyword="key") ) @@ -401,6 +430,7 @@ def scan_runtime_assumptions( relative, source.splitlines(), function_returns=_simple_function_returns(tree), + import_bindings=_runtime_import_bindings(tree), ) visitor.visit(tree) for key, evidence in visitor.runtime.items(): diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index c808787..f27933a 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -952,10 +952,14 @@ def generate_deployment_kit( runtime_sync_blockers = { code for code in plan.risk_gate.blocking_codes - if code == "RUNTIME_SYNC_METADATA_UNSUPPORTED" + if code in {"RUNTIME_SYNC_METADATA_UNSUPPORTED", "LEGACY_LOCK_ROOT_UNIDENTIFIABLE"} } if runtime_sync_blockers: - runtime_sync_code = next(iter(runtime_sync_blockers)) + runtime_sync_code = ( + "RUNTIME_SYNC_METADATA_UNSUPPORTED" + if "RUNTIME_SYNC_METADATA_UNSUPPORTED" in runtime_sync_blockers + else "LEGACY_LOCK_ROOT_UNIDENTIFIABLE" + ) if dry_run: preview = _preview( plan, @@ -980,8 +984,9 @@ def generate_deployment_kit( ) raise PreparationError( f"Deployment planning is blocked: {runtime_sync_code}. The pinned uv 0.12.5 " - "lock workflow does not consume backend-only setup.cfg/setup.py dependency " - "metadata, and PDB will not execute project metadata on the end-user system." + "lock workflow cannot represent this backend-only setup.cfg/setup.py metadata " + "as a provable application dependency graph. Declare standardized [project] " + "metadata and regenerate uv.lock before generation." ) # An escaping setuptools root is outside both the source/provenance # boundary and this kit's standalone staging model. Stop before lock diff --git a/src/python_deployment_builder/planning/lockfile.py b/src/python_deployment_builder/planning/lockfile.py index 2096c4b..25c3e00 100644 --- a/src/python_deployment_builder/planning/lockfile.py +++ b/src/python_deployment_builder/planning/lockfile.py @@ -20,6 +20,40 @@ from python_deployment_builder.planning.index import marker_applies, wheel_matches +def identify_uv_lock_root_name(repository_root: Path) -> str | None: + """Read the single uv 0.12.5 virtual/editable '.' root, without executing code. + + No package records (as emitted for build-system-only legacy projects) means + no root identity. Ambiguous or malformed records raise a controlled error. + The exact lock name is returned; application IDs and directory names are + never distribution-identity evidence. + """ + try: + with (repository_root / "uv.lock").open("rb") as handle: + document = tomllib.load(handle) + except (OSError, ValueError) as exc: + raise ValueError("Cannot read the structural uv.lock root.") from exc + packages = document.get("package", []) + if not isinstance(packages, list) or any(not isinstance(item, dict) for item in packages): + raise ValueError("Malformed uv.lock package records.") + roots = [] + for item in packages: + source = item.get("source", {}) + if not isinstance(source, dict): + raise ValueError("Malformed uv.lock source record.") + if any(source.get(kind) == "." for kind in ("virtual", "editable")): + if source not in ({"virtual": "."}, {"editable": "."}): + raise ValueError("Conflicting uv.lock root source markers.") + name = item.get("name") + if not isinstance(name, str): + raise ValueError("Missing uv.lock root distribution name.") + canonicalize_name(name, validate=True) + roots.append(name) + if len(roots) > 1: + raise ValueError("Ambiguous uv.lock structural roots.") + return roots[0] if roots else None + + def _filename(artifact: dict[str, object]) -> str: url = artifact.get("url") return unquote(Path(urlsplit(url).path).name) if isinstance(url, str) else "" diff --git a/src/python_deployment_builder/planning/planner.py b/src/python_deployment_builder/planning/planner.py index 8048d96..60a617d 100644 --- a/src/python_deployment_builder/planning/planner.py +++ b/src/python_deployment_builder/planning/planner.py @@ -4,6 +4,7 @@ import hashlib import json +import tomllib from datetime import UTC, datetime from pathlib import Path @@ -50,7 +51,7 @@ validate_selected_extras, ) from python_deployment_builder.planning.index import inspect_dependency_wheels -from python_deployment_builder.planning.lockfile import inspect_uv_lock +from python_deployment_builder.planning.lockfile import identify_uv_lock_root_name, inspect_uv_lock from python_deployment_builder.planning.platforms import windows_finding_treatments from python_deployment_builder.planning.policies import ( MinorPythonCompatibility, @@ -587,6 +588,28 @@ def create_deployment_plan( "cannot represent selected dependencies: " + ", ".join(backend_only_dependencies) ) + legacy_root_unresolved = False + if repository_root is not None and {"setup.py", "setup.cfg"} & set( + assessment.project.metadata_files + ): + try: + with (repository_root / "pyproject.toml").open("rb") as handle: + document = tomllib.load(handle) + project = document.get("project", {}) + standardized_name = project.get("name") if isinstance(project, dict) else None + if not standardized_name: + legacy_root_unresolved = ( + identify_uv_lock_root_name(repository_root) is None + if lock_present else True + ) + except (OSError, ValueError): + legacy_root_unresolved = True + if legacy_root_unresolved: + mode_blockers.append( + "LEGACY_LOCK_ROOT_UNIDENTIFIABLE: uv 0.12.5 emits no application root for " + "build-system-only legacy metadata, even with zero dependencies. Declare " + "standardized [project] metadata and regenerate uv.lock before generation." + ) extras = build_extra_plans(assessment, selected_extras, python_version, architecture) configuration = [ ConfigurationPlan( @@ -674,6 +697,14 @@ def create_deployment_plan( ), ] gate = _risk_gate(assessment) + if legacy_root_unresolved: + gate = gate.model_copy(update={ + "outcome": "block", + "blocking_codes": sorted({*gate.blocking_codes, "LEGACY_LOCK_ROOT_UNIDENTIFIABLE"}), + "rationale": ( + gate.rationale + " The staged lock has no provable legacy application root." + ), + }) if entrypoint_extra_blockers: entrypoint_extra_codes = { item.split(":", 1)[0] for item in entrypoint_extra_blockers diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 59c05d4..3f8c98e 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -46,7 +46,7 @@ ValidationHost, ValidationReport, ) -from python_deployment_builder.planning.lockfile import inspect_uv_lock +from python_deployment_builder.planning.lockfile import identify_uv_lock_root_name, inspect_uv_lock from python_deployment_builder.security_policy import ( FORBIDDEN_SHELL, TextContentEncodingError, @@ -128,16 +128,29 @@ def _load_manifest(root: Path) -> DeploymentManifest: def _static_lock_root_name(root: Path, manifest: DeploymentManifest) -> str | None: """Find the staged lock root without re-assessing source packaging metadata.""" - if manifest.application_artifact is not None: - return manifest.application_artifact.distribution_name try: + lock_name = identify_uv_lock_root_name(root) with (root / "pyproject.toml").open("rb") as handle: document = tomllib.load(handle) - except (OSError, tomllib.TOMLDecodeError): + except (OSError, ValueError): + return None + if lock_name is None: return None project = document.get("project") name = project.get("name") if isinstance(project, dict) else None - return name if isinstance(name, str) and name.strip() else None + names = [lock_name] + if name is not None: + if not isinstance(name, str) or not name.strip(): + return None + names.insert(0, name) + if manifest.application_artifact is not None: + names.insert(0, manifest.application_artifact.distribution_name) + try: + if len({canonicalize_name(value, validate=True) for value in names}) != 1: + return None + except ValueError: + return None + return names[0] def _static_lock_plan(root: Path, manifest: DeploymentManifest): diff --git a/tests/test_generation.py b/tests/test_generation.py index 62c4968..89f84c2 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -3710,14 +3710,21 @@ def test_application_wheel_rejects_configured_secret_value( validate_application_wheel(wheel, assessment, plan) +@pytest.mark.parametrize(("imports", "call"), [ + ("import os", "os.getenv"), + ("import os as operating", "operating.getenv"), + ("from os import getenv", "getenv"), + ("from os import getenv as read_env", "read_env"), + ("from os import environ as env", "env.get"), +]) def test_keyword_environment_secret_reaches_application_and_approved_wheel_scans( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, imports: str, call: str ) -> None: source = tmp_path / "source" source.mkdir() _write_mapped_project(source) (source / "code/main.py").write_text( - "import os\nAPI_TOKEN = os.getenv(key='DB_PASSWORD')\ndef main(): return 0\n", + f"{imports}\nAPI_TOKEN = {call}(key='DB_PASSWORD')\ndef main(): return 0\n", encoding="utf-8", ) secret = "keyword-form-secret-that-must-not-ship" @@ -3741,12 +3748,13 @@ def test_keyword_environment_secret_reaches_application_and_approved_wheel_scans ] assert secret not in assessment.model_dump_json() assert secret not in application_plan.model_dump_json() - with pytest.raises(PreparationError, match="security policy"): + with pytest.raises(PreparationError, match="security policy") as application_error: validate_application_wheel( application_wheel, assessment, application_plan, repository_root=source ) - with pytest.raises(PreparationError, match="security policy"): + with pytest.raises(PreparationError, match="security policy") as approved_error: validate_approved_wheel(f"proxy-tools={approved_wheel}", approved_plan) + assert secret not in str(application_error.value) + str(approved_error.value) def test_application_wheel_rejects_configured_secret_in_metadata( diff --git a/tests/test_review_binding_and_lock_root.py b/tests/test_review_binding_and_lock_root.py new file mode 100644 index 0000000..c8742e9 --- /dev/null +++ b/tests/test_review_binding_and_lock_root.py @@ -0,0 +1,490 @@ +"""Regression coverage for PR #9's alias, keyword, and legacy lock-root findings.""" + +import ast +import json +from types import SimpleNamespace + +import pytest + +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.runtime_assumptions import scan_runtime_assumptions +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import configured_secret_values +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.generation.manifest import build_deployment_manifest +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.planning.lockfile import identify_uv_lock_root_name +from python_deployment_builder.planning.planner import create_deployment_plan +from python_deployment_builder.security_policy import text_security_findings +from python_deployment_builder.validation.static import _static_lock_root_name, validate_static_kit + +ENV_BINDINGS = [ + ("import os", "os.getenv"), + ("import os as operating", "operating.getenv"), + ("from os import getenv", "getenv"), + ("from os import getenv as read_env", "read_env"), + ("import os", "os.environ.get"), + ("import os as operating", "operating.environ.get"), + ("from os import environ", "environ.get"), + ("from os import environ as env", "env.get"), +] + + +def repository(root): + return MaterializedRepository(root=root, source=str(root), source_kind="local") + + +def write_project(root, source): + root.mkdir(parents=True, exist_ok=True) + (root / "app.py").write_text(source, encoding="utf-8") + (root / "pyproject.toml").write_text( + "[project]\nname='binding-demo'\nversion='1.0'\nrequires-python='>=3.12'\n" + "[project.scripts]\nbinding-demo='app:main'\n", + encoding="utf-8", + ) + (root / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n" + "[[package]]\nname='binding-demo'\nversion='1.0'\nsource={virtual='.'}\n", + encoding="utf-8", + ) + + +@pytest.mark.parametrize(("imports", "function"), ENV_BINDINGS) +@pytest.mark.parametrize( + "arguments", + [ + "'DB_PASSWORD'", + "key='DB_PASSWORD'", + "'DB_PASSWORD', 'fallback'", + "key='DB_PASSWORD', default='fallback'", + ], +) +def test_environment_import_bindings(tmp_path, imports, function, arguments): + # Imports after the function declaration still provide file-level evidence. + (tmp_path / "app.py").write_text( + f"def main(): return {function}({arguments})\n{imports}\n", encoding="utf-8" + ) + result = scan_runtime_assumptions(tmp_path, ["."]) + assert [(item.name, item.secret) for item in result.configuration_requirements] == [ + ("DB_PASSWORD", True) + ] + + +@pytest.mark.parametrize( + ("imports", "receiver"), + [ + ("import os", "os.environ"), + ("import os as operating", "operating.environ"), + ("from os import environ", "environ"), + ("from os import environ as env", "env"), + ], +) +@pytest.mark.parametrize("key", ["API-KEY", "2FA_TOKEN"]) +def test_environment_alias_subscripts(tmp_path, imports, receiver, key): + (tmp_path / "app.py").write_text(f"{imports}\nvalue={receiver}[{key!r}]\n") + assert [ + item.name for item in scan_runtime_assumptions(tmp_path, ["."]).configuration_requirements + ] == [key] + + +@pytest.mark.parametrize( + "source", + [ + "def getenv(key): return key\ngetenv('DB_PASSWORD')", + "thing.getenv('DB_PASSWORD')", + "from os import *\ngetenv('DB_PASSWORD')", + "from .os import getenv\ngetenv('DB_PASSWORD')", + "from os import getenv as read_env\nread_env(key=name)", + ], +) +def test_unproven_environment_names_are_ignored(tmp_path, source): + (tmp_path / "app.py").write_text(source) + assert not scan_runtime_assumptions(tmp_path, ["."]).configuration_requirements + + +@pytest.mark.parametrize(("imports", "function"), ENV_BINDINGS[1:4] + ENV_BINDINGS[5:]) +def test_alias_secret_manifest_and_scan_chain(tmp_path, monkeypatch, imports, function): + secret = "PDBSyntheticSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + write_project(tmp_path, f"{imports}\ndef main(): return {function}(key='DB_PASSWORD')\n") + assessment = assess_repository(repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + manifest = build_deployment_manifest( + plan, + tmp_path, + bootstrap_mode="online_cmd", + system_certs=False, + approved_artifacts=[], + bundled_uv_sha256=None, + referenced_files=[], + ) + assert manifest.configuration_secret_names == ["DB_PASSWORD"] + assert text_security_findings(secret) == set() + assert "configured_secret" in text_security_findings( + secret, + configured_secret_values=configured_secret_values(manifest.configuration_secret_names), + ) + assert ( + secret + not in assessment.model_dump_json() + plan.model_dump_json() + manifest.model_dump_json() + ) + monkeypatch.setenv("DB_PASSWORD", "1234567") + with pytest.raises(PreparationError, match="SHORT_CONFIGURED_SECRET_UNSCANNABLE") as caught: + configured_secret_values(manifest.configuration_secret_names) + assert "1234567" not in str(caught.value) + + +@pytest.mark.parametrize( + ("imports", "function"), + [ + ("import importlib", "importlib.import_module"), + ("import importlib as il", "il.import_module"), + ("from importlib import import_module", "import_module"), + ("from importlib import import_module as load", "load"), + ("", "__import__"), + ], +) +def test_keyword_dynamic_import_stages_module_and_initializers(tmp_path, imports, function): + write_project( + tmp_path, f"{imports}\ndef main(): return {function}(name='pkg.examples.plugin')\n" + ) + package = tmp_path / "pkg/examples" + package.mkdir(parents=True) + for path in ["pkg/__init__.py", "pkg/examples/__init__.py", "pkg/examples/plugin.py"]: + (tmp_path / path).write_text("VALUE=1\n") + assessment = assess_repository(repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + staged = _staging_files(tmp_path, assessment, plan, include=True) + assert {"pkg/__init__.py", "pkg/examples/__init__.py", "pkg/examples/plugin.py"} <= set(staged) + assert plan.deployment_mode == "source" + + +@pytest.mark.parametrize( + ("imports", "function"), + [ + ("import pkgutil", "pkgutil.get_data"), + ("import pkgutil as pu", "pu.get_data"), + ("from pkgutil import get_data", "get_data"), + ("from pkgutil import get_data as read", "read"), + ], +) +@pytest.mark.parametrize( + "args", + [ + "'pkg', 'nested/defaults.json'", + "'pkg', resource='nested/defaults.json'", + "package='pkg', resource='nested/defaults.json'", + ], +) +def test_pkgutil_keywords_stage_resource(tmp_path, imports, function, args): + write_project(tmp_path, f"{imports}\ndef main(): return {function}({args})\n") + (tmp_path / "pkg/nested").mkdir(parents=True) + (tmp_path / "pkg/__init__.py").write_text("") + (tmp_path / "pkg/nested/defaults.json").write_text("{}\n") + assessment = assess_repository(repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "pkg/nested/defaults.json" in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("kind", ["setup.py", "setup.cfg"]) +@pytest.mark.parametrize("root_present", [True, False]) +def test_legacy_source_kit_static_root_identity(tmp_path, monkeypatch, kind, root_present): + source = tmp_path / "source" + write_project(source, "def main(): return 0\n") + (source / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1']\nbuild-backend='setuptools.build_meta'\n" + ) + if kind == "setup.py": + content = ( + "from setuptools import setup\nsetup(name='binding-demo', version='1.0', " + "py_modules=['app'], entry_points={'console_scripts':['binding-demo=app:main']})\n" + ) + else: + content = ( + "[metadata]\nname=binding-demo\nversion=1.0\n[options]\npy_modules=app\n" + "[options.entry_points]\nconsole_scripts=\n binding-demo=app:main\n" + ) + (source / kind).write_text(content) + if not root_present: + # Actual uv 0.12.5 output for both legacy metadata forms: no package table. + (source / "uv.lock").write_text("version=1\nrevision=3\nrequires-python='>=3.12'\n") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + if not root_present: + plan = create_deployment_plan(assess_repository(repository(source)), repository_root=source) + assert plan.deployment_mode == "source" + assert "LEGACY_LOCK_ROOT_UNIDENTIFIABLE" in plan.risk_gate.blocking_codes + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" not in plan.risk_gate.blocking_codes + preview = generate_deployment_kit(repository(source), kit, dry_run=True).preview + assert any("LEGACY_LOCK_ROOT_UNIDENTIFIABLE" in item for item in preview.developer_actions) + with pytest.raises(PreparationError, match="LEGACY_LOCK_ROOT_UNIDENTIFIABLE"): + generate_deployment_kit(repository(source), kit, bootstrap_mode="online_cmd") + assert not kit.exists() + return + generate_deployment_kit(repository(source), kit, bootstrap_mode="online_cmd") + report = validate_static_kit(kit) + assert report.final_state.value == "STATIC_VALID", report.model_dump_json() + manifest = json.loads((kit / "deployment/manifest.json").read_text()) + assert manifest["deployment_mode"] == "source" + assert manifest["application_artifact"] is None + + +@pytest.mark.parametrize( + ("project", "artifact", "locked", "expected"), + [ + ("demo", None, "demo", "demo"), + (None, "demo", "demo", "demo"), + (None, None, "Legacy_Demo", "Legacy_Demo"), + ("other", None, "demo", None), + (None, "other", "demo", None), + ("demo", "other", "demo", None), + ], +) +def test_staged_lock_root_identity_cross_checks(tmp_path, project, artifact, locked, expected): + (tmp_path / "pyproject.toml").write_text(f"[project]\nname={project!r}\n" if project else "") + (tmp_path / "uv.lock").write_text( + f"version=1\n[[package]]\nname={locked!r}\nsource={{virtual='.'}}\n" + ) + manifest = SimpleNamespace( + application_artifact=(SimpleNamespace(distribution_name=artifact) if artifact else None) + ) + assert _static_lock_root_name(tmp_path, manifest) == expected + + +def test_keyword_before_positional_resource_is_invalid_python(): + with pytest.raises(SyntaxError): + ast.parse("pkgutil.get_data(package='app', 'defaults.json')") + + +@pytest.mark.parametrize( + ("imports", "call"), + [ + ("import os as operating", "operating.listdir(path='bundle/assets')"), + ("from os import scandir as scan", "scan(path='bundle/assets')"), + ], +) +def test_directory_read_alias_audit(tmp_path, imports, call): + write_project(tmp_path, f"{imports}\ndef main(): return {call}\n") + (tmp_path / "bundle/assets").mkdir(parents=True) + (tmp_path / "bundle/assets/defaults.json").write_text("{}") + assessment = assess_repository(repository(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "bundle/assets/defaults.json" in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("source", ["virtual", "editable"]) +def test_lock_root_preserves_exact_name(tmp_path, source): + (tmp_path / "uv.lock").write_text( + f"version=1\n[[package]]\nname='Legacy_Demo'\nsource={{{source}='.'}}\n" + ) + assert identify_uv_lock_root_name(tmp_path) == "Legacy_Demo" + + +@pytest.mark.parametrize( + "lock", + [ + "version=1\n", + "version=1\npackage=[]\n", + "version=1\n[[package]]\nname='dependency'\nsource={editable='elsewhere'}\n", + ], +) +def test_missing_lock_root_is_not_guessed(tmp_path, lock): + (tmp_path / "uv.lock").write_text(lock) + assert identify_uv_lock_root_name(tmp_path) is None + + +@pytest.mark.parametrize( + "lock", + [ + "[invalid", + "package='wrong'", + "package=[1]", + "[[package]]\nname='bad/name'\nsource={virtual='.'}", + "[[package]]\nsource={virtual='.'}", + "[[package]]\nname='demo'\nsource={virtual='.', editable='.'}", + "[[package]]\nname='demo'\nsource={virtual='.'}\n" + "[[package]]\nname='other'\nsource={editable='.'}", + "[[package]]\nname='demo'\nsource='wrong'", + ], +) +def test_malformed_or_ambiguous_lock_roots_fail_controlled(tmp_path, lock): + (tmp_path / "uv.lock").write_text(lock) + with pytest.raises(ValueError): + identify_uv_lock_root_name(tmp_path) + (tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n") + assert _static_lock_root_name(tmp_path, SimpleNamespace(application_artifact=None)) is None + + +@pytest.mark.parametrize( + "name", + ["module_name", "f'pkg.{name}'", "'.plugin'", "'pkg..plugin'", "'pkg/plugin'", "'pkg-plugin'"], +) +def test_dynamic_keyword_names_are_not_resolved(tmp_path, name): + from python_deployment_builder.analysis.inventory import _imported_modules + + tree = ast.parse(f"import importlib\nimportlib.import_module(name={name})") + assert _imported_modules(tree, tmp_path / "app.py", tmp_path, ["."]) == [("importlib", 1)] + + +@pytest.mark.parametrize( + ("imports", "call", "category", "value"), + [ + ( + "import subprocess as sp", + "sp.run(args=['convert.exe'])", + "external_executable", + "convert.exe", + ), + ( + "from subprocess import run as execute", + "execute(args=['convert.exe'])", + "external_executable", + "convert.exe", + ), + ("import ctypes as ct", "ct.CDLL(name='runtime.dll')", "native_runtime", "runtime.dll"), + ( + "from ctypes import CDLL as load", + "load(name='runtime.dll')", + "native_runtime", + "runtime.dll", + ), + ("from os import getcwd as cwd", "cwd()", "path_assumption", "current working directory"), + ( + "from webbrowser import open as browse", + "browse('https://example.org')", + "external_launcher", + "webbrowser.open", + ), + ], +) +def test_bounded_runtime_binding_audit(tmp_path, imports, call, category, value): + (tmp_path / "app.py").write_text(f"{imports}\n{call}\n") + result = scan_runtime_assumptions(tmp_path, ["."]) + assert any( + item.category == category and item.name == value for item in result.runtime_requirements + ) + + +@pytest.mark.parametrize( + "arguments", + [ + "package=dynamic, resource='defaults.json'", + "package='pkg', resource=dynamic", + "package='pkg', resource='../defaults.json'", + "package='pkg', resource='/defaults.json'", + "package='pkg', resource='nested\\\\defaults.json'", + "package='pkg', resource='defaults.json', unexpected=True", + "'pkg', 'defaults.json', 'extra'", + ], +) +def test_pkgutil_keyword_safety(tmp_path, arguments): + write_project(tmp_path, f"import pkgutil\ndef main(): return pkgutil.get_data({arguments})\n") + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg/__init__.py").write_text("") + (tmp_path / "pkg/defaults.json").write_text("{}") + assessment = assess_repository(repository(tmp_path)) + assert all(item.path != "pkg/defaults.json" for item in assessment.resources) + + +@pytest.mark.parametrize( + "imports", ["from .pkgutil import get_data", "def get_data(package, resource): return None"] +) +def test_pkgutil_resource_binding_must_be_stdlib(tmp_path, imports): + write_project( + tmp_path, + f"{imports}\ndef main(): return get_data(package='pkg', resource='defaults.json')\n", + ) + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg/__init__.py").write_text("") + (tmp_path / "pkg/defaults.json").write_text("{}") + assert all( + item.path != "pkg/defaults.json" + for item in assess_repository(repository(tmp_path)).resources + ) + + +@pytest.mark.parametrize("packaged", [False, True]) +def test_pkgutil_duplicate_bindings_and_package_data(tmp_path, packaged): + write_project( + tmp_path, + "import pkgutil\ndef main(): return pkgutil.get_data(" + "'pkg', 'defaults.json', package='wrong', resource='wrong.json')\n", + ) + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg/__init__.py").write_text("") + (tmp_path / "pkg/defaults.json").write_text("{}") + if packaged: + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text() + "[tool.setuptools]\npackages=['pkg']\n" + "[tool.setuptools.package-data]\npkg=['defaults.json']\n" + ) + assessment = assess_repository(repository(tmp_path)) + resource = next(item for item in assessment.resources if item.path == "pkg/defaults.json") + assert resource.packaging_status == ("packaged" if packaged else "repository_adjacent") + + +def test_pkgutil_keyword_namespace_is_unresolved(tmp_path): + write_project( + tmp_path, + "import pkgutil\ndef main(): return pkgutil.get_data(" + "package='pkg', resource='defaults.json')\n", + ) + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg/defaults.json").write_text("{}") + assessment = assess_repository(repository(tmp_path)) + assert all(item.path != "pkg/defaults.json" for item in assessment.resources) + + +@pytest.mark.parametrize( + "reader", + [ + "open(file='pkg/defaults.json').read()", + "pkgutil.get_data(package='pkg', resource='defaults.json')", + ], +) +def test_alias_secret_in_selected_resource_rejected_before_output( + tmp_path, monkeypatch, capsys, reader +): + source = tmp_path / "source" + write_project( + source, + "from os import getenv as read_env\nimport pkgutil\n" + f"def main():\n read_env(key='DB_PASSWORD')\n return {reader}\n", + ) + (source / "pkg").mkdir() + (source / "pkg/__init__.py").write_text("") + secret = "PDBSyntheticSecret123" + (source / "pkg/defaults.json").write_text(secret) + assert text_security_findings(secret) == set() + monkeypatch.setenv("DB_PASSWORD", secret) + uv = tmp_path / "uv.exe" + uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + with pytest.raises(PreparationError) as caught: + generate_deployment_kit(repository(source), kit, bootstrap_mode="online_cmd") + assert "SECRET" in str(caught.value) + assert secret not in str(caught.value) + str(capsys.readouterr()) + assert not kit.exists() From 40da08522463fc8992d698cf8b187a3e59706270 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:08:49 -0400 Subject: [PATCH 51/59] Fix approved artifact edge context and path ownership --- ...roved-extras-paths-resources-2026-09-11.md | 152 ++++++ scripts/verify_approved_extra_context.py | 85 +++ .../analysis/resources.py | 14 +- .../generation/artifacts.py | 33 +- .../generation/manifest.py | 2 + .../generation/structural.py | 17 + .../validation/static.py | 32 +- tests/test_review_extra_paths_resources.py | 500 ++++++++++++++++++ 8 files changed, 812 insertions(+), 23 deletions(-) create mode 100644 docs/acceptance/pr9-approved-extras-paths-resources-2026-09-11.md create mode 100644 scripts/verify_approved_extra_context.py create mode 100644 tests/test_review_extra_paths_resources.py diff --git a/docs/acceptance/pr9-approved-extras-paths-resources-2026-09-11.md b/docs/acceptance/pr9-approved-extras-paths-resources-2026-09-11.md new file mode 100644 index 0000000..8b35017 --- /dev/null +++ b/docs/acceptance/pr9-approved-extras-paths-resources-2026-09-11.md @@ -0,0 +1,152 @@ +# PR #9: approved-extra context, artifact paths, legacy resource keywords + +Starting head: `bc682d75b5d77580d5e76b7f17c3a375843368fe`. +Exact-head review submitted: `2026-09-11T14:06:18Z`. + +| P2 finding | Thread | Comment | +| --- | --- | --- | +| Match root-extra edges when validating approved wheels | PRRT_kwDOT9hvCc6hf6Wv | 3989921446 | +| Reject duplicate approved-artifact filenames | PRRT_kwDOT9hvCc6hf6W5 | 3989921458 | +| Bind keyword arguments for legacy resource reads | PRRT_kwDOT9hvCc6hf6W- | 3989921467 | + +Pre-next-review accounting: 114 findings, 111 unresolved inline threads, three +review-level-only findings, and one NOT_APPLICABLE inline finding. No thread +resolution, review dismissal, or merge is part of this correction. + +## Approved dependency edges + +The exact synthetic `map -> pywebview -> proxy-tools -> helper` chain reproduces +the false rejection. The graph includes helper and carries `selected_extra='map'` +on the normal proxy-tools outgoing edge, but no extra is requested of proxy-tools. +Its activated-extra set is empty. Before correction, an otherwise compliant wheel +declaring `Requires-Dist: helper>=1` fails with “no proxy-tools dependency edge”. + +The persisted edge dimensions retain their distinct meanings: + +| Field | Meaning | +| --- | --- | +| selected_extra | Selected application/root extra whose path produced the edge | +| requested_dependency_extras | Extras requested of the child (`to_package`) | +| activated_dependency_extra | Optional group of the parent (`from_package`) | + +One selection helper now gates both approved-parent presence and dependency-extra +proofs. Root lineage must belong to the graph's selected application extras; +parent optional groups must belong to the approved parent's activated extras. +Normal outgoing edges need no parent extra. Same-name root and parent extras do +not activate each other. Incoming requested child extras remain the source of +approved-package activation, using definitely applicable markers. + +The bounded audit of the five requested approved-wheel helpers found one adjacent +gap: incoming-extra collection lacked the selected-root-lineage filter. This is +now checked too. Requires-Dist marker evaluation, version constraints, self +requirements, direct-reference rejection, and child-extra closure remain intact. +No application-wheel proof refactor or persisted field was introduced. + +### Actual uv marker-context experiment + +`scripts/verify_approved_extra_context.py` constructs local synthetic wheels in a +temporary directory and runs the pinned +`uv 0.12.5 (210d1f678 2026-08-14 x86_64-pc-windows-msvc)` with: + +```text +uv lock --offline --no-index --find-links . --python 3.12 +``` + +The experiment does not modify acceptance repositories or execute applications. +Observed uv lock fragments include: + +```toml +# pywebview +dependencies = [{ name = "proxy-tools", extra = ["feature"] }] + +# proxy-tools +dependencies = [{ name = "helper" }] +[package.optional-dependencies] +feature = [{ name = "optional-helper", marker = "sys_platform == 'win32'" }] +``` + +Core Metadata originally declared the optional requirement with +`extra == 'feature' and sys_platform == 'win32'`. uv places it in the feature group +and removes the extra predicate from the edge marker. PDB records root lineage +`map` on both outgoing edges, with parent activation `None` on helper and +`feature` on optional-helper. The observed marker has only the platform condition, +so this correction preserves the existing marker-evaluation contract and changes +edge selection. APPLIES can prove presence/extras; false, malformed, or UNPROVABLE +markers cannot. Tests explicitly set optimistic `edge.applicable=True` to prove +the stricter marker check is not bypassed. + +## One-to-one approved wheel materialization + +A valid generated kit required both `foo==1` and `bar==1`, with two independently +validated wheels. The reproduction then changed foo's record to bar's filename +and SHA, removed foo's file/index entry/reference, retained both lock identities +and suppression pairs, and refreshed the manifest index hash. With bar last, the +old path-keyed identity dictionary silently retained only bar. The self-consistent +tampered kit was **STATIC_VALID before correction**. + +`approved_artifacts_by_path` now validates safe kit-relative approved paths and +requires uniqueness under Windows case folding before returning a dictionary. +Static validation reports `APPROVED_ARTIFACT_PATH_UNIQUENESS` and uses that one +validated mapping for both expected identities and wheel-dependency validation. +An invalid set never supplies an overwritten/surviving identity map. + +Tests cover exact/case-equivalent duplicate paths, equal SHA values, PEP 440 +equivalent version spellings, repeated identity with distinct paths, unsafe +filenames, and valid distinct artifacts. Existing identity and reverse lock +completeness checks remain independent. Application and approved artifacts retain +different materialization directories even when basenames match; installed-path +collision validation remains independent. Canonical sync construction is unchanged. + +Generation audit: CLI wheel validation proves filename and METADATA distribution +identity, and `validate_artifact_set` rejects repeated distributions. Thus two +different validated identities cannot ordinarily share a case-equivalent wheel +filename. However, the independently callable manifest builder accepts artifact +records directly. It now invokes the same path invariant before constructing +sync arguments or returning a manifest. No new manifest fields are needed. + +## Legacy resource keywords + +The exact-head legacy resolver recognizes the function binding but requires two +positional arguments. Running it in memory against the corrected source-mode +fixture reproduced four keyword/mixed-form failures while two positional cases +passed. No production source was reverted for that comparison. + +All four already-supported functions (`read_text`, `read_binary`, `open_text`, +`open_binary`) now bind package/resource through the existing `call_argument`. +The 48-case staging matrix covers canonical modules, module aliases, direct +function imports, direct aliases, and positional/mixed/keyword forms. Text forms +retain encoding/errors keywords; binary forms reject them. Positional arguments +still win invalid duplicate bindings. + +The existing package-anchor resolver, direct-member restriction, source roots, +containment, and namespace rules remain unchanged. Dynamic/unsafe/nested members, +unrelated functions, and unknown keywords do not produce concrete resource paths. +All four keyword APIs also have promoted-resource configured-secret rejection +tests, including assertions that exceptions do not expose the synthetic value. +Modern importlib.resources and pkgutil semantics are not broadened or changed. + +## Read-only acceptance and schemas + +| Repository | Unchanged SHA | Result | +| --- | --- | --- | +| SimpleGeorefGUI | `f484570d89fb1f9e9170fac915475358dfc1234e` | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python, 77 data members | +| Geo Map Explanation Extractor | `5e7b321d0aeb9ba1d586bfc548c79793d84c6033` | source / SOURCE_COMPATIBLE | +| TN Coordinate Converter | `e1e7a1588c37a99c2d02efaf3eef3d04636f12f0` | source / SOURCE_COMPATIBLE; existing reviewed kit STATIC_VALID | + +Before/after Git working-tree states match. Configuration-name lists are unchanged. +TN's existing kit retains `proxy-tools==0.1.0`; the synthetic helper chain exists +only in disposable PDB fixtures. No wheels, kits, environments, or uv temporary +projects are committed. Analysis schema 1.4 and planning schema 1.3 are unchanged. + +## Quality + +Focused regression file: **127 passed**. The single complete suite passed: +**1125 passed, 3 skipped in 262.00 seconds**, up from 998 passed, 3 skipped. +`ruff check .` and `git diff --check` passed. The final diff was inspected for +unrelated changes. + +The complete suite retains the preceding alias/environment, legacy lock-root, +dynamic-import and pkgutil corrections; MANIFEST/package-surface protections; +exact sync and mode-aware pip contracts; entry-point, short-secret, LOCALAPPDATA, +wheel/dependency, resource/inventory, wheel integrity/security/collision, and +provenance/workflow regressions. Persisted schema versions remain unchanged. diff --git a/scripts/verify_approved_extra_context.py b/scripts/verify_approved_extra_context.py new file mode 100644 index 0000000..27bee9d --- /dev/null +++ b/scripts/verify_approved_extra_context.py @@ -0,0 +1,85 @@ +"""Explicit offline uv 0.12.5 experiment: root lineage versus package extras.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import subprocess +import sys +import tempfile +import tomllib +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from python_deployment_builder.planning.lockfile import inspect_uv_lock + + +def wheel(root: Path, name: str, requirements: list[str], extras: list[str]) -> str: + normalized = name.replace("-", "_") + info = f"{normalized}-1.dist-info" + files = { + f"{info}/METADATA": ( + f"Metadata-Version: 2.1\nName: {name}\nVersion: 1\n" + + "".join(f"Provides-Extra: {extra}\n" for extra in extras) + + "".join(f"Requires-Dist: {item}\n" for item in requirements) + + "\n" + ).encode(), + f"{info}/WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + f"{normalized}/__init__.py": b"", + } + record = "".join( + f"{path},sha256={base64.urlsafe_b64encode(hashlib.sha256(data).digest()).decode().rstrip('=')},{len(data)}\n" + for path, data in files.items() + ) + files[f"{info}/RECORD"] = (record + f"{info}/RECORD,,\n").encode() + filename = f"{normalized}-1-py3-none-any.whl" + with zipfile.ZipFile(root / filename, "w") as bundle: + for path, data in files.items(): + bundle.writestr(path, data) + return filename + + +def main() -> None: + uv = Path(sys.argv[1]).resolve() + version = subprocess.check_output([str(uv), "--version"], text=True).strip() + assert version.startswith("uv 0.12.5 "), version + root = Path(tempfile.mkdtemp(prefix="pdb-approved-extra-context-")) + print(version, root, flush=True) + for name, requirements, extras in [ + ("pywebview", ["proxy-tools[feature]>=1"], []), + ( + "proxy-tools", + ["helper>=1", "optional-helper>=1; extra == 'feature' and sys_platform == 'win32'"], + ["feature"], + ), + ("helper", [], []), + ("optional-helper", [], []), + ]: + wheel(root, name, requirements, extras) + (root / "pyproject.toml").write_text( + "[project]\nname='app'\nversion='1'\nrequires-python='>=3.12'\n" + "[project.optional-dependencies]\nmap=['pywebview']\n", + encoding="utf-8", + ) + subprocess.run( + [str(uv), "lock", "--offline", "--no-index", "--find-links", ".", "--python", "3.12"], + cwd=root, + check=True, + ) + document = tomllib.loads((root / "uv.lock").read_text(encoding="utf-8")) + print(json.dumps(document), flush=True) + graph = inspect_uv_lock(root, "app", "3.12", "x86_64", ["map"]) + print(json.dumps([item.model_dump() for item in graph.edges]), flush=True) + normal = next(item for item in graph.edges if item.to_package == "helper") + optional = next(item for item in graph.edges if item.to_package == "optional-helper") + assert normal.selected_extra == optional.selected_extra == "map" + assert normal.activated_dependency_extra is None + assert optional.activated_dependency_extra == "feature" + assert optional.marker == "sys_platform == 'win32'" + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index e42a53c..bb1ef74 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -740,13 +740,19 @@ def _legacy_importlib_resource_path_values( return None # Python 3.12 legacy resource calls address one direct member of a # package. Keep dynamic, nested, and traversal-like members unresolved. - if len(node.args) != 2: + if len(node.args) > 2: return function, [] - allowed_keywords = {"encoding", "errors"} if function.endswith("text") else set() + allowed_keywords = {"package", "resource"} + if function.endswith("text"): + allowed_keywords.update({"encoding", "errors"}) if any(keyword.arg not in allowed_keywords for keyword in node.keywords): return function, [] + package = call_argument(node, position=0, keyword="package") + resource = call_argument(node, position=1, keyword="resource") + if package is None or resource is None: + return function, [] package_roots = _resource_package_anchor_values( - node.args[0], + package, root=root, source_path=source_path, source_roots=source_roots, @@ -755,7 +761,7 @@ def _legacy_importlib_resource_path_values( returns=returns, ) members = _path_values( - node.args[1], + resource, root=root, source_path=source_path, assignments=assignments, diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index ba1291f..8a0d0e8 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -1029,9 +1029,15 @@ def _approved_package_activated_extras(graph, plan: DeploymentPlan, package_name """Return extras definitely requested of an approved package by incoming edges.""" activated: set[str] = set() + selected_root_extras = {canonicalize_name(extra) for extra in graph.selected_extras} for edge in graph.edges: if canonicalize_name(edge.to_package) != package_name: continue + if ( + edge.selected_extra + and canonicalize_name(edge.selected_extra) not in selected_root_extras + ): + continue try: applicability = target_marker_applicability( edge.marker, @@ -1046,6 +1052,23 @@ def _approved_package_activated_extras(graph, plan: DeploymentPlan, package_name return activated +def _approved_parent_edge_is_selected(edge, graph, activated_parent_extras: set[str]) -> bool: + """Keep the three extra dimensions separate for approved-parent proofs. + + selected_extra is root/application lineage; activated_dependency_extra is + the optional group of from_package; requested_dependency_extras are extras + requested of to_package (checked separately after edge eligibility). + """ + selected_root_extras = {canonicalize_name(extra) for extra in graph.selected_extras} + return ( + edge.selected_extra is None + or canonicalize_name(edge.selected_extra) in selected_root_extras + ) and ( + edge.activated_dependency_extra is None + or canonicalize_name(edge.activated_dependency_extra) in activated_parent_extras + ) + + def _approved_requirement_applies( requirement: Requirement, plan: DeploymentPlan, activated_extras: set[str] ) -> bool: @@ -1092,10 +1115,7 @@ def _parent_dependency_presence_proven( or canonicalize_name(edge.to_package) != dependency_name ): continue - if ( - edge.selected_extra - and canonicalize_name(edge.selected_extra) not in activated_parent_extras - ): + if not _approved_parent_edge_is_selected(edge, graph, activated_parent_extras): continue matching = True try: @@ -1145,10 +1165,7 @@ def _parent_dependency_extras_proven( or canonicalize_name(edge.to_package) != canonicalize_name(requirement.name) ): continue - if ( - edge.selected_extra - and canonicalize_name(edge.selected_extra) not in activated_parent_extras - ): + if not _approved_parent_edge_is_selected(edge, graph, activated_parent_extras): continue try: applicability = target_marker_applicability( diff --git a/src/python_deployment_builder/generation/manifest.py b/src/python_deployment_builder/generation/manifest.py index 36b8a48..ebd22f2 100644 --- a/src/python_deployment_builder/generation/manifest.py +++ b/src/python_deployment_builder/generation/manifest.py @@ -10,6 +10,7 @@ from python_deployment_builder import __version__ from python_deployment_builder.backends.uv_managed import uv_sync_arguments from python_deployment_builder.generation.acquisition import PreparationError, sha256_file +from python_deployment_builder.generation.structural import approved_artifacts_by_path from python_deployment_builder.models import ( ApplicationArtifact, ApprovedArtifact, @@ -48,6 +49,7 @@ def build_deployment_manifest( application_artifact: ApplicationArtifact | None = None, generated_at: datetime | None = None, ) -> DeploymentManifest: + approved_artifacts_by_path(approved_artifacts) repository_root = repository_root.resolve() pyproject = repository_root / "pyproject.toml" lockfile = repository_root / "uv.lock" diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index 26845cc..e73b684 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -9,6 +9,7 @@ from python_deployment_builder.generation.acquisition import PreparationError from python_deployment_builder.models import ( + ApprovedArtifact, DeploymentManifest, FindingStatus, RiskFinding, @@ -53,6 +54,22 @@ def manifest_artifact_wheel_path(directory: str, filename: str) -> str | None: return f"deployment/{directory}/{filename}" +def approved_artifacts_by_path(artifacts: list[ApprovedArtifact]) -> dict[str, ApprovedArtifact]: + """Require one approved record per safe, Windows-distinct materialization path.""" + result: dict[str, ApprovedArtifact] = {} + windows_paths: set[str] = set() + for artifact in artifacts: + relative = manifest_artifact_wheel_path("wheels", artifact.filename) + if relative is None: + raise PreparationError(f"Unsafe approved artifact filename: {artifact.filename}") + key = relative.casefold() + if key in windows_paths: + raise PreparationError(f"Duplicate approved artifact materialization path: {relative}") + windows_paths.add(key) + result[relative] = artifact + return result + + def trusted_artifact_wheel_paths(manifest: DeploymentManifest) -> set[str]: """Return exact manifest-owned wheel paths with dedicated validation. diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 3f8c98e..65c38a7 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -34,6 +34,7 @@ validate_wheel_target_compatibility, ) from python_deployment_builder.generation.structural import ( + approved_artifacts_by_path, manifest_artifact_wheel_path, trusted_artifact_wheel_paths, ) @@ -383,6 +384,21 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=unexpected_paths, ) ) + approved_path_failures: list[str] = [] + try: + approved_by_relative = approved_artifacts_by_path(manifest.approved_artifacts) + except PreparationError as exc: + approved_by_relative = {} + approved_path_failures.append(str(exc)) + checks.append( + _check( + "APPROVED_ARTIFACT_PATH_UNIQUENESS", + not approved_path_failures, + "Each approved artifact owns one safe, Windows-distinct wheel path.", + "Approved artifacts have unsafe or duplicate materialization paths.", + evidence=approved_path_failures, + ) + ) trusted_wheels = trusted_artifact_wheel_paths(manifest) secret_scanability_failures: list[str] = [] try: @@ -495,12 +511,11 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR manifest.application_artifact.distribution_name, manifest.application_artifact.version, ) - for artifact in manifest.approved_artifacts: - if relative := manifest_artifact_wheel_path("wheels", artifact.filename): - expected_wheel_identities[relative] = ( - artifact.distribution_name, - artifact.version, - ) + for relative, artifact in approved_by_relative.items(): + expected_wheel_identities[relative] = ( + artifact.distribution_name, + artifact.version, + ) wheel_metadata_failures: list[str] = [] wheel_metadata_by_path = {} for path in safe_trusted_wheel_paths: @@ -656,11 +671,6 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR if manifest.application_artifact is not None else None ) - approved_by_relative = { - relative: artifact - for artifact in manifest.approved_artifacts - if (relative := manifest_artifact_wheel_path("wheels", artifact.filename)) - } for path in dependency_wheels: relative = path.relative_to(root).as_posix() try: diff --git a/tests/test_review_extra_paths_resources.py b/tests/test_review_extra_paths_resources.py new file mode 100644 index 0000000..226b94d --- /dev/null +++ b/tests/test_review_extra_paths_resources.py @@ -0,0 +1,500 @@ +"""PR #9 regressions: approved-edge context, artifact ownership, legacy keywords.""" + +import ast +import json +from types import SimpleNamespace + +import pytest +from packaging.requirements import Requirement +from packaging.version import Version +from test_generation import _make_wheel, _plan, _update_indexed_hashes + +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import ( + _approved_package_activated_extras, + _parent_dependency_extras_proven, + _parent_dependency_presence_proven, + validate_approved_requires_dist, + validate_approved_wheel, +) +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.generation.manifest import build_deployment_manifest +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.generation.structural import ( + approved_artifacts_by_path, + trusted_artifact_wheel_paths, +) +from python_deployment_builder.models import ApprovedArtifact +from python_deployment_builder.planning.lockfile import inspect_uv_lock +from python_deployment_builder.planning.planner import create_deployment_plan +from python_deployment_builder.validation.static import validate_static_kit + + +def repo(root): + return MaterializedRepository(root=root, source=str(root), source_kind="local") + + +def write_source(root, source="def main(): return 0\n", dependencies="[]"): + root.mkdir(parents=True, exist_ok=True) + (root / "main.py").write_text(source, encoding="utf-8") + (root / "pyproject.toml").write_text( + "[project]\nname='review-demo'\nversion='1'\nrequires-python='>=3.12'\n" + f"dependencies={dependencies}\n[project.scripts]\nreview-demo='main:main'\n" + ) + (root / "uv.lock").write_text( + "version=1\nrequires-python='>=3.12'\n[[package]]\n" + "name='review-demo'\nversion='1'\nsource={virtual='.'}\n" + ) + + +@pytest.fixture +def fake_preparation(tmp_path, monkeypatch): + uv = tmp_path / "uv.exe" + uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + + +def extra_plan(root): + (root / "uv.lock").write_text( + "version=1\n[[package]]\nname='app'\nsource={virtual='.'}\n" + "[package.optional-dependencies]\nmap=[{name='pywebview'}]\n" + "[[package]]\nname='pywebview'\nversion='1'\ndependencies=[{name='proxy-tools'}]\n" + "[[package]]\nname='proxy-tools'\nversion='0.1.0'\n" + "sdist={url='https://example.invalid/proxy_tools-0.1.0.tar.gz'}\n" + "dependencies=[{name='helper'}]\n" + "[[package]]\nname='helper'\nversion='1'\n" + "wheels=[{url='https://example.invalid/helper-1-py3-none-any.whl'}]\n" + ) + plan = _plan().model_copy(deep=True) + plan.lock_graph = inspect_uv_lock(root, "app", "3.12", "x86_64", ["map"]) + plan.runtime.selected_extras = ["map"] + return plan + + +def test_approved_normal_dependency_beneath_root_extra(tmp_path): + plan = extra_plan(tmp_path) + graph = plan.lock_graph + assert _approved_package_activated_extras(graph, plan, "proxy-tools") == set() + edge = next(item for item in graph.edges if item.from_package == "proxy-tools") + assert edge.selected_extra == "map" + assert edge.activated_dependency_extra is None + assert any(item.name == "helper" for item in graph.dependencies) + wheel = _make_wheel(tmp_path, requires_dist_values=["helper>=1"]) + assert ( + validate_approved_wheel(f"proxy-tools={wheel}", plan)[0].distribution_name == "proxy-tools" + ) + + +def two_artifact_kit(tmp_path): + source = tmp_path / "source" + write_source(source, dependencies="['foo==1', 'bar==1']") + with (source / "uv.lock").open("a") as stream: + stream.write("dependencies=[{name='foo'}, {name='bar'}]\n") + for name in ["foo", "bar"]: + stream.write( + f"[[package]]\nname='{name}'\nversion='1'\n" + f"sdist={{url='https://example.invalid/{name}-1.tar.gz'}}\n" + ) + wheels = [_make_wheel(tmp_path, name=name, version="1") for name in ["foo", "bar"]] + kit = tmp_path / "kit" + generate_deployment_kit( + repo(source), + kit, + bootstrap_mode="online_cmd", + artifact_values=[ + f"{name}={wheel}" for name, wheel in zip(["foo", "bar"], wheels, strict=True) + ], + ) + assert validate_static_kit(kit).final_state.value == "STATIC_VALID" + return kit + + +@pytest.mark.parametrize("case_variant", [False, True]) +@pytest.mark.parametrize("version", ["1", "1.0.0"]) +def test_duplicate_approved_path_rejected_after_consistent_reindex( + tmp_path, fake_preparation, case_variant, version +): + kit = two_artifact_kit(tmp_path) + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text()) + records = {item["distribution_name"]: item for item in manifest["approved_artifacts"]} + records["foo"]["filename"] = records["bar"]["filename"] + if case_variant: + records["foo"]["filename"] = records["foo"]["filename"].upper() + records["foo"]["version"] = version + records["foo"]["sha256"] = records["bar"]["sha256"] + # Preserve both lock identities and suppression pairs, with bar last in the old map. + manifest["approved_artifacts"] = [records["foo"], records["bar"]] + removed = "deployment/wheels/foo-1-py3-none-any.whl" + (kit / removed).unlink() + manifest["referenced_files"] = [x for x in manifest["referenced_files"] if x != removed] + manifest_path.write_text(json.dumps(manifest)) + index_path = kit / "deployment/generated-files.json" + index = json.loads(index_path.read_text()) + index["files"] = [x for x in index["files"] if x["path"] != removed] + index_path.write_text(json.dumps(index)) + _update_indexed_hashes(kit, "deployment/manifest.json") + report = validate_static_kit(kit) + checks = {item.code: item for item in report.static_checks} + assert checks["APPROVED_ARTIFACT_LOCK_IDENTITY"].status.value == "PASS" + assert checks["SYNC_ARGUMENTS_CONTRACT"].status.value == "PASS" + assert checks["GENERATED_FILE_HASHES"].status.value == "PASS" + assert report.final_state.value == "FAILED", "one physical wheel cannot own foo and bar" + assert checks["APPROVED_ARTIFACT_PATH_UNIQUENESS"].status.value == "FAIL" + assert checks["WHEEL_METADATA_SEMANTICS"].status.value == "FAIL" + + +def test_static_identity_uniqueness_separate_from_path_uniqueness(tmp_path, fake_preparation): + kit = two_artifact_kit(tmp_path) + path = kit / "deployment/manifest.json" + manifest = json.loads(path.read_text()) + manifest["approved_artifacts"][0]["distribution_name"] = manifest["approved_artifacts"][1][ + "distribution_name" + ] + manifest["approved_artifacts"][0]["version"] = "1.0.0" + path.write_text(json.dumps(manifest)) + _update_indexed_hashes(kit, "deployment/manifest.json") + checks = {item.code: item for item in validate_static_kit(kit).static_checks} + assert checks["APPROVED_ARTIFACT_PATH_UNIQUENESS"].status.value == "PASS" + assert checks["APPROVED_ARTIFACT_LOCK_IDENTITY"].status.value == "FAIL" + assert any("repeats" in item for item in checks["APPROVED_ARTIFACT_LOCK_IDENTITY"].evidence) + + +@pytest.mark.parametrize("function", ["read_text", "read_binary", "open_text", "open_binary"]) +@pytest.mark.parametrize("binding", ["module", "module_alias", "direct", "direct_alias"]) +@pytest.mark.parametrize( + "arguments", + [ + "'app', 'defaults.json'", + "'app', resource='defaults.json'", + "package='app', resource='defaults.json'", + ], +) +def test_legacy_keyword_resource_staged(tmp_path, function, binding, arguments): + write_source(tmp_path, "def main(): return 0\n") + project = tmp_path / "pyproject.toml" + project.write_text(project.read_text().replace("main:main", "app.main:main")) + (tmp_path / "src/app").mkdir(parents=True) + (tmp_path / "src/app/__init__.py").write_text("") + imports, call = { + "module": ("import importlib.resources", f"importlib.resources.{function}"), + "module_alias": ("import importlib.resources as ir", f"ir.{function}"), + "direct": (f"from importlib.resources import {function}", function), + "direct_alias": (f"from importlib.resources import {function} as read", "read"), + }[binding] + source = f"{imports}\ndef main():\n return {call}({arguments})\n" + (tmp_path / "src/app/main.py").write_text(source) + (tmp_path / "src/app/defaults.json").write_text("{}") + call = next(node for node in ast.walk(ast.parse(source)) if isinstance(node, ast.Call)) + if arguments.startswith("package="): + assert call.args == [] + assessment = assess_repository(repo(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert plan.deployment_mode == "source" + assert "src/app/defaults.json" in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize( + ("root_extra", "selected", "parent_extra", "activated", "accepted"), + [ + (None, [], None, [], True), + ("map", ["map"], None, [], True), + ("map", [], None, ["map"], False), + ("map", ["map"], "map", [], False), + ("map", ["map"], "feature", ["feature"], True), + (None, [], "feature", [], False), + (None, [], "feature", ["feature"], True), + ("MAP", ["map"], "FEATURE", ["feature"], True), + ], +) +def test_parent_edge_context_dimensions( + tmp_path, root_extra, selected, parent_extra, activated, accepted +): + plan = extra_plan(tmp_path) + graph = plan.lock_graph + graph.selected_extras = selected + edge = next(item for item in graph.edges if item.from_package == "proxy-tools") + edge.selected_extra = root_extra + edge.activated_dependency_extra = parent_extra + edge.requested_dependency_extras = ["child"] + for proof, args in [ + ( + _parent_dependency_presence_proven, + (graph, plan, "proxy-tools", "helper", set(activated)), + ), + ( + _parent_dependency_extras_proven, + (Requirement("helper[child]"), graph, plan, "proxy-tools", set(activated)), + ), + ]: + if accepted: + proof(*args) + else: + with pytest.raises(PreparationError): + proof(*args) + + +@pytest.mark.parametrize( + ("marker", "accepted"), + [ + (None, True), + ("sys_platform == 'win32'", True), + ("sys_platform == 'linux'", False), + ("python_full_version >= '3.12.5'", False), + ("not a valid marker", False), + ], +) +def test_parent_edge_strict_marker_proof(tmp_path, marker, accepted): + plan = extra_plan(tmp_path) + graph = plan.lock_graph + edge = next(item for item in graph.edges if item.from_package == "proxy-tools") + edge.marker = marker + edge.applicable = True # This optimistic summary must not replace tri-state proof. + edge.requested_dependency_extras = ["child"] + for proof, args in [ + (_parent_dependency_presence_proven, (graph, plan, "proxy-tools", "helper", set())), + ( + _parent_dependency_extras_proven, + (Requirement("helper[child]"), graph, plan, "proxy-tools", set()), + ), + ]: + if accepted: + proof(*args) + else: + with pytest.raises(PreparationError): + proof(*args) + + +@pytest.mark.parametrize( + ("selected", "requested", "expected"), + [ + (["map"], [], set()), + (["map"], ["feature"], {"feature"}), + (["map"], ["map"], {"map"}), + ([], ["feature"], set()), + ], +) +def test_incoming_extras_are_not_root_extras(tmp_path, selected, requested, expected): + plan = extra_plan(tmp_path) + graph = plan.lock_graph + graph.selected_extras = selected + incoming = next(item for item in graph.edges if item.to_package == "proxy-tools") + incoming.requested_dependency_extras = requested + assert _approved_package_activated_extras(graph, plan, "proxy-tools") == expected + + +@pytest.mark.parametrize("requested", [[], ["feature"]]) +def test_incoming_extra_activates_parent_optional_dependency(tmp_path, requested): + plan = extra_plan(tmp_path) + graph = plan.lock_graph + incoming = next(item for item in graph.edges if item.to_package == "proxy-tools") + incoming.requested_dependency_extras = requested + outgoing = next(item for item in graph.edges if item.from_package == "proxy-tools") + outgoing.activated_dependency_extra = "feature" + outgoing.requested_dependency_extras = ["child"] + next( + item for item in graph.dependencies if item.name == "helper" + ).available_dependency_extras = ["child"] + if requested: + validate_approved_requires_dist(["helper[child]>=1"], plan, "proxy-tools", Version("0.1.0")) + else: + with pytest.raises(PreparationError, match="no proxy-tools dependency edge"): + validate_approved_requires_dist( + ["helper[child]>=1"], plan, "proxy-tools", Version("0.1.0") + ) + + +@pytest.mark.parametrize("marker", ["sys_platform == 'linux'", "python_full_version >= '3.12.5'"]) +def test_unproven_incoming_edges_do_not_activate_extras(tmp_path, marker): + plan = extra_plan(tmp_path) + incoming = next(item for item in plan.lock_graph.edges if item.to_package == "proxy-tools") + incoming.requested_dependency_extras = ["feature"] + incoming.marker = marker + assert _approved_package_activated_extras(plan.lock_graph, plan, "proxy-tools") == set() + + +@pytest.mark.parametrize( + ("requirement", "accepted"), + [ + ("helper>=1", True), + ("helper>=2", False), + ("proxy-tools==0.1", True), + ("proxy-tools>=1", False), + ("helper @ https://example.invalid/helper.whl", False), + ("helper>=2; extra == 'map'", True), # Root map does not activate parent map. + ], +) +def test_approved_requirement_semantics_after_selection(tmp_path, requirement, accepted): + plan = extra_plan(tmp_path) + if accepted: + validate_approved_requires_dist([requirement], plan, "proxy-tools", Version("0.1.0")) + else: + with pytest.raises(PreparationError): + validate_approved_requires_dist([requirement], plan, "proxy-tools", Version("0.1.0")) + + +def artifact(name, filename, version="1"): + return ApprovedArtifact( + distribution_name=name, + version=version, + filename=filename, + sha256="0" * 64, + wheel_tags=["py3-none-any"], + ) + + +@pytest.mark.parametrize( + "filenames", + [ + ["foo-1-py3-none-any.whl", "foo-1-py3-none-any.whl"], + ["Foo-1-py3-none-any.whl", "foo-1-py3-none-any.whl"], + ["foo-1-py3-none-any.whl", "FOO-1-PY3-NONE-ANY.WHL"], + ], +) +@pytest.mark.parametrize("version", ["1", "1.0.0"]) +def test_approved_materialization_paths_are_windows_unique(tmp_path, filenames, version): + records = [artifact("foo", filenames[0]), artifact("bar", filenames[1], version)] + with pytest.raises(PreparationError, match="Duplicate approved artifact materialization path"): + approved_artifacts_by_path(records) + # The independently callable generation manifest API enforces the same invariant. + with pytest.raises(PreparationError, match="Duplicate approved artifact materialization path"): + build_deployment_manifest( + _plan(), + tmp_path, + bootstrap_mode="online_cmd", + system_certs=False, + approved_artifacts=records, + bundled_uv_sha256=None, + referenced_files=[], + ) + + +@pytest.mark.parametrize( + "filenames", + [[], ["foo-1-py3-none-any.whl"], ["foo-1-py3-none-any.whl", "bar-1-py3-none-any.whl"]], +) +def test_distinct_approved_paths(filenames): + records = [artifact(str(index), name) for index, name in enumerate(filenames)] + assert len(approved_artifacts_by_path(records)) == len(records) + + +def test_application_and_approved_basename_are_separate_paths(): + record = artifact("foo", "foo-1-py3-none-any.whl") + assert len(approved_artifacts_by_path([record])) == 1 + manifest = SimpleNamespace(approved_artifacts=[record], application_artifact=record) + assert trusted_artifact_wheel_paths(manifest) == { + "deployment/wheels/foo-1-py3-none-any.whl", + "deployment/application/foo-1-py3-none-any.whl", + } + + +@pytest.mark.parametrize( + "filename", ["../foo.whl", "nested/foo.whl", "nested\\foo.whl", "C:\\foo.whl"] +) +def test_approved_path_safety(filename): + with pytest.raises(PreparationError, match="Unsafe approved artifact filename"): + approved_artifacts_by_path([artifact("foo", filename)]) + + +@pytest.mark.parametrize("function", ["read_text", "read_binary", "open_text", "open_binary"]) +@pytest.mark.parametrize( + "arguments", + [ + "package=dynamic, resource='defaults.json'", + "package='app', resource=dynamic", + "package='app', resource='../defaults.json'", + "package='app', resource='templates/defaults.json'", + "package='app', resource='defaults.json', unknown=True", + ], +) +def test_legacy_keyword_resource_safety(tmp_path, function, arguments): + write_source( + tmp_path, + "import importlib.resources\n" + f"def main(): return importlib.resources.{function}({arguments})\n", + ) + (tmp_path / "app/templates").mkdir(parents=True) + (tmp_path / "app/__init__.py").write_text("") + (tmp_path / "app/defaults.json").write_text("{}") + (tmp_path / "app/templates/defaults.json").write_text("{}") + assert all( + item.kind == "unresolved_path_reference" + for item in assess_repository(repo(tmp_path)).resources + ) + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary"]) +def test_legacy_binary_encoding_keyword_is_unresolved(tmp_path, function): + write_source( + tmp_path, + f"from importlib.resources import {function}\n" + f"def main(): return {function}(package='app', " + "resource='defaults.json', encoding='utf-8')\n", + ) + (tmp_path / "app").mkdir() + (tmp_path / "app/__init__.py").write_text("") + (tmp_path / "app/defaults.json").write_text("{}") + assert all( + item.kind == "unresolved_path_reference" + for item in assess_repository(repo(tmp_path)).resources + ) + + +@pytest.mark.parametrize("function", ["read_text", "read_binary", "open_text", "open_binary"]) +def test_unrelated_legacy_resource_function_untouched(tmp_path, function): + write_source( + tmp_path, + f"def {function}(package, resource): return None\n" + f"def main(): return {function}(package='app', resource='defaults.json')\n", + ) + (tmp_path / "app").mkdir() + (tmp_path / "app/__init__.py").write_text("") + (tmp_path / "app/defaults.json").write_text("{}") + assert not assess_repository(repo(tmp_path)).resources + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +def test_legacy_keyword_text_options_and_positional_wins(tmp_path, function): + write_source( + tmp_path, + f"from importlib.resources import {function} as read\n" + "def main(): return read('app', 'defaults.json', package='wrong', " + "resource='wrong.json', encoding='utf-8', errors='strict')\n", + ) + (tmp_path / "app").mkdir() + (tmp_path / "app/__init__.py").write_text("") + (tmp_path / "app/defaults.json").write_text("{}") + assert [item.path for item in assess_repository(repo(tmp_path)).resources] == [ + "app/defaults.json" + ] + + +@pytest.mark.parametrize("function", ["read_text", "read_binary", "open_text", "open_binary"]) +def test_legacy_keyword_resource_security(tmp_path, fake_preparation, monkeypatch, function): + source = tmp_path / "source" + write_source( + source, + "from os import getenv as read_env\nimport importlib.resources\n" + "def main():\n read_env(key='DB_PASSWORD')\n" + f" return importlib.resources.{function}(package='app', resource='defaults.json')\n", + ) + (source / "app").mkdir() + (source / "app/__init__.py").write_text("") + secret = "PDBSyntheticSecret123" + (source / "app/defaults.json").write_text(secret) + monkeypatch.setenv("DB_PASSWORD", secret) + with pytest.raises(PreparationError, match="SECRET") as caught: + generate_deployment_kit(repo(source), tmp_path / "kit", bootstrap_mode="online_cmd") + assert secret not in str(caught.value) + assert not (tmp_path / "kit").exists() From bd5ea6081a0337c89d90ffdf14244a51a5458db1 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:15:08 -0400 Subject: [PATCH 52/59] Respect static project dependency and entry-point authority --- .../pr9-dependency-authority-2026-09-11.md | 138 +++++++++ scripts/verify_dependency_authority.py | 147 ++++++++++ .../analysis/assessor.py | 19 ++ .../analysis/metadata.py | 57 +++- tests/test_dependency_authority.py | 267 ++++++++++++++++++ 5 files changed, 626 insertions(+), 2 deletions(-) create mode 100644 docs/acceptance/pr9-dependency-authority-2026-09-11.md create mode 100644 scripts/verify_dependency_authority.py create mode 100644 tests/test_dependency_authority.py diff --git a/docs/acceptance/pr9-dependency-authority-2026-09-11.md b/docs/acceptance/pr9-dependency-authority-2026-09-11.md new file mode 100644 index 0000000..79b2eec --- /dev/null +++ b/docs/acceptance/pr9-dependency-authority-2026-09-11.md @@ -0,0 +1,138 @@ +# PR #9: standardized dependency-field authority + +Starting head: `40da08522463fc8992d698cf8b187a3e59706270`. +Exact-head review submitted: `2026-09-11T15:17:07Z`. + +P2: **Ignore legacy dependencies overridden by [project]**. +Thread: `PRRT_kwDOT9hvCc6hhp8n`; comment: `3990594314`. +Pre-next-review accounting: 115 findings, 112 unresolved inline threads, +three review-level-only findings, one NOT_APPLICABLE inline finding. +No thread resolution, review dismissal, or merge is authorized by this correction. + +## Failing reproduction and field-level correction + +Regression tests were added before production changes. Static nonempty and empty +PEP 621 dependency fixtures with stale setup.cfg or literal setup.py requirements +failed: assessment retained obsolete, selected_dependencies included it, and +planning added RUNTIME_SYNC_METADATA_UNSUPPORTED despite a lock excluding it. +Equivalent legacy requirements polluted authoritative evidence; conflicting +requirements contributed stale evidence instead of being ignored at ingestion. + +Inspection now distinguishes key presence from truthiness. A valid dependencies +list is authoritative only when present and not listed in project.dynamic. +Explicit [] is present. An invalid list type or non-string member raises a +controlled ValueError before any legacy fallback. Malformed dynamic declarations +also fail rather than changing the field's authority accidentally. + +For authoritative static dependencies, only install_requires ingestion from +setup.cfg and setup.py is suppressed. Both files remain metadata/provenance and +packaging-surface inputs; package/module selection and unrelated supported fields +are still read. A retained standardized dependency has exactly pyproject.toml +evidence with detail `Declared in [project].dependencies.` Neither equivalent nor +conflicting legacy constraints are merged into it. + +Absent/dynamic dependencies do not receive the override. Genuine setup.cfg-only +and setup.py-only runtime dependencies still trigger the existing blocker. +The planner's backend-only filter is unchanged; corrected metadata now makes its +input meaningful. Optional dependency groups and entry-point-extra requirements +are unaffected, including the distinction between unselected and selected extras. + +## Pinned toolchain evidence + +`scripts/verify_dependency_authority.py` creates disposable local synthetic modern +and obsolete wheels and ten projects (five shapes for each legacy file type). +It runs actual isolated builds with setuptools 79.0.1 and: + +```text +uv 0.12.5 (210d1f678 2026-08-14 x86_64-pc-windows-msvc) +uv lock --python 3.12 --find-links +uv build --wheel --python 3.12 --out-dir /dist +``` + +The wheel Generator field confirms setuptools 79.0.1. The script asserts lock, +wheel, assessment, and plan results. It executes only explicitly created synthetic +build fixtures, not acceptance applications. Generated projects/wheels remain +outside the repository and are not committed. + +| Project dependency shape | Actual uv lock | Actual wheel / build | Corrected PDB | +| --- | --- | --- | --- | +| Static modern>=1 | demo + modern; obsolete absent | Requires-Dist: modern>=1 | modern only; no false blocker | +| Static [] | demo only | No Requires-Dist | Empty runtime set; no false blocker | +| Static modern + dynamic dependencies | Lock fails | setuptools rejects simultaneous static/dynamic field | Retain evidence; RUNTIME_SYNC_METADATA_UNSUPPORTED | +| Dynamic-only with legacy obsolete>=1 | demo + obsolete | Requires-Dist: obsolete>=1 | Retain obsolete; existing conservative M6.1 blocker | + +The current [PyPA pyproject specification](https://packaging.python.org/en/latest/specifications/pyproject-toml/) +allows static list/table entries together with dynamic append-only extension. +This must not be confused with the observed capabilities of setuptools 79.0.1: +that pinned backend rejects the simultaneous dependency shape, so no reliable +appended-dependency lock contract exists for it. PDB does not treat the static +subset as complete. A transient MetadataResult evidence list emits the existing +blocking risk even when legacy declarations duplicate the static requirement or +no literal backend additions can be inspected. Dry-run exposes the blocker; +generation stops before uv acquisition or output creation. + +Dynamic-only deserves a separate qualification: uv 0.12.5 can call the backend and +lock its supplied dependencies in this experiment. It is not accurate to claim uv +never reads legacy requirements. This correction preserves M6.1's requested +conservative treatment rather than introducing backend execution or a new dynamic +metadata proof into production assessment. + +## Bounded core-field precedence audit + +| Modeled field | Audit result | +| --- | --- | +| name | Existing standardized-value guards prevent stale legacy override; unchanged | +| version | Existing standardized-value guards prevent stale legacy override; unchanged | +| requires-python | Existing standardized-value guards prevent stale legacy override; unchanged | +| dependencies | Ingestion now obeys present-and-not-dynamic authority | +| scripts / gui-scripts | One adjacent launcher defect reproduced and corrected at group level | + +The launcher reproduction declares a standardized console entry app:main and an +explicit empty project.gui-scripts table, with obsolete=old:main in legacy GUI +entry points. Both pinned builds contain only the standardized console launcher. +Before correction PDB preferred the stale GUI target old:main. Now the static +standardized group suppresses only its corresponding legacy group, including +explicit empty tables. Other groups and dynamically extensible groups are not +silently declared overridden. Tests cover console/GUI groups, empty/nonempty +tables, both legacy sources, and preservation of the other group. No full metadata +precedence engine or extras redesign was added. + +## Acceptance and compatibility + +All three repositories were assessed read-only. Their metadata dependency objects +(including constraints and evidence) were compared against inspect_metadata from +the exact starting commit and are identical. Git status before/after is unchanged. + +| Repository | Unchanged SHA | Result | +| --- | --- | --- | +| SimpleGeorefGUI | f484570d89fb1f9e9170fac915475358dfc1234e | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python, 77 package-data members | +| Geo Map Explanation Extractor | 5e7b321d0aeb9ba1d586bfc548c79793d84c6033 | source / SOURCE_COMPATIBLE; nine dependency records, all pyproject provenance | +| TN Coordinate Converter | e1e7a1588c37a99c2d02efaf3eef3d04636f12f0 | source / SOURCE_COMPATIBLE; six dependency records unchanged | + +TN's existing `pdb-m61-regression-20260901/tn-kit` is STATIC_VALID with reviewed +proxy-tools==0.1.0 (`proxy_tools-0.1.0-py3-none-any.whl`) unchanged. +Geo's seven selected runtime names and constraints also exactly match its static +project.dependencies list; the nine total records include unselected groups. +Geo and TN assessment-only plans retain their existing pending lock-currentness +verification; this correction does not claim a new runtime acceptance run. + +No persisted fields were added. ANALYSIS_SCHEMA_VERSION stays 1.4 and +PLANNING_SCHEMA_VERSION stays 1.3. The new evidence list is transient inspection +state feeding an existing risk code and existing generation guard. + +## Quality + +Fifty new focused regression cases pass. The single complete-suite run reports +**1175 passed, 3 skipped in 252.32 seconds**, up from 1125 passed, 3 skipped. +Ruff and `git diff --check` pass. Final diff inspection found only the two +production analysis files, focused regression tests, the synthetic toolchain +verifier, and this evidence report. No generated acceptance/build artifacts are +included. + +The full suite preserves the prior approved-extra edge-context separation, +approved path uniqueness, legacy resource keywords, alias-aware environment reads, +lock-root guards, dynamic imports, pkgutil handling, MANIFEST/package surfaces, +sync contract, reverse approved-artifact completeness, mode-aware pip checks, +entry-point extras, short-secret/LOCALAPPDATA behavior, wheel policies and +dependency proofs, resource/inventory analysis, provenance, rollback, and +deterministic packaging. Persisted schema constants are unchanged. diff --git a/scripts/verify_dependency_authority.py b/scripts/verify_dependency_authority.py new file mode 100644 index 0000000..bac1c2a --- /dev/null +++ b/scripts/verify_dependency_authority.py @@ -0,0 +1,147 @@ +"""Explicit synthetic setuptools 79.0.1 / uv 0.12.5 dependency-authority experiment.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import tomllib +import zipfile +from email.parser import BytesParser +from pathlib import Path + +from verify_approved_extra_context import wheel + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.planning.planner import create_deployment_plan + + +def main() -> None: + uv = Path(sys.argv[1]).resolve() + version = subprocess.check_output([str(uv), "--version"], text=True).strip() + assert version.startswith("uv 0.12.5 "), version + workspace = Path(tempfile.mkdtemp(prefix="pdb-dependency-authority-")) + artifacts = workspace / "input-wheels" + artifacts.mkdir() + for name in ("modern", "obsolete"): + wheel(artifacts, name, [], []) + environment = os.environ.copy() + environment.update(UV_SYSTEM_CERTS="true", UV_NO_ENV_FILE="1", UV_NO_PROGRESS="1") + print(version, workspace, flush=True) + for legacy in ("setup.cfg", "setup.py"): + for kind in ("nonempty", "empty", "static-dynamic", "dynamic-only", "scripts"): + if len(sys.argv) > 2 and kind not in sys.argv[2:]: + continue + root = workspace / (legacy + "-" + kind) + root.mkdir() + (root / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + field = "dependencies=['modern>=1']\n" + if kind == "empty": + field = "dependencies=[]\n" + if kind == "static-dynamic": + field += "dynamic=['dependencies']\n" + if kind == "dynamic-only": + field = "dynamic=['dependencies']\n" + (root / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='demo'\nversion='1.0.0'\nrequires-python='>=3.12'\n" + + field + + ( + "[project.scripts]\ndemo='app:main'\n[project.gui-scripts]\n" + if kind == "scripts" + else "[project.gui-scripts]\ndemo='app:main'\n" + ), + encoding="utf-8", + ) + if legacy == "setup.cfg": + contents = "[options]\npy_modules=app\ninstall_requires=\n obsolete>=1\n" + if kind == "scripts": + contents += "[options.entry_points]\ngui_scripts=\n obsolete=old:main\n" + else: + extra = ( + ", entry_points={'gui_scripts':['obsolete=old:main']}" + if kind == "scripts" + else "" + ) + contents = ( + "from setuptools import setup\n" + "setup(py_modules=['app'], install_requires=['obsolete>=1']" + extra + ")\n" + ) + (root / legacy).write_text(contents, encoding="utf-8") + result = {"legacy": legacy, "kind": kind} + lock = subprocess.run( + [str(uv), "lock", "--python", "3.12", "--find-links", str(artifacts)], + cwd=root, + env=environment, + capture_output=True, + text=True, + ) + result["lock_exit"] = lock.returncode + if lock.returncode: + result["lock_error"] = lock.stderr[-2200:] + else: + document = tomllib.loads((root / "uv.lock").read_text(encoding="utf-8")) + result["locked_packages"] = [p["name"] for p in document.get("package", [])] + build = subprocess.run( + [str(uv), "build", "--wheel", "--python", "3.12", "--out-dir", str(root / "dist")], + cwd=root, + env=environment, + capture_output=True, + text=True, + ) + result["build_exit"] = build.returncode + if build.returncode: + result["build_error"] = build.stderr[-2200:] + else: + with zipfile.ZipFile(next((root / "dist").glob("*.whl"))) as bundle: + names = bundle.namelist() + metadata = BytesParser().parsebytes( + bundle.read(next(n for n in names if n.endswith(".dist-info/METADATA"))) + ) + result["requires_dist"] = metadata.get_all("Requires-Dist", []) + result["wheel"] = bundle.read( + next(n for n in names if n.endswith(".dist-info/WHEEL")) + ).decode() + result["entry_points"] = bundle.read( + next(n for n in names if n.endswith(".dist-info/entry_points.txt")) + ).decode() + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=root) + result["pdb_dependencies"] = [d.distribution_name for d in assessment.dependencies] + result["pdb_blockers"] = plan.risk_gate.blocking_codes + result["pdb_entry_point"] = plan.entry_point.target if plan.entry_point else None + print(json.dumps(result), flush=True) + blocked = "RUNTIME_SYNC_METADATA_UNSUPPORTED" in result["pdb_blockers"] + if kind == "static-dynamic": + assert lock.returncode and build.returncode, result + assert "project.dynamic" in lock.stderr, result + assert blocked and "obsolete" in result["pdb_dependencies"], result + else: + assert lock.returncode == build.returncode == 0, result + expected = ( + [] + if kind == "empty" + else ["obsolete"] + if kind == "dynamic-only" + else ["modern"] + ) + assert sorted(result["locked_packages"]) == sorted(["demo", *expected]), result + assert result["requires_dist"] == [f"{name}>=1" for name in expected], result + assert "setuptools (79.0.1)" in result["wheel"], result + assert result["pdb_dependencies"] == expected, result + assert blocked == (kind == "dynamic-only"), result + assert result["pdb_entry_point"] == "app:main", result + if kind == "scripts": + assert "obsolete" not in result["entry_points"], result + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index d73c19f..fe7adde 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -205,6 +205,25 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen evidence=metadata.uv_workspace_evidence, ) ) + if metadata.dynamic_dependency_evidence: + risks.append( + RiskFinding( + code="RUNTIME_SYNC_METADATA_UNSUPPORTED", + title="Dynamic dependencies lack a complete static runtime contract", + severity=RiskSeverity.BLOCKING, + status=FindingStatus.DETECTED, + description=( + "M6.1 cannot treat a dynamically supplied or extensible dependency list " + "as complete standardized metadata. The pinned setuptools backend also " + "rejects simultaneously static and dynamic dependencies." + ), + recommendation=( + "Declare the complete [project].dependencies list without dependencies " + "in [project].dynamic, then regenerate uv.lock." + ), + evidence=metadata.dynamic_dependency_evidence, + ) + ) if metadata.setuptools_surface_unresolved: risks.append( RiskFinding( diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 9b26cba..b636f4f 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -39,6 +39,7 @@ class MetadataResult: setuptools_surface_evidence: list[Evidence] = field(default_factory=list) setuptools_external_packaging_roots: list[str] = field(default_factory=list) setuptools_external_packaging_root_evidence: list[Evidence] = field(default_factory=list) + dynamic_dependency_evidence: list[Evidence] = field(default_factory=list) @dataclass(frozen=True) @@ -749,6 +750,9 @@ def inspect_metadata(root: Path) -> MetadataResult: uv_workspace = False uv_workspace_source = False uv_workspace_evidence: list[Evidence] = [] + project_dependencies_static_authoritative = False + static_entry_point_groups: set[str] = set() + dynamic_dependency_evidence: list[Evidence] = [] pyproject_path = root / "pyproject.toml" if pyproject_path.is_file(): @@ -756,6 +760,39 @@ def inspect_metadata(root: Path) -> MetadataResult: with pyproject_path.open("rb") as handle: document: dict[str, Any] = tomllib.load(handle) project = document.get("project") if isinstance(document.get("project"), dict) else {} + project_dynamic = project.get("dynamic", []) + if not isinstance(project_dynamic, list) or any( + not isinstance(value, str) for value in project_dynamic + ): + raise ValueError("[project].dynamic must be a list of field names.") + project_dependencies_present = "dependencies" in project + project_dependencies_dynamic = "dependencies" in project_dynamic + project_dependencies_static_authoritative = ( + project_dependencies_present and not project_dependencies_dynamic + ) + if project_dependencies_present and ( + not isinstance(project["dependencies"], list) + or any(not isinstance(value, str) for value in project["dependencies"]) + ): + raise ValueError("[project].dependencies must be a list of requirement strings.") + if project_dependencies_dynamic: + dynamic_dependency_evidence.append( + _evidence( + root, + pyproject_path, + "[project].dynamic includes dependencies; the static list is not a " + "complete dependency contract. M6.1 does not prove backend additions, " + "and setuptools 79.0.1 rejects simultaneous static/dynamic dependencies.", + _line_number(pyproject_path, "dynamic"), + ) + ) + static_entry_point_groups = { + legacy_group + for group, legacy_group in ( + ("scripts", "console_scripts"), ("gui-scripts", "gui_scripts") + ) + if group in project and group not in project_dynamic + } build_system = ( document.get("build-system") if isinstance(document.get("build-system"), dict) else {} ) @@ -1058,7 +1095,13 @@ def inspect_metadata(root: Path) -> MetadataResult: distribution_name = parser.get("metadata", "name", fallback=None) project_version = parser.get("metadata", "version", fallback=None) requires_python = parser.get("options", "python_requires", fallback=None) - install_requires = parser.get("options", "install_requires", fallback="") + # Field presence, not truthiness: an explicit [] overrides legacy data. + # A dynamically extensible list is not authoritative on its own. + install_requires = ( + "" + if project_dependencies_static_authoritative + else parser.get("options", "install_requires", fallback="") + ) for specification in _multiline_values(install_requires): parsed = _dependency( specification, @@ -1074,6 +1117,8 @@ def inspect_metadata(root: Path) -> MetadataResult: dependencies.append(parsed) if parser.has_section("options.entry_points"): for group in ("console_scripts", "gui_scripts"): + if group in static_entry_point_groups: + continue for specification in _multiline_values( parser.get("options.entry_points", group, fallback="") ): @@ -1237,7 +1282,12 @@ def inspect_metadata(root: Path) -> MetadataResult: project_version = setup_values["version"] if requires_python is None and isinstance(setup_values.get("python_requires"), str): requires_python = setup_values["python_requires"] - for specification in _literal_string_sequence(setup_values.get("install_requires")) or []: + legacy_runtime_specs = ( + [] + if project_dependencies_static_authoritative + else _literal_string_sequence(setup_values.get("install_requires")) or [] + ) + for specification in legacy_runtime_specs: parsed = _dependency( specification, "runtime", @@ -1253,6 +1303,8 @@ def inspect_metadata(root: Path) -> MetadataResult: setup_entry_points = setup_values.get("entry_points", {}) if isinstance(setup_entry_points, dict): for group in ("console_scripts", "gui_scripts"): + if group in static_entry_point_groups: + continue values = _literal_string_sequence(setup_entry_points.get(group)) if values is None: continue @@ -1688,4 +1740,5 @@ def safe_roots(values: list[str]) -> list[str]: setuptools_surface_evidence=setuptools_surface_evidence, setuptools_external_packaging_roots=sorted(set(setuptools_external_packaging_roots)), setuptools_external_packaging_root_evidence=setuptools_external_packaging_root_evidence, + dynamic_dependency_evidence=dynamic_dependency_evidence, ) diff --git a/tests/test_dependency_authority.py b/tests/test_dependency_authority.py new file mode 100644 index 0000000..b80c3f7 --- /dev/null +++ b/tests/test_dependency_authority.py @@ -0,0 +1,267 @@ +"""Static PEP 621 dependency authority versus stale legacy install_requires.""" + +import pytest + +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.metadata import inspect_metadata +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.generator import generate_deployment_kit +from python_deployment_builder.planning.extras import selected_dependencies +from python_deployment_builder.planning.planner import create_deployment_plan + + +def write_project(root, legacy, dependencies, obsolete="obsolete>=1", dynamic=None): + root.mkdir(parents=True, exist_ok=True) + (root / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + field = f"dependencies={dependencies}\n" if dependencies is not None else "" + if dynamic is not None: + field += f"dynamic={dynamic}\n" + (root / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='demo'\nversion='1.0.0'\nrequires-python='>=3.12'\n" + + field + + "[project.scripts]\ndemo='app:main'\n", + encoding="utf-8", + ) + if legacy == "setup.cfg": + contents = "[options]\npy_modules=app\ninstall_requires=\n " + obsolete + "\n" + else: + contents = ( + "from setuptools import setup\n" + f"setup(py_modules=['app'], install_requires=[{obsolete!r}])\n" + ) + (root / legacy).write_text(contents, encoding="utf-8") + (root / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n" + "[[package]]\nname='demo'\nversion='1.0.0'\nsource={editable='.'}\n" + + ( + "dependencies=[{name='modern'}]\n[[package]]\nname='modern'\nversion='2'\n" + "wheels=[{url='https://example.invalid/modern-2-py3-none-any.whl'}]\n" + if dependencies and "modern" in dependencies + else "" + ), + encoding="utf-8", + ) + + +def assess(root): + return assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("dependencies", ["['modern>=1']", "[]"]) +def test_static_dependencies_override_stale_install_requires(tmp_path, legacy, dependencies): + write_project(tmp_path, legacy, dependencies) + assessment = assess(tmp_path) + chosen = selected_dependencies(assessment, [], "3.12", "x86_64") + expected = ["modern"] if "modern" in dependencies else [] + assert [item.distribution_name for item in chosen] == expected + assert legacy in assessment.project.metadata_files + assert assessment.project.py_modules == ["app"] + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" not in plan.risk_gate.blocking_codes + assert [item.name for item in plan.lock_graph.dependencies] == expected + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("obsolete", ["modern>=2", "modern<2"]) +def test_static_dependency_evidence_excludes_legacy_duplicate(tmp_path, legacy, obsolete): + write_project(tmp_path, legacy, "['modern>=2']", obsolete) + metadata = inspect_metadata(tmp_path) + assert len(metadata.dependencies) == 1 + dependency = metadata.dependencies[0] + assert dependency.declared_constraint == ">=2" + assert [(item.file, item.detail) for item in dependency.evidence] == [ + ("pyproject.toml", "Declared in [project].dependencies.") + ] + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("dependencies", ["'obsolete>=1'", "{}", "42", "[1]", "['modern', 1]"]) +def test_malformed_standardized_dependencies_fail_without_legacy_fallback( + tmp_path, legacy, dependencies +): + write_project(tmp_path, legacy, dependencies) + with pytest.raises(ValueError, match=r"\[project\].dependencies"): + inspect_metadata(tmp_path) + + +@pytest.mark.parametrize("dependencies", ["[]", "['modern>=1']", None]) +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_dynamic_dependencies_retain_legacy_and_block(tmp_path, dependencies, legacy): + write_project(tmp_path, legacy, dependencies, dynamic="['dependencies']") + assessment = assess(tmp_path) + assert "obsolete" in [d.distribution_name for d in assessment.dependencies] + assert ( + "RUNTIME_SYNC_METADATA_UNSUPPORTED" + in create_deployment_plan(assessment, repository_root=tmp_path).risk_gate.blocking_codes + ) + + +def test_static_dynamic_cannot_be_mistaken_for_complete_static_set(tmp_path): + write_project(tmp_path, "setup.py", "['modern>=1']", "modern>=1", "['dependencies']") + plan = create_deployment_plan(assess(tmp_path), repository_root=tmp_path) + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" in plan.risk_gate.blocking_codes + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_explicit_empty_gui_scripts_override_stale_legacy_launcher(tmp_path, legacy): + write_project(tmp_path, legacy, "[]") + with (tmp_path / "pyproject.toml").open("a") as stream: + stream.write("[project.gui-scripts]\n") + if legacy == "setup.cfg": + with (tmp_path / legacy).open("a") as stream: + stream.write("[options.entry_points]\ngui_scripts=\n obsolete=old:main\n") + else: + path = tmp_path / legacy + path.write_text( + path.read_text().replace( + "install_requires=['obsolete>=1']", + "install_requires=['obsolete>=1'], " + "entry_points={'gui_scripts':['obsolete=old:main']}", + ) + ) + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert plan.entry_point.target == "app:main" + assert len(assessment.project.entry_points) == 1 + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("build_system_only", [False, True]) +def test_genuine_legacy_dependencies_still_block(tmp_path, legacy, build_system_only): + write_project(tmp_path, legacy, None) + if build_system_only: + path = tmp_path / "pyproject.toml" + path.write_text(path.read_text().split("[project]")[0]) + path = tmp_path / legacy + if legacy == "setup.cfg": + path.write_text("[metadata]\nname=demo\nversion=1.0.0\n" + path.read_text()) + else: + path.write_text( + path.read_text().replace("setup(", "setup(name='demo', version='1.0.0', ") + ) + assessment = assess(tmp_path) + assert [d.distribution_name for d in assessment.dependencies] == ["obsolete"] + assert {e.file for e in assessment.dependencies[0].evidence} == {legacy} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" in plan.risk_gate.blocking_codes + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_unrelated_dynamic_field_does_not_reactivate_legacy_dependencies(tmp_path, legacy): + write_project(tmp_path, legacy, "[]", dynamic="['description']") + assert inspect_metadata(tmp_path).dependencies == [] + + +@pytest.mark.parametrize("dynamic", ["'dependencies'", "{}", "[1]"]) +def test_malformed_dynamic_declaration_fails_controlled(tmp_path, dynamic): + write_project(tmp_path, "setup.cfg", "[]", dynamic=dynamic) + with pytest.raises(ValueError, match=r"\[project\].dynamic"): + inspect_metadata(tmp_path) + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_selected_optional_and_entry_point_extras_are_unchanged(tmp_path, legacy): + write_project(tmp_path, legacy, "[]") + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text().replace("app:main", "app:main [map]") + + "[project.optional-dependencies]\nmap=['modern>=1']\n" + ) + (tmp_path / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n" + "[[package]]\nname='demo'\nversion='1.0.0'\nsource={editable='.'}\n" + "[package.optional-dependencies]\nmap=[{name='modern'}]\n" + "[[package]]\nname='modern'\nversion='2'\n" + "wheels=[{url='https://example.invalid/modern-2-py3-none-any.whl'}]\n" + ) + assessment = assess(tmp_path) + assert selected_dependencies(assessment, [], "3.12", "x86_64") == [] + assert [ + d.distribution_name for d in selected_dependencies(assessment, ["map"], "3.12", "x86_64") + ] == ["modern"] + unselected = create_deployment_plan(assessment, repository_root=tmp_path) + assert "ENTRYPOINT_EXTRA_NOT_SELECTED" in unselected.risk_gate.blocking_codes + selected = create_deployment_plan(assessment, repository_root=tmp_path, selected_extras=["map"]) + assert "ENTRYPOINT_EXTRA_NOT_SELECTED" not in selected.risk_gate.blocking_codes + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" not in selected.risk_gate.blocking_codes + assert [d.name for d in selected.lock_graph.dependencies] == ["modern"] + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_standardized_core_identity_fields_already_win(tmp_path, legacy): + write_project(tmp_path, legacy, "[]") + path = tmp_path / legacy + if legacy == "setup.cfg": + path.write_text( + "[metadata]\nname=old\nversion=0.1\n" + path.read_text() + "python_requires=<3.10\n" + ) + else: + path.write_text( + path.read_text().replace( + "setup(", "setup(name='old', version='0.1', python_requires='<3.10', " + ) + ) + metadata = inspect_metadata(tmp_path) + project = metadata.project + assert project.distribution_name == "demo" + assert project.version == "1.0.0" + assert metadata.python.requires_python == ">=3.12" + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("group", ["console_scripts", "gui_scripts"]) +@pytest.mark.parametrize("empty", [False, True]) +def test_static_script_group_suppresses_only_its_legacy_group(tmp_path, legacy, group, empty): + write_project(tmp_path, legacy, "[]") + field = "scripts" if group == "console_scripts" else "gui-scripts" + other = "gui_scripts" if group == "console_scripts" else "console_scripts" + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text().split("[project.scripts]")[0] + + f"[project.{field}]\n" + + ("" if empty else "demo='app:main'\n") + ) + path = tmp_path / legacy + if legacy == "setup.cfg": + path.write_text( + path.read_text() + + "[options.entry_points]\n" + + f"{group}=\n stale=old:main\n{other}=\n retained=app:main\n" + ) + else: + path.write_text( + path.read_text().replace( + "setup(", + f"setup(entry_points={{{group!r}: ['stale=old:main'], " + f"{other!r}: ['retained=app:main']}}, ", + ) + ) + names = {entry.name for entry in inspect_metadata(tmp_path).project.entry_points} + assert names == ({"retained"} if empty else {"demo", "retained"}) + + +@pytest.mark.parametrize("dependencies", ["['modern>=1']", None]) +def test_dynamic_contract_without_literal_additions_blocks_before_generation( + tmp_path, monkeypatch, dependencies +): + source = tmp_path / "source" + write_project(source, "setup.cfg", dependencies, dynamic="['dependencies']") + (source / "setup.cfg").write_text("[options]\npy_modules=app\n") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + output = tmp_path / "kit" + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail("dynamic metadata must block before uv acquisition"), + ) + preview = generate_deployment_kit(repository, output, dry_run=True).preview + assert any("RUNTIME_SYNC_METADATA_UNSUPPORTED" in item for item in preview.developer_actions) + assert not output.exists() + with pytest.raises(PreparationError, match="RUNTIME_SYNC_METADATA_UNSUPPORTED"): + generate_deployment_kit(repository, output, prepare_lock=True) + assert not output.exists() From 2a7556358e4d909dc58a4715ef31d6850f9a9ff1 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:06:51 -0400 Subject: [PATCH 53/59] Honor omitted project fields and compatible resource anchors --- ...omitted-fields-files-keyword-2026-09-12.md | 162 +++++++++++++++++ scripts/verify_files_keyword.py | 94 ++++++++++ scripts/verify_omitted_metadata.py | 144 +++++++++++++++ .../analysis/assessor.py | 18 ++ .../analysis/metadata.py | 45 ++++- .../analysis/resources.py | 33 ++-- .../generation/generator.py | 19 +- tests/test_analysis_scope.py | 2 +- tests/test_dependency_authority.py | 20 ++- tests/test_files_package_keyword.py | 162 +++++++++++++++++ tests/test_omitted_metadata.py | 168 ++++++++++++++++++ 11 files changed, 832 insertions(+), 35 deletions(-) create mode 100644 docs/acceptance/pr9-omitted-fields-files-keyword-2026-09-12.md create mode 100644 scripts/verify_files_keyword.py create mode 100644 scripts/verify_omitted_metadata.py create mode 100644 tests/test_files_package_keyword.py create mode 100644 tests/test_omitted_metadata.py diff --git a/docs/acceptance/pr9-omitted-fields-files-keyword-2026-09-12.md b/docs/acceptance/pr9-omitted-fields-files-keyword-2026-09-12.md new file mode 100644 index 0000000..fc6c1a4 --- /dev/null +++ b/docs/acceptance/pr9-omitted-fields-files-keyword-2026-09-12.md @@ -0,0 +1,162 @@ +# PR #9: omitted project fields and files(package=...) + +Starting head: `bd5ea6081a0337c89d90ffdf14244a51a5458db1`. +Exact-head review: `2026-09-11T23:20:26Z`. + +| P2 finding | Thread | Comment | +| --- | --- | --- | +| Treat omitted dependencies as an authoritative empty field | PRRT_kwDOT9hvCc6hqtuf | 3994149075 | +| Suppress legacy entry points when static groups are omitted | PRRT_kwDOT9hvCc6hqtui | 3994149079 | +| Resolve the compatible files(package=...) spelling | PRRT_kwDOT9hvCc6hqtuj | 3994149081 | + +Pre-next-review accounting: 118 findings/comments, 115 unresolved inline threads, +three review-level-only findings, one NOT_APPLICABLE inline finding. No historical +thread resolution, review dismissal, or merge belongs to this correction. + +## Reproduction before production changes + +Eleven initial regression cases failed against the starting head: omitted +dependencies with both legacy sources, four omitted console/GUI legacy launcher +cases, a stale setup.py python_requires constraint, and four files(package=...) +binding forms. Eight additional dynamic script-group cases failed before the +new conservative blocker was implemented. + +`scripts/verify_omitted_metadata.py` builds fourteen disposable synthetic projects +with the exact pinned uv 0.12.5 and setuptools 79.0.1. The script runs uv lock and +isolated uv build --wheel with Python 3.12, records actual lock/wheel metadata, and +asserts post-correction outcomes. Successful wheels report Generator: +`setuptools (79.0.1)`. Generated projects, wheels and environments are outside the +repository; acceptance repositories are never built or modified by this script. + +| Shape, tested with setup.cfg and literal setup.py | Actual pinned result | Starting-head PDB result | +| --- | --- | --- | +| Existing project; dependencies/scripts/gui-scripts omitted; stale install_requires and both launchers | Lock contains demo only; no Requires-Dist or entry_points.txt | Retains obsolete and both stale launchers; false RUNTIME_SYNC_METADATA_UNSUPPORTED | +| Dynamic-only scripts or gui-scripts | Lock succeeds; tested wheel emits neither legacy launcher | Retains both groups without a blocker | +| Static plus dynamic scripts or gui-scripts | uv lock succeeds; setuptools build rejects simultaneous static/dynamic field | Merges static and stale legacy launchers without a blocker | +| Omitted requires-python plus legacy python_requires <3.10 | uv lock ignores legacy constraint; setuptools build crashes clearing it (NoneType error) | setup.py constraint is adopted and prevents Python policy selection | +| Omitted optional-dependencies plus legacy extras_require | Lock contains demo only; wheel has no optional requirement | No legacy extras merged; no corresponding PDB defect | + +The requires-python experiment is not a successful wheel-build claim: the pinned +backend has its own clearing-field failure. The proven PDB defect is borrowing a +non-authoritative legacy constraint and rejecting Python selection even though +the standardized lock does not impose that constraint. + +## Authority and bounded audit + +The current [PyPA pyproject specification](https://packaging.python.org/en/latest/specifications/pyproject-toml/) +distinguishes no project table from an existing project whose optional metadata is +not dynamic. Absence of an optional, non-dynamic field under an existing project +is authoritative absence. Static list/table entries may also coexist with dynamic +append-only extensions; pinned backend capabilities must be established separately. + +`project_table_present` is now separate from field presence. Dependencies and each +launcher group are statically authoritative when that table exists and the +corresponding field is not dynamic. This covers omitted, explicit empty, and +explicit nonempty fields. Dependency key presence is still used for type checking, +not authority. Malformed dependency/dynamic values still fail controlled. + +Only overridden fields are suppressed. Legacy files remain provenance and +packaging-surface inputs. No-project legacy dependencies retain the existing +RUNTIME_SYNC_METADATA_UNSUPPORTED blocker; no-project legacy launchers remain +available. Dynamic dependency evidence and the preceding conservative dependency +blocker are unchanged. + +| Audited field | Omission under existing project | Legacy refill / dynamic outcome | +| --- | --- | --- | +| dependencies | Empty when non-dynamic | Corrected both legacy sources; dynamic evidence retained and blocked conservatively | +| scripts | Empty when non-dynamic | Corrected console_scripts only; dynamic evidence retained | +| gui-scripts | Empty when non-dynamic | Corrected gui_scripts independently; dynamic evidence retained | +| name | Required static identity, not a meaningful optional empty value | Existing explicit identity guards unchanged; no general malformed-project validator introduced | +| version | Required static or dynamic identity, not a meaningful optional empty value | Existing explicit-value and bounded dynamic-version logic unchanged | +| requires-python | No standardized Python constraint when omitted/non-dynamic | Corrected setup.py/Pipfile refill; setup.cfg path also guarded. No-project and existing dynamic fallback behavior retained | +| optional-dependencies | No standardized optional groups when omitted/non-dynamic | PDB does not merge legacy extras_require into this field, so no refill correction needed; selected extras unchanged | + +Generic project.entry-points custom groups are not modeled as console/GUI +launchers. No custom-group parser, suppression rule or new API family was added. + +Because the tested pinned backend cannot reliably materialize dynamic launcher +groups, transient inspection evidence now emits ENTRYPOINT_METADATA_UNSUPPORTED. +Both dynamic-only and static+dynamic forms keep inspectable launcher evidence but +cannot be represented as a proven deployment. Dry-run reports the blocker and +generation stops before uv acquisition, lock preparation or output writes. The +existing static groups still win over conflicting legacy targets. Focused tests +also validate a correct console-entry application wheel with a stale omitted GUI +group present in either legacy source. + +Two previous test assumptions were deliberately corrected, not preserved as false +contracts: an omitted dependency field under project is not legacy-owned, and an +omitted second launcher group is not implicitly dynamic. Coverage now explicitly +distinguishes no-project, static-empty and dynamic group behavior. + +## Resource spelling and target evidence + +`scripts/verify_files_keyword.py` creates a disposable package and probes actual +selected interpreters. Python 3.11.16 was acquired in uv's managed cache for this +test; the configured PyCharm SDK was not changed. + +| Interpreter | files('app') | files(package='app') | files(anchor='app') | files() inside package module | +| --- | --- | --- | --- | --- | +| Python 3.11.16 | Works | Works, no warning | TypeError | TypeError | +| Python 3.12.14 | Works | Works, DeprecationWarning | Works | Works | +| Python 3.13.7 | Works | Works, DeprecationWarning | Works | Works | + +The policy order remains 3.12, 3.13, 3.11, 3.14. The +[Python 3.14 documentation](https://docs.python.org/3.14/library/importlib.resources.html#importlib.resources.files) +also retains package= compatibility with a warning. No target-policy or schema +change is necessary for this spelling. Analysis is not a universal cross-version +API validator: existing anchor= and implicit-caller recognition is not a claim +that those forms execute on Python 3.11. + +One bounded branch selects position zero or the single recognized anchor/package +keyword through call_argument. Positional wins a single duplicate binding; +conflicting anchor plus package, unknown keywords, **kwargs, and extra positional +arguments remain unresolved. Only no arguments and no keywords select the +implicit caller. Explicit values use the existing package-anchor resolver. + +The exact pre/post staged-source experiment used the starting-head resolver and +the corrected resolver against the same disposable source: before correction the +resource was not staged and launching raised FileNotFoundError; after correction +it was staged and the read returned `{}`. Tests cover module/direct aliases, +static variable resolution, unresolved dynamic values, custom roots, exact and +parent package-dir mappings, traversal rejection, role-aware staging and normal +release secret scanning. Existing containment, symlink, modern/legacy resource +and pkgutil paths are unchanged. The existing three-call resource-evidence test +now expects three pieces of evidence rather than two. + +## Acceptance and schemas + +Read-only comparisons against inspect_metadata from the exact starting SHA prove +that all three repositories retain identical dependency and entry-point records, +including provenance. Their Git status is unchanged. + +| Repository | Unchanged SHA | Contract | +| --- | --- | --- | +| SimpleGeorefGUI | f484570d89fb1f9e9170fac915475358dfc1234e | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python, 77 package-data members | +| Geo Map Explanation Extractor | 5e7b321d0aeb9ba1d586bfc548c79793d84c6033 | source / SOURCE_COMPATIBLE | +| TN Coordinate Converter | e1e7a1588c37a99c2d02efaf3eef3d04636f12f0 | source / SOURCE_COMPATIBLE; existing kit STATIC_VALID | + +SGG retains its explicit GUI entry +`simple-georef-gui = simple_georef_gui_app.georef_main:main`; its omitted console +group gains nothing from legacy configuration. Geo retains both standardized +console entries. TN retains its standardized console and GUI entries, and the +existing kit's reviewed proxy-tools==0.1.0 artifact is unchanged. No new runtime +acceptance execution is claimed for these repositories. + +ANALYSIS_SCHEMA_VERSION stays 1.4; PLANNING_SCHEMA_VERSION stays 1.3. The launcher +evidence list is a transient MetadataResult field. A new precise risk-code value +uses the existing risk model and does not add persisted model fields. + +## Quality + +The single complete-suite run passed: **1249 passed, 3 skipped in 292.23 seconds**, +up 74 from the 1175-pass baseline. Ruff and `git diff --check` pass. Final diff +inspection covers only the four production files, four regression-test files, +two synthetic verification scripts, and this report. No wheels, kits, temporary +projects, managed environments, or acceptance artifacts are committed. + +The full suite retains approved root-extra separation, approved-path uniqueness, +legacy resource keywords, environment aliases, structural lock-root policy, +dynamic imports, pkgutil keywords, MANIFEST/include-package-data guards, exact +sync arguments, reverse artifact completeness, mode-aware pip checks, entry-point +extras, short-secret and LOCALAPPDATA behavior, wheel policies, dependency proofs, +security/provenance, rollback, and deterministic packaging. diff --git a/scripts/verify_files_keyword.py b/scripts/verify_files_keyword.py new file mode 100644 index 0000000..9b4234c --- /dev/null +++ b/scripts/verify_files_keyword.py @@ -0,0 +1,94 @@ +"""Probe files() keyword compatibility on explicitly selected Python runtimes.""" + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +PROBE = """ +import json, sys, warnings +from importlib.resources import files +from app.probe import implicit +result = {'version': sys.version.split()[0]} +for name, call in [ + ('positional', lambda: files('app')), + ('package', lambda: files(package='app')), + ('anchor', lambda: files(anchor='app')), + ('implicit', implicit), +]: + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter('always') + try: + result[name] = {'read': call().joinpath('defaults.json').read_text()} + except Exception as error: + result[name] = {'error': type(error).__name__} + result[name]['warnings'] = [type(w.message).__name__ for w in seen] +assert result['package']['read'] == '{}' +if sys.version_info[:2] == (3, 11): + assert result['anchor']['error'] == 'TypeError' + assert result['implicit']['error'] == 'TypeError' +else: + assert result['anchor']['read'] == result['implicit']['read'] == '{}' + assert result['package']['warnings'] == ['DeprecationWarning'] +print(json.dumps(result)) +""" + + +def main(): + uv = str(Path(sys.argv[1]).resolve()) + root = Path(tempfile.mkdtemp(prefix="pdb-files-keyword-")) + package = root / "app" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "defaults.json").write_text("{}") + (package / "probe.py").write_text( + "from importlib.resources import files\ndef implicit(): return files()\n" + ) + environment = dict(os.environ, PYTHONPATH=str(root), UV_SYSTEM_CERTS="true") + for version in ("3.11", "3.12"): + run = subprocess.run( + [ + uv, + "run", + "--no-project", + "--no-config", + "--managed-python", + "--python", + version, + "python", + "-B", + "-c", + PROBE, + ], + cwd=root, + env=environment, + capture_output=True, + text=True, + ) + print( + json.dumps( + { + "requested": version, + "exit": run.returncode, + "stdout": run.stdout, + "stderr": run.stderr, + } + ), + flush=True, + ) + assert run.returncode == 0 + run = subprocess.run( + [sys.executable, "-B", "-c", PROBE], + cwd=root, + env=environment, + capture_output=True, + text=True, + ) + print(run.stdout, run.stderr, flush=True) + assert run.returncode == 0 + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_omitted_metadata.py b/scripts/verify_omitted_metadata.py new file mode 100644 index 0000000..6ecae0f --- /dev/null +++ b/scripts/verify_omitted_metadata.py @@ -0,0 +1,144 @@ +"""Pinned synthetic builds for omitted project fields and dynamic script groups.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import tomllib +import zipfile +from email.parser import BytesParser +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.planning.planner import create_deployment_plan + + +def main(): + uv = Path(sys.argv[1]).resolve() + version = subprocess.check_output([str(uv), "--version"], text=True).strip() + assert version.startswith("uv 0.12.5 "), version + workspace = Path(tempfile.mkdtemp(prefix="pdb-omitted-authority-")) + environment = dict(os.environ, UV_SYSTEM_CERTS="true", UV_NO_ENV_FILE="1", UV_NO_PROGRESS="1") + print(version, workspace, flush=True) + for legacy in ("setup.cfg", "setup.py"): + for kind in ( + "omitted", + "dynamic-console", + "dynamic-gui", + "both-console", + "both-gui", + "requires-python", + "optional", + ): + root = workspace / (legacy + "-" + kind) + root.mkdir() + (root / "app.py").write_text("def main(): return 0\n", encoding="utf-8") + field = "scripts" if kind.endswith("console") else "gui-scripts" + extra = f"dynamic=['{field}']\n" if kind.startswith(("dynamic", "both")) else "" + if kind.startswith("both"): + extra += f"[project.{field}]\nmodern='app:main'\n" + (root / "pyproject.toml").write_text( + "[build-system]\nrequires=['setuptools==79.0.1']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='demo'\nversion='1.0.0'\n" + extra, + encoding="utf-8", + ) + if legacy == "setup.cfg": + contents = "[options]\npy_modules=app\n" + if kind == "omitted": + contents += "install_requires=obsolete>=1\n" + if kind == "requires-python": + contents += "python_requires=<3.10\n" + contents += ( + "[options.entry_points]\nconsole_scripts=\n stale-tool=app:main\n" + "gui_scripts=\n stale-gui=app:main\n" + ) + if kind == "optional": + contents += "[options.extras_require]\nmap=obsolete>=1\n" + else: + keywords = ", install_requires=['obsolete>=1']" if kind == "omitted" else "" + if kind == "requires-python": + keywords += ", python_requires='<3.10'" + if kind == "optional": + keywords += ", extras_require={'map':['obsolete>=1']}" + contents = ( + "from setuptools import setup\nsetup(py_modules=['app'], entry_points={" + "'console_scripts':['stale-tool=app:main'], " + "'gui_scripts':['stale-gui=app:main']}" + + keywords + + ")\n" + ) + (root / legacy).write_text(contents, encoding="utf-8") + result = {"legacy": legacy, "kind": kind} + for operation, arguments in ( + ("lock", ["lock", "--python", "3.12"]), + ( + "build", + ["build", "--wheel", "--python", "3.12", "--out-dir", str(root / "dist")], + ), + ): + run = subprocess.run( + [str(uv), *arguments], cwd=root, env=environment, capture_output=True, text=True + ) + result[operation + "_exit"] = run.returncode + if run.returncode: + result[operation + "_error"] = run.stderr[-1300:] + if (root / "uv.lock").exists(): + lock = tomllib.loads((root / "uv.lock").read_text(encoding="utf-8")) + result["lock_packages"] = [p["name"] for p in lock.get("package", [])] + if result["build_exit"] == 0: + with zipfile.ZipFile(next((root / "dist").glob("*.whl"))) as bundle: + names = bundle.namelist() + metadata = BytesParser().parsebytes( + bundle.read(next(n for n in names if n.endswith(".dist-info/METADATA"))) + ) + result["requires_dist"] = metadata.get_all("Requires-Dist", []) + result["requires_python"] = metadata.get("Requires-Python") + result["generator"] = BytesParser().parsebytes(bundle.read(next( + n for n in names if n.endswith(".dist-info/WHEEL") + ))).get("Generator") + entries = next((n for n in names if n.endswith("/entry_points.txt")), None) + result["wheel_entries"] = bundle.read(entries).decode() if entries else "" + try: + assessment = assess_repository( + MaterializedRepository(root=root, source=str(root), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=root) + result["pdb_dependencies"] = [d.distribution_name for d in assessment.dependencies] + result["pdb_entries"] = [e.name for e in assessment.project.entry_points] + result["pdb_python"] = assessment.python.requires_python + result["pdb_blockers"] = plan.risk_gate.blocking_codes + result["pdb_entry"] = plan.entry_point.target if plan.entry_point else None + except ValueError as error: + result["pdb_error"] = str(error) + print(json.dumps(result), flush=True) + assert result["lock_exit"] == 0 and result["lock_packages"] == ["demo"], result + assert "pdb_error" not in result, result + assert result["pdb_dependencies"] == [] and result["pdb_python"] is None, result + if kind.startswith(("dynamic", "both")): + assert "ENTRYPOINT_METADATA_UNSUPPORTED" in result["pdb_blockers"], result + expected_stale = "stale-tool" if field == "scripts" else "stale-gui" + assert expected_stale in result["pdb_entries"], result + else: + assert result["pdb_entries"] == [], result + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" not in result["pdb_blockers"], result + if kind.startswith("both"): + assert result["build_exit"] != 0 and "project.dynamic" in result["build_error"] + elif kind == "requires-python": + # Pinned setuptools itself crashes while clearing the stale field; + # uv still creates a standardized lock without the legacy constraint. + assert result["build_exit"] != 0 and "NoneType" in result["build_error"] + else: + assert result["build_exit"] == 0, result + assert result["generator"] == "setuptools (79.0.1)", result + assert result["requires_dist"] == [] and result["wheel_entries"] == "", result + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index fe7adde..9a0002b 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -224,6 +224,24 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen evidence=metadata.dynamic_dependency_evidence, ) ) + if metadata.dynamic_entry_point_evidence: + risks.append( + RiskFinding( + code="ENTRYPOINT_METADATA_UNSUPPORTED", + title="Dynamic launcher metadata lacks a complete static contract", + severity=RiskSeverity.BLOCKING, + status=FindingStatus.DETECTED, + description=( + "M6.1 cannot prove the complete backend-generated scripts/gui-scripts " + "groups. A uv lock alone does not establish the wheel's launchers." + ), + recommendation=( + "Declare complete static [project.scripts] and [project.gui-scripts] " + "groups without listing them in [project].dynamic." + ), + evidence=metadata.dynamic_entry_point_evidence, + ) + ) if metadata.setuptools_surface_unresolved: risks.append( RiskFinding( diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index b636f4f..8b98d93 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -40,6 +40,7 @@ class MetadataResult: setuptools_external_packaging_roots: list[str] = field(default_factory=list) setuptools_external_packaging_root_evidence: list[Evidence] = field(default_factory=list) dynamic_dependency_evidence: list[Evidence] = field(default_factory=list) + dynamic_entry_point_evidence: list[Evidence] = field(default_factory=list) @dataclass(frozen=True) @@ -751,15 +752,18 @@ def inspect_metadata(root: Path) -> MetadataResult: uv_workspace_source = False uv_workspace_evidence: list[Evidence] = [] project_dependencies_static_authoritative = False + project_python_static_authoritative = False static_entry_point_groups: set[str] = set() dynamic_dependency_evidence: list[Evidence] = [] + dynamic_entry_point_evidence: list[Evidence] = [] pyproject_path = root / "pyproject.toml" if pyproject_path.is_file(): metadata_files.append("pyproject.toml") with pyproject_path.open("rb") as handle: document: dict[str, Any] = tomllib.load(handle) - project = document.get("project") if isinstance(document.get("project"), dict) else {} + project_table_present = isinstance(document.get("project"), dict) + project = document["project"] if project_table_present else {} project_dynamic = project.get("dynamic", []) if not isinstance(project_dynamic, list) or any( not isinstance(value, str) for value in project_dynamic @@ -768,7 +772,10 @@ def inspect_metadata(root: Path) -> MetadataResult: project_dependencies_present = "dependencies" in project project_dependencies_dynamic = "dependencies" in project_dynamic project_dependencies_static_authoritative = ( - project_dependencies_present and not project_dependencies_dynamic + project_table_present and not project_dependencies_dynamic + ) + project_python_static_authoritative = ( + project_table_present and "requires-python" not in project_dynamic ) if project_dependencies_present and ( not isinstance(project["dependencies"], list) @@ -791,8 +798,20 @@ def inspect_metadata(root: Path) -> MetadataResult: for group, legacy_group in ( ("scripts", "console_scripts"), ("gui-scripts", "gui_scripts") ) - if group in project and group not in project_dynamic + if project_table_present and group not in project_dynamic } + for group in ("scripts", "gui-scripts"): + if group in project_dynamic: + dynamic_entry_point_evidence.append( + _evidence( + root, + pyproject_path, + f"[project].{group} is dynamic; M6.1 cannot prove the backend's " + "complete launcher group. Pinned setuptools rejects simultaneous " + "static/dynamic groups and may omit legacy-only launchers.", + _line_number(pyproject_path, "dynamic"), + ) + ) build_system = ( document.get("build-system") if isinstance(document.get("build-system"), dict) else {} ) @@ -1094,9 +1113,10 @@ def inspect_metadata(root: Path) -> MetadataResult: if distribution_name is None: distribution_name = parser.get("metadata", "name", fallback=None) project_version = parser.get("metadata", "version", fallback=None) - requires_python = parser.get("options", "python_requires", fallback=None) - # Field presence, not truthiness: an explicit [] overrides legacy data. - # A dynamically extensible list is not authoritative on its own. + if not project_python_static_authoritative: + requires_python = parser.get("options", "python_requires", fallback=None) + # Under an existing [project], omitted non-dynamic fields are empty too. + # Without [project], legacy metadata remains authoritative. install_requires = ( "" if project_dependencies_static_authoritative @@ -1280,7 +1300,11 @@ def inspect_metadata(root: Path) -> MetadataResult: distribution_name = setup_values["name"] if project_version is None and isinstance(setup_values.get("version"), str): project_version = setup_values["version"] - if requires_python is None and isinstance(setup_values.get("python_requires"), str): + if ( + not project_python_static_authoritative + and requires_python is None + and isinstance(setup_values.get("python_requires"), str) + ): requires_python = setup_values["python_requires"] legacy_runtime_specs = ( [] @@ -1520,7 +1544,11 @@ def inspect_metadata(root: Path) -> MetadataResult: pipfile_requires = ( pipfile.get("requires") if isinstance(pipfile.get("requires"), dict) else {} ) - if requires_python is None and isinstance(pipfile_requires.get("python_version"), str): + if ( + not project_python_static_authoritative + and requires_python is None + and isinstance(pipfile_requires.get("python_version"), str) + ): requires_python = f"=={pipfile_requires['python_version']}.*" if not source_roots: @@ -1741,4 +1769,5 @@ def safe_roots(values: list[str]) -> list[str]: setuptools_external_packaging_roots=sorted(set(setuptools_external_packaging_roots)), setuptools_external_packaging_root_evidence=setuptools_external_packaging_root_evidence, dynamic_dependency_evidence=dynamic_dependency_evidence, + dynamic_entry_point_evidence=dynamic_entry_point_evidence, ) diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index bb1ef74..864274a 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -794,27 +794,24 @@ def _importlib_resource_path_values( if _is_resource_files_call(node, module_bindings, files_bindings): if not isinstance(node, ast.Call): return [] - if not node.args: - # ``anchor=`` is the Python 3.12 spelling. Deliberately leave - # deprecated ``package=`` unresolved rather than treating either - # keyword form as the zero-argument implicit caller anchor. - if not node.keywords: - return _implicit_resource_root(root, source_path) - if len(node.keywords) == 1 and node.keywords[0].arg == "anchor": - return _resource_package_anchor_values( - node.keywords[0].value, - root=root, - source_path=source_path, - source_roots=source_roots, - project=project, - assignments=assignments, - returns=returns, - ) + keywords = [keyword.arg for keyword in node.keywords] + if ( + len(node.args) > 1 + or any(name not in {"anchor", "package"} for name in keywords) + or len(keywords) > 1 + ): return [] - if len(node.args) != 1 or node.keywords: + if not node.args and not node.keywords: + return _implicit_resource_root(root, source_path) + # package= is the 3.11 spelling, retained compatibly in 3.12+. + # Never confuse either explicit keyword with the implicit caller. + anchor = call_argument( + node, position=0, keyword="package" if "package" in keywords else "anchor" + ) + if anchor is None: return [] return _resource_package_anchor_values( - node.args[0], + anchor, root=root, source_path=source_path, source_roots=source_roots, diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index f27933a..a1a2c1a 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -952,12 +952,17 @@ def generate_deployment_kit( runtime_sync_blockers = { code for code in plan.risk_gate.blocking_codes - if code in {"RUNTIME_SYNC_METADATA_UNSUPPORTED", "LEGACY_LOCK_ROOT_UNIDENTIFIABLE"} + if code in { + "RUNTIME_SYNC_METADATA_UNSUPPORTED", "LEGACY_LOCK_ROOT_UNIDENTIFIABLE", + "ENTRYPOINT_METADATA_UNSUPPORTED", + } } if runtime_sync_blockers: runtime_sync_code = ( "RUNTIME_SYNC_METADATA_UNSUPPORTED" if "RUNTIME_SYNC_METADATA_UNSUPPORTED" in runtime_sync_blockers + else "ENTRYPOINT_METADATA_UNSUPPORTED" + if "ENTRYPOINT_METADATA_UNSUPPORTED" in runtime_sync_blockers else "LEGACY_LOCK_ROOT_UNIDENTIFIABLE" ) if dry_run: @@ -974,7 +979,11 @@ def generate_deployment_kit( ) preview.developer_actions.insert( 0, - f"Stop: {runtime_sync_code} prevents immutable dependency synchronization.", + ( + f"Stop: {runtime_sync_code} prevents a provable launcher metadata contract." + if runtime_sync_code == "ENTRYPOINT_METADATA_UNSUPPORTED" + else f"Stop: {runtime_sync_code} prevents immutable dependency synchronization." + ), ) return GenerationResult( output_directory=str(output_root), @@ -982,6 +991,12 @@ def generate_deployment_kit( generated=False, preview=preview, ) + if runtime_sync_code == "ENTRYPOINT_METADATA_UNSUPPORTED": + raise PreparationError( + "Deployment planning is blocked: ENTRYPOINT_METADATA_UNSUPPORTED. " + "Declare complete static scripts/gui-scripts groups before generation; " + "the pinned lock does not prove backend-generated launchers." + ) raise PreparationError( f"Deployment planning is blocked: {runtime_sync_code}. The pinned uv 0.12.5 " "lock workflow cannot represent this backend-only setup.cfg/setup.py metadata " diff --git a/tests/test_analysis_scope.py b/tests/test_analysis_scope.py index 1a32df5..6250654 100644 --- a/tests/test_analysis_scope.py +++ b/tests/test_analysis_scope.py @@ -1334,7 +1334,7 @@ def test_importlib_resources_implicit_anchor_honors_source_root_and_keywords( for evidence in resource.evidence if "importlib.resources.files" in evidence.detail ] - ) == 2 + ) == 3 def test_importlib_resources_rejects_unproven_or_escaping_resource_paths(tmp_path: Path) -> None: diff --git a/tests/test_dependency_authority.py b/tests/test_dependency_authority.py index b80c3f7..357f15c 100644 --- a/tests/test_dependency_authority.py +++ b/tests/test_dependency_authority.py @@ -133,7 +133,7 @@ def test_explicit_empty_gui_scripts_override_stale_legacy_launcher(tmp_path, leg @pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) @pytest.mark.parametrize("build_system_only", [False, True]) -def test_genuine_legacy_dependencies_still_block(tmp_path, legacy, build_system_only): +def test_dependency_ownership_follows_project_table(tmp_path, legacy, build_system_only): write_project(tmp_path, legacy, None) if build_system_only: path = tmp_path / "pyproject.toml" @@ -146,10 +146,15 @@ def test_genuine_legacy_dependencies_still_block(tmp_path, legacy, build_system_ path.read_text().replace("setup(", "setup(name='demo', version='1.0.0', ") ) assessment = assess(tmp_path) - assert [d.distribution_name for d in assessment.dependencies] == ["obsolete"] - assert {e.file for e in assessment.dependencies[0].evidence} == {legacy} + assert [d.distribution_name for d in assessment.dependencies] == ( + ["obsolete"] if build_system_only else [] + ) + if build_system_only: + assert {e.file for e in assessment.dependencies[0].evidence} == {legacy} plan = create_deployment_plan(assessment, repository_root=tmp_path) - assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" in plan.risk_gate.blocking_codes + assert ( + "RUNTIME_SYNC_METADATA_UNSUPPORTED" in plan.risk_gate.blocking_codes + ) == build_system_only @pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) @@ -217,13 +222,16 @@ def test_standardized_core_identity_fields_already_win(tmp_path, legacy): @pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) @pytest.mark.parametrize("group", ["console_scripts", "gui_scripts"]) @pytest.mark.parametrize("empty", [False, True]) -def test_static_script_group_suppresses_only_its_legacy_group(tmp_path, legacy, group, empty): +@pytest.mark.parametrize("other_dynamic", [False, True]) +def test_script_group_authority_is_independent(tmp_path, legacy, group, empty, other_dynamic): write_project(tmp_path, legacy, "[]") field = "scripts" if group == "console_scripts" else "gui-scripts" other = "gui_scripts" if group == "console_scripts" else "console_scripts" + other_field = "gui-scripts" if group == "console_scripts" else "scripts" path = tmp_path / "pyproject.toml" path.write_text( path.read_text().split("[project.scripts]")[0] + + (f"dynamic=[{other_field!r}]\n" if other_dynamic else "") + f"[project.{field}]\n" + ("" if empty else "demo='app:main'\n") ) @@ -243,7 +251,7 @@ def test_static_script_group_suppresses_only_its_legacy_group(tmp_path, legacy, ) ) names = {entry.name for entry in inspect_metadata(tmp_path).project.entry_points} - assert names == ({"retained"} if empty else {"demo", "retained"}) + assert names == ((set() if empty else {"demo"}) | ({"retained"} if other_dynamic else set())) @pytest.mark.parametrize("dependencies", ["['modern>=1']", None]) diff --git a/tests/test_files_package_keyword.py b/tests/test_files_package_keyword.py new file mode 100644 index 0000000..e5e4cb2 --- /dev/null +++ b/tests/test_files_package_keyword.py @@ -0,0 +1,162 @@ +"""Compatible explicit files(package=...) anchors use the existing resolver.""" + +import pytest +from test_dependency_authority import assess + +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.planning.planner import create_deployment_plan + + +def resource_project(root, imports, call, source_root="src", mapping=None): + package = root / source_root / "app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + (package / "defaults.json").write_text("{}\n") + (package / "main.py").write_text( + f"{imports}\ndef main():\n return {call}.joinpath('defaults.json').read_text()\n" + ) + (root / "pyproject.toml").write_text( + "[project]\nname='demo'\nversion='1.0'\nrequires-python='>=3.12'\n" + "[project.scripts]\ndemo='app.main:main'\n" + + (f"[tool.setuptools]\npackages=['app']\npackage-dir={mapping}\n" if mapping else "") + ) + (root / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n" + "[[package]]\nname='demo'\nversion='1.0'\nsource={virtual='.'}\n" + ) + return (package / "defaults.json").relative_to(root).as_posix() + + +@pytest.mark.parametrize( + ("imports", "call"), + [ + ("import importlib.resources", "importlib.resources.files(package='app')"), + ("import importlib.resources as resources", "resources.files(package='app')"), + ("from importlib import resources", "resources.files(package='app')"), + ("from importlib import resources as resources", "resources.files(package='app')"), + ("from importlib.resources import files", "files(package='app')"), + ("from importlib.resources import files as rf", "rf(package='app')"), + ], +) +def test_package_keyword_promotes_and_stages_resource(tmp_path, imports, call): + path = resource_project(tmp_path, imports, call) + assessment = assess(tmp_path) + assert any(item.path == path for item in assessment.resources) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert path in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("argument", ["'app'", "anchor='app'", "package='app'", ""]) +@pytest.mark.parametrize( + ("source_root", "mapping"), + [ + (".", None), + ("src", None), + ("lib", "{''='lib'}"), + ("lib", "{'app'='lib/app'}"), + ], +) +def test_files_anchor_forms_share_source_root_resolution(tmp_path, argument, source_root, mapping): + path = resource_project( + tmp_path, + "from importlib.resources import files", + f"files({argument})", + source_root, + mapping, + ) + assessment = assess(tmp_path) + assert any(item.path == path for item in assessment.resources) + + +@pytest.mark.parametrize( + "argument", + [ + "anchor='app', package='other'", + "foo='app'", + "package=unknown", + "**{'package':'app'}", + "'app', 'other'", + "package='../app'", + "package='/app'", + "package='app', foo='ignored'", + ], +) +def test_unknown_or_conflicting_files_anchor_is_unresolved(tmp_path, argument): + path = resource_project(tmp_path, "from importlib.resources import files", f"files({argument})") + assert all(item.path != path for item in assess(tmp_path).resources) + + +@pytest.mark.parametrize("argument", ["'app', package='other'", "'app', anchor='other'"]) +def test_files_positional_argument_wins_duplicate_binding(tmp_path, argument): + path = resource_project(tmp_path, "from importlib.resources import files", f"files({argument})") + assert any(item.path == path for item in assess(tmp_path).resources) + + +def test_files_package_keyword_uses_static_assignment_resolver(tmp_path): + path = resource_project( + tmp_path, "from importlib.resources import files\nPACKAGE='app'", "files(package=PACKAGE)" + ) + assert any(item.path == path for item in assess(tmp_path).resources) + + +def test_user_defined_files_is_not_promoted(tmp_path): + path = resource_project(tmp_path, "def files(**kwargs): return None", "files(package='app')") + assert all(item.path != path for item in assess(tmp_path).resources) + + +def test_files_package_keyword_parent_mapping(tmp_path): + path = resource_project( + tmp_path, + "from importlib.resources import files", + "files(package='app')", + "lib", + "{'app'='lib/app'}", + ) + parent = tmp_path / "lib/app" + child = parent / "child" + child.mkdir() + (child / "__init__.py").write_text("") + (child / "defaults.json").write_text("{}") + (parent / "main.py").write_text( + "from importlib.resources import files\n" + "def main(): return files(package='app.child').joinpath('defaults.json').read_text()\n" + ) + resources = {item.path for item in assess(tmp_path).resources} + assert "lib/app/child/defaults.json" in resources + assert path not in resources + + +def test_files_package_keyword_rejects_joinpath_traversal(tmp_path): + path = resource_project( + tmp_path, "from importlib.resources import files", "files(package='app')" + ) + main = tmp_path / "src/app/main.py" + main.write_text(main.read_text().replace("'defaults.json'", "'../app/defaults.json'")) + assert all(item.path != path for item in assess(tmp_path).resources) + + +def test_keyword_promoted_resource_receives_release_security_scan(tmp_path, monkeypatch): + source = tmp_path / "source" + path = resource_project(source, "from importlib.resources import files", "files(package='app')") + (source / path).write_text("API_KEY = 'sk-abcdefghijklmnop'\n") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + with pytest.raises(PreparationError, match="NO_SECRET_VALUES"): + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + ) + assert not (tmp_path / "kit").exists() diff --git a/tests/test_omitted_metadata.py b/tests/test_omitted_metadata.py new file mode 100644 index 0000000..1088bcf --- /dev/null +++ b/tests/test_omitted_metadata.py @@ -0,0 +1,168 @@ +"""Existing project tables own omitted non-dynamic metadata fields too.""" + +import pytest +from test_dependency_authority import assess, write_project +from test_generation import _make_application_wheel, _write_mapped_project + +from python_deployment_builder.analysis.metadata import inspect_metadata +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import validate_application_wheel +from python_deployment_builder.generation.generator import generate_deployment_kit +from python_deployment_builder.planning.planner import create_deployment_plan + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_omitted_dependencies_are_authoritative_empty(tmp_path, legacy): + write_project(tmp_path, legacy, None) + assessment = assess(tmp_path) + assert assessment.dependencies == [] + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "RUNTIME_SYNC_METADATA_UNSUPPORTED" not in plan.risk_gate.blocking_codes + assert plan.lock_graph.dependencies == [] + + +def add_legacy_entry(root, legacy, group): + path = root / legacy + if legacy == "setup.cfg": + path.write_text(path.read_text() + f"[options.entry_points]\n{group}=\n stale=old:main\n") + else: + path.write_text( + path.read_text().replace( + "setup(", f"setup(entry_points={{{group!r}: ['stale=old:main']}}, " + ) + ) + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("group", ["console_scripts", "gui_scripts"]) +def test_omitted_script_groups_suppress_stale_launchers(tmp_path, legacy, group): + write_project(tmp_path, legacy, "[]") + path = tmp_path / "pyproject.toml" + path.write_text(path.read_text().split("[project.scripts]")[0]) + add_legacy_entry(tmp_path, legacy, group) + assert inspect_metadata(tmp_path).project.entry_points == [] + + +def test_omitted_requires_python_does_not_use_stale_setup_py(tmp_path): + write_project(tmp_path, "setup.py", "[]") + path = tmp_path / "pyproject.toml" + path.write_text(path.read_text().replace("requires-python='>=3.12'\n", "")) + path = tmp_path / "setup.py" + path.write_text(path.read_text().replace("setup(", "setup(python_requires='<3.10', ")) + assert inspect_metadata(tmp_path).python.requires_python is None + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("field", ["scripts", "gui-scripts"]) +@pytest.mark.parametrize("static", [False, True]) +def test_dynamic_script_evidence_is_retained_but_not_promised(tmp_path, legacy, field, static): + write_project(tmp_path, legacy, "[]", dynamic=f"[{field!r}]") + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text().split("[project.scripts]")[0] + + (f"[project.{field}]\nmodern='app:main'\n" if static else "") + ) + group = "console_scripts" if field == "scripts" else "gui_scripts" + add_legacy_entry(tmp_path, legacy, group) + assessment = assess(tmp_path) + assert "stale" in {entry.name for entry in assessment.project.entry_points} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "ENTRYPOINT_METADATA_UNSUPPORTED" in plan.risk_gate.blocking_codes + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +@pytest.mark.parametrize("group", ["console_scripts", "gui_scripts"]) +def test_no_project_table_retains_legacy_launchers(tmp_path, legacy, group): + write_project(tmp_path, legacy, None) + path = tmp_path / "pyproject.toml" + path.write_text(path.read_text().split("[project]")[0]) + add_legacy_entry(tmp_path, legacy, group) + assert {entry.name for entry in inspect_metadata(tmp_path).project.entry_points} == {"stale"} + + +@pytest.mark.parametrize("field", ["scripts", "gui-scripts"]) +def test_dynamic_script_contract_stops_generation_before_uv(tmp_path, monkeypatch, field): + source = tmp_path / "source" + write_project(source, "setup.py", "[]", dynamic=f"[{field!r}]") + repository = MaterializedRepository(root=source, source=str(source), source_kind="local") + output = tmp_path / "kit" + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: pytest.fail("must stop before acquisition"), + ) + preview = generate_deployment_kit(repository, output, dry_run=True).preview + assert any("ENTRYPOINT_METADATA_UNSUPPORTED" in item for item in preview.developer_actions) + with pytest.raises(PreparationError, match="ENTRYPOINT_METADATA_UNSUPPORTED"): + generate_deployment_kit(repository, output, prepare_lock=True) + assert not output.exists() + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_omitted_optional_groups_do_not_gain_legacy_extras(tmp_path, legacy): + write_project(tmp_path, legacy, "[]") + path = tmp_path / legacy + if legacy == "setup.cfg": + path.write_text(path.read_text() + "[options.extras_require]\nmap=obsolete>=1\n") + else: + path.write_text( + path.read_text().replace("setup(", "setup(extras_require={'map':['obsolete']}, ") + ) + metadata = inspect_metadata(tmp_path) + assert metadata.project.optional_dependency_groups == {} + assert metadata.dependencies == [] + + +def test_custom_entry_point_group_is_not_a_console_or_gui_launcher(tmp_path): + write_project(tmp_path, "setup.py", "[]") + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text().split("[project.scripts]")[0] + + "[project.entry-points.'some.group']\nplugin='app:main'\n" + ) + assert inspect_metadata(tmp_path).project.entry_points == [] + + +@pytest.mark.parametrize("project_present", [False, True]) +def test_requires_python_pipfile_fallback_obeys_project_authority(tmp_path, project_present): + write_project(tmp_path, "setup.py", "[]") + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text().replace("requires-python='>=3.12'\n", "") + if project_present + else path.read_text().split("[project]")[0] + ) + (tmp_path / "Pipfile").write_text("[requires]\npython_version='3.11'\n") + assert inspect_metadata(tmp_path).python.requires_python == ( + None if project_present else "==3.11.*" + ) + + +def test_dynamic_requires_python_keeps_literal_backend_evidence(tmp_path): + write_project(tmp_path, "setup.py", "[]", dynamic="['requires-python']") + path = tmp_path / "pyproject.toml" + path.write_text(path.read_text().replace("requires-python='>=3.12'\n", "")) + path = tmp_path / "setup.py" + path.write_text(path.read_text().replace("setup(", "setup(python_requires='>=3.11', ")) + assert inspect_metadata(tmp_path).python.requires_python == ">=3.11" + + +@pytest.mark.parametrize("legacy", ["setup.cfg", "setup.py"]) +def test_omitted_gui_group_cannot_reject_correct_console_wheel(tmp_path, legacy): + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source, entry_group="scripts") + if legacy == "setup.cfg": + (source / legacy).write_text("[options.entry_points]\ngui_scripts=\n stale=old:main\n") + else: + (source / legacy).write_text( + "from setuptools import setup\nsetup(entry_points={'gui_scripts':['stale=old:main']})\n" + ) + assessment = assess(source) + plan = create_deployment_plan(assessment, repository_root=source) + assert plan.entry_point.declared_group == "console_scripts" + wheel = _make_application_wheel(tmp_path, entry_group="console_scripts") + artifact, _ = validate_application_wheel( + wheel, assessment, plan, repository_root=source, validate_locked_dependencies=False + ) + assert artifact.distribution_name == "mapped-app" From 0d26ebe9b536335d278fabf44477a22fa13a42c1 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:51:02 -0400 Subject: [PATCH 54/59] Preserve legacy secret scanning and resolve module resource anchors --- ...egacy-secrets-module-anchors-2026-09-12.md | 161 +++++++++++ scripts/verify_module_anchors.py | 155 +++++++++++ .../analysis/assessor.py | 1 + .../analysis/inventory.py | 40 +-- .../analysis/module_resolution.py | 73 +++++ .../analysis/resources.py | 42 ++- .../generation/manifest.py | 13 + .../validation/runtime.py | 3 +- .../validation/static.py | 3 +- tests/test_legacy_manifest_secrets.py | 195 ++++++++++++++ tests/test_module_resource_anchors.py | 250 ++++++++++++++++++ 11 files changed, 889 insertions(+), 47 deletions(-) create mode 100644 docs/acceptance/pr9-legacy-secrets-module-anchors-2026-09-12.md create mode 100644 scripts/verify_module_anchors.py create mode 100644 src/python_deployment_builder/analysis/module_resolution.py create mode 100644 tests/test_legacy_manifest_secrets.py create mode 100644 tests/test_module_resource_anchors.py diff --git a/docs/acceptance/pr9-legacy-secrets-module-anchors-2026-09-12.md b/docs/acceptance/pr9-legacy-secrets-module-anchors-2026-09-12.md new file mode 100644 index 0000000..5063cd2 --- /dev/null +++ b/docs/acceptance/pr9-legacy-secrets-module-anchors-2026-09-12.md @@ -0,0 +1,161 @@ +# PR #9: legacy manifest secrets and explicit module resource anchors + +Starting head: `2a7556358e4d909dc58a4715ef31d6850f9a9ff1` on +`milestone-6-1-generation-contract`. This records the corrections to the exact-head +review submitted at `2026-09-12T00:14:54Z`: + +| Priority | Finding | Thread | Comment | +| --- | --- | --- | --- | +| P1 | Preserve secret scanning for legacy manifests | PRRT_kwDOT9hvCc6hrPQK | 3994357474 | +| P2 | Resolve explicit module anchors for resource files | PRRT_kwDOT9hvCc6hrPQN | 3994357479 | + +Accounting remains 120 findings/comments, 117 unresolved inline threads, three +review-level-only findings, and one NOT_APPLICABLE inline finding. The latter is +the automatic flat-package plus loose-module review premise. The three +review-level-only items remain historical #35, approved-wheel PEP 440 semantic +version comparison, and complete locked sync-command validation. No historical +thread is resolved, no review dismissed, and no merge authorized by this work. + +## Legacy configured-secret compatibility + +Failing regressions were added before production changes. A valid generated +source kit was re-indexed after physically removing `configuration_secret_names` +from its JSON and setting `configuration_presence_names` to `DB_PASSWORD`. An +ordinary indexed source comment, and separately an indexed runtime-helper +comment, contained a synthetic configured value longer than eight characters. +Both received STATIC_VALID before correction: the generic secret patterns did +not independently recognize the fixture. The actual loader returned an empty +default secret list despite retaining the presence name. + +Both static and runtime JSON loaders call `DeploymentManifest.model_validate_json` +directly. Tests prove Pydantic's `model_fields_set` excludes the physically absent +field but includes an explicitly serialized empty list. One helper, +`effective_configuration_secret_names`, uses this transient distinction: + +| Serialized state | Effective configured-secret names | +| --- | --- | +| secret-name field absent | legacy presence names | +| explicit empty secret-name list | empty; no fallback | +| explicit secret-name subset | exactly that subset | + +Static validation computes configured values once and supplies the same values +to indexed source/generated-text scanning and approved/application wheel-member +scanning. Re-indexed, RECORD-consistent approved and first-party wheels are both +rejected when they contain the legacy configured value. The runtime diagnostic +privacy check now uses the same helper; runtime environment isolation itself is +unchanged and continues to remove presence names before restoring harness-owned +LOCALAPPDATA. + +Unset and empty legacy values remain benign. Nonempty values shorter than eight +characters fail CONFIGURED_SECRET_SCANABILITY with +SHORT_CONFIGURED_SECRET_UNSCANNABLE; they are not substring-scanned. Regression +assertions verify that values are absent from validation diagnostics, captured +logs, error messages, and manifests. Deliberately contaminated test inputs are +not rewritten or echoed as evidence. Current explicit-empty/subset manifests do +not scan the unrelated DISPLAY_THEME value. Current generation already uses +`model_dump_json(indent=2)` without default exclusion, and its explicit empty +secret-name serialization is now directly tested. Existing old-source-manifest +static validity and release packaging tests remain applicable. + +The bounded compatibility audit compared DeploymentManifest with pre-M6.1 commit +`2fc5ec1`. Only `application_artifact` and `configuration_secret_names` were +added. For application_artifact, absent and explicit null intentionally have the +same meaning: accepted for source mode, rejected for package mode. Four focused +tests cover both spellings in both modes. No other defaulted M6.1 manifest field +requires an absence distinction. Reverse approved-artifact completeness and +strict sync-command validation are not bypassed. + +## Module resource anchors + +The three initial failing resource tests covered positional, anchor=, and +package= string anchors naming `app.config`, physically `src/app/config.py`. +The starting resolver recognized files() but considered package directories only; +the adjacent defaults.json was not promoted. + +`scripts/verify_module_anchors.py` runs the actual managed Python 3.12.14 +interpreter in disposable fixtures. Observed results: + +| Physical anchor | Resulting container | +| --- | --- | +| app/__init__.py | app/ | +| app/config.py | app/ | +| config.py | source root | +| namespace/config.py, without namespace/__init__.py | namespace/ | +| both app/both.py and app/both/__init__.py | app/both/ (regular package wins) | + +All four tested runtime forms (positional string, anchor= string, package= +string, and imported module object) read the expected file. package= emits +DeprecationWarning; the other explicit forms do not. These observations agree +with the [Python 3.12 files() documentation](https://docs.python.org/3.12/library/importlib.resources.html#importlib.resources.files). + +The script also compares the starting-head anchor function with corrected +analysis against the same disposable source project. Before correction, source +staging omits defaults.json and an isolated Python 3.12.14 read fails with +FileNotFoundError. After correction, the staged source includes the file and the +same read succeeds. + +A small shared `analysis/module_resolution.py` now supplies physical locations +to inventory, packaging-directory resolution, and explicit modern resource +anchors. It retains exact, longest-parent and empty package-dir mappings, +configured source roots, and repository containment. Inventory retains regular +initializer promotion without manufacturing namespace initializers. Modern +files() selects a regular package before a same-location module; a module uses +its parent directory. Namespace containers are a fallback, not an extra root +when a concrete package/module is selected. Unsafe concrete anchors cannot be +reinterpreted as namespaces. + +Tests cover package/subpackage/module/top-level anchors, namespace parents, +dot/src/lib roots, exact and longest-parent mappings, all established files +bindings and explicit spellings, safe static assignments, missing/dynamic/invalid +anchors, containment, symlink policy, staging, and configured/generic secret +scanning of promoted resources. An installed-only mapping plus an unpackaged +resource still yields DEPLOYMENT_MODE_CONFLICT through the existing planner; +source-compatible layouts stage the resource normally. + +The bounded module-object audit found that the resource value engine tracks +literal assignments/returns, not local import-object identities. Its existing +import-binding sets establish the resource API callable, not a general module +object/type environment. Module-object anchors remain explicitly outside this +bounded static model (with negative tests); no generic name/dataflow resolver +was added. Literal and statically assigned string anchors are supported. + +Implicit files() is untouched. Legacy functional resource reads and pkgutil +still use their package-only anchor rules; this module-anchor expansion applies +only to modern files(). Python 3.11 remains package-only: package= works, while +anchor= and implicit files() do not. Recognition of Python 3.12 module semantics +does not claim those calls work on 3.11 or redesign target selection. + +## Acceptance, schemas, and quality + +Read-only starting-head comparisons prove unchanged dependency, entry-point, +configuration, and resource records for all three acceptance repositories. Git +SHAs and working-tree status were preserved. + +| Repository | Unchanged SHA | Result | +| --- | --- | --- | +| SimpleGeorefGUI | f484570d89fb1f9e9170fac915475358dfc1234e | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python, 77 data members | +| Geo Map Explanation Extractor | 5e7b321d0aeb9ba1d586bfc548c79793d84c6033 | source / SOURCE_COMPATIBLE | +| TN Coordinate Converter | e1e7a1588c37a99c2d02efaf3eef3d04636f12f0 | source / SOURCE_COMPATIBLE; existing kit STATIC_VALID with reviewed proxy-tools==0.1.0 | + +SGG's GUI target remains `simple_georef_gui_app.georef_main:main`. Geo retains +LOCALAPPDATA and OPENAI_API_KEY configuration evidence, with no new evidence or +persisted configuration values. No acceptance runtime execution is claimed. + +ANALYSIS_SCHEMA_VERSION remains 1.4 and PLANNING_SCHEMA_VERSION remains 1.3. +No persisted field, compatibility flag, or schema migration was added. + +The single complete suite passed: **1316 passed, 4 skipped in 316.42 seconds**, +up 67 passes from the 1249-pass baseline. The fourth skip is the new real-symlink +test on this Windows host without symlink-creation permission; independent +mocked-path regressions exercise rejection of unsafe concrete anchors. Ruff and +`git diff --check` pass. Final diff inspection contains only seven production +files, two regression-test files, the synthetic runtime probe, and this report. +No generated wheels, kits, environments, or acceptance artifacts are committed. + +The full suite retains PEP 621 omitted/explicit/dynamic field authority, approved +extra-context separation, artifact-path uniqueness, reverse lock completeness, +exact sync arguments, mode-aware pip checks, entry-point extras, short-secret and +LOCALAPPDATA handling, environment aliases, structural lock-root handling, +dynamic imports, modern/legacy/pkgutil resources, MANIFEST/package-data guards, +wheel dependency/security/integrity/collision rules, provenance, rollback, and +deterministic packaging. diff --git a/scripts/verify_module_anchors.py b/scripts/verify_module_anchors.py new file mode 100644 index 0000000..fd87bbc --- /dev/null +++ b/scripts/verify_module_anchors.py @@ -0,0 +1,155 @@ +"""Disposable direct-interpreter evidence for Python 3.12 module resource anchors.""" + +import argparse +import ast +import json +import subprocess +import tempfile +from pathlib import Path +from unittest.mock import patch + +import python_deployment_builder.analysis.resources as resources +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.generator import _staging_files +from python_deployment_builder.planning.planner import create_deployment_plan + + +def staging_probe(root: Path, python: Path) -> dict: + """Compare exact starting-head anchor analysis with corrected source staging.""" + + source = root / "source" + package = source / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + (package / "config.py").write_text( + "from importlib.resources import files\n" + "def load(): return files('app.config').joinpath('defaults.json').read_text()\n" + ) + (package / "defaults.json").write_text("MODULE_RESOURCE") + (source / "pyproject.toml").write_text( + "[project]\nname='demo'\nversion='1.0'\nrequires-python='>=3.12'\n" + "[project.scripts]\ndemo='app.config:load'\n" + ) + (source / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n" + "[[package]]\nname='demo'\nversion='1.0'\nsource={virtual='.'}\n" + ) + original = subprocess.check_output( + [ + "git", + "show", + "2a7556358e4d909dc58a4715ef31d6850f9a9ff1:" + "src/python_deployment_builder/analysis/resources.py", + ], + text=True, + ) + node = next( + item + for item in ast.parse(original).body + if isinstance(item, ast.FunctionDef) and item.name == "_resource_package_anchor_values" + ) + namespace = dict(resources.__dict__) + exec(compile(ast.Module(body=[node], type_ignores=[]), "", "exec"), namespace) + original_anchor = namespace[node.name] + current_anchor = resources._resource_package_anchor_values + result = {} + for label, resolver in ( + ("before", lambda *args, allow_module_anchor=False, **kw: original_anchor(*args, **kw)), + ("after", current_anchor), + ): + with patch.object(resources, "_resource_package_anchor_values", resolver): + assessment = assess_repository( + MaterializedRepository(root=source, source=str(source), source_kind="local") + ) + plan = create_deployment_plan(assessment, repository_root=source) + files = _staging_files(source, assessment, plan, include=True) + kit = root / label + for relative, content in files.items(): + path = kit / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + run = subprocess.run( + [str(python), "-B", "-E", "-s", "-c", "from app.config import load; print(load())"], + cwd=kit / "src", + capture_output=True, + text=True, + check=False, + ) + result[label] = { + "staged": "src/app/defaults.json" in files, + "exit": run.returncode, + "result": run.stdout.strip() if run.returncode == 0 else run.stderr.splitlines()[-1], + } + assert not result["before"]["staged"] + assert "FileNotFoundError" in result["before"]["result"] + assert result["after"] == {"staged": True, "exit": 0, "result": "MODULE_RESOURCE"} + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("python", type=Path) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="pdb-module-anchors-") as temporary: + root = Path(temporary) + for relative, content in { + "app/__init__.py": "", + "app/config.py": "", + "app/defaults.json": "MODULE", + "app/both.py": "", + "app/both/__init__.py": "", + "app/both/defaults.json": "PACKAGE", + "config.py": "", + "defaults.json": "TOP", + "namespace/config.py": "", + "namespace/defaults.json": "NAMESPACE", + }.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + probe = """ +import importlib, importlib.resources as resources, json, sys, warnings +result = {'version': sys.version.split()[0], 'anchors': {}} +for name in ['app', 'app.config', 'app.both', 'config', 'namespace.config']: + entries = {} + for spelling in ['positional', 'anchor', 'package', 'object']: + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter('always') + if spelling == 'positional': + container = resources.files(name) + elif spelling == 'object': + container = resources.files(importlib.import_module(name)) + else: + container = resources.files(**{spelling: name}) + entries[spelling] = { + 'text': container.joinpath('defaults.json').read_text(), + 'warnings': [type(item.message).__name__ for item in captured], + } + result['anchors'][name] = entries +print(json.dumps(result, indent=2)) +""" + result = subprocess.run( + [str(args.python), "-B", "-c", probe], + cwd=root, + capture_output=True, + text=True, + check=True, + ) + evidence = json.loads(result.stdout) + assert evidence["version"] == "3.12.14" + for name, expected in { + "app": "MODULE", + "app.config": "MODULE", + "app.both": "PACKAGE", + "config": "TOP", + "namespace.config": "NAMESPACE", + }.items(): + assert all(item["text"] == expected for item in evidence["anchors"][name].values()) + assert evidence["anchors"][name]["package"]["warnings"] == ["DeprecationWarning"] + evidence["staging"] = staging_probe(root / "staging", args.python) + print(json.dumps(evidence, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/assessor.py b/src/python_deployment_builder/analysis/assessor.py index 9a0002b..fcc54c8 100644 --- a/src/python_deployment_builder/analysis/assessor.py +++ b/src/python_deployment_builder/analysis/assessor.py @@ -76,6 +76,7 @@ def assess_repository(repository: MaterializedRepository) -> RepositoryAssessmen inventory.items, inventory.application_files, metadata.project.source_roots, + metadata.project.package_directories, ) imports = scan_imports( root, diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index 40e4b6e..9b722fa 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -13,6 +13,7 @@ from pathspec import PathSpec from python_deployment_builder.analysis.ast_utils import call_argument +from python_deployment_builder.analysis.module_resolution import module_locations, safe_local_path from python_deployment_builder.models import ( AnalysisScopeSummary, DependencyAssessment, @@ -377,7 +378,12 @@ def _imported_modules( return modules -def _module_files(root: Path, module: str, source_roots: list[str]) -> list[Path]: +def _module_files( + root: Path, + module: str, + source_roots: list[str], + package_directories: dict[str, str] | None = None, +) -> list[Path]: """Resolve a simple absolute import beneath every safe configured source root. This deliberately mirrors the packaging/inventory source-root model instead @@ -386,31 +392,24 @@ def _module_files(root: Path, module: str, source_roots: list[str]) -> list[Path initializers, never imported or executed. """ - relative = Path(*module.split(".")) - resolved_root = root.resolve() candidates: list[Path] = [] - for source_root in source_roots: - candidate_root = root / source_root - if candidate_root.is_symlink() or not candidate_root.is_dir(): - continue - try: - candidate_root.resolve().relative_to(resolved_root) - except ValueError: - continue + for location in module_locations(root, module, source_roots, package_directories): candidates.extend( ( - candidate_root / relative.with_suffix(".py"), - candidate_root / relative / "__init__.py", + location.with_suffix(".py"), + location / "__init__.py", ) ) - # Importing a dotted local module executes every existing regular - # package initializer on its path. Preserve those files as application - # source without fabricating namespace-package initializers. + # Importing a dotted local module executes every existing regular + # package initializer on its path. Preserve those files as application + # source without fabricating namespace-package initializers. + for index in range(1, len(module.split("."))): + parent = ".".join(module.split(".")[:index]) candidates.extend( - candidate_root / Path(*relative.parts[:index]) / "__init__.py" - for index in range(1, len(relative.parts)) + location / "__init__.py" + for location in module_locations(root, parent, source_roots, package_directories) ) - return sorted({path for path in candidates if path.is_file()}) + return sorted({path for path in candidates if path.is_file() and safe_local_path(path, root)}) def promote_imported_application_files( @@ -418,6 +417,7 @@ def promote_imported_application_files( items: list[RepositoryFileInventoryItem], application_files: list[Path], source_roots: list[str], + package_directories: dict[str, str] | None = None, ) -> None: """Promote non-ignored Python modules imported by production source.""" @@ -435,7 +435,7 @@ def promote_imported_application_files( except (OSError, SyntaxError, UnicodeError): continue for module, line in _imported_modules(tree, source_path, root, source_roots): - for imported_path in _module_files(root, module, source_roots): + for imported_path in _module_files(root, module, source_roots, package_directories): relative = imported_path.relative_to(root).as_posix() item = by_path.get(relative) if item is None: diff --git a/src/python_deployment_builder/analysis/module_resolution.py b/src/python_deployment_builder/analysis/module_resolution.py new file mode 100644 index 0000000..cdaae85 --- /dev/null +++ b/src/python_deployment_builder/analysis/module_resolution.py @@ -0,0 +1,73 @@ +"""Shared bounded physical locations for local import/package identities. + +No code is imported. Package-dir mappings use exact/longest-parent precedence; +otherwise candidates follow the supplied source-root order. Callers decide +whether they need a package directory, a module, or initializer promotion. +""" + +from pathlib import Path + + +def safe_local_path(path: Path, root: Path) -> bool: + """Reject symlink leaves and any resolved escape from the repository.""" + + if path.is_symlink(): + return False + try: + path.resolve().relative_to(root.resolve()) + except (OSError, ValueError): + return False + return True + + +def module_locations( + root: Path, + module: str, + source_roots: list[str], + package_directories: dict[str, str] | None = None, +) -> list[Path]: + """Return safe unsuffixed module/package locations, without executing imports.""" + + if not module or not all(part.isidentifier() for part in module.split(".")): + return [] + mappings = package_directories or {} + parents = [key for key in mappings if key and (module == key or module.startswith(key + "."))] + if parents: + parent = max(parents, key=lambda key: len(key.split("."))) + remainder = module.split(".")[len(parent.split(".")) :] + candidates = [root / mappings[parent] / Path(*remainder)] + elif "" in mappings: + candidates = [root / mappings[""] / Path(*module.split("."))] + else: + candidates = [ + root / source_root / Path(*module.split(".")) + for source_root in source_roots + if (root / source_root).is_dir() and safe_local_path(root / source_root, root) + ] + return list(dict.fromkeys(path for path in candidates if safe_local_path(path, root))) + + +def module_resource_roots( + root: Path, + module: str, + source_roots: list[str], + package_directories: dict[str, str] | None = None, +) -> list[Path]: + """Resolve Python 3.12 explicit files() anchors to one importable container. + + At a physical location a regular package wins over a same-named .py module. + Namespace directories are considered only in the absence of a concrete + package/module, matching the import system's namespace fallback rule. + """ + + namespaces: list[Path] = [] + for location in module_locations(root, module, source_roots, package_directories): + initializer = location / "__init__.py" + module_file = location.with_suffix(".py") + if initializer.is_file(): + return [location] if safe_local_path(initializer, root) else [] + if module_file.is_file(): + return [module_file.parent] if safe_local_path(module_file, root) else [] + if location.is_dir(): + namespaces.append(location) + return namespaces diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 864274a..d1a1690 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -9,6 +9,10 @@ from python_deployment_builder.analysis.ast_utils import call_argument from python_deployment_builder.analysis.imports import EXCLUDED_DIRECTORIES +from python_deployment_builder.analysis.module_resolution import ( + module_locations, + module_resource_roots, +) from python_deployment_builder.models import ( ConfigurationRequirement, Evidence, @@ -66,31 +70,9 @@ def _physical_package_roots( ) -> list[Path]: """Resolve a declared installed package name to existing source directories.""" - candidates: list[Path] = [] - explicit = project.package_directories.get(package) - if explicit is not None: - candidates.append(root / explicit) - else: - package_parts = package.split(".") - parent_mappings = [ - mapping - for mapping in project.package_directories - if mapping and (package == mapping or package.startswith(f"{mapping}.")) - ] - if parent_mappings: - parent = max(parent_mappings, key=lambda mapping: len(mapping.split("."))) - remainder = package_parts[len(parent.split(".")) :] - candidates.append(root / project.package_directories[parent] / Path(*remainder)) - else: - base = project.package_directories.get("") - if base is not None: - candidates.append(root / base / Path(*package_parts)) - else: - candidates.extend( - root / source_root / Path(*package_parts) - for source_root in project.source_roots - ) - candidates.append(root / Path(*package_parts)) + candidates = module_locations( + root, package, [*project.source_roots, "."], project.package_directories + ) resolved_root = root.resolve() roots: list[Path] = [] @@ -645,6 +627,7 @@ def _resource_package_anchor_values( assignments: dict[str, ast.AST], returns: dict[str, ast.AST], require_initializer: bool = False, + allow_module_anchor: bool = False, ) -> list[str]: package_values = _path_values( node, @@ -655,6 +638,14 @@ def _resource_package_anchor_values( ) if len(package_values) != 1: return [] + if allow_module_anchor: + roots = module_resource_roots( + root, + package_values[0], + [*source_roots, "."], + project.package_directories if project else None, + ) + return [path.relative_to(root.resolve()).as_posix() for path in roots] return [ package_root.relative_to(root.resolve()).as_posix() for package_root in _resource_package_roots( @@ -818,6 +809,7 @@ def _importlib_resource_path_values( project=project, assignments=assignments, returns=returns, + allow_module_anchor=True, ) if ( isinstance(node, ast.Call) diff --git a/src/python_deployment_builder/generation/manifest.py b/src/python_deployment_builder/generation/manifest.py index ebd22f2..7c73ae3 100644 --- a/src/python_deployment_builder/generation/manifest.py +++ b/src/python_deployment_builder/generation/manifest.py @@ -20,6 +20,19 @@ from python_deployment_builder.security_policy import is_valid_environment_name +def effective_configuration_secret_names(manifest: DeploymentManifest) -> list[str]: + """Preserve legacy scanning without broadening an explicit current secret set. + + The JSON loader retains serialized key presence in Pydantic's fields set. + Pre-M6.1 manifests only recorded presence names; an explicitly empty secret + list in a current manifest is intentional and must not select this fallback. + """ + + if "configuration_secret_names" in manifest.model_fields_set: + return manifest.configuration_secret_names + return manifest.configuration_presence_names + + def source_roots_from_plan(plan: DeploymentPlan) -> list[str]: raw = plan.runtime.environment_variables.get("PYTHONPATH", "") roots: list[str] = [] diff --git a/src/python_deployment_builder/validation/runtime.py b/src/python_deployment_builder/validation/runtime.py index 3da3f31..4279645 100644 --- a/src/python_deployment_builder/validation/runtime.py +++ b/src/python_deployment_builder/validation/runtime.py @@ -17,6 +17,7 @@ from python_deployment_builder.analysis import assess_repository from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.manifest import effective_configuration_secret_names from python_deployment_builder.generation.security import redact_secrets from python_deployment_builder.models import ( DeploymentManifest, @@ -907,7 +908,7 @@ def fail(code: str, phase: str, detail: str, evidence: list[str] | None = None) and "Managed application Python is unavailable" in broken.stdout and all( value not in healthy.stdout + broken.stdout - for name in manifest.configuration_secret_names + for name in effective_configuration_secret_names(manifest) if (value := os.environ.get(name)) ) ) diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 65c38a7..74bb2cd 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -33,6 +33,7 @@ validate_wheel_static_safety, validate_wheel_target_compatibility, ) +from python_deployment_builder.generation.manifest import effective_configuration_secret_names from python_deployment_builder.generation.structural import ( approved_artifacts_by_path, manifest_artifact_wheel_path, @@ -402,7 +403,7 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR trusted_wheels = trusted_artifact_wheel_paths(manifest) secret_scanability_failures: list[str] = [] try: - secret_values = configured_secret_values(manifest.configuration_secret_names) + secret_values = configured_secret_values(effective_configuration_secret_names(manifest)) except PreparationError as exc: secret_values = () secret_scanability_failures.append(str(exc)) diff --git a/tests/test_legacy_manifest_secrets.py b/tests/test_legacy_manifest_secrets.py new file mode 100644 index 0000000..87c2be9 --- /dev/null +++ b/tests/test_legacy_manifest_secrets.py @@ -0,0 +1,195 @@ +"""Serialized absence, not an empty value, selects legacy secret compatibility.""" + +import json + +import pytest +from test_generation import ( + _make_application_wheel, + _make_wheel, + _refresh_manifest_wheel_hash, + _repository, + _rewrite_application_wheel, + _update_indexed_hashes, + _write_mapped_project, +) +from test_validation import _kit, _refresh_manifest_index, _status + +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.generator import generate_deployment_kit +from python_deployment_builder.generation.manifest import effective_configuration_secret_names +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.models import ValidationCheckStatus +from python_deployment_builder.validation.runtime import _manifest, _runtime_environment +from python_deployment_builder.validation.static import _load_manifest, validate_static_kit + + +def configure_manifest(kit, presence, secrets=None): + path = kit / "deployment/manifest.json" + payload = json.loads(path.read_text()) + payload["configuration_presence_names"] = presence + if secrets is None: + payload.pop("configuration_secret_names", None) + else: + payload["configuration_secret_names"] = secrets + path.write_text(json.dumps(payload, indent=2) + "\n") + _refresh_manifest_index(kit) + + +@pytest.mark.parametrize("relative", ["src/prepared_gui/main.py", "deployment/runtime/launch.py"]) +def test_legacy_configured_value_is_scanned_in_indexed_text(tmp_path, monkeypatch, relative): + kit = _kit(monkeypatch, tmp_path) + # Select a real indexed application source without depending on fixture module spelling. + if relative.startswith("src/"): + relative = next( + p.relative_to(kit).as_posix() + for p in kit.rglob("*.py") + if "deployment" not in p.relative_to(kit).parts + ) + configure_manifest(kit, ["DB_PASSWORD"]) + manifest = _load_manifest(kit) + assert manifest.configuration_secret_names == [] + assert "configuration_secret_names" not in manifest.model_fields_set + secret = "PDBLegacyConfiguredSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + path = kit / relative + path.write_text(path.read_text() + f"\n# ordinary fixture {secret}\n") + _update_indexed_hashes(kit, relative) + report = validate_static_kit(kit) + assert _status(report, "NO_SECRET_CONTENT") == ValidationCheckStatus.FAIL + assert secret not in report.model_dump_json() + + +@pytest.mark.parametrize("value", [None, "", "482731"]) +def test_legacy_unset_empty_and_short_values(tmp_path, monkeypatch, caplog, value): + kit = _kit(monkeypatch, tmp_path) + configure_manifest(kit, ["DEMO_PIN"]) + monkeypatch.delenv("DEMO_PIN", raising=False) + if value is not None: + monkeypatch.setenv("DEMO_PIN", value) + report = validate_static_kit(kit) + if value: + assert _status(report, "CONFIGURED_SECRET_SCANABILITY") == ValidationCheckStatus.FAIL + assert "SHORT_CONFIGURED_SECRET_UNSCANNABLE" in report.model_dump_json() + assert value not in report.model_dump_json() + caplog.text + assert value not in (kit / "deployment/manifest.json").read_text() + else: + assert report.final_state.value == "STATIC_VALID" + + +@pytest.mark.parametrize("secrets", [[], ["DB_PASSWORD"]]) +@pytest.mark.parametrize("embed_secret", [False, True]) +def test_current_secret_set_does_not_fall_back(tmp_path, monkeypatch, secrets, embed_secret): + kit = _kit(monkeypatch, tmp_path) + configure_manifest(kit, ["DISPLAY_THEME", "DB_PASSWORD"], secrets) + manifest = _load_manifest(kit) + assert "configuration_secret_names" in manifest.model_fields_set + assert effective_configuration_secret_names(manifest) == secrets + secret = "PDBLegacyConfiguredSecret123" + monkeypatch.setenv("DISPLAY_THEME", "dark") + monkeypatch.setenv("DB_PASSWORD", secret) + relative = "deployment/runtime/launch.py" + path = kit / relative + path.write_text(path.read_text() + "\n# dark\n" + (f"# {secret}\n" if embed_secret else "")) + _update_indexed_hashes(kit, relative) + report = validate_static_kit(kit) + fails = bool(secrets) and embed_secret + assert (_status(report, "NO_SECRET_CONTENT") == ValidationCheckStatus.FAIL) == fails + if not fails: + assert report.final_state.value == "STATIC_VALID" + assert secret not in report.model_dump_json() + + +def test_current_generation_serializes_explicit_empty_secret_set(tmp_path, monkeypatch): + kit = _kit(monkeypatch, tmp_path) + payload = json.loads((kit / "deployment/manifest.json").read_text()) + assert payload["configuration_secret_names"] == [] + assert "configuration_secret_names" in _load_manifest(kit).model_fields_set + + +@pytest.mark.parametrize("omit", [False, True]) +@pytest.mark.parametrize("mode", ["source", "package"]) +def test_application_artifact_default_has_no_absence_ambiguity(tmp_path, monkeypatch, omit, mode): + kit = _kit(monkeypatch, tmp_path) + path = kit / "deployment/manifest.json" + payload = json.loads(path.read_text()) + payload["deployment_mode"] = mode + if omit: + payload.pop("application_artifact", None) + else: + payload["application_artifact"] = None + path.write_text(json.dumps(payload, indent=2) + "\n") + _refresh_manifest_index(kit) + report = validate_static_kit(kit) + assert (_status(report, "APPLICATION_ARTIFACT_HASH") == ValidationCheckStatus.FAIL) == ( + mode == "package" + ) + if mode == "source": + assert report.final_state.value == "STATIC_VALID" + + +@pytest.mark.parametrize("secrets", [None, []]) +def test_runtime_isolation_still_uses_presence_names(tmp_path, monkeypatch, secrets): + kit = _kit(monkeypatch, tmp_path) + configure_manifest(kit, ["DISPLAY_THEME", "DB_PASSWORD", "LOCALAPPDATA"], secrets) + runtime_manifest = _manifest(kit) + assert effective_configuration_secret_names(runtime_manifest) == ( + ["DISPLAY_THEME", "DB_PASSWORD", "LOCALAPPDATA"] if secrets is None else [] + ) + monkeypatch.setenv("DISPLAY_THEME", "dark") + monkeypatch.setenv("DB_PASSWORD", "PDBLegacyConfiguredSecret123") + harness_root = tmp_path / "harness" + environment = _runtime_environment(_load_manifest(kit), harness_root) + assert "DISPLAY_THEME" not in environment + assert "DB_PASSWORD" not in environment + assert environment["LOCALAPPDATA"] == str(harness_root) + + +@pytest.mark.parametrize("kind", ["approved", "application"]) +def test_legacy_secret_set_reaches_wheel_member_scanners(tmp_path, monkeypatch, caplog, kind): + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + kit = tmp_path / "kit" + if kind == "approved": + wheel = _make_wheel(tmp_path) + generate_deployment_kit( + _repository("optional_map_app"), + kit, + selected_extras=["map"], + artifact_values=[f"proxy-tools={wheel}"], + bootstrap_mode="online_cmd", + ) + relative = f"deployment/wheels/{wheel.name}" + member = "proxy_tools/settings.txt" + else: + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + wheel = _make_application_wheel(tmp_path) + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + application_wheel=wheel, + bootstrap_mode="online_cmd", + ) + relative = f"deployment/application/{wheel.name}" + member = "installed_app/view.html" + assert validate_static_kit(kit).final_state.value == "STATIC_VALID" + secret = "PDBLegacyConfiguredSecret123" + _rewrite_application_wheel(kit / relative, additions={member: f"ordinary text {secret}"}) + _refresh_manifest_wheel_hash(kit, relative, approved=kind == "approved") + configure_manifest(kit, ["DB_PASSWORD"]) + monkeypatch.setenv("DB_PASSWORD", secret) + report = validate_static_kit(kit) + assert _status(report, "WHEEL_SECURITY") == ValidationCheckStatus.FAIL + assert secret not in report.model_dump_json() + caplog.text + assert secret not in (kit / "deployment/manifest.json").read_text() diff --git a/tests/test_module_resource_anchors.py b/tests/test_module_resource_anchors.py new file mode 100644 index 0000000..8c44e4f --- /dev/null +++ b/tests/test_module_resource_anchors.py @@ -0,0 +1,250 @@ +"""Explicit module anchors select the module's containing resource directory.""" + +from pathlib import Path + +import pytest +from test_dependency_authority import assess +from test_files_package_keyword import resource_project + +from python_deployment_builder.analysis.inventory import _module_files +from python_deployment_builder.analysis.module_resolution import module_resource_roots +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.planning.planner import create_deployment_plan + + +@pytest.mark.parametrize( + "argument", ["'app.config'", "anchor='app.config'", "package='app.config'"] +) +def test_literal_module_anchor_promotes_resource(tmp_path, argument): + path = resource_project(tmp_path, "from importlib.resources import files", f"files({argument})") + (tmp_path / "src/app/config.py").write_text("VALUE=1\n") + assessment = assess(tmp_path) + assert any(item.path == path for item in assessment.resources) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert path in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize( + ("imports", "function"), + [ + ("import importlib.resources", "importlib.resources.files"), + ("import importlib.resources as resources", "resources.files"), + ("from importlib import resources as r", "r.files"), + ("from importlib.resources import files", "files"), + ("from importlib.resources import files as rf", "rf"), + ], +) +@pytest.mark.parametrize("keyword", ["", "anchor=", "package="]) +def test_module_anchor_bindings(tmp_path, imports, function, keyword): + path = resource_project(tmp_path, imports, f"{function}({keyword}'app.config')") + (tmp_path / "src/app/config.py").write_text("") + assert path in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize( + ("source_root", "mapping"), + [(".", None), ("src", None), ("lib", "{''='lib'}"), ("lib", "{'app'='lib/app'}")], +) +@pytest.mark.parametrize("namespace", [False, True]) +def test_module_anchor_source_roots_and_namespace_parent(tmp_path, source_root, mapping, namespace): + path = resource_project( + tmp_path, + "from importlib.resources import files", + "files('app.config')", + source_root, + mapping, + ) + package = tmp_path / source_root / "app" + (package / "config.py").write_text("") + if namespace: + (package / "__init__.py").unlink() + assessment = assess(tmp_path) + assert path in {item.path for item in assessment.resources} + assert (package / "__init__.py").exists() != namespace + plan = create_deployment_plan(assessment, repository_root=tmp_path) + if mapping == "{'app'='lib/app'}": + # Exact package-dir relocation is installed-only under the existing + # planner. Resource discovery must not invent a source-mode override. + assert plan.deployment_mode == "package" + assert plan.deployment_mode_condition == "DEPLOYMENT_MODE_CONFLICT" + # The non-wheel-backed adjacent resource creates the established + # installed-only/source-resource conflict, not an omitted resource. + else: + assert plan.deployment_mode == "source" + assert path in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("module", ["app", "app.child", "app.config", "config"]) +def test_package_subpackage_module_and_top_level_anchors(tmp_path, module): + resource_project(tmp_path, "from importlib.resources import files", f"files('{module}')") + (tmp_path / "src/app/config.py").write_text("") + (tmp_path / "src/config.py").write_text("") + (tmp_path / "src/defaults.json").write_text("{}") + child = tmp_path / "src/app/child" + child.mkdir() + (child / "__init__.py").write_text("") + (child / "defaults.json").write_text("{}") + expected = { + "app": "src/app/defaults.json", + "app.child": "src/app/child/defaults.json", + "app.config": "src/app/defaults.json", + "config": "src/defaults.json", + }[module] + assert {item.path for item in assess(tmp_path).resources} == {expected} + + +@pytest.mark.parametrize( + ("mappings", "module", "location"), + [ + ({"app": "lib"}, "app.config", "lib/config"), + ({"app": "lib", "app.child": "code"}, "app.child.config", "code/config"), + ({"app.child": "code"}, "app.child", "code"), + ], +) +def test_inventory_and_resource_anchors_share_mapping_locations( + tmp_path, mappings, module, location +): + module_file = (tmp_path / location).with_suffix(".py") + module_file.parent.mkdir(parents=True, exist_ok=True) + module_file.write_text("") + assert module_file in _module_files(tmp_path, module, ["."], mappings) + assert module_resource_roots(tmp_path, module, ["."], mappings) == [module_file.parent] + + +def test_longest_parent_mapping_promotes_resource(tmp_path): + resource_project( + tmp_path, + "from importlib.resources import files", + "files('app.child.config')", + "lib", + "{'app'='lib/app', 'app.child'='code'}", + ) + child = tmp_path / "code" + child.mkdir() + (child / "__init__.py").write_text("") + (child / "config.py").write_text("") + (child / "defaults.json").write_text("{}") + assert {item.path for item in assess(tmp_path).resources} == {"code/defaults.json"} + + +def test_regular_package_wins_same_named_module(tmp_path): + path = resource_project( + tmp_path, "from importlib.resources import files", "files('app.config')" + ) + (tmp_path / "src/app/config.py").write_text("") + package = tmp_path / "src/app/config" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "defaults.json").write_text("{}") + resources = {item.path for item in assess(tmp_path).resources} + assert "src/app/config/defaults.json" in resources + assert path not in resources + + +def test_concrete_module_wins_namespace_directory(tmp_path): + resource_project(tmp_path, "from importlib.resources import files", "files('app.config')") + (tmp_path / "src/app/config.py").write_text("") + namespace = tmp_path / "src/app/config" + namespace.mkdir() + (namespace / "defaults.json").write_text("{}") + assert {item.path for item in assess(tmp_path).resources} == {"src/app/defaults.json"} + + +@pytest.mark.parametrize( + "anchor", + ["unknown", "f'app.{unknown}'", "'app.missing'", "'../app.config'", "'app/config'"], +) +def test_unresolved_or_invalid_module_anchor(tmp_path, anchor): + path = resource_project(tmp_path, "from importlib.resources import files", f"files({anchor})") + (tmp_path / "src/app/config.py").write_text("") + assert path not in {item.path for item in assess(tmp_path).resources} + + +def test_static_assignment_module_anchor(tmp_path): + path = resource_project( + tmp_path, "from importlib.resources import files\nANCHOR='app.config'", "files(ANCHOR)" + ) + (tmp_path / "src/app/config.py").write_text("") + assert path in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize( + ("imports", "anchor"), + [("import app.config", "app.config"), ("import app.config as config", "config")], +) +def test_module_object_anchor_remains_outside_bounded_value_model(tmp_path, imports, anchor): + path = resource_project( + tmp_path, f"from importlib.resources import files\n{imports}", f"files({anchor})" + ) + (tmp_path / "src/app/config.py").write_text("") + assert path not in {item.path for item in assess(tmp_path).resources} + + +def test_module_anchor_rejects_external_source_root_and_mapping(tmp_path): + root = tmp_path / "repository" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "config.py").write_text("") + assert module_resource_roots(root, "config", ["../outside"]) == [] + assert module_resource_roots(root, "app.config", ["."], {"app": "../outside"}) == [] + + +def test_module_anchor_rejects_symlink_escape(tmp_path): + root = tmp_path / "repository" + root.mkdir() + outside = tmp_path / "outside.py" + outside.write_text("") + try: + (root / "config.py").symlink_to(outside) + except OSError: + pytest.skip("Host cannot create symlinks") + assert module_resource_roots(root, "config", ["."]) == [] + + +@pytest.mark.parametrize("leaf", ["__init__.py", "module.py"]) +def test_unsafe_concrete_anchor_cannot_fall_back_to_namespace(tmp_path, monkeypatch, leaf): + package = tmp_path / "module" + package.mkdir() + unsafe = package / leaf if leaf == "__init__.py" else tmp_path / leaf + unsafe.write_text("") + original = Path.is_symlink + monkeypatch.setattr(Path, "is_symlink", lambda path: path == unsafe or original(path)) + assert module_resource_roots(tmp_path, "module", ["."]) == [] + + +@pytest.mark.parametrize("configured", [False, True]) +def test_module_adjacent_resource_receives_release_security_scanning( + tmp_path, monkeypatch, configured +): + source = tmp_path / "source" + imports = "from importlib.resources import files" + if configured: + imports += "\nfrom os import getenv\nPASSWORD=getenv(key='DB_PASSWORD')" + path = resource_project(source, imports, "files('app.config')") + (source / "src/app/config.py").write_text("") + secret = "PDBModuleConfiguredSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + (source / path).write_text(secret if configured else "API_KEY = 'sk-abcdefghijklmnop'\n") + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + with pytest.raises(PreparationError, match="NO_SECRET_VALUES") as error: + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + ) + assert secret not in str(error.value) + assert not (tmp_path / "kit").exists() From 94b3fd52e3cfa2a6c4f5ebbb287b6255e291b43a Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:41:01 -0400 Subject: [PATCH 55/59] Recognize as_file resources and relative literal dynamic imports --- ...pr9-as-file-relative-imports-2026-09-12.md | 147 +++++++ scripts/verify_resource_wrappers.py | 218 ++++++++++ .../analysis/inventory.py | 47 +- .../analysis/resources.py | 33 +- tests/test_as_file_relative_imports.py | 404 ++++++++++++++++++ 5 files changed, 841 insertions(+), 8 deletions(-) create mode 100644 docs/acceptance/pr9-as-file-relative-imports-2026-09-12.md create mode 100644 scripts/verify_resource_wrappers.py create mode 100644 tests/test_as_file_relative_imports.py diff --git a/docs/acceptance/pr9-as-file-relative-imports-2026-09-12.md b/docs/acceptance/pr9-as-file-relative-imports-2026-09-12.md new file mode 100644 index 0000000..4732c3f --- /dev/null +++ b/docs/acceptance/pr9-as-file-relative-imports-2026-09-12.md @@ -0,0 +1,147 @@ +# PR #9: as_file resources and relative literal dynamic imports + +Starting head: `0d26ebe9b536335d278fabf44477a22fa13a42c1`, branch +`milestone-6-1-generation-contract`. Exact-head review submitted +`2026-09-12T00:57:36Z`: + +| Priority | Finding | Thread | Comment | +| --- | --- | --- | --- | +| P2 | Recognize resources passed through as_file | PRRT_kwDOT9hvCc6hrm21 | 3994501628 | +| P2 | Resolve relative literal dynamic imports | PRRT_kwDOT9hvCc6hrm27 | 3994501635 | + +Authoritative accounting: 122 findings/comments; 119 unresolved inline threads; +three review-level-only findings (historical #35, approved-wheel PEP 440 semantic +version comparison, and complete locked sync-command validation); one +NOT_APPLICABLE inline finding for the automatic flat-package plus loose-module +premise. No thread closeout, review dismissal, or merge is part of this work. + +## Exact reproductions + +Two failing regressions preceded production edits. For as_file, the test directly +proves that `_importlib_resource_path_values` already resolves the inner +`files('app') / 'model.dat'`, while `_path_uses` supplies no expression for the +outer wrapper. Assessment therefore produced no resource. The relative import +fixture retained `src/app/examples/plugin.py` as `example_or_snippet`: the old +collector rejected the leading dot without consulting the literal package. + +`scripts/verify_resource_wrappers.py` uses disposable, Git-tracked fixtures and +the exact starting-head analysis functions to reproduce generation and execution. +Pinned acquisition/lock preparation are stubbed only for these synthetic kits; +the generated source is executed by managed Python 3.12.14 in isolation. + +| Fixture | Starting-head generated kit | Corrected generated kit | +| --- | --- | --- | +| as_file model.dat | STATIC_VALID; runtime FileNotFoundError | STATIC_VALID; runtime reads MODEL | +| .examples.plugin anchored at app | STATIC_VALID; runtime ModuleNotFoundError for app.examples | STATIC_VALID; plugin returns 42 | + +No wheel, kit, environment, ZIP, or disposable repository from these experiments +is committed. + +## as_file dispatch and runtime evidence + +The existing resource import-binding collector now records direct/aliased +as_file imports alongside files and legacy resource functions. Proven module +bindings also recognize `.as_file`. A wrapper dispatch sends its AST argument +directly to the existing Traversable resolver and records read evidence. It does +not first reinterpret the expression as a generic path and does not trace the +context manager's yielded variable. Both an unused yielded path and a path +passed only to another library retain the concrete resource. + +Direct probes on Python 3.11.16 and 3.12.14 establish an important signature +limitation: `as_file(traversable=...)` raises TypeError in both interpreters. +The singledispatch wrapper requires a positional argument, despite the displayed +parameter name. PDB therefore accepts the proven one-positional/no-keyword form, +using `call_argument`, and leaves keyword/extra/dynamic/unproven forms unresolved. +No keyword support is inferred solely from signature appearance. + +Slash and joinpath expressions, package and module anchors, anchor=/package= +spellings, custom source roots, and exact/longest-parent package-dir mappings all +reuse the preceding resolver. Existing implicit-caller and anchor containment +semantics are untouched. + +Directory audit: existing filesystem directories work through pathlib-backed +Traversables in both tested runtimes. Zipped directory materialization fails +with IsADirectoryError on 3.11.16 and succeeds on 3.12.14; zipped files work in +both. This agrees with the documented addition of directory Traversable support +in [Python 3.12 as_file](https://docs.python.org/3.12/library/importlib.resources.html#importlib.resources.as_file). +PDB records the concrete directory and uses its existing directory-resource +coverage rules to retain descendants, without inventing individually accessed +filenames. Tests cover a non-conventional directory name and nested model file. + +Release tests reject configured and obvious secrets in referenced text, without +exposing configured values in errors. Git-backed tests reject dirty and +untracked referenced model files. The ordinary shared secret-file, containment, +role, and provenance policies remain in force. + +## Relative literal dynamic imports + +One pure helper consumes name at position zero/name= and, for a relative name, +package at position one/package=. Both use the established positional-wins +`call_argument` policy. Literal relative names are resolved by +`importlib.util.resolve_name`; this function resolves strings without importing +target code. Package anchors and resulting absolute module identities must have +valid dotted identifier components. Above-root resolution errors fail closed. + +Tests prove `.plugin` + app, `.examples.plugin` + app, `..plugin` + app.sub, +positional/keyword packages, name=, module/direct-function aliases, duplicates, +and absolute names with ignored package arguments. Missing/dynamic package +anchors, dynamic/f-string names, and malformed results remain unresolved. +The behavior follows [Python's relative import resolution contract](https://docs.python.org/3.12/library/importlib.html#importlib.util.resolve_name). + +Resolved identities feed the existing `_module_files` / `module_locations` +pipeline. Tests retain examples/docs/tests targets, regular initializers, +namespace ancestors, custom lib roots, exact and longest-parent mappings, and +source staging. Installed-only mappings continue through the existing package +mode constraints; no planner special case was added. + +## Bounded adjacent audit + +The Traversable terminal operations already supported by resources.py continue +to dispatch through the existing path-use machinery. No additional wrapper gap +was established within the requested scope, and no new resource API families +were added beyond as_file. + +The builtin import audit found one adjacent false-evidence case: the old +collector treated an absolute-looking string passed to `__import__` as absolute +even when an explicit level changed its meaning. The bounded correction rejects +nonzero or unresolved level values, while retaining omitted/literal-zero level +absolute calls. Builtin globals/locals/fromlist/relative-context emulation was +not added, and import_module's package argument is never applied to the builtin. + +`package=__package__` remains outside the literal dynamic-import model. Existing +source-package contexts can contain multiple identities for overlapping roots; +physical package-dir relocation and script-versus-package execution add further +ambiguity. They do not provide a generally unique runtime __package__ value. +No new dataflow/context engine was introduced; a regression records this limit. + +## Acceptance, compatibility, and quality + +Read-only comparisons with starting-head analysis confirm identical dependency, +entry-point, configuration, and resource evidence in all acceptance repositories. +Their Git SHAs and working-tree status remain unchanged. + +| Repository | SHA | Contract | +| --- | --- | --- | +| SimpleGeorefGUI | f484570d89fb1f9e9170fac915475358dfc1234e | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python and 77 data members | +| Geo Map Explanation Extractor | 5e7b321d0aeb9ba1d586bfc548c79793d84c6033 | source / SOURCE_COMPATIBLE | +| TN Coordinate Converter | e1e7a1588c37a99c2d02efaf3eef3d04636f12f0 | source / SOURCE_COMPATIBLE; existing kit STATIC_VALID with reviewed proxy-tools==0.1.0 | + +SGG's GUI target remains `simple_georef_gui_app.georef_main:main`. No new resource +or configuration evidence was discovered in these repositories by this change. +No new acceptance-repository runtime execution is claimed. + +ANALYSIS_SCHEMA_VERSION remains 1.4; PLANNING_SCHEMA_VERSION remains 1.3. +No persisted model or compatibility flag changes were required. + +The single complete suite passed: **1401 passed, 4 skipped in 305.91 seconds**, +up 85 passes from the 1316-pass baseline, with no additional skips. Ruff and +`git diff --check` pass, including the staged diff. Inspection confirms only the +two production analysis files, focused regression file, disposable verification +script, and this report changed. + +The full suite preserves legacy-manifest absent-versus-empty secret handling, +source/generated/wheel scanning parity, module resource anchors, PEP 621 field +authority, approved-extra separation, artifact-path uniqueness, exact sync and +reverse lock completeness, mode-aware pip checks, entry-point extras, short +secrets and LOCALAPPDATA ordering, wheel dependency/security policies, +MANIFEST/package-data guards, provenance, rollback, and deterministic packaging. diff --git a/scripts/verify_resource_wrappers.py b/scripts/verify_resource_wrappers.py new file mode 100644 index 0000000..83b5756 --- /dev/null +++ b/scripts/verify_resource_wrappers.py @@ -0,0 +1,218 @@ +"""Disposable interpreter evidence for as_file and relative import_module.""" + +import argparse +import ast +import json +import subprocess +import tempfile +import zipfile +from pathlib import Path +from unittest.mock import patch + +import python_deployment_builder.analysis.inventory as inventory +import python_deployment_builder.analysis.resources as resources +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.generator import generate_deployment_kit +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.validation.static import validate_static_kit + +PROBE = """ +import importlib, importlib.resources as resources, importlib.util, json, sys +result = {'version': sys.version.split()[0]} +for kind, target in [('file', 'model.dat'), ('directory', 'models')]: + for spelling in ['positional', 'keyword']: + try: + traversable = resources.files('app') / target + context = (resources.as_file(traversable) if spelling == 'positional' + else resources.as_file(traversable=traversable)) + with context as path: + text = ((path / 'nested.dat').read_text() if kind == 'directory' + else path.read_text()) + result[kind + '_' + spelling] = text + except Exception as error: + result[kind + '_' + spelling] = type(error).__name__ +result['relative'] = {} +for name, package in [('.examples.plugin', 'app'), ('..examples.plugin', 'app.sub'), + ('...plugin', 'app'), ('app.examples.plugin', 'other')]: + try: + resolved = importlib.util.resolve_name(name, package) + value = importlib.import_module(name, package).run() + result['relative'][name + ':' + package] = [resolved, value] + except Exception as error: + result['relative'][name + ':' + package] = type(error).__name__ +assert result['file_positional'] == 'MODEL' +assert result['file_keyword'] == result['directory_keyword'] == 'TypeError' +assert result['relative']['.examples.plugin:app'] == ['app.examples.plugin', 42] +assert result['relative']['..examples.plugin:app.sub'] == ['app.examples.plugin', 42] +assert result['relative']['...plugin:app'] == 'ImportError' +sys.path.insert(0, 'resources.zip') +for kind, target in [('file', 'model.dat'), ('directory', 'models')]: + try: + with resources.as_file(resources.files('zipped') / target) as path: + result['zip_' + kind] = ((path / 'nested.dat').read_text() if kind == 'directory' + else path.read_text()) + except Exception as error: + result['zip_' + kind] = type(error).__name__ +assert result['zip_file'] == 'MODEL' +assert result['zip_directory'] == ( + 'DIRECTORY' if sys.version_info >= (3,12) else 'IsADirectoryError') +print(json.dumps(result)) +""" + + +def old_function(module, name): + source = subprocess.check_output( + [ + "git", + "show", + "0d26ebe9b536335d278fabf44477a22fa13a42c1:src/" + + module.__name__.replace(".", "/") + + ".py", + ], + text=True, + ) + node = next( + item + for item in ast.parse(source).body + if isinstance(item, ast.FunctionDef) and item.name == name + ) + namespace = dict(module.__dict__) + exec(compile(ast.Module(body=[node], type_ignores=[]), "", "exec"), namespace) + return namespace[name] + + +def deployment_probe(root, python): + old_literal = old_function(resources, "_literal_evidence") + old_literal.__globals__["_resource_import_bindings"] = old_function( + resources, "_resource_import_bindings" + ) + old_imports = old_function(inventory, "_imported_modules") + result = {} + fake_uv = root / "uv.exe" + fake_uv.write_bytes(b"verified uv") + for kind in ["as_file", "relative_import"]: + source = root / kind / "source" + package = source / "src/app" + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + (package / "model.dat").write_text("MODEL") + examples = package / "examples" + examples.mkdir() + (examples / "__init__.py").write_text("") + (examples / "plugin.py").write_text("def run(): return 42\n") + (package / "main.py").write_text( + "from importlib.resources import files, as_file\ndef main():\n" + " with as_file(files('app') / 'model.dat') as path: return path.read_text()\n" + if kind == "as_file" + else "import importlib\ndef main():\n" + " return importlib.import_module('.examples.plugin', package='app').run()\n" + ) + (source / "pyproject.toml").write_text( + "[project]\nname='demo'\nversion='1.0'\nrequires-python='>=3.12'\n" + "[project.scripts]\ndemo='app.main:main'\n" + ) + (source / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n" + "[[package]]\nname='demo'\nversion='1.0'\nsource={virtual='.'}\n" + ) + for args in [ + ("init",), + ("add", "."), + ( + "-c", + "user.name=PDB Test", + "-c", + "user.email=pdb@example.invalid", + "commit", + "-m", + "fixture", + ), + ]: + subprocess.run(["git", "-C", str(source), *args], check=True, capture_output=True) + for label in ["before", "after"]: + kit = root / kind / label + with ( + patch.object( + resources, + "_literal_evidence", + old_literal if label == "before" else resources._literal_evidence, + ), + patch.object( + inventory, + "_imported_modules", + old_imports if label == "before" else inventory._imported_modules, + ), + patch( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + return_value=fake_uv, + ), + patch( + "python_deployment_builder.generation.generator.prepare_lockfile", + side_effect=lambda path, *args, **kw: LockPreparationResult( + path=path / "uv.lock", created=False, checked=True, commands=() + ), + ), + ): + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + bootstrap_mode="online_cmd", + ) + report = validate_static_kit(kit) + assert report.final_state.value == "STATIC_VALID" + run = subprocess.run( + [str(python), "-B", "-E", "-s", "-c", "from app.main import main; print(main())"], + cwd=kit / "src", + capture_output=True, + text=True, + check=False, + ) + result[kind + "_" + label] = { + "static": report.final_state.value, + "exit": run.returncode, + "result": run.stdout.strip() + if run.returncode == 0 + else run.stderr.splitlines()[-1], + } + assert (run.returncode == 0) == (label == "after") + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("python", type=Path, nargs="+") + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="pdb-resource-wrappers-") as temporary: + root = Path(temporary) + for relative, value in { + "app/__init__.py": "", + "app/model.dat": "MODEL", + "app/models/nested.dat": "DIRECTORY", + "app/sub/__init__.py": "", + "app/examples/__init__.py": "", + "app/examples/plugin.py": "def run(): return 42\n", + }.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(value) + with zipfile.ZipFile(root / "resources.zip", "w") as archive: + for name, text in { + "zipped/__init__.py": "", + "zipped/model.dat": "MODEL", + "zipped/models/nested.dat": "DIRECTORY", + }.items(): + archive.writestr(name, text) + for python in args.python: + result = subprocess.run( + [str(python), "-B", "-E", "-s", "-c", PROBE], + cwd=root, + capture_output=True, + text=True, + check=True, + ) + print(result.stdout.strip()) + print(json.dumps(deployment_probe(root, args.python[-1]), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/inventory.py b/src/python_deployment_builder/analysis/inventory.py index 9b722fa..82f65c8 100644 --- a/src/python_deployment_builder/analysis/inventory.py +++ b/src/python_deployment_builder/analysis/inventory.py @@ -7,6 +7,7 @@ import re from collections import Counter from dataclasses import dataclass +from importlib.util import resolve_name from pathlib import Path, PurePosixPath from packaging.requirements import InvalidRequirement, Requirement @@ -310,6 +311,43 @@ def _source_package_contexts( return contexts +def _literal_dynamic_module_name(node: ast.Call, *, builtin: bool) -> str | None: + """Resolve literal import_module strings without importing target code. + + Builtin __import__ uses globals/level, not import_module's package anchor. + Only its existing absolute (level zero) surface is modeled here. + """ + + target = call_argument(node, position=0, keyword="name") + if not isinstance(target, ast.Constant) or not isinstance(target.value, str): + return None + module = target.value + if builtin: + level = call_argument(node, position=4, keyword="level") + if level is not None and not ( + isinstance(level, ast.Constant) and isinstance(level.value, int) and level.value == 0 + ): + return None + if module.startswith("."): + return None + elif module.startswith("."): + package = call_argument(node, position=1, keyword="package") + if not ( + isinstance(package, ast.Constant) + and isinstance(package.value, str) + and package.value + and all(part.isidentifier() for part in package.value.split(".")) + ): + return None + try: + module = resolve_name(module, package.value) + except (ImportError, ValueError): + return None + if module and all(part.isidentifier() for part in module.split(".")): + return module + return None + + def _imported_modules( tree: ast.AST, source_path: Path, @@ -358,7 +396,6 @@ def _imported_modules( if alias.name != "*" ) elif isinstance(node, ast.Call): - target: ast.AST | None = None function = node.func direct_import = isinstance(function, ast.Name) and ( function.id == "__import__" or function.id in import_module_functions @@ -370,10 +407,10 @@ def _imported_modules( and function.value.id in importlib_modules ) if direct_import or module_import: - target = call_argument(node, position=0, keyword="name") - if isinstance(target, ast.Constant) and isinstance(target.value, str): - module = target.value - if module and all(part.isidentifier() for part in module.split(".")): + module = _literal_dynamic_module_name( + node, builtin=isinstance(function, ast.Name) and function.id == "__import__" + ) + if module is not None: modules.append((module, node.lineno)) return modules diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index d1a1690..7d943c9 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -494,7 +494,7 @@ def bind_pattern(target: ast.AST, values: list[ast.AST]) -> None: def _resource_import_bindings( tree: ast.AST, -) -> tuple[set[str], set[str], dict[str, str], set[str], set[str]]: +) -> tuple[set[str], set[str], dict[str, str], set[str], set[str], set[str]]: """Return proven importlib.resources and pkgutil resource bindings.""" modules: set[str] = set() @@ -502,6 +502,7 @@ def _resource_import_bindings( reads: dict[str, str] = {} pkgutil_modules: set[str] = set() pkgutil_get_data: set[str] = set() + as_files: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: @@ -518,13 +519,15 @@ def _resource_import_bindings( for alias in node.names: if alias.name == "files": files.add(alias.asname or alias.name) + elif alias.name == "as_file": + as_files.add(alias.asname or alias.name) elif alias.name in _LEGACY_IMPORTLIB_RESOURCE_READS: reads[alias.asname or alias.name] = alias.name elif node.module == "pkgutil": for alias in node.names: if alias.name == "get_data": pkgutil_get_data.add(alias.asname or alias.name) - return modules, files, reads, pkgutil_modules, pkgutil_get_data + return modules, files, reads, pkgutil_modules, pkgutil_get_data, as_files def _resource_package_roots( @@ -1202,6 +1205,7 @@ def _literal_evidence( read_bindings, pkgutil_modules, pkgutil_get_data, + as_file_bindings, ) = _resource_import_bindings(tree) for node in ast.walk(tree): if not isinstance(node, ast.Call): @@ -1229,7 +1233,30 @@ def _literal_evidence( read_bindings=read_bindings, ) uses: list[tuple[ast.AST, str, list[str] | None, str]] = [] - if pkgutil_resource is not None: + call_name = _qualified_name(node.func) + if call_name in as_file_bindings or any( + call_name == f"{binding}.as_file" for binding in module_bindings + ): + # The 3.11/3.12 singledispatch wrapper requires a positional + # argument: traversable= raises TypeError despite its signature. + if len(node.args) != 1 or node.keywords: + continue + traversable = call_argument(node, position=0, keyword="traversable") + values = _importlib_resource_path_values( + traversable, + root=root, + source_path=path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + module_bindings=module_bindings, + files_bindings=files_bindings, + ) + # Do not reinterpret an unknown Traversable as generic path + # syntax or trace the context manager's yielded variable. + uses.append((node, "read", values or [], "importlib.resources.as_file()")) + elif pkgutil_resource is not None: uses.append((node, "read", pkgutil_resource, "pkgutil.get_data()")) elif legacy_resource is not None: function, values = legacy_resource diff --git a/tests/test_as_file_relative_imports.py b/tests/test_as_file_relative_imports.py new file mode 100644 index 0000000..a66e733 --- /dev/null +++ b/tests/test_as_file_relative_imports.py @@ -0,0 +1,404 @@ +"""Resource wrappers and literal relative dynamic imports reach existing resolvers.""" + +import ast +import subprocess + +import pytest +from test_dependency_authority import assess +from test_files_package_keyword import resource_project + +from python_deployment_builder.analysis.inventory import ( + _imported_modules, + _literal_dynamic_module_name, +) +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.resources import _importlib_resource_path_values, _path_uses +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.planning.planner import create_deployment_plan + + +def as_file_project( + root, + imports="from importlib.resources import as_file, files", + call="as_file(files('app') / 'model.dat')", + source_root="src", + mapping=None, +): + resource_project(root, imports, "files('app')", source_root, mapping) + package = root / source_root / "app" + (package / "main.py").write_text( + f"{imports}\ndef main():\n with {call} as path:\n return path.read_bytes()\n" + ) + (package / "model.dat").write_bytes(b"model fixture") + return (package / "model.dat").relative_to(root).as_posix() + + +def relative_project( + root, + call="importlib.import_module('.examples.plugin', package='app')", + imports="import importlib", + scope="examples", + source_root="src", + mapping=None, +): + resource_project(root, imports, "unused", source_root, mapping) + package = root / source_root / "app" + (package / "main.py").write_text(f"{imports}\ndef main(): return {call}.run()\n") + target = package / scope + target.mkdir() + (target / "__init__.py").write_text("") + (target / "plugin.py").write_text("def run(): return 42\n") + return (target / "plugin.py").relative_to(root).as_posix() + + +def test_as_file_dispatches_already_resolvable_traversable(tmp_path): + relative = as_file_project(tmp_path) + source = tmp_path / "src/app/main.py" + outer = next( + n + for n in ast.walk(ast.parse(source.read_text())) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id == "as_file" + ) + assert _path_uses(outer) == [] + assert _importlib_resource_path_values( + outer.args[0], + root=tmp_path, + source_path=source, + source_roots=["src"], + project=None, + assignments={}, + returns={}, + module_bindings=set(), + files_bindings={"files"}, + ) == [relative] + assessment = assess(tmp_path) + assert relative in {resource.path for resource in assessment.resources} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize( + ("imports", "function"), + [ + ("from importlib.resources import as_file, files", "as_file"), + ( + "import importlib.resources\nfrom importlib.resources import files", + "importlib.resources.as_file", + ), + ( + "import importlib.resources as resources\nfrom importlib.resources import files", + "resources.as_file", + ), + ("from importlib.resources import as_file as materialize, files", "materialize"), + ( + "from importlib import resources as r\nfrom importlib.resources import files", + "r.as_file", + ), + ], +) +@pytest.mark.parametrize( + "expression", ["files('app') / 'model.dat'", "files('app').joinpath('model.dat')"] +) +def test_as_file_binding_and_traversable_matrix(tmp_path, imports, function, expression): + relative = as_file_project(tmp_path, imports, f"{function}({expression})") + assessment = assess(tmp_path) + resource = next(item for item in assessment.resources if item.path == relative) + assert resource.access_mode == "read" + assert any("as_file()" in evidence.detail for evidence in resource.evidence) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("anchor", ["'app'", "anchor='app'", "package='app'", "'app.config'"]) +@pytest.mark.parametrize( + ("source_root", "mapping"), + [("src", None), ("lib", "{''='lib'}"), ("lib", "{'app'='lib/app'}")], +) +def test_as_file_preserves_anchor_and_source_mapping(tmp_path, anchor, source_root, mapping): + relative = as_file_project( + tmp_path, + call=f"as_file(files({anchor}) / 'model.dat')", + source_root=source_root, + mapping=mapping, + ) + (tmp_path / source_root / "app/config.py").write_text("") + assert relative in {item.path for item in assess(tmp_path).resources} + + +def test_as_file_longest_parent_mapping(tmp_path): + as_file_project( + tmp_path, + call="as_file(files('app.child.config') / 'model.dat')", + source_root="lib", + mapping="{'app'='lib/app','app.child'='code'}", + ) + child = tmp_path / "code" + child.mkdir() + (child / "__init__.py").write_text("") + (child / "config.py").write_text("") + (child / "model.dat").write_text("model") + assert {item.path for item in assess(tmp_path).resources} == {"code/model.dat"} + + +@pytest.mark.parametrize( + "call", + [ + "as_file(unknown)", + "as_file(files(unknown) / 'model.dat')", + "as_file(files('app') / unknown)", + "as_file()", + "as_file(traversable=files('app') / 'model.dat')", + "as_file(files('app') / 'model.dat', foo=True)", + "as_file(files('app') / '../model.dat')", + ], +) +def test_as_file_invalid_and_dynamic_inputs_are_unresolved(tmp_path, call): + relative = as_file_project(tmp_path, call=call) + assert relative not in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("body", ["pass", "third_party(path)"]) +def test_as_file_read_evidence_requires_no_later_path_use(tmp_path, body): + relative = as_file_project(tmp_path) + main = tmp_path / "src/app/main.py" + main.write_text(main.read_text().replace("return path.read_bytes()", body)) + resource = next(item for item in assess(tmp_path).resources if item.path == relative) + assert resource.access_mode == "read" + + +@pytest.mark.parametrize( + "mapping", ["{'app.examples'='code/docs'}", "{'app'='lib/app','app.examples'='code/docs'}"] +) +def test_relative_import_exact_longest_mapping(tmp_path, mapping): + relative_project(tmp_path, source_root="lib", mapping=mapping) + target = tmp_path / "code/docs" + target.mkdir(parents=True) + (target / "__init__.py").write_text("") + (target / "plugin.py").write_text("def run(): return 42\n") + roles = {item.path: item.role.value for item in assess(tmp_path).file_inventory} + assert roles["code/docs/plugin.py"] == "application_source" + assert roles["code/docs/__init__.py"] == "application_source" + + +def test_unrelated_as_file_does_not_promote_resource(tmp_path): + relative = as_file_project( + tmp_path, "from importlib.resources import files\ndef as_file(value): return value" + ) + assert relative not in {item.path for item in assess(tmp_path).resources} + + +def test_as_file_directory_promotes_descendants_without_yield_dataflow(tmp_path): + as_file_project(tmp_path, call="as_file(files('app') / 'bundle')") + main = tmp_path / "src/app/main.py" + main.write_text( + main.read_text().replace("return path.read_bytes()", "return third_party(path)") + ) + nested = tmp_path / "src/app/bundle/nested/model.dat" + nested.parent.mkdir(parents=True) + nested.write_text("model") + assessment = assess(tmp_path) + assert "src/app/bundle" in {item.path for item in assessment.resources} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "src/app/bundle/nested/model.dat" in _staging_files( + tmp_path, assessment, plan, include=True + ) + + +@pytest.mark.parametrize( + ("imports", "function"), + [ + ("import importlib", "importlib.import_module"), + ("import importlib as il", "il.import_module"), + ("from importlib import import_module", "import_module"), + ("from importlib import import_module as load", "load"), + ], +) +@pytest.mark.parametrize( + "arguments", + [ + "'.examples.plugin', 'app'", + "'.examples.plugin', package='app'", + "name='.examples.plugin', package='app'", + ], +) +def test_relative_dynamic_aliases_and_argument_forms(tmp_path, imports, function, arguments): + relative = relative_project(tmp_path, f"{function}({arguments})", imports) + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + staged = _staging_files(tmp_path, assessment, plan, include=True) + assert relative in staged + assert "src/app/examples/__init__.py" in staged + assert "src/app/__init__.py" in staged + + +@pytest.mark.parametrize( + ("arguments", "expected"), + [ + ("'.plugin', package='app'", "app.plugin"), + ("'..plugin', package='app.sub'", "app.plugin"), + ("'.', package='app'", "app"), + ("'app.plugin', package='other'", "app.plugin"), + ("'app.plugin', package=unknown", "app.plugin"), + ("'.plugin', 'app', package='other'", "app.plugin"), + ("'.plugin', 'app', name='other'", "app.plugin"), + ("'...plugin', package='app.sub'", None), + ("'.plugin'", None), + ("name=unknown, package='app'", None), + ("'.plugin', package=unknown", None), + ("f'app.{unknown}', package='app'", None), + ("'.bad-name', package='app'", None), + ("'.plugin', package='app..sub'", None), + ("'.plugin', package=__package__", None), + ], +) +def test_literal_relative_resolution_rules(arguments, expected): + node = ast.parse(f"import_module({arguments})", mode="eval").body + assert _literal_dynamic_module_name(node, builtin=False) == expected + + +@pytest.mark.parametrize("scope", ["examples", "docs", "tests"]) +@pytest.mark.parametrize("namespace", [False, True]) +def test_relative_import_promotes_excluded_scope_and_preserves_namespace( + tmp_path, scope, namespace +): + relative = relative_project( + tmp_path, f"importlib.import_module('.{scope}.plugin', 'app')", scope=scope + ) + if namespace: + (tmp_path / "src/app/__init__.py").unlink() + (tmp_path / "src/app" / scope / "__init__.py").unlink() + assessment = assess(tmp_path) + roles = {item.path: item.role.value for item in assessment.file_inventory} + assert roles[relative] == "application_source" + assert (tmp_path / "src/app/__init__.py").exists() != namespace + + +@pytest.mark.parametrize("mapping", ["{''='lib'}", "{'app'='lib/app'}"]) +def test_relative_import_custom_and_parent_mapping(tmp_path, mapping): + relative = relative_project(tmp_path, source_root="lib", mapping=mapping) + assessment = assess(tmp_path) + assert next(item for item in assessment.file_inventory if item.path == relative).role.value == ( + "application_source" + ) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + if mapping == "{''='lib'}": + assert plan.deployment_mode == "source" + assert relative in _staging_files(tmp_path, assessment, plan, include=True) + else: + assert plan.deployment_mode == "package" + + +@pytest.mark.parametrize( + "arguments", + ["'app.examples.plugin'", "name='app.examples.plugin'", "'app.examples.plugin', level=0"], +) +def test_absolute_builtin_import_unchanged(tmp_path, arguments): + relative = relative_project(tmp_path, f"__import__({arguments})", imports="") + assert next( + item for item in assess(tmp_path).file_inventory if item.path == relative + ).role.value == ("application_source") + + +@pytest.mark.parametrize( + "arguments", + [ + "'.examples.plugin', package='app'", + "'app.examples.plugin', level=1", + "'app.examples.plugin', level=unknown", + "'app.examples.plugin', None, None, (), 1", + ], +) +def test_builtin_relative_context_is_not_misread_as_absolute(tmp_path, arguments): + tree = ast.parse(f"__import__({arguments})") + assert _imported_modules(tree, tmp_path / "main.py", tmp_path, ["."]) == [] + + +def test_unrelated_import_module_is_not_proven(tmp_path): + relative = relative_project( + tmp_path, + "import_module('.examples.plugin', 'app')", + "def import_module(*args): return None", + ) + assert next( + item for item in assess(tmp_path).file_inventory if item.path == relative + ).role.value == ("example_or_snippet") + + +@pytest.fixture +def offline_tools(tmp_path, monkeypatch): + fake_uv = tmp_path / "uv.exe" + fake_uv.write_bytes(b"verified uv") + monkeypatch.setattr( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + lambda *args, **kwargs: fake_uv, + ) + monkeypatch.setattr( + "python_deployment_builder.generation.generator.prepare_lockfile", + lambda root, *args, **kwargs: LockPreparationResult( + path=root / "uv.lock", created=False, checked=True, commands=() + ), + ) + + +@pytest.mark.parametrize("configured", [False, True]) +def test_as_file_release_security(tmp_path, monkeypatch, offline_tools, configured): + source = tmp_path / "source" + imports = "from importlib.resources import as_file, files" + if configured: + imports += "\nfrom os import getenv\nPASSWORD=getenv('DB_PASSWORD')" + as_file_project(source, imports, "as_file(files('app') / 'model.txt')") + secret = "PDBWrapperConfiguredSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + (source / "src/app/model.txt").write_text( + secret if configured else "API_KEY='sk-abcdefghijklmnop'" + ) + with pytest.raises(PreparationError, match="NO_SECRET_VALUES") as error: + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + ) + assert secret not in str(error.value) + assert not (tmp_path / "kit").exists() + + +@pytest.mark.parametrize("state", ["dirty", "untracked"]) +def test_as_file_git_provenance_is_enforced(tmp_path, offline_tools, state): + source = tmp_path / "source" + relative = as_file_project(source) + + def git(*args): + subprocess.run(["git", "-C", str(source), *args], check=True, capture_output=True) + + git("init") + git("add", ".") + if state == "untracked": + git("rm", "--cached", relative) + git( + "-c", + "user.name=PDB Test", + "-c", + "user.email=pdb@example.invalid", + "commit", + "-m", + "fixture", + ) + if state == "dirty": + (source / relative).write_bytes(b"changed model") + with pytest.raises(PreparationError): + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + ) + assert not (tmp_path / "kit").exists() + + +def test_relative_dynamic_import_promotes_excluded_target(tmp_path): + relative = relative_project(tmp_path) + assessment = assess(tmp_path) + roles = {item.path: item.role.value for item in assessment.file_inventory} + assert roles[relative] == "application_source" + assert roles["src/app/examples/__init__.py"] == "application_source" + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative in _staging_files(tmp_path, assessment, plan, include=True) From 0ce99790f26e256b1cc4ab1c3879d5db70b8cb1d Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:03:16 -0400 Subject: [PATCH 56/59] Honor named setuptools layouts and positional resource text arguments --- ...named-layout-text-signatures-2026-09-12.md | 164 ++++++++ .../verify_named_layout_text_signatures.py | 245 ++++++++++++ .../analysis/metadata.py | 90 +++-- .../analysis/resources.py | 15 +- tests/test_named_layout_text_signatures.py | 358 ++++++++++++++++++ 5 files changed, 846 insertions(+), 26 deletions(-) create mode 100644 docs/acceptance/pr9-named-layout-text-signatures-2026-09-12.md create mode 100644 scripts/verify_named_layout_text_signatures.py create mode 100644 tests/test_named_layout_text_signatures.py diff --git a/docs/acceptance/pr9-named-layout-text-signatures-2026-09-12.md b/docs/acceptance/pr9-named-layout-text-signatures-2026-09-12.md new file mode 100644 index 0000000..40f6b56 --- /dev/null +++ b/docs/acceptance/pr9-named-layout-text-signatures-2026-09-12.md @@ -0,0 +1,164 @@ +# PR #9: named automatic layouts and positional legacy text arguments + +Starting head: `94b3fd52e3cfa2a6c4f5ebbb287b6255e291b43a`. +Branch: `milestone-6-1-generation-contract`. Do not merge. + +## Review accounting + +Exact-head review submitted `2026-09-12T01:50:25Z`: + +| Finding | Thread | Comment | +| --- | --- | --- | +| P2: Honor named package-dir mappings during auto-discovery | `PRRT_kwDOT9hvCc6hsBoB` | `3994664774` | +| P2: Accept positional encoding arguments in legacy resource reads | `PRRT_kwDOT9hvCc6hsBoH` | `3994664779` | + +Authoritative totals: 124 findings/comments, 121 unresolved inline threads, +three review-level-only findings (historical #35, approved-wheel PEP 440 +comparison, complete locked sync-command validation), and one inline +`NOT_APPLICABLE / REVIEW PREMISE INCORRECT` finding for automatic flat package +plus loose module behavior. GitHub independently returned 121 threads and zero +resolved threads during this correction. No threads or reviews were closed or +dismissed. + +## Pinned setuptools reproduction and discovery correction + +`scripts/verify_named_layout_text_signatures.py` creates disposable projects, +builds wheels with uv 0.12.5 and isolated setuptools **79.0.1**, and compares the +actual installed Python members to PDB's assessed surface. It does not execute +builds in production analysis or touch acceptance repositories. + +Before the correction, `app -> lib` yielded PDB packages `['lib']`, although +setuptools installed exactly `app/__init__.py`, `app/main.py`, and +`app/helpers.py`. Applying the starting-head metadata inspector to the same +correct wheel reproduced the rejection: + +```text +Application wheel is missing authoritative first-party Python source: +lib/__init__.py, lib/helpers.py, lib/main.py +``` + +After correction that actual wheel validates. Seven initial tests failed before +the production edits: three metadata-source variants and four positional text +read variants. + +The complete wheel-build matrix covered each of pyproject, setup.cfg, and +literal setup.py, including **build-system-only legacy projects**: + +| Layout | Observed setuptools result / corrected PDB result | +| --- | --- | +| `app -> lib` | Installed root `app`, not `lib`; physical root initializer belongs to `app` | +| Regular and namespace descendants | `app.sub`, `app.ns`; no fabricated initializer | +| Namespace mapped root | `app/main.py` without `app/__init__.py` | +| `app.plugins -> vendor/plugins` | `app.plugins` and namespace child packages | +| Independent `app` and `other` roots | Both installed identities retained | +| Parent `app` plus specific `app.special` | `app.normal` from parent; `app.special` and child members from specific mapping; shadow parent's `wrong.py` absent | +| Missing mapped directory | Setuptools build fails; PDB marks surface unresolved without fallback | +| Global `"" -> lib` | Src-layout packages, namespace packages, and loose modules all discovered | + +Named mappings now precede generic automatic layout selection only when no +explicit package/module selection or finder is configured. The mapped root is +itself a package identity; existing namespace discovery supplies descendant +names relative to that root. Finder exclusions apply before prefixing, matching +setuptools. The shared `module_locations()` provides safe installed-name +locations, retaining exact/longest-parent precedence for downstream member +resolution. External roots prevent partial automatic authority; missing or +unsafe mapped roots never cause generic rediscovery under a physical name. + +Explicit and wildcard package-data tests require both `app/data/a.json` and +`app/data/b.json`; exclusion removes only the declared member. Correct wheels +pass, missing `app/helpers.py` fails, and missing selected package-data fails. +Local import location agrees with the same installed identity. A physically +source-compatible named mapping remains `source / SOURCE_COMPATIBLE`; a renamed +entry-point layout retains `package / ENTRYPOINT_REQUIRES_PACKAGE_MODE`. + +Bounded layout audit found one additional same-class defect: the old automatic +logic treated a global custom root `"" -> lib` as flat-layout, losing `tests` +namespace/package members and `helper.py`. The pinned build reproduced this in +all three metadata forms. The correction distinguishes src-layout semantics +from the literal directory spelling `src`, preserving flat-layout's existing +package-over-loose-module rule. Explicit empty selections still disable auto +discovery; named mappings take precedence when combined with a global mapping. +No discovery architecture or persisted contract was redesigned. + +The reference implementation is the pinned +[setuptools discovery source](https://github.com/pypa/setuptools/blob/v79.0.1/setuptools/discovery.py). +The report's concrete member results come from the disposable builds, not an +assumption based on newer setuptools behavior. + +## Exact Python resource signatures and correction + +Direct interpreter execution used **Python 3.11.16** and **Python 3.12.14**. +Both produced identical bounded call behavior: + +| API | Accepted positional arguments | Rejected | +| --- | --- | --- | +| `read_text`, `open_text` | package, resource, optional encoding, optional errors | Fifth positional, duplicate encoding, unrelated keyword | +| `read_binary`, `open_binary` | package, resource | Third/fourth positional, encoding/errors keywords | + +Two-, three-, and four-positional text calls returned the fixture content. +Mixed positional package plus keyword resource/encoding/errors also worked. +The probe prints exact signatures and asserts expected successes/TypeErrors. +Keyword-before-bare-positional test cases were not fabricated. + +Pre-fix analysis rejected `len(args) > 2`, so the actual text resource was not +recorded or staged. Text helpers now accept up to four positional arguments; +binary helpers retain their two-argument limit. `call_argument()` still binds +only package/resource. Optional encoding/error expressions need not be literal, +because they do not change resource identity. Duplicate optional text bindings +are unresolved; the previously accepted positional-wins behavior for invalid +duplicate package/resource bindings remains unchanged. + +Direct, module-qualified, module-alias, and directly imported alias forms all +retain resource evidence. Unknown keywords, unrelated functions, dynamic +package/resource values, traversal, and nested legacy members remain unresolved. +Successful resolution uses the existing package-anchor and role-aware staging +path. Release generation rejects a synthetic configured value in the newly +recognized text resource without including its value in the diagnostic. + +The bounded audit of the four legacy resource APIs found no additional +signature discrepancy beyond optional positional text arguments and their +duplicate-binding guard. Modern files(), as_file(), and pkgutil semantics were +not changed. + +## Read-only acceptance checks + +Assessment comparisons against the starting-head metadata/resource recognizers +found identical packaging metadata, dependencies, entry points, configuration +evidence, and resources. Before/after repository statuses and SHAs matched. + +| Repository | Unchanged SHA | Result | +| --- | --- | --- | +| SimpleGeorefGUI | `f484570d89fb1f9e9170fac915475358dfc1234e` | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python, 77 data members | +| Geo Map Explanation Extractor | `5e7b321d0aeb9ba1d586bfc548c79793d84c6033` | source / SOURCE_COMPATIBLE | +| TN Coordinate Converter | `e1e7a1588c37a99c2d02efaf3eef3d04636f12f0` | source / SOURCE_COMPATIBLE; reviewed proxy-tools 0.1.0 | + +SGG **does** have `simple_georef_gui_app -> code`, but explicitly selects +`packages = ['simple_georef_gui_app']`; it does not enter automatic discovery. +Its GUI target remains `simple_georef_gui_app.georef_main:main`. The 13/77 +surface was recomputed, not assumed. + +TN's existing `pdb-m61-regression-20260901/tn-kit` independently remains +`STATIC_VALID` with the reviewed artifact. No acceptance wheels, environments, +source files, locks, or kits were modified. + +## Quality and compatibility + +Analysis schema remains **1.4**; planning schema remains **1.3**. No persisted +fields, manifest defaults, or migrations were introduced. + +Focused new regressions: **123 passed**. The single complete suite finished +with **1524 passed, 4 skipped** in 322.21 seconds, versus the prior +1401-pass/four-skip baseline. `ruff check .` and `git diff --check` passed. +The final diff contains only the two production corrections, their focused +tests, the disposable verification script, and this text evidence report. + +Prior regression coverage includes as_file, relative dynamic imports, legacy +manifest configured-secret compatibility, module anchors, omitted PEP 621 +authority, approved artifact paths/extra contexts/lock identity, exact sync, +mode-aware pip check, entry-point extras, short secrets, LOCALAPPDATA ordering, +MANIFEST/include-package-data guards, wheel security/dependencies/integrity, +provenance, rollback, and deterministic packaging. + +Only the PR Quality pass count may be updated after all gates pass. One focused +commit and one exact-pushed-head `@codex review` request are authorized; no merge +or historical thread resolution is authorized. diff --git a/scripts/verify_named_layout_text_signatures.py b/scripts/verify_named_layout_text_signatures.py new file mode 100644 index 0000000..0df04a4 --- /dev/null +++ b/scripts/verify_named_layout_text_signatures.py @@ -0,0 +1,245 @@ +"""Disposable pinned setuptools discovery and legacy resource signature evidence.""" + +import ast +import json +import os +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from unittest.mock import patch + +import python_deployment_builder.analysis.metadata as metadata +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.metadata import inspect_metadata +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.resources import resolve_packaged_python_sources +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import validate_application_wheel +from python_deployment_builder.planning.planner import create_deployment_plan + + +def starting_inspector(): + source = subprocess.check_output( + [ + "git", + "show", + "94b3fd52e3cfa2a6c4f5ebbb287b6255e291b43a:" + "src/python_deployment_builder/analysis/metadata.py", + ], + text=True, + ) + node = next( + n + for n in ast.parse(source).body + if isinstance(n, ast.FunctionDef) and n.name == "inspect_metadata" + ) + namespace = dict(metadata.__dict__) + exec(compile(ast.Module(body=[node], type_ignores=[]), "", "exec"), namespace) + return namespace["inspect_metadata"] + + +def main(): + uv = sys.argv[1] + old_inspector = starting_inspector() + env = dict(os.environ, UV_SYSTEM_CERTS="true", UV_NO_ENV_FILE="1", UV_NO_PROGRESS="1") + with tempfile.TemporaryDirectory(prefix="pdb-named-layout-") as temporary: + workspace = Path(temporary) + shapes = { + "named": ({"app": "lib"}, ["lib/__init__.py", "lib/main.py", "lib/helpers.py"]), + "descendants": ( + {"app": "lib"}, + ["lib/__init__.py", "lib/sub/__init__.py", "lib/sub/module.py", "lib/ns/module.py"], + ), + "namespace-root": ({"app": "lib"}, ["lib/main.py"]), + "nested": ( + {"app.plugins": "vendor/plugins"}, + ["vendor/plugins/__init__.py", "vendor/plugins/sub/module.py"], + ), + "multiple": ( + {"app": "lib/a", "other": "lib/b"}, + ["lib/a/__init__.py", "lib/b/__init__.py"], + ), + "overlap": ( + {"app": "lib", "app.special": "special-src"}, + [ + "lib/__init__.py", + "lib/normal/__init__.py", + "lib/special/__init__.py", + "lib/special/wrong.py", + "special-src/__init__.py", + "special-src/child/module.py", + ], + ), + "missing": ({"app": "missing-lib"}, ["unrelated/__init__.py"]), + "global": ( + {"": "lib"}, + ["lib/app/__init__.py", "lib/tests/__init__.py", "lib/helper.py"], + ), + } + print(subprocess.check_output([uv, "--version"], text=True).strip()) + for kind in ("pyproject", "setup.cfg", "setup.py"): + for shape, (mapping, members) in shapes.items(): + root = workspace / f"{kind}-{shape}" + root.mkdir() + for member in members: + path = root / member + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("def main(): return 0\n", encoding="utf-8") + config = ( + "[build-system]\nrequires=['setuptools==79.0.1','wheel']\n" + "build-backend='setuptools.build_meta'\n" + ) + if kind == "pyproject": + config += ( + "[project]\nname='mapped-demo'\nversion='1.0.0'\n" + "[tool.setuptools.package-dir]\n" + ) + config += "".join( + f"{json.dumps(k)}={json.dumps(v)}\n" for k, v in mapping.items() + ) + elif kind == "setup.cfg": + (root / kind).write_text( + "[metadata]\nname=mapped-demo\nversion=1.0.0\n[options]\npackage_dir=\n" + + "".join(f" {k} = {v}\n" for k, v in mapping.items()) + ) + else: + (root / kind).write_text( + "from setuptools import setup\n" + f"setup(name='mapped-demo', version='1.0.0', package_dir={mapping!r})\n" + ) + (root / "pyproject.toml").write_text(config) + before = inspect_metadata(root).project + expected = sorted( + item.installed_member_path + for item in resolve_packaged_python_sources(root, before) + ) + result = subprocess.run( + [uv, "build", "--wheel", "--python", "3.12", "--out-dir", str(root / "dist")], + cwd=root, + env=env, + capture_output=True, + text=True, + ) + wheels = list((root / "dist").glob("*.whl")) + installed = [] + if wheels: + with zipfile.ZipFile(wheels[0]) as archive: + installed = sorted(n for n in archive.namelist() if n.endswith(".py")) + assert installed == expected, (kind, shape, installed, expected) + else: + assert shape == "missing", result.stderr + print( + json.dumps( + { + "kind": kind, + "shape": shape, + "success": result.returncode == 0, + "pdb_packages": before.packages, + "pdb_modules": before.py_modules, + "installed": installed, + "error": result.stderr[-500:] if result.returncode else None, + } + ), + flush=True, + ) + if kind == "pyproject" and shape == "named": + # Add the authoritative launcher, then rebuild its correct wheel. + path = root / "pyproject.toml" + path.write_text( + path.read_text() + "[project.scripts]\nmapped-demo='app.main:main'\n" + ) + (root / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n" + "[[package]]\nname='mapped-demo'\nversion='1.0.0'\n" + "source={editable='.'}\n" + ) + subprocess.run( + [ + uv, + "build", + "--wheel", + "--python", + "3.12", + "--out-dir", + str(root / "dist"), + ], + cwd=root, + env=env, + check=True, + capture_output=True, + ) + repository = MaterializedRepository( + root=root, source=str(root), source_kind="local" + ) + assessment = assess_repository(repository) + plan = create_deployment_plan(assessment, repository_root=root) + validate_application_wheel(wheels[0], assessment, plan, repository_root=root) + with patch( + "python_deployment_builder.analysis.assessor.inspect_metadata", + old_inspector, + ): + old_assessment = assess_repository(repository) + old_plan = create_deployment_plan(old_assessment, repository_root=root) + try: + validate_application_wheel( + wheels[0], old_assessment, old_plan, repository_root=root + ) + except PreparationError as error: + print( + json.dumps( + { + "pre_fix_correct_wheel": str(error), + "post_fix_correct_wheel": "PASS", + } + ), + flush=True, + ) + else: + raise AssertionError("Starting-head wrong surface accepted correct wheel") + root = workspace / "runtime" + (root / "app").mkdir(parents=True) + (root / "app/__init__.py").write_text("") + (root / "app/defaults.json").write_text("RESOURCE") + probe = """ +import importlib.resources as r, inspect, json, sys, warnings +warnings.simplefilter('ignore', DeprecationWarning) +result = {'version': sys.version.split()[0]} +for name in ['read_text', 'open_text', 'read_binary', 'open_binary']: + f = getattr(r, name) + records = {'signature': str(inspect.signature(f))} + cases = [(['app','defaults.json'], {}), (['app','defaults.json','utf-8'], {}), + (['app','defaults.json','utf-8','strict'], {}), + (['app','defaults.json','utf-8','strict','extra'], {}), + (['app','defaults.json','utf-8'], {'encoding':'utf-8'}), + (['app'], {'resource':'defaults.json','encoding':'utf-8','errors':'ignore'})] + for i, (args, kwargs) in enumerate(cases): + try: + value = f(*args, **kwargs) + if hasattr(value, 'read'): + with value: value = value.read() + records[str(i)] = repr(value) + except Exception as error: records[str(i)] = type(error).__name__ + result[name] = records + assert records['0'] == ("'RESOURCE'" if name.endswith('text') else "b'RESOURCE'") + if name.endswith('text'): + assert records['1'] == records['2'] == records['5'] == "'RESOURCE'" + else: + assert records['1'] == records['2'] == records['5'] == 'TypeError' + assert records['3'] == records['4'] == 'TypeError' +print(json.dumps(result)) +""" + for version in ("3.11.16", "3.12.14"): + executable = ( + Path(os.environ["APPDATA"]) + / "uv/python" + / f"cpython-{version}-windows-x86_64-none/python.exe" + ) + print( + subprocess.check_output([str(executable), "-B", "-c", probe], cwd=root, text=True) + ) + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/metadata.py b/src/python_deployment_builder/analysis/metadata.py index 8b98d93..de03d14 100644 --- a/src/python_deployment_builder/analysis/metadata.py +++ b/src/python_deployment_builder/analysis/metadata.py @@ -14,6 +14,7 @@ from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name +from python_deployment_builder.analysis.module_resolution import module_locations from python_deployment_builder.models import ( DependencyAssessment, EntryPointAssessment, @@ -481,6 +482,35 @@ def _discover_setuptools_packages( return sorted(discovered) +def _discover_named_setuptools_packages( + root: Path, package_directories: dict[str, str] +) -> list[str] | None: + """Mirror setuptools 79 explicit-layout roots plus PEP420 descendants. + + Discovery supplies installed prefixes; module_locations remains responsible + for exact/longest-parent physical locations. A missing declared root is not + permission to fall back to a different automatic layout. + """ + + packages: set[str] = set() + for package, directory in package_directories.items(): + if not package: + continue + locations = module_locations(root, package, [], package_directories) + if len(locations) != 1 or not locations[0].is_dir(): + return None + packages.add(package) + # The finder exclusions apply to names relative to this root, before + # prefixing, exactly as setuptools' _find_packages_within does. + packages.update( + f"{package}.{descendant}" + for descendant in _discover_setuptools_packages( + root, [directory], {}, ["*"], [], True + ) + ) + return sorted(packages) + + def _discover_setuptools_py_modules( root: Path, search_roots: list[str], @@ -736,6 +766,7 @@ def inspect_metadata(root: Path) -> MetadataResult: python_evidence: list[Evidence] = [] package_discovery_rules: list[tuple[list[str], list[str], list[str], bool]] = [] automatic_setuptools_root: str | None = None + automatic_setuptools_src_layout = False automatic_setuptools_flat_surface_ambiguous = False setuptools_package_selection_configured = False setuptools_surface_unresolved = False @@ -1619,27 +1650,40 @@ def safe_roots(values: list[str]) -> list[str]: and not package_discovery_rules and not setuptools_package_selection_configured and not setuptools_surface_unresolved + and not setuptools_external_packaging_roots ): - automatic_root = ( - source_roots[0] - if len(source_roots) == 1 - else "src" - if (root / "src").is_dir() - else "." - ) - automatic_setuptools_root = automatic_root - package_discovery_rules.append( - ( - [automatic_root], - ["*"], - [] - if automatic_root == "src" - else list(_SETUPTOOLS_79_FLAT_PACKAGE_DEFAULT_EXCLUDES), - # Modern setuptools automatic discovery recognizes implicit - # namespaces; explicit find configuration can still disable it. - True, + if any(package_directories): + named_packages = _discover_named_setuptools_packages(root, package_directories) + if named_packages is None: + setuptools_surface_unresolved = True + setuptools_surface_evidence.append( + _evidence( + root, root_metadata_path, + "Automatic named package-dir discovery cannot establish every " + "declared mapped package root; generic layout fallback is unsafe.", + ) + ) + else: + packages = named_packages + else: + automatic_root = ( + source_roots[0] + if len(source_roots) == 1 + else "src" + if (root / "src").is_dir() + else "." + ) + automatic_setuptools_src_layout = "" in package_directories or automatic_root == "src" + automatic_setuptools_root = automatic_root + package_discovery_rules.append( + ( + [automatic_root], + ["*"], + [] if automatic_setuptools_src_layout + else list(_SETUPTOOLS_79_FLAT_PACKAGE_DEFAULT_EXCLUDES), + True, + ) ) - ) if package_discovery_rules: discovered_packages = { package @@ -1653,7 +1697,7 @@ def safe_roots(values: list[str]) -> list[str]: namespaces, ) } - if automatic_setuptools_root == ".": + if automatic_setuptools_root == "." and not automatic_setuptools_src_layout: top_level_packages = {package.split(".", 1)[0] for package in discovered_packages} if len(top_level_packages) > 1: # Setuptools rejects implicit flat layouts with multiple @@ -1675,7 +1719,7 @@ def safe_roots(values: list[str]) -> list[str]: ) packages = sorted({*packages, *discovered_packages}) if automatic_setuptools_root is not None and ( - automatic_setuptools_root == "src" or not packages + automatic_setuptools_src_layout or not packages ) and not automatic_setuptools_flat_surface_ambiguous: # Setuptools' default source-layout finder discovers top-level modules # as well as packages. Its flat-layout finder selects a package @@ -1688,11 +1732,11 @@ def safe_roots(values: list[str]) -> list[str]: [automatic_setuptools_root], excluded_modules=( list(_SETUPTOOLS_79_FLAT_MODULE_DEFAULT_EXCLUDES) - if automatic_setuptools_root == "." + if not automatic_setuptools_src_layout else None ), ) - if automatic_setuptools_root == "." and len(discovered_modules) > 1: + if not automatic_setuptools_src_layout and len(discovered_modules) > 1: # Mirroring the bounded flat package policy above prevents an # undeclared multi-module distribution from becoming a fabricated # wheel surface. diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 7d943c9..afed75e 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -727,18 +727,27 @@ def _legacy_importlib_resource_path_values( module_bindings: set[str], read_bindings: dict[str, str], ) -> tuple[str, list[str]] | None: - """Resolve the bounded two-argument legacy resource read API statically.""" + """Resolve legacy direct-member reads with Python 3.11/3.12 signatures.""" function = _legacy_resource_function_name(node, module_bindings, read_bindings) if function is None: return None # Python 3.12 legacy resource calls address one direct member of a # package. Keep dynamic, nested, and traversal-like members unresolved. - if len(node.args) > 2: + text_function = function.endswith("text") + if len(node.args) > (4 if text_function else 2): return function, [] allowed_keywords = {"package", "resource"} - if function.endswith("text"): + if text_function: allowed_keywords.update({"encoding", "errors"}) + # Optional text parameters do not affect resource identity, but an + # invalid duplicate binding must not provide read evidence. Preserve + # the established positional-wins policy for package/resource. + if any( + len(node.args) > position and any(item.arg == keyword for item in node.keywords) + for position, keyword in ((2, "encoding"), (3, "errors")) + ): + return function, [] if any(keyword.arg not in allowed_keywords for keyword in node.keywords): return function, [] package = call_argument(node, position=0, keyword="package") diff --git a/tests/test_named_layout_text_signatures.py b/tests/test_named_layout_text_signatures.py new file mode 100644 index 0000000..1457dc5 --- /dev/null +++ b/tests/test_named_layout_text_signatures.py @@ -0,0 +1,358 @@ +"""Named automatic layouts and legacy resource positional text parameters.""" + +import json + +import pytest +from test_as_file_relative_imports import offline_tools # noqa: F401 +from test_dependency_authority import assess +from test_files_package_keyword import resource_project +from test_generation import _make_application_wheel, _rewrite_application_wheel + +from python_deployment_builder.analysis.metadata import inspect_metadata +from python_deployment_builder.analysis.module_resolution import module_locations +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.resources import ( + resolve_package_data_members, + resolve_packaged_python_sources, +) +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import validate_application_wheel +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.planning.planner import create_deployment_plan + + +def mapped_project(root, mapping=None, members=None, legacy=None): + mapping = mapping or {"app": "lib"} + members = members or ["lib/__init__.py", "lib/main.py", "lib/helpers.py"] + for member in members: + path = root / member + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("def main(): return 0\n") + configuration = ( + "[build-system]\nrequires=['setuptools==79.0.1','wheel']\n" + "build-backend='setuptools.build_meta'\n" + "[project]\nname='mapped-demo'\nversion='1.0.0'\nrequires-python='>=3.12'\n" + "[project.scripts]\nmapped-demo='app.main:main'\n" + ) + if legacy == "setup.cfg": + (root / legacy).write_text( + "[options]\npackage_dir=\n" + "".join(f" {k} = {v}\n" for k, v in mapping.items()) + ) + elif legacy == "setup.py": + (root / legacy).write_text( + f"from setuptools import setup\nsetup(package_dir={mapping!r})\n" + ) + else: + configuration += "[tool.setuptools.package-dir]\n" + "".join( + f"{json.dumps(k)}={json.dumps(v)}\n" for k, v in mapping.items() + ) + (root / "pyproject.toml").write_text(configuration) + (root / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.12'\n[[package]]\nname='mapped-demo'\nversion='1.0.0'\nsource={editable='.'}\n" + ) + + +@pytest.mark.parametrize("legacy", [None, "setup.cfg", "setup.py"]) +def test_named_mapping_establishes_installed_root(tmp_path, legacy): + mapped_project(tmp_path, legacy=legacy) + project = inspect_metadata(tmp_path).project + assert project.package_directories == {"app": "lib"} + assert project.packages == ["app"] + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +@pytest.mark.parametrize( + "arguments", ["'app', 'defaults.json', 'utf-8'", "'app', 'defaults.json', 'utf-8', 'strict'"] +) +def test_positional_text_arguments_retain_resource(tmp_path, function, arguments): + path = resource_project(tmp_path, "", "unused") + (tmp_path / "src/app/main.py").write_text( + f"from importlib.resources import {function}\ndef main(): return {function}({arguments})\n" + ) + assessment = assess(tmp_path) + assert path in {item.path for item in assessment.resources} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert path in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize( + ("mapping", "members", "packages"), + [ + ({"": "src"}, ["src/app/__init__.py"], ["app"]), + ( + {"": "lib"}, + ["lib/app/__init__.py", "lib/tests/__init__.py", "lib/helper.py"], + ["app", "tests"], + ), + ({"app": "lib"}, ["lib/main.py"], ["app"]), + ( + {"app": "lib"}, + ["lib/__init__.py", "lib/sub/__init__.py", "lib/sub/module.py", "lib/ns/module.py"], + ["app", "app.ns", "app.sub"], + ), + ( + {"app.plugins": "vendor/plugins"}, + ["vendor/plugins/__init__.py", "vendor/plugins/sub/module.py"], + ["app.plugins", "app.plugins.sub"], + ), + ( + {"app": "lib/a", "other": "lib/b"}, + ["lib/a/__init__.py", "lib/b/__init__.py"], + ["app", "other"], + ), + ( + {"app": "lib", "app.special": "special-src"}, + [ + "lib/__init__.py", + "lib/normal/__init__.py", + "lib/special/__init__.py", + "lib/special/wrong.py", + "special-src/__init__.py", + "special-src/child/module.py", + ], + ["app", "app.normal", "app.special", "app.special.child"], + ), + ( + {"app": "lib", "other": "lib"}, + ["lib/__init__.py", "lib/sub/module.py"], + ["app", "app.sub", "other", "other.sub"], + ), + ], +) +@pytest.mark.parametrize("legacy", [None, "setup.cfg", "setup.py"]) +def test_named_layout_matrix(tmp_path, mapping, members, packages, legacy): + mapped_project(tmp_path, mapping, members, legacy) + metadata = inspect_metadata(tmp_path) + assert not metadata.setuptools_surface_unresolved + assert metadata.project.packages == packages + surface = resolve_packaged_python_sources(tmp_path, metadata.project) + assert all( + item.installed_member_path.startswith(("app/", "other/", "tests/", "helper.py")) + for item in surface + ) + if "app.special" in mapping: + assert "app/special/wrong.py" not in {item.installed_member_path for item in surface} + assert "app/special/child/module.py" in {item.installed_member_path for item in surface} + if mapping == {"": "lib"}: + assert metadata.project.py_modules == ["helper"] + + +@pytest.mark.parametrize("directory", ["missing-lib", "../external"]) +def test_invalid_named_root_never_falls_back(tmp_path, directory): + mapped_project(tmp_path, {"app": directory}, ["unrelated/__init__.py"]) + metadata = inspect_metadata(tmp_path) + assert metadata.project.packages == [] + assert metadata.setuptools_surface_unresolved or metadata.setuptools_external_packaging_roots + plan = create_deployment_plan(assess(tmp_path), repository_root=tmp_path) + assert plan.deployment_mode_condition != "PACKAGE_SURFACE_PROVEN" + + +@pytest.mark.parametrize("selection", ["packages=[]", "py-modules=[]", "packages=['app']"]) +def test_named_mapping_does_not_override_explicit_selection(tmp_path, selection): + mapped_project(tmp_path, members=["lib/__init__.py", "lib/sub/module.py"]) + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text().replace( + "[tool.setuptools.package-dir]", + f"[tool.setuptools]\n{selection}\n[tool.setuptools.package-dir]", + ) + ) + assert inspect_metadata(tmp_path).project.packages == ( + ["app"] if "['app']" in selection else [] + ) + + +@pytest.mark.parametrize("data_key", ["app", "*"]) +@pytest.mark.parametrize("exclude", [False, True]) +def test_named_package_data_and_wheel_completeness(tmp_path, data_key, exclude): + source = tmp_path / "source" + mapped_project(source) + data = source / "lib/data" + data.mkdir() + for name in ("a", "b"): + (data / f"{name}.json").write_text("{}") + path = source / "pyproject.toml" + path.write_text( + path.read_text() + + f"[tool.setuptools.package-data]\n'{data_key}'=['data/*.json']\n" + + ("[tool.setuptools.exclude-package-data]\napp=['data/b.json']\n" if exclude else "") + ) + assessment = assess(source) + assert assessment.project.packages == ["app", "app.data"] + data_members = resolve_package_data_members(source, assessment.project) + expected = {"app/data/a.json"} | (set() if exclude else {"app/data/b.json"}) + assert {item.installed_member_path for item in data_members} == expected + assert module_locations(source, "app.helpers", [], assessment.project.package_directories) == [ + source / "lib/helpers" + ] + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel( + tmp_path, + name="mapped-demo", + version="1.0.0", + package="app", + target="app.main:main", + entry_group="console_scripts", + entry_name="mapped-demo", + requires_python=">=3.12", + ) + _rewrite_application_wheel( + wheel, + removals={"app/view.html"}, + additions={ + "app/helpers.py": "def helper(): return 0\n", + **{name: "{}" for name in expected}, + }, + ) + validate_application_wheel(wheel, assessment, plan, repository_root=source) + _rewrite_application_wheel(wheel, removals={"app/helpers.py"}) + with pytest.raises(PreparationError): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + _rewrite_application_wheel( + wheel, additions={"app/helpers.py": ""}, removals={"app/data/a.json"} + ) + with pytest.raises(PreparationError): + validate_application_wheel(wheel, assessment, plan, repository_root=source) + + +def test_named_mapping_preserves_source_mode_and_imports(tmp_path): + # A mapping alone does not force package mode when physical imports agree. + mapped_project(tmp_path, {"app": "app"}, ["app/__init__.py", "app/main.py", "app/helpers.py"]) + (tmp_path / "app/main.py").write_text("import app.helpers\ndef main(): return 0\n") + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert plan.deployment_mode == "source" + assert plan.deployment_mode_condition == "SOURCE_COMPATIBLE" + assert "app/helpers.py" in _staging_files(tmp_path, assessment, plan, include=True) + + +def test_named_mapping_renamed_entry_still_requires_package_mode(tmp_path): + mapped_project(tmp_path) + plan = create_deployment_plan(assess(tmp_path), repository_root=tmp_path) + assert plan.deployment_mode == "package" + assert plan.deployment_mode_condition == "ENTRYPOINT_REQUIRES_PACKAGE_MODE" + + +def test_named_mapping_excludes_finder_reserved_descendants(tmp_path): + mapped_project( + tmp_path, + members=[ + "lib/__init__.py", + "lib/ez_setup/module.py", + "lib/__pycache__/module.py", + "lib/sub/module.py", + ], + ) + assert inspect_metadata(tmp_path).project.packages == ["app", "app.sub"] + + +def test_external_mapping_cannot_leave_partial_authoritative_surface(tmp_path): + mapped_project(tmp_path, {"app": "lib", "other": "../external"}) + metadata = inspect_metadata(tmp_path) + assert metadata.setuptools_external_packaging_roots == ["../external"] + assert metadata.project.packages == [] + + +def test_global_and_named_mapping_select_explicit_layout_first(tmp_path): + mapped_project( + tmp_path, + {"": "src", "app": "lib"}, + ["src/unrelated/__init__.py", "src/loose.py", "lib/__init__.py"], + ) + project = inspect_metadata(tmp_path).project + assert project.packages == ["app"] + assert project.py_modules == [] + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +@pytest.mark.parametrize("binding", ["module", "module_alias", "direct", "direct_alias"]) +@pytest.mark.parametrize( + "arguments", + [ + "'app', 'defaults.json'", + "'app', 'defaults.json', 'utf-8'", + "'app', 'defaults.json', 'utf-8', 'ignore'", + "'app', 'defaults.json', encoding=selected_encoding, errors=selected_errors", + "package='app', resource='defaults.json', encoding=selected_encoding", + "'app', resource='defaults.json', encoding='utf-8', errors='strict'", + ], +) +def test_text_signature_alias_matrix(tmp_path, function, binding, arguments): + imports, target = { + "module": ("import importlib.resources", f"importlib.resources.{function}"), + "module_alias": ("import importlib.resources as r", f"r.{function}"), + "direct": (f"from importlib.resources import {function}", function), + "direct_alias": (f"from importlib.resources import {function} as load", "load"), + }[binding] + path = resource_project(tmp_path, "", "unused") + (tmp_path / "src/app/main.py").write_text( + f"{imports}\ndef main(): return {target}({arguments})\n" + ) + assert path in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +@pytest.mark.parametrize( + "arguments", + [ + "'app', 'defaults.json', 'utf-8', encoding='ascii'", + "'app', 'defaults.json', 'utf-8', 'strict', errors='ignore'", + "'app', 'defaults.json', 'utf-8', 'strict', 'extra'", + "'app', 'defaults.json', unknown=True", + "'app', '../defaults.json', 'utf-8'", + "'app', 'nested/defaults.json', 'utf-8'", + "'app', resource_variable, 'utf-8'", + "package_variable, 'defaults.json', 'utf-8'", + ], +) +def test_invalid_text_calls_are_unresolved(tmp_path, function, arguments): + path = resource_project(tmp_path, "", "unused") + (tmp_path / "src/app/main.py").write_text( + f"from importlib.resources import {function}\ndef main(): return {function}({arguments})\n" + ) + assert path not in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary"]) +@pytest.mark.parametrize( + "tail", ["", ", 'utf-8'", ", 'utf-8', 'strict'", ", encoding='utf-8'", ", unknown=True"] +) +def test_binary_signatures_remain_separate(tmp_path, function, tail): + path = resource_project(tmp_path, "", "unused") + (tmp_path / "src/app/main.py").write_text( + f"from importlib.resources import {function}\n" + f"def main(): return {function}('app', 'defaults.json'{tail})\n" + ) + assert (path in {item.path for item in assess(tmp_path).resources}) == (not tail) + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +def test_unrelated_text_function_unresolved(tmp_path, function): + path = resource_project(tmp_path, "", "unused") + (tmp_path / "src/app/main.py").write_text( + f"def {function}(*args): pass\n" + f"def main(): return {function}('app', 'defaults.json', 'utf-8')\n" + ) + assert path not in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +@pytest.mark.usefixtures("offline_tools") +def test_positional_text_release_security(tmp_path, monkeypatch, function): + source = tmp_path / "source" + path = resource_project(source, "", "unused") + (source / "src/app/main.py").write_text( + f"from importlib.resources import {function}\nfrom os import getenv\n" + "PASSWORD=getenv('DB_PASSWORD')\n" + f"def main(): return {function}('app', 'defaults.json', 'utf-8')\n" + ) + secret = "PDBPositionalTextSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + (source / path).write_text(secret) + with pytest.raises(PreparationError, match="NO_SECRET_VALUES") as error: + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + tmp_path / "kit", + ) + assert secret not in str(error.value) + assert not (tmp_path / "kit").exists() From 4888ee28710d20dc8e45886ad4d928f0f556aa66 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:47:04 -0400 Subject: [PATCH 57/59] Model functional resource reads across supported Python versions --- ...functional-resource-families-2026-09-12.md | 174 +++++++++ .../verify_functional_resource_families.py | 215 +++++++++++ .../analysis/resources.py | 122 ++++-- tests/test_functional_resource_families.py | 361 ++++++++++++++++++ 4 files changed, 835 insertions(+), 37 deletions(-) create mode 100644 docs/acceptance/pr9-functional-resource-families-2026-09-12.md create mode 100644 scripts/verify_functional_resource_families.py create mode 100644 tests/test_functional_resource_families.py diff --git a/docs/acceptance/pr9-functional-resource-families-2026-09-12.md b/docs/acceptance/pr9-functional-resource-families-2026-09-12.md new file mode 100644 index 0000000..94c4d12 --- /dev/null +++ b/docs/acceptance/pr9-functional-resource-families-2026-09-12.md @@ -0,0 +1,174 @@ +# PR #9: Python 3.13+ functional resource paths + +Starting head: `0ce99790f26e256b1cc4ab1c3879d5db70b8cb1d`. +Branch: `milestone-6-1-generation-contract`. Do not merge. + +## Fresh review and accounting + +Exact-head review submitted `2026-09-12T03:15:07Z` raised P2: +**Support multi-part resource names on Python 3.13+**. +Thread: `PRRT_kwDOT9hvCc6hsp4y`; review comment: `3994907452`. +The comment and its commit identity were independently checked through GitHub. + +Authoritative totals: **125 findings/comments**, **122 unresolved inline +threads**, **three review-level-only findings**, and **one NOT_APPLICABLE inline +finding**. The three review-level-only findings remain historical #35, +approved-wheel PEP 440 comparison, and complete locked sync-command validation. +The NOT_APPLICABLE finding remains automatic flat package plus loose module +behavior. GitHub returned 122 threads, zero resolved. No review dismissal, +historical thread resolution, or merge is authorized by this correction. + +## Direct interpreter evidence + +`scripts/verify_functional_resource_families.py` executes disposable fixtures +under managed CPython **3.11.16**, **3.12.14**, **3.13.15**, and **3.14.7**. +The latter two were acquired using the existing pinned uv **0.12.5** workflow. +No acceptance repository was used as a test fixture. + +| Call behavior | 3.11.16 | 3.12.14 | 3.13.15 | 3.14.7 | +| --- | --- | --- | --- | --- | +| Binary `(package, filename)` | Read succeeds | Read succeeds | Read succeeds | Read succeeds | +| Binary `(anchor, directory, filename)` | TypeError | TypeError | Read succeeds | Read succeeds | +| Binary one slash-containing path name | ValueError: direct filename required | Same | Read succeeds | Read succeeds | +| Text two positional arguments | Read succeeds | Read succeeds | Read succeeds | Read succeeds | +| Text positional encoding / encoding plus errors | Read succeeds | Read succeeds | TypeError: encoding argument required with multiple path names | Same | +| Text multipart names plus `encoding=` | Duplicate encoding TypeError | Same | Read succeeds | Read succeeds | +| Text deeper multipart path plus `encoding=` | Duplicate encoding TypeError | Same | Read succeeds | Read succeeds | +| Text multipart names without `encoding=` | Legacy interpretation; fixture attempts to open a directory | Same | TypeError: encoding argument required | Same | +| `package=` / `resource=` | Read succeeds | Read succeeds | Unexpected keyword TypeError | Same | +| `anchor=` with no path names | Unexpected keyword TypeError | Same | Directory read fails | Same | +| Invented `path_names=` keyword | Unsupported | Unsupported | Unsupported | Unsupported | +| Module / top-level module / namespace-parent module anchor | Not a package TypeError | Read succeeds | Read succeeds | Read succeeds | + +Both read/open variants of binary and text functions were executed. On Windows, +attempting to open a directory in these fixtures raises PermissionError. This +is not evidence for a recursive directory resource read. + +An important bounded audit result is that **3.12's older functional wrappers +already inherit module-anchor behavior from files()**. The resource resolver +therefore uses the existing shared module/package resolver for both families, +not a new module-name-to-path implementation. Regular packages continue to win +over same-named module files; no initializer is invented for namespace parents. + +The [Python 3.13 functional API documentation](https://docs.python.org/3.13/library/importlib.resources.html#functional-api) +describes the signature transition. All patch-version behavior above was also +verified by direct execution rather than inferred from documentation. + +## Private call classifier and target-independent policy + +`_FunctionalResourceCall` is a private frozen dataclass containing only AST +anchor/path expressions and a `legacy_direct`, `multipath`, or `common` family. +It is not a persisted model, and no runtime target is threaded into assessment. + +The bounded classifier recognizes shapes valid for at least one supported +family. For text, multiple positional path names require keyword encoding on +3.13/3.14; that same keyword conflicts with positional encoding on 3.11/3.12. +Consequently the valid families do not assign different file identities to the +same supported call shape. Without keyword encoding, old three-/four-positional +text forms retain the direct-member interpretation. With keyword encoding, +the new multipart interpretation is selected where valid. + +This is resource evidence, **not a promise that every recognized call executes +on every Python minor**. Existing planning and runtime validation remain +responsible for deployment behavior. Python 3.15 or later signature changes +are outside this four-minor policy audit. + +Old `package=`/`resource=` forms remain legacy-only. New `anchor=` has no useful +file-read form with keyword path names: varargs are positional, and supplying +only the anchor selects its directory. Such calls stay unresolved. Starred +argument expansion, unknown keywords, and duplicate keywords are unresolved. +The existing positional-wins compatibility behavior for duplicate legacy +package/resource bindings remains unchanged; it does not manufacture two +anchors or two resource identities. Conflicting optional text bindings are +accepted only if a valid new-family interpretation exists. + +Each required path expression must resolve to exactly one string using the +existing bounded assignment/value engine. Components are flattened and joined +with POSIX resource semantics. Forward-slash subpaths are supported where the +new family applies. Absolute names, backslashes, drive/colon tricks, traversal, +empty/dot segments, and unresolved components are rejected. Old-only keyword +and positional-encoding forms retain the direct-member restriction. Functional +file reads do not promote directories, preventing a missing-encoding text call +from accidentally rescuing all descendants. + +## Exact pre/post deployment reproduction + +The disposable Git-tracked source fixture contains `src/app/models/weights.bin` +and `read_binary('app', 'models', 'weights.bin')`, without package-data rescue. +Its Python requirement selects 3.13. Preparation/acquisition are controlled +offline fixture hooks; independently callable static validation runs normally. +The staged application is executed using managed Python 3.13.15 with `-E -s -B`. + +| Resolver | Resource included | Static validation | Isolated staged execution | +| --- | --- | --- | --- | +| Starting-head function | No | STATIC_VALID | FileNotFoundError for omitted weights.bin | +| Corrected function | Yes | STATIC_VALID | Reads RESOURCE successfully | + +Before implementation, four focused tests failed for read_binary, open_binary, +read_text, and open_text. They demonstrated absent concrete resource evidence. +Post-fix tests prove RUNTIME_RESOURCE promotion and source staging. Additional +text fixtures prove `templates/defaults.txt` staging with explicit functional +read evidence, rather than depending solely on conventional directory names. + +## Safety, package constraints, and bounded audit + +Focused tests cover all four functions, all proven alias forms, one-slash and +multipart paths, one/three/eight directory levels, static assignments, dynamic +encoding/errors, unresolved dynamic components, old keyword calls, conflicting +arguments, package/subpackage/module/top-level anchors, custom roots, exact and +parent mappings, namespace parents, package-over-module precedence, and resolved +repository escapes. + +Release tests reject a configured synthetic value in a nested text resource +without leaking that value in the exception. Dirty and untracked referenced +resources also reject release generation. Existing scanners, provenance, +binary-content policy, and staging machinery are reused without modification. + +A mapped installed-only application with an unbacked nested resource yields +DEPLOYMENT_MODE_CONFLICT. Explicit package-data makes that resource packaged +and preserves ENTRYPOINT_REQUIRES_PACKAGE_MODE. Analysis does not add arbitrary +resources to package_data or weaken first-party wheel completeness. + +The bounded audit covered only read_binary, open_binary, read_text, and open_text. +It closed the same-transition text-signature and module-anchor cases. It did +not add path(), is_resource(), contents(), new library families, general Python +binding/dataflow, or target-conditioned persisted resource fields. + +## Read-only acceptance + +All repository SHAs and before/after statuses matched. Comparisons against the +starting-head recognizer found identical packaging metadata, dependencies, +entry points, configuration evidence, and resource evidence. No proven calls +to these four functional APIs were found in the assessed packaged Python +surfaces of the three acceptance repositories. + +| Repository | Unchanged SHA | Result | +| --- | --- | --- | +| SimpleGeorefGUI | `f484570d89fb1f9e9170fac915475358dfc1234e` | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python / 77 data members | +| Geo Map Explanation Extractor | `5e7b321d0aeb9ba1d586bfc548c79793d84c6033` | source / SOURCE_COMPATIBLE | +| TN Coordinate Converter | `e1e7a1588c37a99c2d02efaf3eef3d04636f12f0` | source / SOURCE_COMPATIBLE; reviewed proxy-tools 0.1.0 | + +SGG's GUI target remains `simple_georef_gui_app.georef_main:main`. TN's existing +`pdb-m61-regression-20260901/tn-kit` remains independently STATIC_VALID. No +acceptance sources, wheels, kits, locks, or environments were changed. + +## Quality and preservation + +New focused tests: **191 passed**. Broader targeted resource/layout suite: +**760 passed, 1 skipped**. The single complete suite finished with +**1715 passed, 4 skipped** in 330.85 seconds, versus the prior 1524-pass/four-skip +baseline. `ruff check .` and `git diff --check` passed. A separate release probe +confirmed the configured-secret rejection is specifically NO_SECRET_VALUES, +with no diagnostic value leakage and no published kit. + +Schemas remain analysis **1.4**, planning **1.3**. Production changes are limited +to analysis/resources.py. Named-layout discovery, old text signatures, modern +files/as_file/module anchors, relative dynamic imports, legacy-manifest secret +compatibility, PEP 621 authority, approved-artifact paths/extra contexts/lock +identity, exact sync, mode-aware pip check, entry-point extras, short secrets, +LOCALAPPDATA, MANIFEST protections, wheel policy/dependency/security, provenance, +rollback, and deterministic packaging retain their regression coverage. + +Only after every gate passes: one correction commit, matching pushed head, +Quality-only PR count update, and exactly one `@codex review` request. No merge, +review dismissal, or historical inline-thread resolution. diff --git a/scripts/verify_functional_resource_families.py b/scripts/verify_functional_resource_families.py new file mode 100644 index 0000000..9b1e7f3 --- /dev/null +++ b/scripts/verify_functional_resource_families.py @@ -0,0 +1,215 @@ +"""Disposable Python 3.11--3.14 functional-resource and staged-kit evidence.""" + +import ast +import json +import os +import subprocess +import tempfile +from pathlib import Path +from unittest.mock import patch + +import python_deployment_builder.analysis.resources as resources +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.generation.generator import generate_deployment_kit +from python_deployment_builder.generation.preparation import LockPreparationResult +from python_deployment_builder.validation.static import validate_static_kit + +VERSIONS = ("3.11.16", "3.12.14", "3.13.15", "3.14.7") +PROBE = """ +import importlib.resources as r, inspect, json, sys, warnings +warnings.simplefilter('ignore', DeprecationWarning) +result = {'version': sys.version.split()[0]} +for name in ['read_binary', 'open_binary', 'read_text', 'open_text']: + f = getattr(r, name) + text = name.endswith('text') + filename = 'defaults.txt' if text else 'weights.bin' + directory = 'templates' if text else 'models' + cases = { + 'common': (['app', filename], {}), + 'multi': (['app', directory, filename], {'encoding':'utf-8'} if text else {}), + 'slash': (['app', directory+'/'+filename], {}), + 'old3': (['app', filename, 'utf-8'], {}), + 'old4': (['app', filename, 'utf-8', 'strict'], {}), + 'missing_encoding': (['app', directory, filename], {}), + 'deep': (['app','templates','defaults','main.txt'], {'encoding':'utf-8'}), + 'legacy_keywords': ([], {'package':'app','resource':filename}), + 'mixed_legacy': (['app'], {'resource':filename}), + 'anchor_keyword': ([], {'anchor':'app'}), + 'invented_path_keyword': ([], {'anchor':'app','path_names':filename}), + 'module_anchor': (['app.config', filename], {}), + 'top_module': (['config', filename], {}), + 'namespace_parent': (['ns.config', filename], {}), + 'package_precedence': (['app.choice', filename], {}), + } + outcomes = {'signature': str(inspect.signature(f))} + for label, (args, kwargs) in cases.items(): + try: + value = f(*args, **kwargs) + if hasattr(value, 'read'): + with value: value = value.read() + outcomes[label] = value.decode() if isinstance(value, bytes) else value + except Exception as error: + outcomes[label] = type(error).__name__ + ': ' + str(error) + assert outcomes['common'] == 'RESOURCE' + modern = sys.version_info >= (3,13) + assert (outcomes['multi'] == 'RESOURCE') == modern + assert (outcomes['slash'] == 'RESOURCE') == modern + assert (outcomes['module_anchor'] == 'RESOURCE') == (sys.version_info >= (3,12)) + assert (outcomes['top_module'] == 'RESOURCE') == (sys.version_info >= (3,12)) + assert (outcomes['namespace_parent'] == 'RESOURCE') == (sys.version_info >= (3,12)) + if text: + assert (outcomes['old3'] == 'RESOURCE') == (not modern) + assert (outcomes['old4'] == 'RESOURCE') == (not modern) + assert (outcomes['deep'] == 'RESOURCE') == modern + result[name] = outcomes +print(json.dumps(result)) +""" + + +def old_resolver(): + source = subprocess.check_output( + [ + "git", + "show", + "0ce99790f26e256b1cc4ab1c3879d5db70b8cb1d:" + "src/python_deployment_builder/analysis/resources.py", + ], + text=True, + ) + node = next( + n + for n in ast.parse(source).body + if isinstance(n, ast.FunctionDef) and n.name == "_legacy_importlib_resource_path_values" + ) + namespace = dict(resources.__dict__) + exec(compile(ast.Module(body=[node], type_ignores=[]), "", "exec"), namespace) + return namespace[node.name] + + +def deployed_probe(root, python): + source = root / "source" + package = source / "src/app" + (package / "models").mkdir(parents=True) + (package / "__init__.py").write_text("") + (package / "models/weights.bin").write_bytes(b"RESOURCE") + (package / "main.py").write_text( + "from importlib.resources import read_binary\n" + "def main(): return read_binary('app', 'models', 'weights.bin').decode()\n" + ) + (source / "pyproject.toml").write_text( + "[project]\nname='demo'\nversion='1.0'\nrequires-python='>=3.13,<3.14'\n" + "[project.scripts]\ndemo='app.main:main'\n" + ) + (source / "uv.lock").write_text( + "version=1\nrevision=3\nrequires-python='>=3.13,<3.14'\n" + "[[package]]\nname='demo'\nversion='1.0'\nsource={virtual='.'}\n" + ) + for args in [ + ("init",), + ("add", "."), + ( + "-c", + "user.name=PDB Test", + "-c", + "user.email=pdb@example.invalid", + "commit", + "-m", + "fixture", + ), + ]: + subprocess.run(["git", "-C", str(source), *args], check=True, capture_output=True) + fake_uv = root / "uv.exe" + fake_uv.write_bytes(b"verified uv") + old = old_resolver() + for label in ("before", "after"): + kit = root / label + with ( + patch.object( + resources, + "_legacy_importlib_resource_path_values", + old if label == "before" else resources._legacy_importlib_resource_path_values, + ), + patch( + "python_deployment_builder.generation.generator.acquire_pinned_uv", + return_value=fake_uv, + ), + patch( + "python_deployment_builder.generation.generator.prepare_lockfile", + side_effect=lambda path, *a, **kw: LockPreparationResult( + path=path / "uv.lock", created=False, checked=True, commands=() + ), + ), + ): + generate_deployment_kit( + MaterializedRepository(root=source, source=str(source), source_kind="local"), + kit, + bootstrap_mode="online_cmd", + ) + report = validate_static_kit(kit) + run = subprocess.run( + [str(python), "-B", "-E", "-s", "-c", "from app.main import main; print(main())"], + cwd=kit / "src", + capture_output=True, + text=True, + ) + print( + json.dumps( + { + "stage": label, + "static": report.final_state.value, + "included": (kit / "src/app/models/weights.bin").exists(), + "runtime": run.stdout.strip() + if not run.returncode + else run.stderr.splitlines()[-1], + } + ), + flush=True, + ) + assert report.final_state.value == "STATIC_VALID" + assert (run.returncode == 0) == (label == "after") + + +def main(): + with tempfile.TemporaryDirectory(prefix="pdb-resource-families-") as directory: + root = Path(directory) + for package in ("app", "app/choice", "ns", ""): + for name in ("weights.bin", "defaults.txt"): + path = root / package / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("RESOURCE") + for name in ( + "app/__init__.py", + "app/config.py", + "app/choice/__init__.py", + "app/choice.py", + "config.py", + "ns/config.py", + ): + (root / name).write_text("") + for name in ( + "app/models/weights.bin", + "app/templates/defaults.txt", + "app/templates/defaults/main.txt", + ): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("RESOURCE") + interpreters = [ + Path(os.environ["APPDATA"]) + / "uv/python" + / f"cpython-{version}-windows-x86_64-none/python.exe" + for version in VERSIONS + ] + for python in interpreters: + print( + subprocess.check_output( + [str(python), "-B", "-E", "-s", "-c", PROBE], cwd=root, text=True + ), + flush=True, + ) + if os.environ.get("PDB_DEPLOYED_PROBE"): + deployed_probe(root, interpreters[2]) + + +if __name__ == "__main__": + main() diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index afed75e..c59c3bf 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -6,6 +6,7 @@ from collections import defaultdict from dataclasses import dataclass from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Literal from python_deployment_builder.analysis.ast_utils import call_argument from python_deployment_builder.analysis.imports import EXCLUDED_DIRECTORIES @@ -715,6 +716,58 @@ def _pkgutil_resource_path_values( return _combine_paths(package_roots, members) +@dataclass(frozen=True) +class _FunctionalResourceCall: + anchor: ast.AST + path_nodes: tuple[ast.AST, ...] + signature_family: Literal["legacy_direct", "multipath", "common"] + + +def _functional_resource_call(node: ast.Call, function: str) -> _FunctionalResourceCall | None: + """Classify only the four supported Python 3.11--3.14 read signatures. + + In 3.13/3.14 multiple text path_names require keyword encoding; in 3.11/ + 3.12 that same keyword conflicts with positional encoding. Thus the two + valid families cannot assign different resource identities to one shape. + Star expansion is outside this bounded argument model. + """ + + keywords = [item.arg for item in node.keywords] + if None in keywords or len(set(keywords)) != len(keywords) or any( + isinstance(argument, ast.Starred) for argument in node.args + ): + return None + text = function.endswith("text") + options = {"encoding", "errors"} if text else set() + modern = ( + len(node.args) >= 2 + and set(keywords) <= options + and (not text or len(node.args) == 2 or "encoding" in keywords) + ) + legacy = ( + len(node.args) <= (4 if text else 2) + and set(keywords) <= options | {"package", "resource"} + and not any( + len(node.args) > position and keyword in keywords + for position, keyword in ((2, "encoding"), (3, "errors")) + ) + ) + anchor = call_argument(node, position=0, keyword="package") + resource = call_argument(node, position=1, keyword="resource") + legacy = legacy and anchor is not None and resource is not None + if modern: + return _FunctionalResourceCall( + node.args[0], tuple(node.args[1:]), "common" if legacy else "multipath" + ) + if legacy and anchor is not None and resource is not None: + # Retain the established positional-wins package/resource extraction. + return _FunctionalResourceCall(anchor, (resource,), "legacy_direct") + # anchor= is accepted in 3.13+, but with no positional path_names it + # addresses a directory, not a readable file. resource=/path_names= are + # not modern keyword parameters; never invent a keyword varargs binder. + return None + + def _legacy_importlib_resource_path_values( node: ast.Call, *, @@ -727,57 +780,52 @@ def _legacy_importlib_resource_path_values( module_bindings: set[str], read_bindings: dict[str, str], ) -> tuple[str, list[str]] | None: - """Resolve legacy direct-member reads with Python 3.11/3.12 signatures.""" + """Resolve functional reads valid in at least one supported signature family.""" function = _legacy_resource_function_name(node, module_bindings, read_bindings) if function is None: return None - # Python 3.12 legacy resource calls address one direct member of a - # package. Keep dynamic, nested, and traversal-like members unresolved. - text_function = function.endswith("text") - if len(node.args) > (4 if text_function else 2): - return function, [] - allowed_keywords = {"package", "resource"} - if text_function: - allowed_keywords.update({"encoding", "errors"}) - # Optional text parameters do not affect resource identity, but an - # invalid duplicate binding must not provide read evidence. Preserve - # the established positional-wins policy for package/resource. - if any( - len(node.args) > position and any(item.arg == keyword for item in node.keywords) - for position, keyword in ((2, "encoding"), (3, "errors")) - ): - return function, [] - if any(keyword.arg not in allowed_keywords for keyword in node.keywords): - return function, [] - package = call_argument(node, position=0, keyword="package") - resource = call_argument(node, position=1, keyword="resource") - if package is None or resource is None: + call = _functional_resource_call(node, function) + if call is None: return function, [] package_roots = _resource_package_anchor_values( - package, + call.anchor, root=root, source_path=source_path, source_roots=source_roots, project=project, assignments=assignments, returns=returns, + # Both 3.12's wrappers and 3.13+ functional helpers delegate to files + # with module semantics. Use its shared package/module precedence. + allow_module_anchor=True, ) - members = _path_values( - resource, - root=root, - source_path=source_path, - assignments=assignments, - returns=returns, - ) - if ( - not package_roots - or len(members) != 1 - or not _safe_resource_member(members) - or len(PurePosixPath(members[0].replace("\\", "/")).parts) != 1 - ): + parts: list[str] = [] + for expression in call.path_nodes: + values = _path_values( + expression, root=root, source_path=source_path, + assignments=assignments, returns=returns, + ) + if len(values) != 1: + return function, [] + value = values[0] + if ( + not _safe_resource_member(values) + or "\\" in value + or ":" in value + or PureWindowsPath(value).drive + or any(part in {"", ".", ".."} for part in value.split("/")) + ): + return function, [] + parts.extend(value.split("/")) + if not package_roots or (call.signature_family == "legacy_direct" and len(parts) != 1): return function, [] - return function, _combine_paths(package_roots, members) + member = PurePosixPath(*parts).as_posix() + candidates = _combine_paths(package_roots, [member]) + # These APIs read files, not directories. In particular a legacy-shaped + # read_text(anchor, directory, filename) must not promote directory trees + # when the modern family rejects the missing encoding keyword. + return function, [path for path in candidates if not (root / path).is_dir()] def _importlib_resource_path_values( diff --git a/tests/test_functional_resource_families.py b/tests/test_functional_resource_families.py new file mode 100644 index 0000000..2eb5b97 --- /dev/null +++ b/tests/test_functional_resource_families.py @@ -0,0 +1,361 @@ +"""Functional resource reads share bounded Python 3.11--3.14 signature evidence.""" + +import ast +import subprocess +from pathlib import Path + +import pytest +from test_as_file_relative_imports import offline_tools # noqa: F401 +from test_dependency_authority import assess +from test_files_package_keyword import resource_project +from test_planning import _write_mode_project + +from python_deployment_builder.analysis.repository import MaterializedRepository +from python_deployment_builder.analysis.resources import _functional_resource_call +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.planning.planner import create_deployment_plan + + +def functional_project( + root, + function="read_binary", + arguments="'app', 'models', 'weights.bin'", + relative="models/weights.bin", + imports=None, +): + resource_project(root, "", "unused") + path = root / "src/app" / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("RESOURCE") + (root / "src/app/main.py").write_text( + (imports or f"from importlib.resources import {function}") + + f"\ndef main(): return {function}({arguments})\n" + ) + return path.relative_to(root).as_posix() + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary", "read_text", "open_text"]) +def test_multipath_promoted_and_staged(tmp_path, function): + arguments = "'app', 'models', 'weights.bin'" + if function.endswith("text"): + arguments += ", encoding='utf-8'" + relative = functional_project(tmp_path, function, arguments) + assessment = assess(tmp_path) + assert relative in {item.path for item in assessment.resources} + roles = {item.path: item.role.value for item in assessment.file_inventory} + assert roles[relative] == "runtime_resource" + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary", "read_text", "open_text"]) +@pytest.mark.parametrize("binding", ["module", "module_alias", "direct", "direct_alias"]) +@pytest.mark.parametrize("components", ["'models', 'weights.bin'", "'models/weights.bin'"]) +def test_multipath_aliases_and_slash(tmp_path, function, binding, components): + imports, target = { + "module": ("import importlib.resources", f"importlib.resources.{function}"), + "module_alias": ("import importlib.resources as r", f"r.{function}"), + "direct": (f"from importlib.resources import {function}", function), + "direct_alias": (f"from importlib.resources import {function} as load", "load"), + }[binding] + arguments = f"'app', {components}" + if function.endswith("text"): + arguments += ", encoding=selected_encoding, errors=selected_errors" + relative = functional_project(tmp_path, target, arguments, imports=imports) + assessment = assess(tmp_path) + assert relative in {item.path for item in assessment.resources} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary", "read_text", "open_text"]) +@pytest.mark.parametrize("depth", [1, 3, 8]) +def test_arbitrary_depth_and_static_components(tmp_path, function, depth): + parts = [f"dir{index}" for index in range(depth)] + ["data.txt"] + relative = "/".join(parts) + arguments = "'app', PREFIX, " + ", ".join(repr(part) for part in parts[1:]) + if function.endswith("text"): + arguments += ", encoding=encoding_choice" + path = functional_project( + tmp_path, + function, + arguments, + relative, + f"from importlib.resources import {function}\nPREFIX={parts[0]!r}", + ) + assert path in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +@pytest.mark.parametrize( + ("arguments", "family", "paths"), + [ + ("'app', 'defaults.txt'", "common", ["defaults.txt"]), + ("'app', 'defaults.txt', 'utf-8'", "legacy_direct", ["defaults.txt"]), + ("'app', 'defaults.txt', 'utf-8', 'strict'", "legacy_direct", ["defaults.txt"]), + ("'app', 'defaults.txt', encoding=choice", "common", ["defaults.txt"]), + ( + "'app', 'templates', 'defaults.txt', encoding=choice", + "multipath", + ["templates", "defaults.txt"], + ), + ( + "'app', 'templates', 'defaults', 'main.txt', encoding=choice", + "multipath", + ["templates", "defaults", "main.txt"], + ), + ( + "package='app', resource='defaults.txt', encoding=choice", + "legacy_direct", + ["defaults.txt"], + ), + ], +) +def test_text_families_are_disjoint(tmp_path, function, arguments, family, paths): + node = ast.parse(f"{function}({arguments})", mode="eval").body + result = _functional_resource_call(node, function) + assert result.signature_family == family + assert [ast.literal_eval(n) for n in result.path_nodes] == paths + # Each family selects exactly one identity; old encoding never becomes a path. + relative = functional_project(tmp_path, function, arguments, "/".join(paths)) + assert relative in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +def test_missing_encoding_does_not_promote_multipath_directory(tmp_path, function): + relative = functional_project(tmp_path, function, "'app', 'models', 'weights.bin'") + assessment = assess(tmp_path) + assert not any(item.path in {relative, "src/app/models"} for item in assessment.resources) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative not in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary", "read_text", "open_text"]) +@pytest.mark.parametrize( + "parts", + [ + "unknown, 'weights.bin'", + "f'{unknown}', 'weights.bin'", + "'models', unknown", + "'../models', 'weights.bin'", + "'/models', 'weights.bin'", + "'C:models', 'weights.bin'", + "r'models\\nested', 'weights.bin'", + "'', 'weights.bin'", + "'.', 'weights.bin'", + "'models//nested', 'weights.bin'", + "'models', '..', 'weights.bin'", + ], +) +def test_unsafe_and_dynamic_components_unresolved(tmp_path, function, parts): + arguments = f"'app', {parts}" + (", encoding='utf-8'" if function.endswith("text") else "") + relative = functional_project(tmp_path, function, arguments) + assert relative not in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary", "read_text", "open_text"]) +@pytest.mark.parametrize( + "arguments", + [ + "anchor='app'", + "anchor='app', resource='weights.bin'", + "anchor='app', path_names='models/weights.bin'", + "'app', 'models', unknown=True", + "'app', 'models', **kwargs", + "'app', *parts", + "'app', 'models', anchor='other'", + ], +) +def test_invalid_functional_keyword_shapes(tmp_path, function, arguments): + relative = functional_project(tmp_path, function, arguments) + assert relative not in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary", "read_text", "open_text"]) +@pytest.mark.parametrize( + "arguments", ["package='app', resource='weights.bin'", "'app', resource='weights.bin'"] +) +def test_old_keywords_still_work(tmp_path, function, arguments): + relative = functional_project(tmp_path, function, arguments, "weights.bin") + assert relative in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("function", ["read_binary", "open_binary", "read_text", "open_text"]) +@pytest.mark.parametrize( + "layout", + [ + "package", + "subpackage", + "module", + "top", + "namespace", + "custom", + "exact", + "parent", + "precedence", + ], +) +def test_functional_module_anchor_shared_resolution(tmp_path, function, layout): + resource_project(tmp_path, "", "unused") + anchor = "app.config" + directory = tmp_path / "src/app" + mapping = None + if layout == "package": + anchor = "app" + elif layout in {"subpackage", "precedence"}: + (directory / "config").mkdir() + (directory / "config/__init__.py").write_text("") + if layout == "precedence": + (directory / "config.py").write_text("") + (directory / "weights.bin").write_text("WRONG") + directory /= "config" + elif layout == "top": + anchor = "config" + directory = tmp_path / "src" + elif layout == "custom": + directory = tmp_path / "lib/app" + mapping = "{''='lib'}" + elif layout in {"exact", "parent"}: + directory = tmp_path / "code" + mapping = "{'app.config'='code/config'}" if layout == "exact" else "{'app'='code'}" + directory.mkdir(parents=True, exist_ok=True) + if layout not in {"package", "subpackage", "precedence"}: + (directory / "config.py").write_text("") + if layout == "namespace": + (tmp_path / "src/app/__init__.py").unlink() + path = directory / "weights.bin" + path.write_text("RESOURCE") + if mapping: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text(pyproject.read_text() + f"[tool.setuptools]\npackage-dir={mapping}\n") + (directory / "main.py" if layout == "custom" else tmp_path / "src/app/main.py").write_text( + f"from importlib.resources import {function}\n" + f"def main(): return {function}({anchor!r}, 'weights.bin')\n" + ) + assessment = assess(tmp_path) + resource_paths = {item.path for item in assessment.resources} + assert path.relative_to(tmp_path).as_posix() in resource_paths + if layout == "precedence": + assert "src/app/weights.bin" not in resource_paths + + +@pytest.mark.parametrize("state", ["secret", "dirty", "untracked"]) +@pytest.mark.usefixtures("offline_tools") +def test_multipath_release_security_and_provenance(tmp_path, monkeypatch, state): + root = tmp_path / "source" + relative = functional_project( + root, + "read_text", + "'app', 'models', 'weights.bin', encoding='utf-8'", + imports=( + "from importlib.resources import read_text\nfrom os import getenv\n" + "PASSWORD=getenv('DB_PASSWORD')" + ), + ) + secret = "PDBMultipathConfiguredSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + if state == "secret": + (root / relative).write_text(secret) + else: + for args in [ + ("init",), + ("add", "."), + ( + "-c", + "user.name=PDB Test", + "-c", + "user.email=pdb@example.invalid", + "commit", + "-m", + "fixture", + ), + ]: + subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True) + if state == "dirty": + (root / relative).write_text("CHANGED") + else: + subprocess.run( + ["git", "-C", str(root), "rm", "--cached", relative], + check=True, + capture_output=True, + ) + with pytest.raises(PreparationError) as error: + generate_deployment_kit( + MaterializedRepository(root=root, source=str(root), source_kind="local"), + tmp_path / "kit", + ) + assert secret not in str(error.value) + assert not (tmp_path / "kit").exists() + + +@pytest.mark.parametrize("function", ["read_text", "open_text"]) +def test_exact_text_templates_multipath_staging(tmp_path, function): + relative = functional_project( + tmp_path, + function, + "'app', 'templates', 'defaults.txt', encoding='utf-8'", + "templates/defaults.txt", + ) + assessment = assess(tmp_path) + resource = next(item for item in assessment.resources if item.path == relative) + assert any(f"importlib.resources.{function}()" in e.detail for e in resource.evidence) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("packaged", [False, True]) +def test_multipath_respects_package_source_constraints(tmp_path, packaged): + _write_mode_project(tmp_path, mapped=True, target="installed_app.main:main") + directory = tmp_path / "code/bundle" + directory.mkdir() + (directory / "weights.bin").write_bytes(b"BINARY") + (tmp_path / "code/main.py").write_text( + "from importlib.resources import read_binary\n" + "def main(): return read_binary('installed_app', 'bundle', 'weights.bin')\n" + ) + if packaged: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + pyproject.read_text() + + "[tool.setuptools.package-data]\ninstalled_app=['bundle/*.bin']\n" + ) + assessment = assess(tmp_path) + resource = next(item for item in assessment.resources if item.path == "code/bundle/weights.bin") + assert resource.packaging_status == ("packaged" if packaged else "repository_adjacent") + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert plan.deployment_mode_condition == ( + "ENTRYPOINT_REQUIRES_PACKAGE_MODE" if packaged else "DEPLOYMENT_MODE_CONFLICT" + ) + if not packaged: + assert not assessment.project.package_data + + +def test_multipath_resolved_escape_cannot_promote_resource(tmp_path, monkeypatch): + root = tmp_path / "source" + relative = functional_project(root) + escaping = root / relative + outside = tmp_path / "outside.bin" + outside.write_bytes(b"OUTSIDE") + original = Path.resolve + monkeypatch.setattr( + Path, "resolve", lambda p, *a, **kw: outside if p == escaping else original(p, *a, **kw) + ) + assessment = assess(root) + assert not any( + item.path == relative and item.role.value == "runtime_resource" + for item in assessment.file_inventory + ) + + +@pytest.mark.parametrize( + "arguments", + [ + "'app', 'model', 'utf-8', 'strict', errors='ignore'", + "'app', 'model', encoding='utf-8', encoding='ascii'", + "'app', 'model', 'utf-8', unknown=True", + ], +) +def test_no_valid_text_family_for_conflicting_arguments(arguments): + call = ast.parse(f"read_text({arguments})", mode="eval").body + assert _functional_resource_call(call, "read_text") is None From 847198a4ac9d9fbd2149a402aa0ef4dbc7b79841 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:59:44 -0400 Subject: [PATCH 58/59] Enforce application wheel authority and complete resource acceptance --- ...urface-assigned-traversables-2026-09-12.md | 159 +++++++ .../pr9-minor-marker-acceptance-2026-09-12.md | 176 +++++++ ...pr9-security-content-context-2026-09-12.md | 84 ++++ .../analysis/resources.py | 28 ++ .../generation/artifacts.py | 79 +++- .../generation/manifest.py | 1 + .../generation/structural.py | 2 +- src/python_deployment_builder/models.py | 2 + .../planning/index.py | 97 +++- .../security_policy.py | 29 +- .../validation/static.py | 23 +- ..._application_surface_assigned_resources.py | 443 ++++++++++++++++++ tests/test_generation.py | 17 + tests/test_minor_marker_proof.py | 254 ++++++++++ tests/test_security_content_context.py | 148 ++++++ 15 files changed, 1525 insertions(+), 17 deletions(-) create mode 100644 docs/acceptance/pr9-application-surface-assigned-traversables-2026-09-12.md create mode 100644 docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md create mode 100644 docs/acceptance/pr9-security-content-context-2026-09-12.md create mode 100644 tests/test_application_surface_assigned_resources.py create mode 100644 tests/test_minor_marker_proof.py create mode 100644 tests/test_security_content_context.py diff --git a/docs/acceptance/pr9-application-surface-assigned-traversables-2026-09-12.md b/docs/acceptance/pr9-application-surface-assigned-traversables-2026-09-12.md new file mode 100644 index 0000000..105603f --- /dev/null +++ b/docs/acceptance/pr9-application-surface-assigned-traversables-2026-09-12.md @@ -0,0 +1,159 @@ +# PR #9: application surface and assigned Traversables + +Starting head: `4888ee28710d20dc8e45886ad4d928f0f556aa66` on +`milestone-6-1-generation-contract`. Changes remain local pending the acceptance +blocker below. No merge, review dismissal, thread resolution, push, PR edit, or +fresh review request was performed. + +## Findings and reproduction + +The exact-head review submitted at `2026-09-12T18:56:17Z` reported: + +- P1 **Reject undeclared executable members in application wheels**: + `PRRT_kwDOT9hvCc6hypCG`, comment `3997215864`. +- P2 **Follow assigned Traversables when resolving resource reads**: + `PRRT_kwDOT9hvCc6hypCJ`, comment `3997215868`. + +Authoritative accounting remains 127 findings/comments, 124 unresolved inline +threads, three review-level-only findings, and one NOT_APPLICABLE inline finding. +GitHub confirmed 124 threads and zero resolved threads during this turn. + +Five regressions were added and run before production changes. All five failed: + +- A correct mapped application wheel plus `requests/__init__.py` was accepted. +- A root `application-hook.pth` was accepted. +- A `.data/purelib/application-hook.pth` relocated to the installed root was accepted. +- `asset = files('app') / 'model.dat'; as_file(asset)` lost the resource. +- The same assignment followed by `asset.read_bytes()` lost the resource. + +The wheel fixtures had valid metadata, tags, entry points and recomputed RECORDs. +The source fixture had no independent package-data/resource-directory promotion. +The pre-fix resource assessment was empty; the wheel checks only required the +modeled files to be a subset of installed files. + +## Source-derived application authority + +`ApplicationArtifact.authoritative_members` is required and nonempty. Generation +stores the sorted union of source-resolved installed Python and package-data +members. It never records wheel inventory as authority. The entry-point module +must also belong to the source-derived Python surface, not merely occur in the +wheel or be inferred from packaged data. + +Generation and independently callable static validation share a relocated +installed-member validator. It requires every authoritative member, rejects +unmodeled `.py` members case-insensitively by suffix, and uses the established +Windows path/collision checks. Required-member spelling remains exact, consistent +with prior completeness checks. Metadata is excluded through the existing +installed application-surface resolver. `.data/purelib` relocation occurs before +enforcement. Extra benign non-executable data is not blanket-rejected. + +Root `.pth` files are forbidden regardless of content. Nested inert `.pth` data +is not a startup destination. The bounded startup audit inspected managed +CPython 3.12.14 `site.py` and executed isolated disposable probes: both +`sitecustomize` and `usercustomize` can be imported as modules **or packages**; +`.pth` processing also executed a harmless probe. Consequently their top-level +`.py` and package `__init__.py` destinations are forbidden even when declared. +No broader startup mechanism was added. + +The static check is `APPLICATION_WHEEL_AUTHORITATIVE_SURFACE`. Re-indexed tests +replace the wheel, recompute RECORD, artifact SHA, deployment fingerprint, +generation ID and generated-file hashes. Foreign Python, startup files, missing +Python and missing package data still fail while metadata/index checks pass. +Legitimate second packages and py-modules pass; existing collision fixtures now +declare their intentionally colliding Python code in their source surfaces. + +The authoritative list participates in the ordinary deployment fingerprint. A +one-member change changes that fingerprint. No sidecar or source reassessment is +needed during static validation. This remains a manifest contract, not a new +cryptographic attestation of source-to-wheel provenance. + +### Serialized contract decision + +Analysis schema stays **1.4**, planning schema stays **1.3**, and the deployment +envelope stays **1.0**. Package mode is an unreleased M6.1 contract; its artifact +shape is strengthened now without an insecure compatibility default. Missing or +empty authoritative lists fail model validation. Old source-mode manifests with +no application artifact retain their existing compatibility behavior. Current +package manifest bytes and fingerprints change, so historical SGG ZIP hashes +cannot be reused as evidence for a regenerated kit. + +## Assigned Traversables + +`_importlib_resource_path_values()` follows the existing `_bindings()` assignment +map directly, with separate cycle keys for assignments and local returns. The +same recursion handles names, stable qualified attribute keys, slash/joinpath +bases, multi-hop aliases and direct no-argument local return expressions. +Assignment and return cycles terminate unresolved. Parameter substitution, +arbitrary object calls and control-flow-sensitive mutation are not modeled. +The existing deterministic `_bindings()` assumptions are unchanged. + +The bounded adjacent audit found attribute keys and local return mappings already +available to ordinary path analysis. Traversable support reuses those mappings; +it does not introduce a call graph or context-manager yielded-variable tracking. + +Package, module, implicit, positional, `anchor=` and `package=` anchors retain +their existing semantics. Tests cover custom/global roots, exact/longest-parent +mappings, namespace parents, directory descendants, dynamic/cyclic inputs and +containment. Assigned `as_file`, `read_bytes` and `read_text` all promote resources. +Generated source-kit tests perform isolated interpreter reads successfully. +Configured-secret content, dirty files and untracked files still block release; +the secret value does not occur in exception diagnostics. + +## Acceptance results and blocker + +All three acceptance repositories retained their exact SHAs and Git status. +Comparing current analysis with the starting-head resource resolver showed no +dependency, entry-point, configuration, packaging or resource drift. + +| Repository | Unchanged SHA | Result | +| --- | --- | --- | +| SimpleGeorefGUI | `f484570d89fb1f9e9170fac915475358dfc1234e` | package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / BLOCKED_PENDING_APPLICATION_WHEEL; 13 Python + 77 package-data members; GUI target unchanged | +| Geo Map Explanation Extractor | `5e7b321d0aeb9ba1d586bfc548c79793d84c6033` | source / SOURCE_COMPATIBLE; no application artifact | +| TN Coordinate Converter | `e1e7a1588c37a99c2d02efaf3eef3d04636f12f0` | source / SOURCE_COMPATIBLE; existing kit STATIC_VALID; proxy-tools 0.1.0 retained | + +The accepted SGG application wheel is: + +`simple_georef_gui-1.3-py3-none-any.whl` + +SHA-256: `e479d668a59e879bbc42ee3d32ffcc89ce3bf40dc65d6c52d703be44703ede03`. + +Its independent **new surface check passes all 90 source-derived members** with +no undeclared Python or startup destinations. However, full regeneration stops +before that check at the existing security policy: + +`Wheel content violates deployment security policy: simple_georef_gui-1.3.dist-info/METADATA` + +The rule is `program_files_write`. METADATA line 352 is descriptive documentation +about copying an ArcGIS Pro environment without copying Program Files security +descriptors. The existing whole-text token rule treats the combination of +`Program Files` and `copy ` as a violation; this is not evidence of an actual +write to Program Files. The starting-head `validate_application_wheel()` function +was executed against the same source/plan/wheel and rejects the same METADATA. +Thus the blocker predates these two corrections. Every saved SGG application +wheel found in the acceptance locations has the same SHA. + +No acceptance source or wheel was edited and no security exemption was added. +The newly created external acceptance output directory has no generated kit. +SGG runtime revalidation, final automated state and twice-packaged deterministic +ZIP evidence therefore remain **not completed**. The old PR hash is historical, +not refreshed evidence. Resolving this requires direction for the pre-existing +metadata policy false positive or an accepted replacement artifact. + +## Quality and historical handoff + +The acceptance blockers recorded above were subsequently resolved under explicit +user direction. See [completed acceptance](pr9-minor-marker-acceptance-2026-09-12.md) +for the refreshed SGG runtime/ZIP evidence and final combined quality gate. + +- New focused regression file: **86 passed**. +- Resource-focused run before the last startup/isolated-read additions: + **407 passed, 1 skipped**. +- One complete suite: **1801 passed, 4 skipped** in 341.25 seconds (baseline + 1715 passed, 4 skipped; 86 additional passing cases). +- Ruff: passed. +- `git diff --check`: passed; final inspection found no unrelated changes. +- No generated wheel, kit, environment or ZIP is part of the local diff. +- Commit/push, PR Quality/evidence changes and the single fresh review request are + withheld because SGG acceptance has not passed. + +PR #9 STILL REQUIRES CORRECTION diff --git a/docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md b/docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md new file mode 100644 index 0000000..bf8dee5 --- /dev/null +++ b/docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md @@ -0,0 +1,176 @@ +# PR #9: minor-marker proof and completed package acceptance + +Starting remote/local HEAD: `4888ee28710d20dc8e45886ad4d928f0f556aa66`. +Branch: `milestone-6-1-generation-contract`. No acceptance repository was modified. + +## Separate accounting + +The pending correction contains **two Codex findings**, unchanged: + +- P1: Reject undeclared executable members in application wheels + (`PRRT_kwDOT9hvCc6hypCG`, comment `3997215864`). +- P2: Follow assigned Traversables when resolving resource reads + (`PRRT_kwDOT9hvCc6hypCJ`, comment `3997215868`). + +Two separately authorized, pre-existing acceptance issues complete their acceptance: + +- `PRE_EXISTING_ACCEPTANCE_POLICY_FALSE_POSITIVE`: descriptive Core METADATA + triggered the Program Files write heuristic. The path-context correction and + original reproduction are recorded in + [the policy report](pr9-security-content-context-2026-09-12.md). +- `PRE_EXISTING_MINOR_MARKER_PROOF_PRECISION_BLOCKER`: minor-invariant + `python_full_version` markers were unnecessarily unprovable. + +Neither acceptance issue is a new review finding. Accounting remains **127 total, +124 unresolved inline, three review-level-only, one NOT_APPLICABLE inline**. +The review-only findings remain historical #35, approved-wheel PEP 440 semantic +version comparison, and complete locked sync-command validation. The automatic +flat package plus loose module review premise remains NOT_APPLICABLE. + +## Before-change proof + +Two regressions were added and run before editing marker production code. Both +failed: `>=3.12` returned UNPROVABLE, and strict direct NumPy presence could not +be proven from the three SGG-shaped root edges. `>=3.12.1` was already +UNPROVABLE, correctly. + +Executing the marker implementation read with `git show` from the exact starting +SHA independently reproduced both atomic UNPROVABLE results. This used trusted +repository implementation code, not target application code or a checkout/reset. + +## Bounded proof implementation + +`target_marker_applicability()` evaluates packaging's parsed marker AST. Ordered +release comparisons and equality/inequality use the existing +`minor_python_compatibility()` interval proof; no target patch is populated. +Equality can reuse the exclusion proof to establish disjoint exact versions. +Major/minor wildcard equality and inequality are supported. Patch-prefix +wildcards and pre/dev/post/local spellings remain conservatively unprovable in +this bounded bridge, as do `in`, `not in`, `~=`, and `===`. + +Actual packaging 25.0 parsed `"3.12" <= python_full_version` as a literal on the +left, operator `<=`, and Variable on the right. Plain numeric comparisons invert +the six supported operators correctly. A wildcard on the left is not treated as +a wildcard specifier. No host `Marker.evaluate()` fallback is used. + +AND groups return false if any term is false, true if every term is true, +otherwise unknown. OR returns true if any group is true, false if every group +is false, otherwise unknown. Nested parsed lists retain parentheses; AND +precedence is retained within each OR group. All 18 binary tri-state combinations +and distinguishing grouped/ungrouped expressions are covered. + +At selected **3.12 / Windows x86_64**: + +| Comparison on python_full_version | Proof | +| --- | --- | +| `>=3.12`, `<3.13`, `==3.12.*`, `!=3.11.*` | APPLIES | +| `<3.12`, `>=3.13`, `==3.11.*`, `!=3.12.*` | DOES_NOT_APPLY | +| `>=3.12.1`, `<3.12.1`, `<3.12.5`, `==3.12.0`, `!=3.12.0` | UNPROVABLE | + +`implementation_version`, `platform_release`, and `platform_version` remain +unselected. They may participate in ordinary boolean dominance, but receive no +invented facts. All four Python policy minors have boundary/patch regressions. + +## Actual SGG uv 0.12.5 graph + +The unchanged SGG root entry was read directly from its TOML lock: + +| Dependency/version | Root-edge marker | 3.12 proof | +| --- | --- | --- | +| numpy 2.2.6 | `python_full_version < '3.11'` | DOES_NOT_APPLY | +| numpy 2.4.6 | `python_full_version == '3.11.*'` | DOES_NOT_APPLY | +| numpy 2.5.2 | `python_full_version >= '3.12'` | APPLIES | +| pyproj 3.7.1 | `python_full_version < '3.11'` | DOES_NOT_APPLY | +| pyproj 3.7.2 | `python_full_version >= '3.11'` | APPLIES | + +Running lock inspection with the starting marker implementation retained all +three NumPy and both pyproj versions. The corrected graph retains only NumPy +2.5.2 and pyproj 3.7.2. Removal occurs only for provably false branches. + +Application and approved-wheel Requires-Dist regressions both prove invariant +edges, still reject incompatible possible versions, still reject patch-sensitive +requirement/presence markers, and retain both sides of a `<3.12.5` / `>=3.12.5` +fork. A separate unconditional edge does not bless an incompatible possible +version. Existing artifact-fork and requested-extra proofs remain in the full +regression suite; their implementation was not changed. + +## SGG acceptance, now completed + +Source: `f484570d89fb1f9e9170fac915475358dfc1234e` (unchanged and clean). +Without artifacts: package / ENTRYPOINT_REQUIRES_PACKAGE_MODE / +BLOCKED_PENDING_APPLICATION_WHEEL. The map-extra flow separately needs its +reviewed proxy-tools artifact before generation, as before. + +- Source surface: **13 Python + 77 package-data = 90 authoritative members**. +- GUI target: `simple_georef_gui_app.georef_main:main`, unchanged. +- Accepted application wheel SHA-256: + `e479d668a59e879bbc42ee3d32ffcc89ce3bf40dc65d6c52d703be44703ede03`. +- Accepted proxy-tools 0.1.0 wheel SHA-256: + `2431971dbca4cf6524f851bf7f9d9e3275ef64f0c5607ad39b348df05ff97bc5`. +- Descriptive METADATA passes; ordinary metadata secret/security scanning remains + active. The 90-member source-derived list exactly matches the manifest; no + unexpected Python or startup-active destinations were accepted. +- Generation succeeds; static validation is **STATIC_VALID**. +- Runtime provisioning used uv **0.12.5** and managed Python **3.12.14**. + The plan still selects only the minor **3.12**; the installed patch is an + observation, not an input to marker proof. +- Every automated runtime check passed: pinned uv, managed Python, helper sibling + imports, locked/no-build setup, application-wheel installation, selected + dependency imports, dev dependency exclusion, configuration isolation, fast + path, controlled staleness, rollback, repair scope, and diagnostics. +- An additional isolated `-I -B` probe imported the authoritative GUI module from + managed `env/Lib/site-packages/simple_georef_gui_app/georef_main.py`, observed + NumPy 2.5.2 and pyproj 3.7.2, and confirmed no ArcPy/osgeo modules or + ArcGIS/GDAL/osgeo/ArcPy/ArcGIS Pro runtime paths. +- Final automated state: **MANUAL_GUI_VALIDATION_REQUIRED**. No unattended GUI + launch is presented as manual visual acceptance. + +Disposable output: sibling `pdb-m61-minor-marker-acceptance-20260912`, with +`static.json`, `runtime.json`, `managed-import.json`, and two package directories. +The isolated runtime is `C:/pdb-m61-sgg-0912`. None is committed. + +Both new ZIPs are byte-identical; both extracted kits validate STATIC_VALID. +`SimpleGeorefGui-Windows-v1.3.zip` is **21,011,577 bytes**, with new SHA-256: + +`69AE1EE3EC8169291F265ACEAE57BEC46035A15EB99D6BD3848C3A5910D6C05B` + +This supersedes the historical `FDCA6A5A...` hash: the required application +surface changed package-mode manifest/fingerprint bytes. The accepted wheel +bytes and SGG source did not change. + +## Source-mode acceptance + +- Geo: `5e7b321d0aeb9ba1d586bfc548c79793d84c6033`, unchanged; + **source / SOURCE_COMPATIBLE**, no application artifact. +- TN: `e1e7a1588c37a99c2d02efaf3eef3d04636f12f0`, unchanged; + **source / SOURCE_COMPATIBLE**. Its existing + `pdb-m61-regression-20260901/tn-kit` remains **STATIC_VALID**, retaining reviewed + **proxy-tools==0.1.0**. +- SGG, Geo, and TN assessment dependencies, entry points, configuration evidence, + and resources compare unchanged against starting-head analysis. The expected + marker-driven graph narrowing is separate from source assessment. + +## Contract and verification + +Analysis/planning schemas remain **1.4 / 1.3**. No marker/policy field is persisted. +The unreleased M6.1 package-only ApplicationArtifact contract requires a nonempty +source-derived authoritative member list and fingerprints it; source manifests +without application artifacts retain compatibility. Deployment envelope version +remains 1.0; no insecure default is added for unpublished package artifacts. + +Focused marker tests: **80 passed**. Broader planning/generation/approved-edge +subset: **712 passed, 3 skipped**. + +One final complete suite: **1933 passed, 4 skipped** in 348.18 seconds. This +includes all 86 application-surface/assigned-Traversable tests, all 52 +content-context policy tests, and all 80 new marker tests. The previous 1801-pass +baseline is not presented as the final count. + +Ruff and `git diff --check` pass. Final diff inspection contains only the two +review corrections, two authorized acceptance corrections, their regression +coverage, and scoped text evidence. Generated wheels, kits, environments, and +ZIPs remain outside the commit. The three acceptance repositories remain clean. + +The single acceptance-completing commit may now be pushed and receive exactly +one fresh exact-head review after PR Quality and the stale SGG hash are updated. +No historical thread is resolved; no review is dismissed; PR #9 is not merged. diff --git a/docs/acceptance/pr9-security-content-context-2026-09-12.md b/docs/acceptance/pr9-security-content-context-2026-09-12.md new file mode 100644 index 0000000..24100f8 --- /dev/null +++ b/docs/acceptance/pr9-security-content-context-2026-09-12.md @@ -0,0 +1,84 @@ +# PR #9: operational write heuristic context + +Remote/local HEAD remains `4888ee28710d20dc8e45886ad4d928f0f556aa66`. +The two prior review corrections remain uncommitted and preserved. +This additional correction is **PRE_EXISTING_ACCEPTANCE_POLICY_FALSE_POSITIVE**, +not another Codex review finding. Accounting remains 127 findings/comments, +124 unresolved inline threads, three review-level-only findings and one +NOT_APPLICABLE inline finding. + +## Reproduction and correction + +Five failing regressions preceded the policy edit. A valid wheel whose METADATA +contained descriptive text about making a data-only copy without copying Program +Files security descriptors was rejected by the original shared scanner. + +The actual accepted SGG wheel retains SHA-256 +`e479d668a59e879bbc42ee3d32ffcc89ce3bf40dc65d6c52d703be44703ede03`. +Its `simple_georef_gui-1.3.dist-info/METADATA` line 352 contains embedded README +documentation about copying a read-only ArcGIS Pro environment without copying +Program Files security descriptors. METADATA is descriptive, not executed write +code. Executing the security-policy module obtained with `git show` from the +starting SHA against those exact bytes returned only `program_files_write`. +The corrected path-aware scanner returns no findings for those same bytes. + +`program_files_write_applicable()` exempts only known descriptive `.md`, `.rst`, +`.txt` paths and standard direct `.dist-info` metadata names from this heuristic. +Python, BAT and CMD are PDB's generated executable forms and remain checked. +HTML application resources, `.pth` startup files, JavaScript and unknown text +paths remain conservatively checked rather than being exempted by a narrow +executable-extension allowlist. Pathless compatibility calls retain the original +conservative behavior. No distribution, filename or sentence special case exists. + +Every production caller now supplies path context: `generation/structural.py` +(rendered/generated and staged source), `generation/artifacts.py` (application +and approved wheel members), and `validation/static.py` (independent static kit +scanning). An AST regression verifies all production calls carry `path`. + +The bounded adjacent audit leaves every other finding untouched. Secrets and +developer paths are meaningful leaks in documentation. No direct acceptance +evidence justifies changing forbidden-shell or permanent-PATH policy, so both +remain active on documentation/METADATA. UTF-8 classification and secret-file +checks are unchanged. Multiline Python writes and realistic BAT/CMD copies into +Program Files still fail. + +## New independent acceptance blocker: historical stop gate + +This stop-point was subsequently resolved under explicit user direction; see +[minor-marker proof and completed acceptance](pr9-minor-marker-acceptance-2026-09-12.md). +The evidence below records the earlier run, not the current acceptance state. + +SGG source is still `f484570d89fb1f9e9170fac915475358dfc1234e`, with the +unchanged 13 Python / 77 package-data surface and package-mode entry point. +The accepted wheel passes security scanning and the 90-member authority check. +Full generation now proceeds to strict locked dependency validation and fails: + +```text +Application wheel Requires-Dist presence cannot be proven from a definitely +applicable direct locked dependency edge for numpy: +python_full_version < '3.11'; +python_full_version == '3.11.*'; +python_full_version >= '3.12'. +``` + +This is distinct from the corrected documentation policy false positive. Per the +explicit instruction to stop on another acceptance blocker, no dependency-proof +changes, further acceptance attempts, full-suite run, runtime validation or ZIP +packaging were undertaken after this failure. The historical deterministic SGG +ZIP hash has not been represented as regenerated evidence. + +## Verification and state + +- New policy tests: **52 passed** after correcting a test-call keyword typo. +- The prior 86 review-correction cases also passed in the focused combined run; + its four policy-test failures were the same subsequently corrected test typo. +- Last complete suite remains the preceding local baseline: **1801 passed, + 4 skipped**. A new complete suite was not run because the acceptance stop gate + was reached. +- Analysis/planning schemas remain **1.4 / 1.3**; no new persisted policy field. +- Geo/TN acceptance results from the preceding turn remain recorded separately; + acceptance was not repeated after the SGG stop gate. +- No commit, push, PR Quality/evidence edit, review request, thread resolution, + review dismissal or merge. + +PR #9 STILL REQUIRES CORRECTION diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index c59c3bf..3841527 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -839,9 +839,35 @@ def _importlib_resource_path_values( returns: dict[str, ast.AST], module_bindings: set[str], files_bindings: set[str], + seen: frozenset[str] = frozenset(), ) -> list[str] | None: """Resolve a bounded ``importlib.resources.files`` path expression statically.""" + # Share _bindings' deterministic assignment model, not general control flow. + binding = _qualified_name(node) if isinstance(node, (ast.Name, ast.Attribute)) else "" + value = assignments.get(binding) + key = f"assignment:{binding}" + if value is None and isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + # Only direct no-argument local returns: no parameter substitution or call graph. + binding = node.func.id + if not node.args and not node.keywords: + value = returns.get(binding) + key = f"return:{binding}" + if value is not None: + if key in seen: + return [] + return _importlib_resource_path_values( + value, + root=root, + source_path=source_path, + source_roots=source_roots, + project=project, + assignments=assignments, + returns=returns, + module_bindings=module_bindings, + files_bindings=files_bindings, + seen=seen | {key}, + ) if _is_resource_files_call(node, module_bindings, files_bindings): if not isinstance(node, ast.Call): return [] @@ -886,6 +912,7 @@ def _importlib_resource_path_values( returns=returns, module_bindings=module_bindings, files_bindings=files_bindings, + seen=seen, ) if base is None: return None @@ -915,6 +942,7 @@ def _importlib_resource_path_values( returns=returns, module_bindings=module_bindings, files_bindings=files_bindings, + seen=seen, ) if base is None: return None diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 8a0d0e8..00f5995 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -766,7 +766,7 @@ def _validate_wheel_security( if text is None: continue if text_security_findings( - text, configured_secret_values=configured_secret_values + text, path=member_path, configured_secret_values=configured_secret_values ): failures.append(name) if failures: @@ -813,6 +813,74 @@ def _validate_application_wheel_content_policy( ) +def _validate_application_member_surface( + installed_names: set[str], authoritative_members: Iterable[str], entry_point_module: str +) -> None: + """Prove source/manifest authority against relocated, non-metadata destinations.""" + + authoritative = set(authoritative_members) + for member in authoritative: + if ( + _normalized_wheel_path(member) != member + or PurePosixPath(member).parts[0].endswith((".dist-info", ".data")) + ): + raise PreparationError("Invalid authoritative application member: " + member) + _validate_regular_file_path_collisions( + [(member, member) for member in sorted(authoritative)], domain="authoritative" + ) + module = "/".join(entry_point_module.split(".")) + if not {f"{module}.py", f"{module}/__init__.py"}.intersection(authoritative): + raise PreparationError( + "Application entry-point module is outside the authoritative packaging surface." + ) + startup = sorted( + member + for member in installed_names + if ( + "/" not in member and ( + member.casefold().endswith(".pth") + or member.casefold() in {"sitecustomize.py", "usercustomize.py"} + ) + ) or member.casefold() in {"sitecustomize/__init__.py", "usercustomize/__init__.py"} + ) + if startup: + raise PreparationError( + "Application wheel has startup-active destinations: " + ", ".join(startup) + ) + missing = sorted(authoritative - installed_names) + if missing: + raise PreparationError( + "Application wheel is missing authoritative members: " + ", ".join(missing) + ) + unexpected = sorted( + member for member in installed_names + if member.casefold().endswith(".py") and member not in authoritative + ) + if unexpected: + raise PreparationError( + "Application wheel has Python members outside the authoritative surface: " + + ", ".join(unexpected) + ) + + +def validate_application_wheel_surface( + path: Path, authoritative_members: Iterable[str], entry_point_module: str +) -> None: + """Independently enforce the manifest's source-derived installed surface.""" + + try: + with zipfile.ZipFile(path) as bundle: + members = _member_map(_safe_wheel_members(bundle)) + _dist_info_members(members, path) + _validate_application_member_surface( + installed_wheel_member_paths(members, path), + authoritative_members, + entry_point_module, + ) + except zipfile.BadZipFile as exc: + raise PreparationError(f"Malformed application wheel: {path.name}") from exc + + def validate_application_wheel_content_policy(path: Path) -> None: """Validate first-party content rules without source or plan state.""" @@ -1724,6 +1792,14 @@ def validate_application_wheel( "Application wheel is missing authoritative first-party Python source: " + ", ".join(missing_python_members) ) + if not module_candidates.intersection(expected_python_members): + raise PreparationError( + "Application entry-point module is outside the authoritative Python surface." + ) + authoritative_members = sorted(expected_members | expected_python_members) + _validate_application_member_surface( + installed_names, authoritative_members, entry_point.module + ) except (zipfile.BadZipFile, UnicodeDecodeError, configparser.Error) as exc: raise PreparationError(f"Malformed application wheel: {path.name}") from exc @@ -1736,6 +1812,7 @@ def validate_application_wheel( wheel_tags=sorted(str(item) for item in filename_tags), entry_point_name=entry_point.name, entry_point_target=entry_point.target, + authoritative_members=authoritative_members, ), path, ) diff --git a/src/python_deployment_builder/generation/manifest.py b/src/python_deployment_builder/generation/manifest.py index 7c73ae3..5b3cade 100644 --- a/src/python_deployment_builder/generation/manifest.py +++ b/src/python_deployment_builder/generation/manifest.py @@ -117,6 +117,7 @@ def build_deployment_manifest( "version": application_artifact.version, "sha256": application_artifact.sha256, "entry_point": application_artifact.entry_point_target, + "authoritative_members": application_artifact.authoritative_members, } deployment_fingerprint = _deployment_fingerprint(fingerprint_payload) return DeploymentManifest( diff --git a/src/python_deployment_builder/generation/structural.py b/src/python_deployment_builder/generation/structural.py index e73b684..cb985ee 100644 --- a/src/python_deployment_builder/generation/structural.py +++ b/src/python_deployment_builder/generation/structural.py @@ -224,7 +224,7 @@ def validate_rendered_files( if text is None: continue findings = text_security_findings( - text, configured_secret_values=secret_values or [] + text, path=path, configured_secret_values=secret_values or [] ) if "forbidden_shell" in findings: forbidden_hits.append(relative) diff --git a/src/python_deployment_builder/models.py b/src/python_deployment_builder/models.py index d66cf7d..0c0d921 100644 --- a/src/python_deployment_builder/models.py +++ b/src/python_deployment_builder/models.py @@ -634,6 +634,8 @@ class ApplicationArtifact(StrictModel): wheel_tags: list[str] = Field(default_factory=list) entry_point_name: str entry_point_target: str + # Required for unreleased M6.1 package mode; never inferred from wheel contents. + authoritative_members: list[str] = Field(min_length=1) class DeploymentManifest(StrictModel): diff --git a/src/python_deployment_builder/planning/index.py b/src/python_deployment_builder/planning/index.py index c787762..7dd2dcf 100644 --- a/src/python_deployment_builder/planning/index.py +++ b/src/python_deployment_builder/planning/index.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from collections.abc import Callable from datetime import UTC, datetime from enum import StrEnum @@ -82,6 +83,90 @@ def _marker_variables(value: object) -> set[str]: return set() +def _full_version_marker_applicability( + atom: tuple, python_version: str +) -> TargetMarkerApplicability: + """Prove a version atom over the same minor interval as Requires-Python.""" + + left, operator, right = atom + operation = operator.value + inverses = {"<": ">", "<=": ">=", ">": "<", ">=": "<=", "==": "==", "!=": "!="} + if operation not in inverses or isinstance(left, Variable) == isinstance(right, Variable): + return TargetMarkerApplicability.UNPROVABLE + if isinstance(left, Variable): + value = right.value + else: + value = left.value + # A wildcard on the left is a candidate version, not a specifier. + if "*" in value: + return TargetMarkerApplicability.UNPROVABLE + operation = inverses[operation] + # Limit this bridge to release comparisons with PEP 440 interval semantics. + # In particular, do not let patch-prefix wildcards use the minor-prefix helper. + if value.endswith(".*"): + prefix = value[:-2] + if operation not in {"==", "!="} or not re.fullmatch(r"[0-9]+(?:\.[0-9]+)?", prefix): + return TargetMarkerApplicability.UNPROVABLE + value = ".".join(str(int(part)) for part in prefix.split(".")) + ".*" + elif not re.fullmatch(r"[0-9]+(?:\.[0-9]+)*", value): + return TargetMarkerApplicability.UNPROVABLE + result = minor_python_compatibility(python_version, operation + value) + # The existing equality proof is conservative for exact versions below the + # minor. Its complement can prove those disjoint cases without new bounds. + if ( + operation == "==" + and result == MinorPythonCompatibility.UNPROVABLE + and minor_python_compatibility(python_version, "!=" + value) + == MinorPythonCompatibility.COMPATIBLE + ): + return TargetMarkerApplicability.DOES_NOT_APPLY + return { + MinorPythonCompatibility.COMPATIBLE: TargetMarkerApplicability.APPLIES, + MinorPythonCompatibility.INCOMPATIBLE: TargetMarkerApplicability.DOES_NOT_APPLY, + MinorPythonCompatibility.UNPROVABLE: TargetMarkerApplicability.UNPROVABLE, + }[result] + + +def _target_marker_expression( + markers: list, environment: dict[str, str] +) -> TargetMarkerApplicability: + """Evaluate packaging's grouped AST with AND precedence and tri-state facts.""" + + state = TargetMarkerApplicability + groups: list[list[TargetMarkerApplicability]] = [[]] + for item in markers: + if item == "or": + groups.append([]) + elif item == "and": + continue + elif isinstance(item, list): + groups[-1].append(_target_marker_expression(item, environment)) + elif isinstance(item, tuple): + variables = _marker_variables(item) + if variables == {"python_full_version"}: + result = _full_version_marker_applicability(item, environment["python_version"]) + elif variables - set(environment): + result = state.UNPROVABLE + else: + # Never use Marker.evaluate(): it fills missing facts from the host. + result = ( + state.APPLIES if _evaluate_markers([item], environment) + else state.DOES_NOT_APPLY + ) + groups[-1].append(result) + conjunctions = [ + state.DOES_NOT_APPLY if state.DOES_NOT_APPLY in group + else state.APPLIES if all(value == state.APPLIES for value in group) + else state.UNPROVABLE + for group in groups + ] + if state.APPLIES in conjunctions: + return state.APPLIES + if all(value == state.DOES_NOT_APPLY for value in conjunctions): + return state.DOES_NOT_APPLY + return state.UNPROVABLE + + def target_marker_applicability( marker: str | None, python_version: str, @@ -98,17 +183,7 @@ def target_marker_applicability( except InvalidMarker as exc: raise TargetMarkerEnvironmentError(f"Malformed environment marker: {marker!r}") from exc environment = target_marker_environment(python_version, architecture, extra=extra) - unprovable = sorted(_marker_variables(parsed._markers) - set(environment)) - if unprovable: - return TargetMarkerApplicability.UNPROVABLE - return ( - TargetMarkerApplicability.APPLIES - # ``Marker.evaluate`` begins from the builder host's default environment - # before applying overrides. The target environment must be complete for - # the variables we use and contain no host-derived fallback values. - if _evaluate_markers(parsed._markers, environment) - else TargetMarkerApplicability.DOES_NOT_APPLY - ) + return _target_marker_expression(parsed._markers, environment) def target_marker_applies( diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index ce8c775..a4ad4a0 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -146,8 +146,28 @@ def is_textual_wheel_member(path: PurePosixPath, content: bytes | None = None) - return is_textual_content(path, content) +def program_files_write_applicable(path: PurePosixPath) -> bool: + """Exclude known descriptive surfaces, not arbitrary operational text. + + Python/BAT/CMD are the generated runtime forms. HTML application resources, + startup .pth files and unknown script suffixes stay conservative too: an + executable extension allowlist would silently exempt these other surfaces. + This classification affects no content-leak or forbidden-shell finding. + """ + + if path.suffix.casefold() in {".md", ".rst", ".txt"}: + return False + return not ( + path.parent.name.casefold().endswith(".dist-info") + and path.name.casefold() in TEXTUAL_WHEEL_METADATA_FILENAMES + ) + + def text_security_findings( - text: str, *, configured_secret_values: Iterable[str] = () + text: str, + *, + path: PurePosixPath | None = None, + configured_secret_values: Iterable[str] = (), ) -> set[str]: """Return the deployment security rules violated by application text.""" @@ -159,8 +179,10 @@ def text_security_findings( findings.add("developer_path") if "setx" in lowered and "path" in lowered: findings.add("permanent_path") - if "program files" in lowered and any( - token in lowered for token in PROGRAM_FILES_WRITE_TOKENS + # Pathless compatibility callers retain the original conservative rule. + if (path is None or program_files_write_applicable(path)) and ( + "program files" in lowered + and any(token in lowered for token in PROGRAM_FILES_WRITE_TOKENS) ): findings.add("program_files_write") if OBVIOUS_SECRET.search(text): @@ -183,5 +205,6 @@ def text_security_findings( "is_valid_environment_name", "is_probably_utf8_text", "is_textual_wheel_member", + "program_files_write_applicable", "text_security_findings", ] diff --git a/src/python_deployment_builder/validation/static.py b/src/python_deployment_builder/validation/static.py index 74bb2cd..9c407b8 100644 --- a/src/python_deployment_builder/validation/static.py +++ b/src/python_deployment_builder/validation/static.py @@ -25,6 +25,7 @@ installed_wheel_member_paths, validate_application_requires_dist, validate_application_wheel_content_policy, + validate_application_wheel_surface, validate_approved_artifact_lock_identity, validate_approved_requires_dist, validate_combined_wheel_installation_paths, @@ -589,6 +590,25 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR evidence=application_content_failures, ) ) + application_surface_failures: list[str] = [] + if application_wheel is not None and application_wheel in safe_trusted_wheel_paths: + try: + validate_application_wheel_surface( + application_wheel, + manifest.application_artifact.authoritative_members, + manifest.entry_point_module, + ) + except PreparationError as exc: + application_surface_failures.append(str(exc)) + checks.append( + _check( + "APPLICATION_WHEEL_AUTHORITATIVE_SURFACE", + not application_surface_failures, + "Application wheel matches its source-derived executable and required-data surface.", + "Application wheel violates its authoritative installed surface.", + evidence=application_surface_failures, + ) + ) static_plan = None static_lock_failures: list[str] = [] try: @@ -897,7 +917,8 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR if text is None: continue findings = text_security_findings( - text, configured_secret_values=secret_values + text, path=PurePosixPath(path.relative_to(root).as_posix()), + configured_secret_values=secret_values, ) if "forbidden_shell" in findings: lowered = text.lower() diff --git a/tests/test_application_surface_assigned_resources.py b/tests/test_application_surface_assigned_resources.py new file mode 100644 index 0000000..48a3075 --- /dev/null +++ b/tests/test_application_surface_assigned_resources.py @@ -0,0 +1,443 @@ +"""First-party executable authority and bounded assigned Traversable regressions.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from pydantic import ValidationError +from test_as_file_relative_imports import as_file_project +from test_dependency_authority import assess +from test_generation import ( + _make_application_wheel, + _refresh_manifest_wheel_hash, + _rewrite_application_wheel, + _update_indexed_hashes, + _write_mapped_project, +) +from test_review_extra_paths_resources import fake_preparation, repo # noqa: F401 + +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import ( + validate_application_wheel, + validate_application_wheel_surface, +) +from python_deployment_builder.generation.generator import _staging_files, generate_deployment_kit +from python_deployment_builder.generation.manifest import build_deployment_manifest +from python_deployment_builder.models import ApplicationArtifact, DeploymentManifest +from python_deployment_builder.planning.planner import create_deployment_plan +from python_deployment_builder.validation.static import validate_static_kit + + +@pytest.mark.parametrize( + "extra", + [ + "requests/__init__.py", + "application-hook.pth", + "mapped_app-1.2.3.data/purelib/application-hook.pth", + "requests/api.py", + "foreign.py", + "docs/runtime.py", + "foreign.PY", + "HOOK.PTH", + "sitecustomize.py", + "usercustomize.py", + "sitecustomize/__init__.py", + "usercustomize/__init__.py", + "mapped_app-1.2.3.data/purelib/foreign.py", + ], +) +def test_undeclared_executable_wheel_rejected(tmp_path, extra): + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + assessment = assess(source) + plan = create_deployment_plan(assessment, repository_root=source) + wheel = _make_application_wheel(tmp_path) + validate_application_wheel(wheel, assessment, plan) + _rewrite_application_wheel(wheel, additions={extra: "# benign fixture\n"}) + with pytest.raises(PreparationError, match="authoritative|startup-active"): + validate_application_wheel(wheel, assessment, plan) + + +@pytest.mark.parametrize( + "consumer", + [ + "with as_file(asset) as path:\n return path.read_bytes()", + "return asset.read_bytes()", + "return asset.read_text()", + ], +) +def test_assigned_traversable_is_staged(tmp_path, consumer): + relative = as_file_project(tmp_path) + (tmp_path / "src/app/main.py").write_text( + "from importlib.resources import files, as_file\n" + "asset = files('app') / 'model.dat'\n" + f"def main():\n {consumer}\n" + ) + assessment = assess(tmp_path) + assert relative in {item.path for item in assessment.resources} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert relative in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("missing", ["installed_app/main.py", "installed_app/view.html"]) +def test_surface_missing_member_generation_and_independent_validation(tmp_path, missing): + _write_mapped_project(tmp_path) + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + wheel = _make_application_wheel(tmp_path) + artifact, _ = validate_application_wheel(wheel, assessment, plan) + assert artifact.authoritative_members == [ + "installed_app/__init__.py", + "installed_app/main.py", + "installed_app/view.html", + ] + _rewrite_application_wheel(wheel, removals={missing}) + with pytest.raises(PreparationError, match="missing|entry-point module"): + validate_application_wheel(wheel, assessment, plan) + with pytest.raises(PreparationError, match="missing authoritative"): + validate_application_wheel_surface( + wheel, artifact.authoritative_members, "installed_app.main" + ) + + +@pytest.mark.parametrize( + "extra", + [ + "installed_app/data/example.pth", + "installed_app/extra.txt", + "mapped_app-1.2.3.dist-info/example.py", + ], +) +def test_benign_non_executable_data_and_metadata_allowed(tmp_path, extra): + _write_mapped_project(tmp_path) + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + wheel = _make_application_wheel(tmp_path) + _rewrite_application_wheel(wheel, additions={extra: "# inert fixture\n"}) + artifact, _ = validate_application_wheel(wheel, assessment, plan) + assert extra not in artifact.authoritative_members # Never bless observed wheel inventory. + validate_application_wheel_surface(wheel, artifact.authoritative_members, "installed_app.main") + + +def test_modeled_second_package_and_module_are_authoritative(tmp_path): + _write_mapped_project(tmp_path) + path = tmp_path / "pyproject.toml" + path.write_text( + path.read_text().replace( + 'packages = ["installed_app"]', + 'packages = ["installed_app", "plugins"]\npy-modules = ["helper"]', + ) + ) + (tmp_path / "plugins").mkdir() + (tmp_path / "plugins/__init__.py").write_text("") + (tmp_path / "helper.py").write_text("") + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + wheel = _make_application_wheel(tmp_path) + _rewrite_application_wheel(wheel, additions={"plugins/__init__.py": "", "helper.py": ""}) + artifact, _ = validate_application_wheel(wheel, assessment, plan) + assert {"plugins/__init__.py", "helper.py"} <= set(artifact.authoritative_members) + + +@pytest.mark.parametrize( + "startup", + [ + "sitecustomize.py", + "usercustomize.py", + "hook.pth", + "sitecustomize/__init__.py", + "usercustomize/__init__.py", + ], +) +def test_declared_startup_destination_still_forbidden(tmp_path, startup): + wheel = _make_application_wheel(tmp_path) + _rewrite_application_wheel(wheel, additions={startup: "# startup\n"}) + with pytest.raises(PreparationError, match="startup-active"): + validate_application_wheel_surface( + wheel, ["installed_app/main.py", startup], "installed_app.main" + ) + + +def test_entrypoint_cannot_be_blessed_by_package_data_or_wheel(tmp_path): + _write_mapped_project(tmp_path) + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + # The supplied wheel contains main.py, but source assessment cannot model it. + (tmp_path / "code/main.py").unlink() + with pytest.raises(PreparationError, match="outside the authoritative Python surface"): + validate_application_wheel(_make_application_wheel(tmp_path), assessment, plan) + + +@pytest.mark.usefixtures("fake_preparation") +@pytest.mark.parametrize( + "mutation", + [ + "requests/__init__.py", + "hook.pth", + "mapped_app-1.2.3.data/purelib/hook.pth", + "sitecustomize.py", + "usercustomize.py", + "missing_python", + "missing_data", + ], +) +def test_reindexed_static_kit_enforces_source_surface(tmp_path, mutation): + source = tmp_path / "source" + source.mkdir() + _write_mapped_project(source) + wheel = _make_application_wheel(tmp_path) + kit = tmp_path / "kit" + generate_deployment_kit(repo(source), kit, application_wheel=wheel, bootstrap_mode="online_cmd") + assert validate_static_kit(kit).final_state.value == "STATIC_VALID" + relative = f"deployment/application/{wheel.name}" + wheel = kit / relative + if mutation.startswith("missing"): + missing = ( + "installed_app/main.py" if mutation == "missing_python" else "installed_app/view.html" + ) + _rewrite_application_wheel(wheel, removals={missing}) + else: + _rewrite_application_wheel(wheel, additions={mutation: "# fixture\n"}) + _refresh_manifest_wheel_hash(kit, relative) + manifest_path = kit / "deployment/manifest.json" + manifest = DeploymentManifest.model_validate_json(manifest_path.read_text()) + plan = create_deployment_plan(assess(source), repository_root=source) + rebuilt = build_deployment_manifest( + plan, + source, + bootstrap_mode=manifest.bootstrap_mode, + system_certs=manifest.system_certs, + approved_artifacts=manifest.approved_artifacts, + bundled_uv_sha256=manifest.bundled_uv_sha256, + referenced_files=manifest.referenced_files, + application_artifact=manifest.application_artifact, + generated_at=manifest.generated_at, + ) + manifest.deployment_fingerprint = rebuilt.deployment_fingerprint + manifest.generation_id = rebuilt.generation_id + manifest_path.write_text(manifest.model_dump_json(indent=2)) + _update_indexed_hashes(kit, "deployment/manifest.json") + report = validate_static_kit(kit) + checks = {item.code: item for item in report.static_checks} + assert checks["GENERATED_FILE_HASHES"].status.value == "PASS" + assert checks["WHEEL_METADATA_SEMANTICS"].status.value == "PASS" + assert checks["APPLICATION_WHEEL_AUTHORITATIVE_SURFACE"].status.value == "FAIL" + assert report.final_state.value == "FAILED" + + +def test_surface_required_and_fingerprinted(tmp_path): + _write_mapped_project(tmp_path) + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + artifact, _ = validate_application_wheel(_make_application_wheel(tmp_path), assessment, plan) + arguments = dict( + bootstrap_mode="online_cmd", + system_certs=False, + approved_artifacts=[], + bundled_uv_sha256=None, + referenced_files=[], + ) + first = build_deployment_manifest(plan, tmp_path, application_artifact=artifact, **arguments) + changed = artifact.model_copy( + update={"authoritative_members": [*artifact.authoritative_members, "new.txt"]} + ) + second = build_deployment_manifest(plan, tmp_path, application_artifact=changed, **arguments) + assert first.deployment_fingerprint != second.deployment_fingerprint + data = artifact.model_dump() + del data["authoritative_members"] + with pytest.raises(ValidationError): + ApplicationArtifact.model_validate(data) + data["authoritative_members"] = [] + with pytest.raises(ValidationError): + ApplicationArtifact.model_validate(data) + + +@pytest.mark.parametrize( + "assignments", + [ + "asset = files('app') / 'model.dat'", + "base = files('app')\nasset = base / 'model.dat'", + "base = files('app')\nasset = base.joinpath('model.dat')", + "base = files('app') / 'model.dat'\nother = base\nasset = other", + "holder.asset = files('app') / 'model.dat'\nasset = holder.asset", + "def resource():\n return files('app') / 'model.dat'\nasset = resource()", + ], +) +@pytest.mark.parametrize("consumer", ["asset.read_bytes()", "asset.read_text()", "as_file(asset)"]) +def test_bounded_assignment_forms(tmp_path, assignments, consumer): + relative = as_file_project(tmp_path) + (tmp_path / "src/app/main.py").write_text( + f"from importlib.resources import files, as_file\n{assignments}\n" + f"def main(): return {consumer}\n" + ) + assert relative in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize( + "assignments", + [ + "asset = other\nother = asset", + "asset = asset / 'model.dat'", + "asset = files(unknown) / 'model.dat'", + "asset = files('app') / unknown", + "def resource():\n return resource()\nasset = resource()", + "def resource():\n return asset\nasset = resource()", + ], +) +def test_assigned_cycles_and_unknowns_unresolved(tmp_path, assignments): + relative = as_file_project(tmp_path) + (tmp_path / "src/app/main.py").write_text( + f"from importlib.resources import files\n{assignments}\n" + "def main(): return asset.read_bytes()\n" + ) + assert relative not in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("anchor", ["'app'", "'app.config'", "anchor='app'", "package='app'", ""]) +@pytest.mark.parametrize( + "source_root,mapping", + [ + ("src", None), + ("lib", "{''='lib'}"), + ("lib", "{'app'='lib/app'}"), + ], +) +def test_assigned_anchors_and_mappings(tmp_path, anchor, source_root, mapping): + relative = as_file_project(tmp_path, source_root=source_root, mapping=mapping) + package = tmp_path / source_root / "app" + (package / "config.py").write_text("") + (package / "main.py").write_text( + f"from importlib.resources import files\nbase = files({anchor})\n" + "asset = base / 'model.dat'\ndef main(): return asset.read_bytes()\n" + ) + assert relative in {item.path for item in assess(tmp_path).resources} + + +@pytest.mark.parametrize("state", ["secret", "dirty", "untracked"]) +@pytest.mark.usefixtures("fake_preparation") +def test_assigned_resource_security_and_provenance(tmp_path, monkeypatch, state): + source = tmp_path / "source" + relative = as_file_project(source) + (source / "src/app/main.py").write_text( + "from importlib.resources import files, as_file\nfrom os import getenv\n" + "PASSWORD=getenv('DB_PASSWORD')\nasset=files('app') / 'model.dat'\n" + "def main(): return as_file(asset)\n" + ) + secret = "PDBAssignedResourceSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + if state == "secret": + (source / relative).write_text(secret) + else: + + def git(*args): + subprocess.run(["git", "-C", str(source), *args], check=True, capture_output=True) + + git("init") + git("add", ".") + git( + "-c", + "user.name=PDB Test", + "-c", + "user.email=pdb@example.invalid", + "commit", + "-m", + "fixture", + ) + if state == "dirty": + (source / relative).write_text("CHANGED") + else: + git("rm", "--cached", relative) + with pytest.raises(PreparationError) as error: + generate_deployment_kit(repo(source), tmp_path / "kit", bootstrap_mode="online_cmd") + assert secret not in str(error.value) + assert not (tmp_path / "kit").exists() + + +@pytest.mark.usefixtures("fake_preparation") +@pytest.mark.parametrize("consumer", ["asset.read_bytes()", "as_file(asset)"]) +def test_assigned_resource_generation_and_static_parity(tmp_path, consumer): + source = tmp_path / "source" + relative = as_file_project(source) + (source / "src/app/main.py").write_text( + "from importlib.resources import files, as_file\n" + f"asset=files('app') / 'model.dat'\ndef main(): return {consumer}\n" + ) + kit = tmp_path / "kit" + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + assert (kit / relative).read_bytes() == b"model fixture" + assert validate_static_kit(kit).final_state.value == "STATIC_VALID" + assert ( + json.loads((kit / "deployment/manifest.json").read_text())["application_artifact"] is None + ) + code = "import sys; sys.path.insert(0, sys.argv[1]); from app.main import main; " + if consumer == "asset.read_bytes()": + code += "assert main() == b'model fixture'" + else: + code += "exec(\"with main() as path:\\n assert path.read_bytes() == b'model fixture'\")" + result = subprocess.run( + [sys.executable, "-I", "-B", "-c", code, str(kit / "src")], + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "consumer", + [ + "as_file(directory)", + "asset.read_bytes()", + "directory.joinpath('weights.bin').read_bytes()", + ], +) +def test_assigned_directory_and_chained_descendants(tmp_path, consumer): + as_file_project(tmp_path) + directory = tmp_path / "src/app/bundle" + directory.mkdir() + (directory / "weights.bin").write_bytes(b"MODEL") + (tmp_path / "src/app/main.py").write_text( + "from importlib.resources import files, as_file\n" + "base=files('app')\ndirectory=base / 'bundle'\nasset=directory / 'weights.bin'\n" + f"def main(): return {consumer}\n" + ) + assessment = assess(tmp_path) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "src/app/bundle/weights.bin" in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize("namespace", [False, True]) +def test_assigned_longest_parent_mapping(tmp_path, namespace): + as_file_project(tmp_path, source_root="lib", mapping="{'app'='lib/app','app.child'='code'}") + directory = tmp_path / "code" + directory.mkdir() + (directory / "config.py").write_text("") + if not namespace: + (directory / "__init__.py").write_text("") + (directory / "model.dat").write_text("MODEL") + (tmp_path / "lib/app/main.py").write_text( + "from importlib.resources import files\nbase=files('app.child.config')\n" + "asset=base / 'model.dat'\ndef main(): return asset.read_bytes()\n" + ) + assert "code/model.dat" in {item.path for item in assess(tmp_path).resources} + + +def test_assigned_resource_containment(tmp_path, monkeypatch): + source = tmp_path / "source" + relative = as_file_project(source) + (source / "src/app/main.py").write_text( + "from importlib.resources import files\nasset=files('app') / 'model.dat'\n" + "def main(): return asset.read_bytes()\n" + ) + target = source / relative + outside = tmp_path / "outside.dat" + outside.write_text("OUTSIDE") + original = Path.resolve + monkeypatch.setattr( + Path, "resolve", lambda p, *a, **kw: outside if p == target else original(p, *a, **kw) + ) + assert not any( + item.path == relative and item.role.value == "runtime_resource" + for item in assess(source).file_inventory + ) diff --git a/tests/test_generation.py b/tests/test_generation.py index 89f84c2..5663c10 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -2170,6 +2170,20 @@ def test_application_and_approved_wheels_must_have_combined_installation_paths( source = tmp_path / "source" source.mkdir() _write_mapped_project(source) + if application_member.endswith(".py"): + # This test reaches cross-wheel collisions with legitimately modeled code. + extra_source = source / application_member + extra_source.parent.mkdir(parents=True) + extra_source.write_text("application\n", encoding="utf-8") + project_file = source / "pyproject.toml" + extra_package = extra_source.parent.name + project_file.write_text( + project_file.read_text().replace( + 'packages = ["installed_app"]', + f'packages = ["installed_app", "{extra_package}"]', + ), + encoding="utf-8", + ) assessment = assess_repository( MaterializedRepository(root=source, source=str(source), source_kind="local") ) @@ -3254,6 +3268,7 @@ def test_application_wheel_requires_nested_package_data_from_parent_mapping( plan = create_deployment_plan(assessment, repository_root=source) incomplete = _rewrite_application_wheel( _make_application_wheel(tmp_path, package="app", target="app.sub.main:main"), + removals={"app/main.py"}, additions={"app/sub/__init__.py": "", "app/sub/main.py": "def main(): return 0\n"}, ) @@ -6835,6 +6850,7 @@ def test_validated_artifact_models_are_the_generated_destination_authority() -> sha256="b" * 64, entry_point_name="mapped-app", entry_point_target="installed_app.main:main", + authoritative_members=["installed_app/__init__.py", "installed_app/main.py"], ) paths = _planned_generated_paths( @@ -7165,6 +7181,7 @@ def test_rendered_files_allows_exact_manifest_declared_artifact_wheels(tmp_path: sha256=hashlib.sha256(application_wheel.read_bytes()).hexdigest(), entry_point_name="mapped-app", entry_point_target="installed_app.main:main", + authoritative_members=["installed_app/__init__.py", "installed_app/main.py"], ) } ) diff --git a/tests/test_minor_marker_proof.py b/tests/test_minor_marker_proof.py new file mode 100644 index 0000000..dc6950e --- /dev/null +++ b/tests/test_minor_marker_proof.py @@ -0,0 +1,254 @@ +"""Minor-invariant full-version markers are proofs, not invented patch values.""" + +from types import SimpleNamespace + +import pytest +from packaging.markers import Marker +from packaging.version import Version +from test_generation import _plan + +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import ( + _direct_dependency_presence_proven, + validate_application_requires_dist, + validate_approved_requires_dist, +) +from python_deployment_builder.planning.index import ( + TargetMarkerApplicability as State, +) +from python_deployment_builder.planning.index import ( + target_marker_applicability, + target_marker_environment, +) +from python_deployment_builder.planning.lockfile import inspect_uv_lock + + +def test_minor_boundary_is_provable_but_patch_boundary_is_not(): + assert ( + target_marker_applicability("python_full_version >= '3.12.1'", "3.12", "x86_64") + == State.UNPROVABLE + ) + assert ( + target_marker_applicability("python_full_version >= '3.12'", "3.12", "x86_64") + == State.APPLIES + ) + + +def test_sgg_numpy_root_edge_presence(): + graph = SimpleNamespace( + selected_extras=[], + edges=[ + SimpleNamespace( + from_package="simple-georef-gui", + to_package="numpy", + selected_extra=None, + marker=f"python_full_version {condition}", + ) + for condition in ["< '3.11'", "== '3.11.*'", ">= '3.12'"] + ], + ) + _direct_dependency_presence_proven(graph, _plan(), "simple-georef-gui", "numpy") + + +@pytest.mark.parametrize( + ("comparison", "expected"), + [ + (">= '3.12'", State.APPLIES), + ("< '3.13'", State.APPLIES), + ("== '3.12.*'", State.APPLIES), + ("!= '3.11.*'", State.APPLIES), + ("< '3.12'", State.DOES_NOT_APPLY), + (">= '3.13'", State.DOES_NOT_APPLY), + ("== '3.11.*'", State.DOES_NOT_APPLY), + ("!= '3.12.*'", State.DOES_NOT_APPLY), + (">= '3.12.1'", State.UNPROVABLE), + ("< '3.12.1'", State.UNPROVABLE), + ("< '3.12.5'", State.UNPROVABLE), + ("== '3.12.0'", State.UNPROVABLE), + ("!= '3.12.0'", State.UNPROVABLE), + ("> '3.11'", State.APPLIES), + ("<= '3.13'", State.APPLIES), + ("> '3.13'", State.DOES_NOT_APPLY), + ("<= '3.11'", State.DOES_NOT_APPLY), + ("> '3.12'", State.UNPROVABLE), + ("<= '3.12'", State.UNPROVABLE), + ("== '3.11.9'", State.DOES_NOT_APPLY), + ("!= '3.11.9'", State.APPLIES), + ("== '3.13.0'", State.DOES_NOT_APPLY), + ("== '3.*'", State.APPLIES), + ("== '03.012.*'", State.APPLIES), + ("== '3.12.0.*'", State.UNPROVABLE), + ("!= '3.12.0.*'", State.UNPROVABLE), + ("in '3.12'", State.UNPROVABLE), + ("not in '3.11'", State.UNPROVABLE), + ("~= '3.12'", State.UNPROVABLE), + ("=== '3.12.0'", State.UNPROVABLE), + (">= '3.12rc1'", State.UNPROVABLE), + ("== 'nonsense'", State.UNPROVABLE), + ], +) +def test_atomic_full_version_matrix(comparison, expected): + assert ( + target_marker_applicability(f"python_full_version {comparison}", "3.12", "x86_64") + == expected + ) + + +@pytest.mark.parametrize( + ("marker", "expected"), + [ + ('"3.12" <= python_full_version', State.APPLIES), + ('"3.13" > python_full_version', State.APPLIES), + ('"3.12" > python_full_version', State.DOES_NOT_APPLY), + ('"3.13" <= python_full_version', State.DOES_NOT_APPLY), + ('"3.12.1" <= python_full_version', State.UNPROVABLE), + ('"3.12.0" == python_full_version', State.UNPROVABLE), + ('"3.12.0" != python_full_version', State.UNPROVABLE), + ('"3.12.*" == python_full_version', State.UNPROVABLE), + ], +) +def test_packaging_right_hand_variable(marker, expected): + assert Marker(marker)._markers[0][2].value == "python_full_version" + assert target_marker_applicability(marker, "3.12", "x86_64") == expected + + +@pytest.mark.parametrize("operator", ["and", "or"]) +@pytest.mark.parametrize("left", list(State)) +@pytest.mark.parametrize("right", list(State)) +def test_all_tristate_boolean_combinations(operator, left, right): + atoms = { + State.APPLIES: "python_full_version >= '3.12'", + State.DOES_NOT_APPLY: "sys_platform == 'linux'", + State.UNPROVABLE: "python_full_version >= '3.12.1'", + } + pair = {left, right} + if operator == "and": + expected = ( + State.DOES_NOT_APPLY + if State.DOES_NOT_APPLY in pair + else State.APPLIES + if pair == {State.APPLIES} + else State.UNPROVABLE + ) + else: + expected = ( + State.APPLIES + if State.APPLIES in pair + else State.DOES_NOT_APPLY + if pair == {State.DOES_NOT_APPLY} + else State.UNPROVABLE + ) + assert ( + target_marker_applicability(f"{atoms[left]} {operator} {atoms[right]}", "3.12", "x86_64") + == expected + ) + + +@pytest.mark.parametrize( + ("marker", "expected"), + [ + ("python_full_version >= '3.12' and python_full_version < '3.13'", State.APPLIES), + ("python_full_version >= '3.12' and sys_platform == 'win32'", State.APPLIES), + ("python_full_version >= '3.12.1' or sys_platform == 'win32'", State.APPLIES), + ("python_full_version >= '3.12.1' or sys_platform == 'linux'", State.UNPROVABLE), + ( + "(python_full_version >= '3.12.1' or sys_platform == 'win32') " + "and sys_platform == 'linux'", + State.DOES_NOT_APPLY, + ), + ( + "python_full_version >= '3.12.1' or sys_platform == 'win32' " + "and sys_platform == 'linux'", + State.UNPROVABLE, + ), + ], +) +def test_grouping_and_platform_context(marker, expected): + assert target_marker_applicability(marker, "3.12", "x86_64") == expected + + +@pytest.mark.parametrize( + "field", ["platform_release", "platform_version", "implementation_version"] +) +def test_unselected_fields_remain_unknown(field): + assert field not in target_marker_environment("3.12", "x86_64") + marker = f"{field} >= '3.12'" + assert target_marker_applicability(marker, "3.12", "x86_64") == State.UNPROVABLE + assert ( + target_marker_applicability(marker + " and sys_platform == 'linux'", "3.12", "x86_64") + == State.DOES_NOT_APPLY + ) + assert ( + target_marker_applicability(marker + " or sys_platform == 'win32'", "3.12", "x86_64") + == State.APPLIES + ) + + +def lock_plan(root, branches): + edges = ",".join( + f'{{name="helper", version="{version}", marker="python_full_version {condition}"}}' + for version, condition in branches + ) + packages = "\n".join( + f'[[package]]\nname="helper"\nversion="{version}"\n' + f'wheels=[{{url="https://example.invalid/helper-{version}-py3-none-any.whl"}}]' + for version, _ in branches + ) + (root / "uv.lock").write_text( + 'version=1\n[[package]]\nname="app"\nversion="1"\n' + f'source={{virtual="."}}\ndependencies=[{edges}]\n{packages}\n', + encoding="utf-8", + ) + plan = _plan().model_copy(deep=True) + plan.lock_graph = inspect_uv_lock(root, "app", "3.12", "x86_64", []) + return plan + + +@pytest.mark.parametrize( + "proof", [validate_application_requires_dist, validate_approved_requires_dist] +) +def test_invariant_lock_branches_prove_wheel_requirements(tmp_path, proof): + plan = lock_plan( + tmp_path, [("2.2.6", "< '3.11'"), ("2.4.6", "== '3.11.*'"), ("2.5.2", ">= '3.12'")] + ) + assert [(d.name, d.version) for d in plan.lock_graph.dependencies] == [("helper", "2.5.2")] + proof(["helper>=2.5", "helper>=2.5; python_full_version >= '3.12'"], plan, "app", Version("1")) + with pytest.raises(PreparationError, match="target-possible"): + proof(["helper<2.5"], plan, "app", Version("1")) + + +@pytest.mark.parametrize( + "proof", [validate_application_requires_dist, validate_approved_requires_dist] +) +def test_patch_sensitive_forks_remain_possible_and_unproven(tmp_path, proof): + plan = lock_plan(tmp_path, [("1", "< '3.12.5'"), ("2", ">= '3.12.5'")]) + assert {d.version for d in plan.lock_graph.dependencies} == {"1", "2"} + with pytest.raises(PreparationError, match="definitely applicable"): + proof(["helper>=1"], plan, "app", Version("1")) + # Even a separate unconditional presence proof cannot bless a bad possible version. + plan.lock_graph.edges[0].marker = None + with pytest.raises(PreparationError, match="target-possible"): + proof(["helper>=2"], plan, "app", Version("1")) + + +@pytest.mark.parametrize( + "proof", [validate_application_requires_dist, validate_approved_requires_dist] +) +def test_patch_sensitive_wheel_requirement_marker_is_not_guessed(tmp_path, proof): + plan = lock_plan(tmp_path, [("1", ">= '3.12'")]) + with pytest.raises(PreparationError, match="cannot be proven"): + proof(["helper; python_full_version >= '3.12.1'"], plan, "app", Version("1")) + + +@pytest.mark.parametrize("minor", ["3.11", "3.12", "3.13", "3.14"]) +def test_all_policy_minors_use_interval_proof_not_a_patch(minor): + for marker in [f"python_full_version >= '{minor}'", f"python_full_version == '{minor}.*'"]: + assert target_marker_applicability(marker, minor, "x86_64") == State.APPLIES + for marker in [f"python_full_version >= '{minor}.1'", f"python_full_version == '{minor}.0'"]: + assert target_marker_applicability(marker, minor, "x86_64") == State.UNPROVABLE + + +def test_sgg_pyproj_minor_boundary_graph(tmp_path): + plan = lock_plan(tmp_path, [("3.7.1", "< '3.11'"), ("3.7.2", ">= '3.11'")]) + assert {d.version for d in plan.lock_graph.dependencies} == {"3.7.2"} + validate_application_requires_dist(["helper==3.7.2"], plan, "app", Version("1")) diff --git a/tests/test_security_content_context.py b/tests/test_security_content_context.py new file mode 100644 index 0000000..e6e2879 --- /dev/null +++ b/tests/test_security_content_context.py @@ -0,0 +1,148 @@ +"""Operational write heuristics must not classify descriptive metadata as code.""" + +import ast +from pathlib import Path, PurePosixPath + +import pytest +from test_as_file_relative_imports import as_file_project +from test_generation import _make_application_wheel, _rewrite_application_wheel +from test_review_extra_paths_resources import fake_preparation, repo # noqa: F401 + +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import validate_wheel_static_safety +from python_deployment_builder.generation.generator import generate_deployment_kit +from python_deployment_builder.security_policy import text_security_findings +from python_deployment_builder.validation.static import validate_static_kit + +DESCRIPTION = ( + "The launcher makes a data-only copy without copying Program Files\nsecurity descriptors." +) +METADATA_HEADER = "Metadata-Version: 2.1\nName: mapped-app\nVersion: 1.2.3\n\n" + + +def test_descriptive_core_metadata_is_not_an_operational_write(tmp_path): + wheel = _make_application_wheel(tmp_path) + _rewrite_application_wheel( + wheel, + replacements={ + "mapped_app-1.2.3.dist-info/METADATA": METADATA_HEADER + DESCRIPTION, + }, + ) + validate_wheel_static_safety(wheel) + + +@pytest.mark.parametrize( + "path", ["demo-1.dist-info/METADATA", "README.md", "guide.rst", "guide.txt"] +) +def test_descriptive_text_does_not_trigger_write(path): + assert "program_files_write" not in text_security_findings( + DESCRIPTION, path=PurePosixPath(path) + ) + + +@pytest.mark.parametrize( + "path", + [ + "main.py", + "Run.bat", + "bootstrap.cmd", + "page.html", + "script.js", + "hook.pth", + "unknown", + "main.PY", + ], +) +@pytest.mark.parametrize("operation", ["mkdir", "write_text"]) +def test_operational_and_unknown_text_keeps_multiline_write_detection(path, operation): + text = 'from pathlib import Path\ntarget = Path(r"C:\\Program Files\\Example")\n' + text += f'target.{operation}("x")\n' + assert "program_files_write" in text_security_findings(text, path=PurePosixPath(path)) + + +@pytest.mark.parametrize("suffix", ["bat", "cmd"]) +def test_actual_batch_copy_into_program_files(suffix): + text = '@echo off\nset "DEST=C:\\Program Files\\Example"\ncopy state.txt "%DEST%\\state.txt"\n' + assert "program_files_write" in text_security_findings( + text, path=PurePosixPath(f"run.{suffix}") + ) + + +def test_pathless_call_retains_conservative_compatibility(): + assert "program_files_write" in text_security_findings(DESCRIPTION) + + +@pytest.mark.parametrize( + "path", ["demo-1.dist-info/METADATA", "README.md", "manual.rst", "guide.txt"] +) +@pytest.mark.parametrize( + "text,expected", + [ + ("sk-abcdefghijklmnopqrstuv", "obvious_secret"), + ("PDBContextConfiguredSecret123", "configured_secret"), + ("powershell.exe", "forbidden_shell"), + (r"C:\Users\Developer\project", "developer_path"), + ("setx PATH example", "permanent_path"), + ], +) +def test_documentation_keeps_all_other_text_rules(path, text, expected): + findings = text_security_findings( + DESCRIPTION + "\n" + text, + path=PurePosixPath(path), + configured_secret_values=("PDBContextConfiguredSecret123",), + ) + assert findings == {expected} + + +@pytest.mark.parametrize( + "payload", + [ + "sk-abcdefghijklmnopqrstuv", + "PDBContextConfiguredSecret123", + "powershell.exe", + r"C:\Users\Developer\project", + ], +) +def test_wheel_metadata_is_still_security_scanned(tmp_path, payload): + wheel = _make_application_wheel(tmp_path) + _rewrite_application_wheel( + wheel, + replacements={ + "mapped_app-1.2.3.dist-info/METADATA": METADATA_HEADER + payload, + }, + ) + with pytest.raises(PreparationError, match="METADATA") as error: + validate_wheel_static_safety( + wheel, configured_secret_values=("PDBContextConfiguredSecret123",) + ) + assert payload not in str(error.value) + + +@pytest.mark.parametrize("filename", ["README.md", "guide.rst", "guide.txt"]) +@pytest.mark.usefixtures("fake_preparation") +def test_staged_documentation_generation_static_parity(tmp_path, filename): + source = tmp_path / "source" + as_file_project(source, call=f"as_file(files('app') / {filename!r})") + (source / "src/app" / filename).write_text(DESCRIPTION) + kit = tmp_path / "kit" + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + assert (kit / "src/app" / filename).is_file() + assert validate_static_kit(kit).final_state.value == "STATIC_VALID" + + +def test_all_production_scanner_calls_supply_path_context(): + source = Path(__file__).parents[1] / "src/python_deployment_builder" + callers = [] + for path in source.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8-sig"))): + if ( + isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + and node.func.id == "text_security_findings" + ): + assert "path" in {keyword.arg for keyword in node.keywords}, str(path) + callers.append(path.relative_to(source).as_posix()) + assert sorted(callers) == [ + "generation/artifacts.py", + "generation/structural.py", + "validation/static.py", + ] From 83dcead905269bd67cb3a559e727603a3af3e670 Mon Sep 17 00:00:00 2001 From: alwunder <89945503+alwunder@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:34:49 -0400 Subject: [PATCH 59/59] Validate Windows kit paths and preserve explicit resource reads --- .../pr9-minor-marker-acceptance-2026-09-12.md | 36 ++ ...windows-paths-explicit-reads-2026-09-16.md | 142 +++++ .../analysis/resources.py | 32 +- .../generation/artifacts.py | 97 +--- .../generation/generator.py | 44 ++ .../security_policy.py | 2 +- .../validation/static.py | 33 ++ .../windows_paths.py | 76 +++ tests/test_windows_staging_explicit_reads.py | 489 ++++++++++++++++++ 9 files changed, 857 insertions(+), 94 deletions(-) create mode 100644 docs/acceptance/pr9-windows-paths-explicit-reads-2026-09-16.md create mode 100644 src/python_deployment_builder/windows_paths.py create mode 100644 tests/test_windows_staging_explicit_reads.py diff --git a/docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md b/docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md index bf8dee5..7b36bf3 100644 --- a/docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md +++ b/docs/acceptance/pr9-minor-marker-acceptance-2026-09-12.md @@ -138,6 +138,42 @@ This supersedes the historical `FDCA6A5A...` hash: the required application surface changed package-mode manifest/fingerprint bytes. The accepted wheel bytes and SGG source did not change. +### Regenerated acceptance for the Windows-path/read-intent correction + +The historical acceptance ZIP above remains valid evidence for its historical +generated kit. After the three subsequent P2 corrections, the unchanged SGG +source and the same reviewed wheels were used for another generation run. +The regeneration-hash stop gate was reviewed and cleared on 2026-09-16. + +- Historical acceptance ZIP: + `69AE1EE3EC8169291F265ACEAE57BEC46035A15EB99D6BD3848C3A5910D6C05B`. +- Current exact-head regenerated acceptance ZIP: + `31875A58658E11AFC968FA9224258B2180A30858B81659DDB6CE1E87BEE37BD6`. +- Each current ZIP is **21,011,577 bytes**. Packaging the current generated kit + twice produced byte-identical ZIPs, with successful extracted-kit validation. +- The current hash is the authoritative acceptance artifact for the new + Windows-path/read-intent correction head; it does not invalidate the historical + hash for the earlier kit. + +Only `deployment/manifest.json` (`generated_at` and `generation_id`) and the +corresponding manifest digest in `deployment/generated-files.json` differ. +Every other ZIP member is byte-identical. Both manifests have the same stable +deployment fingerprint: +`862a5faff7702b2bf392c6a6e1a00a667319000c9f76d73ee0bd2033699cd817`. + +Generation deliberately records its timestamp and timestamp-derived ID. +Deterministic **packaging of one generated kit** is required; byte-identical +**generation at different times** is not. No code was changed to recover the +historical hash or suppress intentional generation metadata. + +The regenerated kit again passes generation, the 13/77/90 member proofs, +STATIC_VALID, and runtime automation through MANUAL_GUI_VALIDATION_REQUIRED. +The authoritative module imports from managed site-packages with no prohibited +GIS runtime paths. Outputs are in the uncommitted sibling +`pdb-m61-windows-path-acceptance-20260912`; the isolated runtime is +`C:/pdb-m61-path-0912`. The [subsequent correction report](pr9-windows-paths-explicit-reads-2026-09-16.md) +records the three findings, source-mode checks, and new full-suite result. + ## Source-mode acceptance - Geo: `5e7b321d0aeb9ba1d586bfc548c79793d84c6033`, unchanged; diff --git a/docs/acceptance/pr9-windows-paths-explicit-reads-2026-09-16.md b/docs/acceptance/pr9-windows-paths-explicit-reads-2026-09-16.md new file mode 100644 index 0000000..b1d6b51 --- /dev/null +++ b/docs/acceptance/pr9-windows-paths-explicit-reads-2026-09-16.md @@ -0,0 +1,142 @@ +# PR #9: Windows kit paths, explicit reads, and developer-path separators + +Starting HEAD: `847198a4ac9d9fbd2149a402aa0ef4dbc7b79841`. +Branch: `milestone-6-1-generation-contract`. + +## Review accounting + +This focused correction addresses exactly three fresh P2 findings: + +- Validate staged paths for Windows before copying: + `PRRT_kwDOT9hvCc6h0gwN`, comment `3997952892`. +- Preserve explicit reads with arbitrary filenames: + `PRRT_kwDOT9hvCc6h0gwP`, comment `3997952896`. +- Match forward-slash Windows developer paths: + `PRRT_kwDOT9hvCc6h0gwS`, comment `3997952900`. + +Authoritative accounting: **130 total findings/comments, 127 unresolved inline +threads, three review-level-only findings, one NOT_APPLICABLE inline finding**. +The review-only findings are historical #35, approved-wheel PEP 440 semantic +version comparison, and complete locked sync-command validation. Automatic flat +package plus loose-module behavior remains NOT_APPLICABLE / REVIEW PREMISE +INCORRECT. No threads are resolved, reviews dismissed, or merge performed. + +## Windows materialization and bounded audit + +Six focused regressions were added before production edits and failed as +expected: three selected Windows-invalid/colliding source-path cases, two +arbitrary-filename reads, and forward-slash developer-path detection. + +For physical reserved-name evidence, an NT extended-namespace disposable fixture +created `CON.py` without ordinary Win32 normalization. Running trusted generator +and static-validator code read from the starting SHA accepted and indexed the +file and returned STATIC_VALID. Current static validation rejects that same kit +with WINDOWS_PATH_MATERIALIZATION. Current generation rejects the selected file +before output creation. A non-selected `docs/CON.txt` does not block generation. +Pinned acquisition is stubbed in this focused reproduction to isolate path +policy, not to claim a runtime-provisioning test. + +Forbidden colon and case-equivalent names are covered by a case-sensitive-host +selected-file adapter; this Windows host does not supply a native POSIX filesystem +for those two physical coexistence cases. Tests assert no staging bytes are read +before rejection. This distinction is not presented as a native Linux run. + +`windows_paths.py` extracts the existing wheel component/reserved-name and +regular-file collision rules into one host-independent policy. It rejects raw +absolute/dot/parent paths, forbidden/control characters, trailing dot/space, +reserved device basenames with extensions, case-equivalent destinations, and +regular-file ancestors. Differently named files under case-equivalent directory +spellings remain valid when their full destinations do not collide. + +Source staging validates selected inputs and planned generated/artifact paths +before content reads. Output preview and writing share the same guard. Existing +exact source/generated collision preview behavior is retained; differently cased +collisions cannot collapse in a dictionary. Non-selected odd repository entries +are not treated as deployment inputs. + +Static validation checks raw index declarations, manifest references, and actual +kit files. Unsafe paths are rejected before referenced-file access. Re-indexing +cannot bypass the target-filesystem check. Packaging requires STATIC_VALID before +creating output and validates the extracted ZIP again, so no redundant packager +policy was introduced. Wheels reuse the shared rules while retaining archive, +purelib relocation, installed collision, and content-specific checks. + +## Explicit reads and bounded audit + +Recognized read/read_write intent now permits safe existing repository files or +directories regardless of suffix. Tests cover `.dat`, extensionless paths, +positional/keyword builtin open, Path readers/open, already-supported image and +document readers, and unconventional directory readers with descendants. + +The filename heuristic remains for weaker evidence and unresolved conventional +names. No new dataflow or reader API family was added. Write-only paths do not +become immutable read resources; read_write evidence still passes through the +existing mutable-state filtering. Dynamic/nonexistent paths are not fabricated. + +The bounded consumer audit also distinguishes `Path(...).open()` from builtin +open for receiver/mode binding. Literal containment no longer strips leading +slashes or parent syntax into apparently local names; os.path.join retains its +absolute prefix for rejection. Existing source-root and repository/symlink +containment remain active. Staged fixtures execute successfully in isolation; +dirty/untracked inputs and configured-secret content still block release without +leaking the configured value. + +## Developer-path separator audit + +The shared drive-letter Users/home detector accepts backslash, forward slash, +and mixed separators, with a lexical left boundary protecting ordinary URL +fixtures. Its scope was not expanded to UNC/device paths or other policy classes. +Source, application-wheel, approved-wheel, and independently re-indexed static-kit +tests use the shared detector. Program Files operational-content context, +obvious/configured secrets, and other existing text-security policies remain +unchanged. + +## Acceptance and generation variance + +SGG source remains clean at `f484570d89fb1f9e9170fac915475358dfc1234e`: +package / ENTRYPOINT_REQUIRES_PACKAGE_MODE, 13 Python plus 77 package-data members, +90 source-derived authoritative artifact members. Generation, STATIC_VALID, +managed-site-packages import, and runtime automation pass; final automated state +is MANUAL_GUI_VALIDATION_REQUIRED, not a claim of manual visual acceptance. + +The [detailed SGG report](pr9-minor-marker-acceptance-2026-09-12.md#regenerated-acceptance-for-the-windows-pathread-intent-correction) +retains both historical and current hashes and the accepted generation-variance +explanation. Current ZIP SHA-256: +`31875A58658E11AFC968FA9224258B2180A30858B81659DDB6CE1E87BEE37BD6`. +The same current kit packaged twice is byte-identical. Different generation runs +retain intentional timestamp/ID differences; no generation code was altered. + +Geo remains clean at `5e7b321d0aeb9ba1d586bfc548c79793d84c6033`, source / +SOURCE_COMPATIBLE. Against the exact starting resource analyzer, requirements +remain **3 -> 3**, runtime-resource inventory files **10 -> 10**, and staged paths +**26 -> 26**. There are no added/removed resource or staged paths. Dependencies, +project/entry-point metadata, and configuration evidence are unchanged. Actual +staging still passes security/provenance checks. + +TN remains clean at `e1e7a1588c37a99c2d02efaf3eef3d04636f12f0`, source / +SOURCE_COMPATIBLE. Requirements remain **1 -> 1**, runtime-resource inventory +files **0 -> 0**, and staged paths **6 -> 6**. Its existing +`pdb-m61-regression-20260901/tn-kit` revalidates STATIC_VALID with reviewed +`proxy-tools==0.1.0`. Both source plans retain their ordinary pre-lock-verification +readiness; no new acceptance blocker or dependency/configuration change appears. + +## Quality and compatibility + +Analysis/planning schemas stay **1.4 / 1.3**; no persisted fields change. +Accepted application-artifact authority, assigned Traversables, legacy manifest +security, PEP 621 authority, importlib resource families, relative imports, +minor-invariant/patch-sensitive marker proofs, approved dependencies, exact sync, +provenance, rollback, and deterministic packaging remain in the complete suite. + +Focused results before the cleared hash gate: **532 passed, 3 skipped** and +**453 passed, 1 skipped**. The new regression file adds 98 test cases. +One complete suite after acceptance: **2031 passed, 4 skipped** in 386.07 seconds, +up from the committed-head baseline of 1933 passed, 4 skipped. Ruff and +`git diff --check` pass. Final diff inspection is limited to the three corrections, +their shared policy/regressions, and these acceptance reports. Generated wheels, +kits, environments, and ZIPs remain outside the commit. + +All three acceptance repositories remain at their required clean SHAs. The +single focused correction commit can now be pushed and receive exactly one +fresh exact-head review after PR Quality/current acceptance evidence is updated. +The 127 inline threads remain unresolved; PR #9 is not merged. diff --git a/src/python_deployment_builder/analysis/resources.py b/src/python_deployment_builder/analysis/resources.py index 3841527..c5b937c 100644 --- a/src/python_deployment_builder/analysis/resources.py +++ b/src/python_deployment_builder/analysis/resources.py @@ -394,8 +394,14 @@ def _looks_like_resource_literal(value: str) -> bool: def _resolve_literal(root: Path, source_path: Path, value: str) -> list[Path]: - normalized = value.replace("\\", "/").strip().lstrip("./") - if not normalized or Path(normalized).is_absolute(): + normalized = value.replace("\\", "/") + if ( + not normalized or normalized.startswith("/") + or PureWindowsPath(normalized).drive + or ".." in normalized.split("/") + or "\x00" in normalized + or normalized.startswith(("http://", "https://")) + ): return [] relative = Path(normalized) candidates = [root / relative, source_path.parent / relative] @@ -408,9 +414,10 @@ def _resolve_literal(root: Path, source_path: Path, value: str) -> list[Path]: for candidate in candidates: try: candidate.resolve().relative_to(root.resolve()) - except ValueError: + exists = candidate.exists() + except (ValueError, OSError): continue - if candidate.exists() and candidate not in found: + if exists and candidate not in found: found.append(candidate) return found @@ -1109,7 +1116,7 @@ def _path_values( if not part: return [] values = _combine_paths(values, part) - return [value.lstrip("/") for value in values] + return values if isinstance(node.func, ast.Attribute) and method in {"joinpath", "with_name"}: base = _path_values( node.func.value, @@ -1160,9 +1167,10 @@ def _path_values( def _open_access(name: str, node: ast.Call) -> str: mode_node: ast.AST | None = None - if name == "open" and len(node.args) >= 2: + builtin = name == "open" and isinstance(node.func, ast.Name) + if builtin and len(node.args) >= 2: mode_node = node.args[1] - elif name != "open" and node.args: + elif not builtin and node.args: mode_node = node.args[0] mode_node = next( (keyword.value for keyword in node.keywords if keyword.arg == "mode"), @@ -1202,7 +1210,7 @@ def _path_uses( ) -> list[tuple[ast.AST, str]]: name = _qualified_name(node.func) method = name.split(".")[-1].lower() - if name == "open": + if name == "open" and isinstance(node.func, ast.Name): path = call_argument(node, position=0, keyword="file") return [(path, _open_access(name, node))] if path is not None else [] if method == "open" and isinstance(node.func, ast.Attribute): @@ -1384,7 +1392,11 @@ def _literal_evidence( ) ) for value in values: - if resource_values is None and not _looks_like_resource_literal(value): + conventional = _looks_like_resource_literal(value) + if ( + resource_values is None and not conventional + and mode not in {"read", "read_write"} + ): continue resolved_any = False for resolved in _resolve_literal(root, path, value): @@ -1405,7 +1417,7 @@ def _literal_evidence( excerpt=lines[node.lineno - 1].strip(), ) ) - if not resolved_any: + if not resolved_any and (resource_values is not None or conventional): unresolved.add(value.replace("\\", "/")) access[value.replace("\\", "/")].add(mode) found[value.replace("\\", "/")].append( diff --git a/src/python_deployment_builder/generation/artifacts.py b/src/python_deployment_builder/generation/artifacts.py index 00f5995..c023b0f 100644 --- a/src/python_deployment_builder/generation/artifacts.py +++ b/src/python_deployment_builder/generation/artifacts.py @@ -59,6 +59,11 @@ is_textual_wheel_member, text_security_findings, ) +from python_deployment_builder.windows_paths import ( + WindowsPathError, + validate_windows_file_collisions, + validate_windows_relative_path, +) MAX_WHEEL_MEMBERS = 10_000 MAX_WHEEL_MEMBER_SIZE = 256 * 1024 * 1024 @@ -77,25 +82,6 @@ class WheelStaticMetadata: requires_python: str | None requires_dist: tuple[str, ...] -_WINDOWS_FORBIDDEN_COMPONENT_CHARACTERS = frozenset('<>:"|?*') -_WINDOWS_RESERVED_DEVICE_BASENAMES = frozenset( - { - "CON", - "PRN", - "AUX", - "NUL", - "CONIN$", - "CONOUT$", - *(f"COM{number}" for number in range(1, 10)), - *(f"LPT{number}" for number in range(1, 10)), - "COM¹", - "COM²", - "COM³", - "LPT¹", - "LPT²", - "LPT³", - } -) def parse_artifact_argument(value: str) -> tuple[str, Path]: @@ -105,76 +91,21 @@ def parse_artifact_argument(value: str) -> tuple[str, Path]: return canonicalize_name(name.strip()), Path(raw_path.strip()).expanduser() -def _windows_materializable_component(component: str) -> None: - """Reject components Windows cannot create through ordinary file APIs.""" - - if ( - not component - or component.endswith((".", " ")) - or any( - character in _WINDOWS_FORBIDDEN_COMPONENT_CHARACTERS - or character == "\x00" - or 1 <= ord(character) <= 31 - for character in component - ) - ): - raise PreparationError(f"Wheel contains a Windows-invalid path component: {component!r}") - # Windows reserves device basenames even when an ordinary-looking suffix - # follows (for example CON.py or NUL.tar.gz). - basename = component.split(".", 1)[0].upper() - if basename in _WINDOWS_RESERVED_DEVICE_BASENAMES: - raise PreparationError(f"Wheel contains a Windows-reserved path component: {component!r}") - - def _normalized_wheel_path(value: str) -> str: - if ( - not value - or "\x00" in value - or "\\" in value - or value.startswith("/") - or re.match(r"^[A-Za-z]:", value) - ): - raise PreparationError(f"Wheel contains an unsafe member: {value}") - path = PurePosixPath(value) - if not path.parts or ".." in path.parts: - raise PreparationError(f"Wheel contains an unsafe member: {value}") - for component in path.parts: - _windows_materializable_component(component) - return path.as_posix() + try: + # ZIP directory entries may end in one slash; regular kit paths may not. + return validate_windows_relative_path(value.removesuffix("/")) + except WindowsPathError as exc: + raise PreparationError(f"Wheel contains an unsafe member: {value!r}; {exc}") from exc def _validate_regular_file_path_collisions( paths: list[tuple[str, str]], *, domain: str ) -> None: - """Reject Windows-equivalent file paths that cannot coexist on disk. - - ``paths`` contain materialized regular-file destinations and their archive - provenance. Archive paths and relocated ``purelib`` paths are different - domains, but a file can never also be a component ancestor in either one. - """ - - regular_paths: dict[str, str] = {} - for normalized, provenance in paths: - collision_key = normalized.casefold() - if previous := regular_paths.get(collision_key): - raise PreparationError( - f"Wheel contains colliding regular files in {domain} paths: " - f"{previous}, {provenance}" - ) - for index in range(1, len(PurePosixPath(normalized).parts)): - ancestor = "/".join(PurePosixPath(normalized).parts[:index]).casefold() - if previous := regular_paths.get(ancestor): - raise PreparationError( - f"Wheel contains a regular-file ancestor collision in {domain} paths: " - f"{previous}, {provenance}" - ) - for existing_key, existing_name in regular_paths.items(): - if existing_key.startswith(collision_key + "/"): - raise PreparationError( - f"Wheel contains a regular-file ancestor collision in {domain} paths: " - f"{existing_name}, {provenance}" - ) - regular_paths[collision_key] = provenance + try: + validate_windows_file_collisions(paths, domain=domain) + except WindowsPathError as exc: + raise PreparationError(f"Wheel contains {exc}") from exc def _safe_wheel_members(bundle: zipfile.ZipFile) -> list[zipfile.ZipInfo]: diff --git a/src/python_deployment_builder/generation/generator.py b/src/python_deployment_builder/generation/generator.py index a1a2c1a..4ffdd86 100644 --- a/src/python_deployment_builder/generation/generator.py +++ b/src/python_deployment_builder/generation/generator.py @@ -52,6 +52,11 @@ RepositoryFileRole, ) from python_deployment_builder.planning import create_deployment_plan +from python_deployment_builder.windows_paths import ( + WindowsPathError, + validate_windows_materialized_paths, + validate_windows_relative_path, +) GENERATED_INDEX = "deployment/generated-files.json" PYTHON_CACHE_DIRECTORY = re.compile(r"^__pycache__(?:\s*\(\d+\))?$", re.IGNORECASE) @@ -430,6 +435,13 @@ def _tracked_deployment_paths( return selected +def _validate_kit_windows_paths(paths) -> None: + try: + validate_windows_materialized_paths(paths) + except WindowsPathError as exc: + raise PreparationError(f"Windows kit materialization is unsafe: {exc}") from exc + + def _staging_files( repository_root: Path, assessment, @@ -438,6 +450,7 @@ def _staging_files( include: bool, allow_missing_lock: bool = False, created_lock: Path | None = None, + generated_paths: list[str] | tuple[str, ...] = (), ) -> dict[str, bytes]: """Stage inventory-approved runtime inputs, never a broad repository copy.""" @@ -450,6 +463,11 @@ def _staging_files( created_lock=created_lock, allow_missing_lock=allow_missing_lock, ) + # Before reading bytes, validate selected source and the generated namespace + # together. Do not let a host filesystem normalize or collapse target names. + # Exact source/generated overlaps retain the existing preview collision + # diagnostic below; case-equivalent and ancestor overlaps must not collapse. + _validate_kit_windows_paths(sorted(selected | set(generated_paths))) files: dict[str, bytes] = {} for relative_text in sorted(selected): relative = Path(relative_text) @@ -553,7 +571,25 @@ def _classify_output( create: list[str] = [] replace: list[str] = [] collisions: list[str] = [] + existing_files = { + path.relative_to(output_root).as_posix() + for path in output_root.rglob("*") if path.is_file() + } for relative in planned_paths: + conflicts = [] + for existing in sorted(existing_files - {relative}): + try: + validate_windows_relative_path(existing) + except WindowsPathError: + # Non-selected repository entries are not materialized inputs. + continue + try: + validate_windows_materialized_paths([relative, existing]) + except WindowsPathError: + conflicts.append(existing) + if conflicts: + collisions.append(f"{relative} (Windows destination conflicts with {conflicts})") + continue path = output_root / Path(relative) if not path.exists(): create.append(relative) @@ -682,6 +718,7 @@ def _generated_index(application_id: str, owned: dict[str, bytes]) -> bytes: def _write_files(output_root: Path, files: dict[str, bytes]) -> None: + _validate_kit_windows_paths(files) output_root.mkdir(parents=True, exist_ok=True) for relative, data in files.items(): destination = output_root / Path(relative) @@ -714,6 +751,7 @@ def _preview( if prepare_lock and "uv.lock" not in paths: paths.append("uv.lock") paths.sort() + _validate_kit_windows_paths(paths) previous = _load_previous_index(output_root) create, replace, collisions = _classify_output(output_root, paths, previous) obsolete, obsolete_collisions = _obsolete_owned_paths(output_root, paths, previous) @@ -1113,6 +1151,9 @@ def generate_deployment_kit( plan, include=output_root != repository_root, allow_missing_lock=allow_missing_lock_for_analysis, + generated_paths=_planned_generated_paths( + plan, bootstrap_mode, approved, application_artifact + ), ) preview = _preview( plan, @@ -1221,6 +1262,9 @@ def generate_deployment_kit( plan, include=output_root != repository_root, created_lock=(lock_result.path if prepare_lock and lock_result.created else None), + generated_paths=_planned_generated_paths( + plan, bootstrap_mode, approved, application_artifact + ), ) bundled_uv = uv_executable if bootstrap_mode == "bundled_uv" else None owned, manifest = _render_owned_files( diff --git a/src/python_deployment_builder/security_policy.py b/src/python_deployment_builder/security_policy.py index a4ad4a0..9fc5f02 100644 --- a/src/python_deployment_builder/security_policy.py +++ b/src/python_deployment_builder/security_policy.py @@ -7,7 +7,7 @@ from pathlib import PurePosixPath FORBIDDEN_SHELL = ("powershell.exe", "pwsh.exe", "executionpolicy") -WINDOWS_ABSOLUTE = re.compile(r"(?i)[a-z]:\\(?:users|home)\\[^\r\n\"]+") +WINDOWS_ABSOLUTE = re.compile(r"(?i)(? Path | None: + try: + validate_windows_relative_path(relative) + except WindowsPathError: + return None candidate = (root / Path(relative)).resolve() try: candidate.relative_to(root) @@ -376,6 +385,30 @@ def validate_static_kit(kit_root: Path, *, dry_run: bool = False) -> ValidationR for path in root.rglob("*") if path.is_file() and path != index_path } + windows_path_failures: list[str] = [] + raw_index_paths = [ + item["path"] for item in index.get("files", []) + if isinstance(item, dict) and isinstance(item.get("path"), str) + ] + # Validate each declaration list before deduplicating legitimate overlaps + # between the index, references, and actual files. + for paths in ( + raw_index_paths, + manifest.referenced_files, + sorted(actual_paths | indexed_paths | set(manifest.referenced_files) + | {"deployment/generated-files.json"}), + ): + try: + validate_windows_materialized_paths(paths) + except WindowsPathError as exc: + windows_path_failures.append(str(exc)) + checks.append(_check( + "WINDOWS_PATH_MATERIALIZATION", + not windows_path_failures, + "Indexed, referenced, and materialized kit files have safe Windows destinations.", + "Kit paths cannot be safely materialized on Windows.", + evidence=windows_path_failures, + )) unexpected_paths = sorted(actual_paths - indexed_paths) checks.append( _check( diff --git a/src/python_deployment_builder/windows_paths.py b/src/python_deployment_builder/windows_paths.py new file mode 100644 index 0000000..c89de98 --- /dev/null +++ b/src/python_deployment_builder/windows_paths.py @@ -0,0 +1,76 @@ +"""Host-independent Windows materialization rules for kit and wheel destinations.""" + +from collections.abc import Iterable + + +class WindowsPathError(ValueError): + """A relative destination cannot be materialized safely on Windows.""" + + +_FORBIDDEN = frozenset('<>:"|?*') +_RESERVED = frozenset( + { + "CON", + "PRN", + "AUX", + "NUL", + "CONIN$", + "CONOUT$", + *(f"COM{n}" for n in range(1, 10)), + *(f"LPT{n}" for n in range(1, 10)), + "COM¹", + "COM²", + "COM³", + "LPT¹", + "LPT²", + "LPT³", + } +) + + +def validate_windows_relative_path(value: str) -> str: + """Validate raw POSIX components before any host Path can normalize them.""" + if not value or "\\" in value or any(p in {"", ".", ".."} for p in value.split("/")): + raise WindowsPathError(f"Windows-unsafe relative path: {value!r}") + for component in value.split("/"): + if component.endswith((".", " ")) or any( + character in _FORBIDDEN or ord(character) < 32 for character in component + ): + raise WindowsPathError(f"Windows-invalid path component: {component!r}") + if component.split(".", 1)[0].upper() in _RESERVED: + raise WindowsPathError(f"Windows-reserved path component: {component!r}") + return value + + +def windows_path_key(value: str) -> str: + return validate_windows_relative_path(value).casefold() + + +def validate_windows_file_collisions(paths: Iterable[tuple[str, str]], *, domain: str) -> None: + """Check regular-file destinations while retaining their caller's provenance.""" + regular_paths: dict[str, str] = {} + for path, provenance in paths: + key = windows_path_key(path) + if key in regular_paths: + raise WindowsPathError( + f"colliding regular files in {domain} paths: {regular_paths[key]}, {provenance}" + ) + parts = path.split("/") + for index in range(1, len(parts)): + ancestor = "/".join(parts[:index]).casefold() + if ancestor in regular_paths: + raise WindowsPathError( + f"regular-file ancestor collision in {domain} paths: " + f"{regular_paths[ancestor]}, {provenance}" + ) + for existing_key, existing_name in regular_paths.items(): + if existing_key.startswith(key + "/"): + raise WindowsPathError( + f"regular-file ancestor collision in {domain} paths: " + f"{existing_name}, {provenance}" + ) + regular_paths[key] = provenance + + +def validate_windows_materialized_paths(paths: Iterable[str]) -> None: + validate_windows_file_collisions(((path, path) for path in paths), domain="Windows kit") diff --git a/tests/test_windows_staging_explicit_reads.py b/tests/test_windows_staging_explicit_reads.py new file mode 100644 index 0000000..525f9f8 --- /dev/null +++ b/tests/test_windows_staging_explicit_reads.py @@ -0,0 +1,489 @@ +"""Windows kit paths, explicit read intent, and portable developer-path spellings.""" + +import json +import os +import subprocess +import sys +from pathlib import Path, PurePosixPath + +import pytest +from test_generation import ( + _make_application_wheel, + _make_wheel, + _rewrite_application_wheel, + _update_indexed_hashes, +) +from test_review_extra_paths_resources import fake_preparation, repo, write_source # noqa: F401 + +from python_deployment_builder.analysis.assessor import assess_repository +from python_deployment_builder.analysis.resources import inspect_resources +from python_deployment_builder.generation.acquisition import PreparationError +from python_deployment_builder.generation.artifacts import validate_wheel_static_safety +from python_deployment_builder.generation.generator import ( + _classify_output, + _staging_files, + _write_files, + generate_deployment_kit, +) +from python_deployment_builder.packaging.packager import PackageError, package_deployment_kit +from python_deployment_builder.planning.planner import create_deployment_plan +from python_deployment_builder.security_policy import text_security_findings +from python_deployment_builder.validation.static import validate_static_kit +from python_deployment_builder.windows_paths import ( + WindowsPathError, + validate_windows_materialized_paths, + validate_windows_relative_path, +) + + +@pytest.mark.parametrize( + "names", + [ + ["app/CON.py"], + ["data/foo:bar.json"], + ["data/Config.json", "data/config.json"], + ], +) +def test_selected_posix_names_rejected_before_staging_reads(tmp_path, monkeypatch, names): + """Model case-sensitive-host selected files without Win32 normalizing their names.""" + write_source(tmp_path) + assessment = assess_repository(repo(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + monkeypatch.setattr( + "python_deployment_builder.generation.generator._tracked_deployment_paths", + lambda *args, **kwargs: {"pyproject.toml", "uv.lock", *names}, + ) + original_is_file, original_read = Path.is_file, Path.read_bytes + + def relative(path): + return path.relative_to(tmp_path).as_posix() if path.is_relative_to(tmp_path) else "" + + monkeypatch.setattr(Path, "is_file", lambda p: relative(p) in names or original_is_file(p)) + reads = [] + + def read(path): + reads.append(relative(path)) + return b"fixture" if relative(path) in names else original_read(path) + + monkeypatch.setattr(Path, "read_bytes", read) + with pytest.raises(PreparationError, match="Windows|collid"): + _staging_files(tmp_path, assessment, plan, include=True) + assert not reads + + +@pytest.mark.parametrize( + ("name", "call"), + [ + ("weights.dat", "open('weights.dat', 'rb')"), + ("weights", "Path('weights').read_bytes()"), + ], +) +def test_explicit_read_promotes_unconventional_name(tmp_path, name, call): + write_source(tmp_path, f"from pathlib import Path\ndef main(): return {call}\n") + (tmp_path / name).write_bytes(b"resource fixture") + assessment = assess_repository(repo(tmp_path)) + assert name in {r.path for r in assessment.resources if r.access_mode == "read"} + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert name in _staging_files(tmp_path, assessment, plan, include=True) + + +def test_forward_slash_developer_path_is_detected(): + assert "developer_path" in text_security_findings( + "C:/Users/alice/project/data.json", path=PurePosixPath("main.py") + ) + + +@pytest.mark.parametrize( + "name", + [ + "app/CON.py", + "NUL.txt", + "COM1.cfg", + "LPT9.txt", + "COM¹.py", + "data/foo:bar.json", + "foo*.dat", + "foo?.dat", + 'foo".dat', + "foo<.dat", + "foo>.dat", + "foo|.dat", + "trailing.", + "trailing ", + "dir./file", + "dir /file", + "a\x00b", + "a\x1fb", + "/root/file", + "C:/root/file", + "../file", + "a/../file", + "./file", + "a/./file", + "a//file", + "a\\file", + "", + "a/", + ], +) +def test_shared_windows_component_policy(name): + with pytest.raises(WindowsPathError): + validate_windows_relative_path(name) + + +@pytest.mark.parametrize( + "paths", + [ + ["data/Config.json", "data/config.json"], + ["a", "A/child"], + ["A/child", "a"], + ["same", "same"], + ["Deployment/manifest.json", "deployment/manifest.json"], + ], +) +def test_shared_windows_regular_destinations_collide(paths): + with pytest.raises(WindowsPathError, match="colli"): + validate_windows_materialized_paths(paths) + + +def test_different_files_in_case_equivalent_directories_are_valid(): + validate_windows_materialized_paths(["Data/first.json", "data/SECOND.json", "main.py"]) + + +@pytest.mark.parametrize("name", ["Deployment/manifest.json"]) +def test_source_generated_collision_before_read(tmp_path, monkeypatch, name): + write_source(tmp_path) + assessment = assess_repository(repo(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + monkeypatch.setattr( + "python_deployment_builder.generation.generator._tracked_deployment_paths", + lambda *args, **kwargs: {"pyproject.toml", "uv.lock", name}, + ) + with pytest.raises(PreparationError, match="collid"): + _staging_files( + tmp_path, assessment, plan, include=True, generated_paths=["deployment/manifest.json"] + ) + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_generation_blocks_target_paths_without_output_mutation(tmp_path, monkeypatch, dry_run): + source, kit = tmp_path / "source", tmp_path / "kit" + write_source(source) + monkeypatch.setattr( + "python_deployment_builder.generation.generator._selected_deployment_paths", + lambda *args: {"pyproject.toml", "uv.lock", "app/CON.py"}, + ) + with pytest.raises(PreparationError, match="Windows-reserved"): + generate_deployment_kit(repo(source), kit, dry_run=dry_run, bootstrap_mode="online_cmd") + assert not kit.exists() + + +def test_writer_checks_before_creating_output(tmp_path): + with pytest.raises(PreparationError, match="Windows-reserved"): + _write_files(tmp_path / "kit", {"NUL.txt": b"fixture"}) + assert not (tmp_path / "kit").exists() + + +def test_existing_output_case_collision_is_not_overwritten(tmp_path): + (tmp_path / "Config.json").write_text("user owned") + _, _, collisions = _classify_output(tmp_path, ["config.json"], {}) + assert collisions + assert (tmp_path / "Config.json").read_text() == "user owned" + + +@pytest.mark.usefixtures("fake_preparation") +def test_nonselected_odd_repository_path_does_not_block(tmp_path): + source = tmp_path / "source" + write_source(source) + # ':' is a stream separator on Windows, so use a real ordinary but unselected + # filename here; host-independent selected-name cases cover forbidden names. + (source / "unselected.odd").write_text("not runtime input") + kit = tmp_path / "kit" + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + assert not (kit / "unselected.odd").exists() + + +@pytest.mark.usefixtures("fake_preparation") +@pytest.mark.parametrize( + ("path", "declaration"), + [ + ("app/CON.py", "index"), + ("foo:bar.json", "index"), + ("MAIN.py", "index"), + ("a/../main.py", "index"), + ("NUL.txt", "references"), + ("MAIN.py", "references"), + ], +) +def test_static_windows_paths_cannot_be_bypassed_by_index(tmp_path, path, declaration): + source, kit = tmp_path / "source", tmp_path / "kit" + write_source(source) + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + if declaration == "index": + index_path = kit / "deployment/generated-files.json" + index = json.loads(index_path.read_text()) + digest = next(item["sha256"] for item in index["files"] if item["path"] == "main.py") + index["files"].append({"path": path, "sha256": digest}) + index_path.write_text(json.dumps(index)) + else: + manifest_path = kit / "deployment/manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["referenced_files"].append(path) + manifest_path.write_text(json.dumps(manifest)) + report = validate_static_kit(kit) + checks = {c.code: c for c in report.static_checks} + assert checks["WINDOWS_PATH_MATERIALIZATION"].status.value == "FAIL" + with pytest.raises(PackageError, match="WINDOWS_PATH_MATERIALIZATION"): + package_deployment_kit(kit, output_directory=tmp_path / "zip") + assert not (tmp_path / "zip").exists() + + +@pytest.mark.parametrize( + "call", + [ + "open('weights.dat', 'rb')", + "open(file='weights.dat', mode='rb')", + "Path('weights.dat').read_bytes()", + "Path('weights.dat').read_text()", + "Path('weights.dat').open('rb')", + "Image.open('weights.dat')", + "fitz.open('weights.dat')", + ], +) +def test_all_supported_explicit_file_readers(tmp_path, call): + write_source(tmp_path, f"from pathlib import Path\ndef main(): return {call}\n") + (tmp_path / "weights.dat").write_bytes(b"\x00\xffMODEL") + assessment = assess_repository(repo(tmp_path)) + assert any(r.path == "weights.dat" and r.access_mode == "read" for r in assessment.resources) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert ( + _staging_files(tmp_path, assessment, plan, include=True)["weights.dat"] == b"\x00\xffMODEL" + ) + + +@pytest.mark.parametrize( + "call", + [ + "os.listdir('oddstore')", + "os.scandir(path='oddstore')", + "Path('oddstore').iterdir()", + ], +) +def test_arbitrary_directory_read_retains_descendants(tmp_path, call): + write_source(tmp_path, f"import os\nfrom pathlib import Path\ndef main(): return {call}\n") + (tmp_path / "oddstore").mkdir() + (tmp_path / "oddstore/blob").write_bytes(b"resource") + assessment = assess_repository(repo(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert any(r.path == "oddstore" and r.access_mode == "read" for r in assessment.resources) + assert "oddstore/blob" in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.parametrize( + ("mode", "access"), + [ + ("rb", "read"), + ("wb", None), + ("ab", None), + ("r+", "read_write"), + ("a+", "read_write"), + ("w+", "read_write"), + ], +) +def test_read_write_classification_is_preserved(tmp_path, mode, access): + write_source(tmp_path, f"def main(): return open('weights.dat', {mode!r})\n") + (tmp_path / "weights.dat").write_text("state") + assessment = assess_repository(repo(tmp_path)) + resources = {r.path: r for r in inspect_resources(tmp_path, ["."])[0]} + if access is None: + assert "weights.dat" not in resources + else: + assert resources["weights.dat"].access_mode == access + if access == "read_write": + assert "weights.dat" not in {r.path for r in assessment.resources} + assert any( + item.path == "weights.dat" and item.role.value == "mutable_state_candidate" + for item in assessment.file_inventory + ) + + +@pytest.mark.parametrize( + "expression", + [ + "filename", + "'missing.dat'", + "'../weights.dat'", + "'/weights.dat'", + "'C:/external/weights.dat'", + "os.path.join('/outside', 'weights.dat')", + "'https://example.invalid/weights.dat'", + ], +) +def test_unproven_or_external_reads_are_not_materialized(tmp_path, expression): + write_source(tmp_path, f"import os\ndef main(): return open({expression}, 'rb')\n") + (tmp_path / "weights.dat").write_bytes(b"unrelated") + assessment = assess_repository(repo(tmp_path)) + plan = create_deployment_plan(assessment, repository_root=tmp_path) + assert "weights.dat" not in _staging_files(tmp_path, assessment, plan, include=True) + + +@pytest.mark.usefixtures("fake_preparation") +@pytest.mark.parametrize( + "reader", ["open('weights.dat', 'rb').read()", "Path('weights').read_bytes()"] +) +def test_explicit_read_kit_executes_after_staging(tmp_path, reader): + source, kit = tmp_path / "source", tmp_path / "kit" + write_source(source, f"from pathlib import Path\ndef main(): return {reader}\n") + filename = "weights.dat" if "weights.dat" in reader else "weights" + (source / filename).write_bytes(b"MODEL") + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + assert validate_static_kit(kit).final_state.value == "STATIC_VALID" + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + "import sys;sys.path.insert(0,sys.argv[1]);" + "from main import main;assert main()==b'MODEL'", + str(kit), + ], + cwd=kit, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "value", + [ + r"C:\Users\alice\project\data.json", + "C:/Users/alice/project/data.json", + r"C:\Users/alice/project/data.json", + r"C:/Users\alice/project\data.json", + "c:/users/alice/data", + "C:/home/alice/data", + ], +) +def test_developer_path_separator_matrix(value): + assert "developer_path" in text_security_findings(value, path=PurePosixPath("main.py")) + + +@pytest.mark.parametrize( + "value", + [ + "users/alice/data", + "/home/alice/data", + "https://example.invalid/users/alice/data", + "https://example.invalid/C:/Users/alice/data", + "https://c:/Users/alice/data", + ], +) +def test_non_windows_paths_and_urls_not_flagged(value): + assert "developer_path" not in text_security_findings(value, path=PurePosixPath("README.md")) + + +@pytest.mark.parametrize("application", [True, False]) +def test_forward_slash_wheel_security_parity(tmp_path, application): + wheel = _make_application_wheel(tmp_path) if application else _make_wheel(tmp_path) + _rewrite_application_wheel(wheel, additions={"data/readme.txt": "C:/Users/alice/data.json"}) + with pytest.raises(PreparationError, match="security"): + validate_wheel_static_safety(wheel) + + +@pytest.mark.usefixtures("fake_preparation") +@pytest.mark.parametrize("state", ["secret", "developer_path", "dirty", "untracked"]) +def test_arbitrary_read_security_and_provenance(tmp_path, monkeypatch, state): + source, kit = tmp_path / "source", tmp_path / "kit" + write_source( + source, + "from os import getenv\nPASSWORD=getenv('DB_PASSWORD')\n" + "def main(): return open('weights.dat', 'rb').read()\n", + ) + resource = source / "weights.dat" + secret = "PDBExplicitReadSecret123" + monkeypatch.setenv("DB_PASSWORD", secret) + resource.write_text( + secret + if state == "secret" + else "C:/Users/alice/data.json" + if state == "developer_path" + else "MODEL" + ) + if state in {"dirty", "untracked"}: + + def git(*args): + subprocess.run(["git", "-C", str(source), *args], check=True, capture_output=True) + + git("init") + git("add", ".") + if state == "untracked": + git("rm", "--cached", "weights.dat") + git( + "-c", + "user.name=PDB Test", + "-c", + "user.email=pdb@example.invalid", + "commit", + "-m", + "fixture", + ) + if state == "dirty": + resource.write_text("CHANGED") + with pytest.raises(PreparationError) as error: + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + assert secret not in str(error.value) + assert not kit.exists() + + +@pytest.mark.usefixtures("fake_preparation") +def test_static_reindexed_forward_slash_developer_path(tmp_path): + source, kit = tmp_path / "source", tmp_path / "kit" + write_source(source) + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + (kit / "main.py").write_text("PATH='C:/Users/alice/project/data.json'\n") + _update_indexed_hashes(kit, "main.py") + checks = {c.code: c for c in validate_static_kit(kit).static_checks} + assert checks["GENERATED_FILE_HASHES"].status.value == "PASS" + assert checks["NO_DEVELOPER_PATHS"].status.value == "FAIL" + + +@pytest.mark.usefixtures("fake_preparation") +@pytest.mark.parametrize("selected", [False, True]) +def test_physical_reserved_filename_selected_only(tmp_path, selected): + # NT's extended namespace permits creating this POSIX-valid test filename, + # even though ordinary Windows ZIP materialization must reject it. + root = Path("//?/" + str(tmp_path).replace("\\", "/")) if os.name == "nt" else tmp_path + source, kit = root / "source", root / "kit" + write_source(source) + target = source / ("CON.py" if selected else "docs/CON.txt") + target.parent.mkdir(exist_ok=True) + target.write_text("VALUE=1\n") + try: + if selected: + with pytest.raises(PreparationError, match="Windows-reserved"): + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + assert not kit.exists() + else: + generate_deployment_kit(repo(source), kit, bootstrap_mode="online_cmd") + assert validate_static_kit(kit).final_state.value == "STATIC_VALID" + assert not (kit / "docs/CON.txt").exists() + finally: + target.unlink() + + +def test_explicit_arbitrary_read_respects_symlink_containment(tmp_path, monkeypatch): + write_source(tmp_path, "def main(): return open('weights.dat', 'rb')\n") + target = tmp_path / "weights.dat" + target.write_bytes(b"MODEL") + original = Path.resolve + monkeypatch.setattr( + Path, + "resolve", + lambda p, *a, **k: tmp_path.parent / "outside.dat" if p == target else original(p, *a, **k), + ) + assessment = assess_repository(repo(tmp_path)) + assert not any( + r.path == "weights.dat" and r.packaging_status != "unknown" for r in assessment.resources + )