Skip to content

Replaceing the metadata parser from packaging.metadata #607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LalatenduMohanty wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Replaceing the metadata parser from packaging.metadata #607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ dependencies = [
"elfdeps>=0.2.0",
"license-expression",
"packaging",
"pkginfo",
"psutil",
"pydantic",
"pypi_simple",
Expand Down Expand Up @@ -204,7 +203,7 @@ exclude = [

[[tool.mypy.overrides]]
# packages without typing annotations and stubs
module = ["license_expression", "pyproject_hooks", "requests_mock", "resolver", "stevedore"]
module = ["hatchling", "hatchling.build", "license_expression", "pyproject_hooks", "requests_mock", "resolver", "stevedore"]
ignore_missing_imports = true

[tool.basedpyright]
Expand Down
7 changes: 2 additions & 5 deletions src/fromager/bootstrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import tempfile
import typing
import zipfile
from email.parser import BytesParser
from urllib.parse import urlparse

from packaging.requirements import Requirement
Expand Down Expand Up @@ -1242,10 +1241,8 @@ def _get_version_from_package_metadata(
config_settings=pbi.config_settings,
)
metadata_filename = source_dir.parent / metadata_dir_base / "METADATA"
with open(metadata_filename, "rb") as f:
p = BytesParser()
metadata = p.parse(f, headersonly=True)
return Version(metadata["Version"])
metadata = dependencies.parse_metadata(metadata_filename)
Copy link
Collaborator

@tiran tiran Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

parse_metadata validates the metadata by default. This will raise an exception if the metadata version does not match the metadata content, e.g. license-file field in Metadata < 2.4.

return metadata.version

def _resolve_prebuilt_with_history(
self,
Expand Down
45 changes: 20 additions & 25 deletions src/fromager/candidate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@
import datetime
import logging
import typing
from email.message import EmailMessage, Message
from email.parser import BytesParser
from io import BytesIO
from typing import TYPE_CHECKING
from zipfile import ZipFile

from packaging.metadata import Metadata
from packaging.requirements import Requirement
from packaging.utils import BuildTag, canonicalize_name
from packaging.version import Version
Expand All @@ -16,13 +14,6 @@

logger = logging.getLogger(__name__)

# fix for runtime errors caused by inheriting classes that are generic in stubs but not runtime
# https://mypy.readthedocs.io/en/latest/runtime_troubles.html#using-classes-that-are-generic-in-stubs-but-not-at-runtime
if TYPE_CHECKING:
Metadata = Message[str, str]
else:
Metadata = Message


@dataclasses.dataclass(frozen=True, order=True, slots=True, repr=False, kw_only=True)
class Candidate:
Expand Down Expand Up @@ -73,11 +64,10 @@ def metadata(self) -> Metadata:
return self._metadata

def _get_dependencies(self) -> typing.Iterable[Requirement]:
deps = self.metadata.get_all("Requires-Dist", [])
deps = self.metadata.requires_dist or []
extras = self.extras if self.extras else [""]

for d in deps:
r = Requirement(d)
for r in deps:
if r.marker is None:
yield r
else:
Expand All @@ -95,19 +85,22 @@ def dependencies(self) -> list[Requirement]:

@property
def requires_python(self) -> str | None:
return self.metadata.get("Requires-Python")
spec = self.metadata.requires_python
return str(spec) if spec is not None else None


def get_metadata_for_wheel(url: str, metadata_url: str | None = None) -> Metadata:
"""
Get metadata for a wheel, supporting PEP 658 metadata endpoints.
def get_metadata_for_wheel(
url: str, metadata_url: str | None = None, *, validate: bool = True
) -> Metadata:
"""Get metadata for a wheel, supporting PEP 658 metadata endpoints.

Args:
url: URL of the wheel file
metadata_url: Optional URL of the metadata file (PEP 658)
validate: Whether to validate metadata (default: True)

Returns:
Parsed metadata as a Message object
Parsed metadata as a Metadata object
"""
# Try PEP 658 metadata endpoint first if available
if metadata_url:
Expand All @@ -118,9 +111,9 @@ def get_metadata_for_wheel(url: str, metadata_url: str | None = None) -> Metadat
response = session.get(metadata_url)
response.raise_for_status()

# Parse metadata directly from the response content
p = BytesParser()
metadata = p.parse(BytesIO(response.content), headersonly=True)
# Parse metadata directly using packaging.metadata.Metadata
# (avoiding circular import with dependencies module)
metadata = Metadata.from_email(response.content, validate=validate)
logger.debug(f"Successfully retrieved metadata via PEP 658 for {url}")
return metadata

Expand All @@ -136,8 +129,10 @@ def get_metadata_for_wheel(url: str, metadata_url: str | None = None) -> Metadat
with ZipFile(BytesIO(data)) as z:
for n in z.namelist():
if n.endswith(".dist-info/METADATA"):
p = BytesParser()
return p.parse(z.open(n), headersonly=True)
metadata_content = z.read(n)
# Parse metadata directly using packaging.metadata.Metadata
# (avoiding circular import with dependencies module)
return Metadata.from_email(metadata_content, validate=validate)

# If we didn't find the metadata, return an empty dict
return EmailMessage()
# If we didn't find the metadata, raise an error
raise ValueError(f"Could not find METADATA file in wheel: {url}")
85 changes: 76 additions & 9 deletions src/fromager/dependencies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
import pathlib
import tempfile
import typing
import zipfile

import pkginfo
import pyproject_hooks
import tomlkit
from packaging.metadata import Metadata
Expand Down Expand Up @@ -344,14 +344,23 @@ def default_get_install_dependencies_of_sdist(
return set(metadata.requires_dist)


def parse_metadata(metadata_file: pathlib.Path, *, validate: bool = True) -> Metadata:
"""Parse a dist-info/METADATA file
def parse_metadata(
metadata_source: pathlib.Path | bytes, *, validate: bool = True
) -> Metadata:
"""Parse metadata from a file path or bytes.

Args:
metadata_source: Path to METADATA file or bytes containing metadata
validate: Whether to validate metadata (default: True)

The default parse mode is 'strict'. It even fails for a mismatch of field
and core metadata version, e.g. a package with metadata 2.2 and
license-expression field (added in 2.4).
Returns:
Parsed Metadata object
"""
return Metadata.from_email(metadata_file.read_bytes(), validate=validate)
if isinstance(metadata_source, pathlib.Path):
metadata_bytes = metadata_source.read_bytes()
else:
metadata_bytes = metadata_source
return Metadata.from_email(metadata_bytes, validate=validate)


def pep517_metadata_of_sdist(
Expand Down Expand Up @@ -418,16 +427,74 @@ def validate_dist_name_version(
def get_install_dependencies_of_wheel(
req: Requirement, wheel_filename: pathlib.Path, requirements_file_dir: pathlib.Path
) -> set[Requirement]:
"""Get install dependencies from a wheel file.

Extracts and parses the METADATA file from the wheel to get the
Requires-Dist entries.

Args:
req: The requirement being processed
wheel_filename: Path to the wheel file
requirements_file_dir: Directory to write the requirements file

Returns:
Set of requirements from the wheel's metadata
"""
logger.info(f"getting installation dependencies from {wheel_filename}")
wheel = pkginfo.Wheel(str(wheel_filename))
deps = _filter_requirements(req, wheel.requires_dist)
# Disable validation because many third-party packages have metadata version
# mismatches (e.g., setuptools declares Metadata-Version: 2.2 but uses
# license-file which was introduced in 2.4). The old pkginfo library
# didn't validate this, so we maintain backward compatibility.
metadata = _get_metadata_from_wheel(wheel_filename, validate=False)
requires_dist = metadata.requires_dist or []
deps = _filter_requirements(req, requires_dist)
_write_requirements_file(
deps,
requirements_file_dir / INSTALL_REQ_FILE_NAME,
)
return deps


def _get_metadata_from_wheel(
wheel_filename: pathlib.Path, *, validate: bool = True
) -> Metadata:
"""Extract and parse METADATA from a wheel file.

Args:
wheel_filename: Path to the wheel file
validate: Whether to validate metadata (default: True)

Returns:
Parsed Metadata object

Raises:
ValueError: If no METADATA file is found in the wheel
"""
# Predict the dist-info directory name from the wheel filename
# Wheel format: {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl
# Dist-info format: {distribution}-{version}.dist-info
# Note: We extract from the filename directly rather than using parse_wheel_filename
# because the dist-info directory uses the original (non-normalized) name
wheel_name_parts = wheel_filename.stem.split("-")
dist_name = wheel_name_parts[0]
dist_version = wheel_name_parts[1]
predicted_dist_info = f"{dist_name}-{dist_version}.dist-info/METADATA"
Comment on lines +478 to +481
Copy link
Collaborator

@tiran tiran Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's not invent our own wheel parsing algorithm. The function add_extra_metadata_to_wheels has some code to get the dist-info directory of a wheel file. Perhaps move the code into a common, shared helper?


with zipfile.ZipFile(wheel_filename) as whl:
# Try predicted path first for efficiency
if predicted_dist_info in whl.namelist():
metadata_content = whl.read(predicted_dist_info)
return parse_metadata(metadata_content, validate=validate)
Comment on lines +485 to +487
Copy link
Collaborator

@tiran tiran Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This constructs a list of all files in a wheel and then searches through the list of files. It's inefficient.

try:
    metadata_content = whl.read(metadata_file)
except ...


# Fallback to iterating if prediction fails (e.g., non-standard naming)
for entry in whl.namelist():
if entry.endswith(".dist-info/METADATA"):
metadata_content = whl.read(entry)
return parse_metadata(metadata_content, validate=validate)
Comment on lines +489 to +493
Copy link
Collaborator

@tiran tiran Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This check is not necessary. A wheel file MUST contain a correct dist-info directory. If the dist-info directory does not match the wheel file name, then pip and uv will refuse to install the file.

$ mv fromager-0.75.0-py3-none-any.whl cheesemaker-0.75.0-py3-none-any.whl
$ pip install ./cheesemaker-0.75.0-py3-none-any.whl 
Processing ./cheesemaker-0.75.0-py3-none-any.whl

ERROR: cheesemaker has an invalid wheel, .dist-info directory 'fromager-0.75.0.dist-info' does not start with 'cheesemaker'


raise ValueError(f"Could not find METADATA file in wheel: {wheel_filename}")


def get_pyproject_contents(sdist_root_dir: pathlib.Path) -> dict[str, typing.Any]:
pyproject_toml_filename = sdist_root_dir / "pyproject.toml"
if not os.path.exists(pyproject_toml_filename):
Expand Down
Loading
Loading