diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0fad2df..6ee7b4e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,13 +31,13 @@ jobs:
run: uv python install ${{ env.PYTHON_VERSION }}
- name: Install dependencies
- run: uv sync --python ${{ env.PYTHON_VERSION }} --dev
+ run: uv sync --locked --python ${{ env.PYTHON_VERSION }} --dev
- name: Run ruff check
- run: uv run ruff check src/ tests/
+ run: uv run ruff check src/ tests/ scripts/
- name: Run ruff format check
- run: uv run ruff format --check src/ tests/
+ run: uv run ruff format --check src/ tests/ scripts/
typecheck:
name: Type Check
@@ -54,7 +54,7 @@ jobs:
run: uv python install ${{ env.PYTHON_VERSION }}
- name: Install dependencies
- run: uv sync --python ${{ env.PYTHON_VERSION }} --dev
+ run: uv sync --locked --python ${{ env.PYTHON_VERSION }} --dev
- name: Run ty check
run: uv run ty check
@@ -110,7 +110,7 @@ jobs:
allow-prereleases: true
- name: Install dependencies
- run: uv sync --python python --dev
+ run: uv sync --locked --python python --dev
- name: Import package
run: uv run --python python python -c "import ml4t.specs"
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 068f06f..42bca88 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -3,48 +3,54 @@ name: Docs
on:
push:
branches: [main]
+ pull_request:
+ branches: [main]
workflow_dispatch:
-permissions:
- contents: read
+permissions: {}
concurrency:
- group: docs
+ group: docs-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
+env:
+ PYTHON_VERSION: "3.12"
+
jobs:
build:
- name: Build and deploy docs
+ name: Build revision-identified documentation
runs-on: ubuntu-latest
+ permissions:
+ contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
- fetch-depth: 0
-
- - name: Install uv
- uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+ persist-credentials: false
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
with:
version: "0.10.9"
-
- - name: Set up Python
- run: uv python install 3.12
-
- - name: Install dependencies
- run: uv sync --python 3.12 --dev --extra docs
-
- - name: Build docs
- run: uv run mkdocs build --strict
-
- - name: Deploy to website repository
- uses: cpina/github-action-push-to-another-repository@55306faa4ed53b815ae49e564af8cfb359d32ae2 # v1.7.3
- env:
- SSH_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_KEY }}
+ - run: uv python install ${{ env.PYTHON_VERSION }}
+ - run: uv sync --locked --python ${{ env.PYTHON_VERSION }} --dev --extra docs
+ - name: Record source identity
+ shell: bash
+ run: |
+ echo "ML4T_DOCS_VERSION=$(uv run python -c 'from ml4t.specs import __version__; print(__version__)')" >> "$GITHUB_ENV"
+ echo "ML4T_DOCS_COMMIT=${GITHUB_SHA}" >> "$GITHUB_ENV"
+ - name: Execute the README quick start
+ run: uv run python scripts/run_readme_quickstart.py
+ - name: Check every README link
+ run: uv run python scripts/check_readme_links.py README.md
+ - name: Build and verify documentation
+ run: |
+ uv run mkdocs build --strict
+ uv run python scripts/verify_documentation_identity.py \
+ --site site \
+ --expected-library specs \
+ --expected-version "$ML4T_DOCS_VERSION" \
+ --expected-commit "$ML4T_DOCS_COMMIT"
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- source-directory: site/
- destination-github-username: ml4t
- destination-repository-name: website
- target-directory: static/docs/specs/
- target-branch: main
- commit-message: "docs(specs): update from ml4t/specs@${{ github.sha }}"
- user-name: ml4t-bot
- user-email: bot@ml4trading.io
+ name: docs-${{ github.sha }}
+ path: site/
+ if-no-files-found: error
+ retention-days: 14
diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml
index eb01e36..a643392 100644
--- a/.github/workflows/ecosystem.yml
+++ b/.github/workflows/ecosystem.yml
@@ -17,6 +17,6 @@ jobs:
qualification:
permissions:
contents: read
- uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@457678d8860d0c816b1dc376e7e2691daa93af4d # v0.1.0
+ uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@da2cbb9a9d0b4d2f166ad630470ff7d7a1aa2643 # 2026-09-19 policy snapshot
with:
import-package: ml4t.specs
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 390de92..5329717 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,55 +1,316 @@
name: Release
on:
- push:
- tags: ["v*"]
+ workflow_dispatch:
+ inputs:
+ version:
+ description: Stable package version without the v prefix
+ required: true
+ type: string
+ candidate-sha:
+ description: Exact current main commit to publish
+ required: true
+ type: string
+
+permissions:
+ contents: read
+
+concurrency:
+ group: release
+ cancel-in-progress: false
+
+env:
+ PYTHON_VERSION: "3.12"
jobs:
+ preflight:
+ name: Validate unpublished main candidate
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ outputs:
+ commit: ${{ steps.identity.outputs.commit }}
+ tree: ${{ steps.identity.outputs.tree }}
+ version: ${{ steps.identity.outputs.version }}
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ ref: ${{ inputs.candidate-sha }}
+ - run: git fetch --no-tags origin main
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+ with:
+ version: "0.10.9"
+ - name: Check main identity and public version absence
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: >-
+ uv run --no-project --with packaging
+ python scripts/check_release_preflight.py
+ --version "${{ inputs.version }}"
+ --candidate-sha "${{ inputs.candidate-sha }}"
+ --workflow-sha "${{ github.sha }}"
+ --repository "${{ github.repository }}"
+ - name: Record accepted candidate
+ id: identity
+ run: |
+ {
+ echo "commit=${{ inputs.candidate-sha }}"
+ echo "tree=$(git rev-parse 'HEAD^{tree}')"
+ echo "version=${{ inputs.version }}"
+ } >> "$GITHUB_OUTPUT"
+
ecosystem-qualification:
name: Ecosystem Qualification
+ needs: preflight
permissions:
contents: read
- uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@457678d8860d0c816b1dc376e7e2691daa93af4d # v0.1.0
+ uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@da2cbb9a9d0b4d2f166ad630470ff7d7a1aa2643 # 2026-09-19 policy snapshot
with:
import-package: ml4t.specs
qualification:
- name: Qualify Release
+ name: Release Qualification
+ needs: preflight
permissions:
contents: read
uses: ./.github/workflows/ci.yml
+ build:
+ name: Build package and documentation once
+ runs-on: ubuntu-latest
+ needs: [preflight, ecosystem-qualification, qualification]
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ ref: ${{ needs.preflight.outputs.commit }}
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+ with:
+ version: "0.10.9"
+ - run: uv python install ${{ env.PYTHON_VERSION }}
+ - run: uv sync --locked --dev --extra docs
+ - name: Build distributions once
+ run: uv build --out-dir candidate/dist
+ - name: Validate and bind the release candidate
+ run: |
+ uv run twine check candidate/dist/*
+ uv run python scripts/release_candidate.py create candidate \
+ --commit-sha "${{ needs.preflight.outputs.commit }}" \
+ --git-tree "${{ needs.preflight.outputs.tree }}"
+ uv run python scripts/release_candidate.py verify candidate \
+ --expected-commit "${{ needs.preflight.outputs.commit }}" \
+ --expected-tree "${{ needs.preflight.outputs.tree }}" \
+ --expected-tag "v${{ needs.preflight.outputs.version }}"
+ - name: Install and exercise the candidate wheel
+ shell: bash
+ run: |
+ wheel=$(find candidate/dist -name '*.whl' -print -quit)
+ uv run --isolated --no-project --with "$wheel" python -I \
+ scripts/run_readme_quickstart.py
+ - name: Build and verify revision-identified documentation
+ env:
+ ML4T_DOCS_VERSION: ${{ needs.preflight.outputs.version }}
+ ML4T_DOCS_COMMIT: ${{ needs.preflight.outputs.commit }}
+ run: |
+ uv run mkdocs build --strict --site-dir candidate/site
+ uv run python scripts/verify_documentation_identity.py \
+ --site candidate/site \
+ --expected-library specs \
+ --expected-version "$ML4T_DOCS_VERSION" \
+ --expected-commit "$ML4T_DOCS_COMMIT"
+ - name: Upload immutable release candidate
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: release-candidate-${{ needs.preflight.outputs.commit }}
+ path: |
+ candidate/dist/
+ candidate/candidate.json
+ if-no-files-found: error
+ retention-days: 90
+ - name: Upload immutable documentation
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: docs-${{ needs.preflight.outputs.commit }}
+ path: candidate/site/
+ if-no-files-found: error
+ retention-days: 90
+
+ documentation:
+ name: Deploy and verify qualified documentation
+ runs-on: ubuntu-latest
+ needs: [preflight, build]
+ environment: documentation
+ permissions:
+ actions: read
+ contents: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ ref: ${{ needs.preflight.outputs.commit }}
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: docs-${{ needs.preflight.outputs.commit }}
+ path: candidate-site/
+ - name: Verify documentation before deployment
+ run: >-
+ python scripts/verify_documentation_identity.py
+ --site candidate-site
+ --expected-library specs
+ --expected-version "${{ needs.preflight.outputs.version }}"
+ --expected-commit "${{ needs.preflight.outputs.commit }}"
+ - name: Require the documentation deploy credential
+ env:
+ DOCS_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_KEY }}
+ run: test -n "$DOCS_DEPLOY_KEY"
+ - name: Deploy the qualified documentation artifact
+ uses: cpina/github-action-push-to-another-repository@55306faa4ed53b815ae49e564af8cfb359d32ae2 # v1.7.3
+ env:
+ SSH_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_KEY }}
+ with:
+ source-directory: candidate-site/
+ destination-github-username: ml4t
+ destination-repository-name: website
+ target-directory: static/docs/specs/
+ target-branch: main
+ commit-message: "docs(specs): release ${{ needs.preflight.outputs.version }} from ${{ needs.preflight.outputs.commit }}"
+ user-name: ml4t-bot
+ user-email: bot@ml4trading.io
+ - name: Verify deployed documentation identity
+ run: >-
+ python scripts/verify_documentation_identity.py
+ --url https://www.ml4trading.io/docs/specs/
+ --url https://www.ml4trading.io/docs/specs/getting-started/
+ --url https://www.ml4trading.io/docs/specs/api/
+ --expected-library specs
+ --expected-version "${{ needs.preflight.outputs.version }}"
+ --expected-commit "${{ needs.preflight.outputs.commit }}"
+ --attempts 24
+ --delay 10
+
publish:
- name: Publish to PyPI
+ name: Publish exact candidate to PyPI
runs-on: ubuntu-latest
- needs: [ecosystem-qualification, qualification]
+ needs: [preflight, documentation]
environment: pypi
permissions:
+ actions: read
+ contents: read
id-token: write
steps:
- - name: Download build artifacts
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
- name: dist
- path: dist/
-
- - name: Publish to PyPI
+ persist-credentials: false
+ ref: ${{ needs.preflight.outputs.commit }}
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: release-candidate-${{ needs.preflight.outputs.commit }}
+ path: candidate/
+ - name: Verify retained candidate
+ run: >-
+ python scripts/release_candidate.py verify candidate
+ --expected-commit "${{ needs.preflight.outputs.commit }}"
+ --expected-tree "${{ needs.preflight.outputs.tree }}"
+ --expected-tag "v${{ needs.preflight.outputs.version }}"
+ - name: Publish through trusted publishing
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+ with:
+ packages-dir: candidate/dist/
+ verify-metadata: true
+ print-hash: true
+ attestations: true
github-release:
- name: Create GitHub Release
+ name: Create tag and GitHub release
runs-on: ubuntu-latest
- needs: publish
+ needs: [preflight, publish]
permissions:
+ actions: read
contents: write
steps:
- - name: Create GitHub Release
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: release-candidate-${{ needs.preflight.outputs.commit }}
+ path: candidate/
+ - name: Create release from the published candidate
env:
GH_TOKEN: ${{ github.token }}
- GH_REPO: ${{ github.repository }}
+ run: >-
+ gh release create "v${{ needs.preflight.outputs.version }}"
+ candidate/dist/* candidate/candidate.json
+ --generate-notes
+ --repo "${{ github.repository }}"
+ --target "${{ needs.preflight.outputs.commit }}"
+
+ post-publication:
+ name: Verify package, release, and documentation
+ runs-on: ubuntu-latest
+ needs: [preflight, github-release]
+ permissions:
+ actions: read
+ contents: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ ref: ${{ needs.preflight.outputs.commit }}
+ - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+ with:
+ version: "0.10.9"
+ - run: uv python install ${{ env.PYTHON_VERSION }}
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: release-candidate-${{ needs.preflight.outputs.commit }}
+ path: candidate/
+ - name: Verify PyPI metadata and GitHub release digests
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: >-
+ uv run --no-project
+ python scripts/verify_published_release.py
+ --manifest candidate/candidate.json
+ --repository "${{ github.repository }}"
+ - name: Install and exercise the published wheel
+ shell: bash
run: |
- flags=(--generate-notes --verify-tag)
- if [[ "${GITHUB_REF_NAME}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(a|b|rc)[0-9]+$ ]]; then
- flags+=(--prerelease)
- fi
- gh release create "${GITHUB_REF_NAME}" "${flags[@]}"
+ uv venv --clear --python "${{ env.PYTHON_VERSION }}" "${RUNNER_TEMP}/published-venv"
+ uv pip install \
+ --python "${RUNNER_TEMP}/published-venv/bin/python" \
+ --no-cache \
+ --index-url https://pypi.org/simple \
+ "ml4t-specs==${{ needs.preflight.outputs.version }}"
+ "${RUNNER_TEMP}/published-venv/bin/python" -I scripts/run_readme_quickstart.py
+ - name: Verify deployed documentation again
+ run: >-
+ python scripts/verify_documentation_identity.py
+ --url https://www.ml4trading.io/docs/specs/
+ --url https://www.ml4trading.io/docs/specs/api/
+ --expected-library specs
+ --expected-version "${{ needs.preflight.outputs.version }}"
+ --expected-commit "${{ needs.preflight.outputs.commit }}"
+ --attempts 6
+ --delay 10
+
+ recovery:
+ name: Record a post-publication recovery issue
+ runs-on: ubuntu-latest
+ needs: [preflight, publish, github-release, post-publication]
+ if: >-
+ ${{ always() && needs.publish.result == 'success' &&
+ (needs.github-release.result != 'success' || needs.post-publication.result != 'success') }}
+ permissions:
+ issues: write
+ steps:
+ - name: Record the failed release stage
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: >-
+ gh issue create
+ --repo "${{ github.repository }}"
+ --title "Recover release v${{ needs.preflight.outputs.version }}"
+ --body "Publication of v${{ needs.preflight.outputs.version }} from ${{ needs.preflight.outputs.commit }} crossed the PyPI boundary, but GitHub release or post-publication verification failed. Inspect workflow run ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. Do not reuse this version for different artifacts."
diff --git a/README.md b/README.md
index 3824996..d7b92db 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,27 @@ Today it is used by:
pip install ml4t-specs
```
+## Quick Start
+
+Normalize provider-specific field names into a portable feed contract:
+
+```python
+from ml4t.specs import FeedSpec
+
+feed = FeedSpec.from_mapping(
+ {
+ "time_col": "date",
+ "symbol_col": "ticker",
+ "price_col": "settle",
+ "close_col": "settle",
+ "calendar": "NYSE",
+ }
+)
+assert feed.timestamp_col == "date"
+assert feed.entity_col == "ticker"
+assert feed.price_col == "settle"
+```
+
## Main Types
### FeedSpec
@@ -172,6 +193,13 @@ uv run pytest tests/ -q
uv build
```
+## Resources
+
+- [Documentation](https://www.ml4trading.io/docs/specs/)
+- [Issue tracker](https://github.com/ml4t/specs/issues)
+- [Release notes](https://github.com/ml4t/specs/releases)
+- [License](LICENSE)
+
## License
MIT
diff --git a/mkdocs.yml b/mkdocs.yml
index 8489a3e..6759d02 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -6,6 +6,7 @@ repo_name: ml4t/specs
theme:
name: material
+ custom_dir: overrides
font: false
features:
- navigation.indexes
@@ -33,6 +34,12 @@ plugins:
show_root_full_path: false
members_order: source
+extra:
+ release:
+ library: specs
+ version: !ENV [ML4T_DOCS_VERSION, dev]
+ commit: !ENV [ML4T_DOCS_COMMIT, 0000000000000000000000000000000000000000]
+
nav:
- Home: index.md
- Getting started: getting-started.md
diff --git a/overrides/main.html b/overrides/main.html
new file mode 100644
index 0000000..51d48cc
--- /dev/null
+++ b/overrides/main.html
@@ -0,0 +1,8 @@
+{% extends "base.html" %}
+
+{% block extrahead %}
+ {{ super() }}
+
+
+
+{% endblock %}
diff --git a/pyproject.toml b/pyproject.toml
index 2ea5c28..5dc0e7e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -97,6 +97,7 @@ dev = [
[tool.pytest.ini_options]
testpaths = ["tests"]
+pythonpath = ["."]
python_files = ["test_*.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
@@ -128,4 +129,4 @@ ignore = [
[tool.ty.environment]
python-version = "3.12"
-root = ["src"]
+root = ["src", "."]
diff --git a/scripts/check_readme_links.py b/scripts/check_readme_links.py
new file mode 100644
index 0000000..771e22b
--- /dev/null
+++ b/scripts/check_readme_links.py
@@ -0,0 +1,80 @@
+"""Validate every local and HTTP link in the root README."""
+
+from __future__ import annotations
+
+import argparse
+import re
+import time
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.parse import unquote, urlparse
+from urllib.request import Request, urlopen
+
+LINK = re.compile(r"(?[^)]+)\)")
+IMAGE = re.compile(r"!\[[^]]*\]\((?P[^)]+)\)")
+USER_AGENT = "ml4t-specs-readme-check/1.0"
+
+
+def targets(readme: Path) -> tuple[str, ...]:
+ """Return distinct README link targets in source order."""
+ content = readme.read_text()
+ matches = [
+ (match.start("target"), match["target"].strip())
+ for pattern in (LINK, IMAGE)
+ for match in pattern.finditer(content)
+ ]
+ return tuple(dict.fromkeys(target for _, target in sorted(matches)))
+
+
+def validate_local(root: Path, target: str) -> None:
+ """Require a relative link to resolve inside the repository."""
+ parsed = urlparse(target)
+ path = (root / unquote(parsed.path)).resolve()
+ try:
+ path.relative_to(root.resolve())
+ except ValueError as error:
+ raise ValueError(f"README link escapes the repository: {target}") from error
+ if not path.exists():
+ raise ValueError(f"README link does not exist: {target}")
+
+
+def validate_http(target: str, *, attempts: int = 3, retry_seconds: float = 1.0) -> None:
+ """Require an HTTP target to return a non-error response."""
+ last_error: Exception | None = None
+ for attempt in range(attempts):
+ request = Request(target, headers={"User-Agent": USER_AGENT})
+ try:
+ with urlopen(request, timeout=20) as response:
+ if response.status < 400:
+ return
+ except (HTTPError, URLError, TimeoutError) as error:
+ last_error = error
+ if attempt + 1 < attempts:
+ time.sleep(retry_seconds)
+ raise RuntimeError(f"README link failed: {target}: {last_error}")
+
+
+def check(readme: Path, *, check_http: bool = True) -> None:
+ """Validate every Markdown link in *readme*."""
+ root = readme.resolve().parent
+ for target in targets(readme):
+ scheme = urlparse(target).scheme
+ if scheme in {"http", "https"}:
+ if check_http:
+ validate_http(target)
+ elif not scheme:
+ validate_local(root, target)
+ else:
+ raise ValueError(f"README uses unsupported link scheme: {target}")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("readme", type=Path, nargs="?", default=Path("README.md"))
+ parser.add_argument("--local-only", action="store_true")
+ args = parser.parse_args()
+ check(args.readme, check_http=not args.local_only)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/check_release_preflight.py b/scripts/check_release_preflight.py
new file mode 100644
index 0000000..4a248cc
--- /dev/null
+++ b/scripts/check_release_preflight.py
@@ -0,0 +1,217 @@
+"""Reject an ambiguous or already-published ml4t-specs release candidate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+from packaging.version import InvalidVersion, Version
+
+try:
+ from scripts.release_candidate import (
+ EXPECTED_DESCRIPTION,
+ EXPECTED_GITHUB_HOMEPAGE,
+ EXPECTED_GITHUB_TOPICS,
+ )
+except ModuleNotFoundError: # Direct execution adds scripts/, not the repository root, to sys.path.
+ from release_candidate import (
+ EXPECTED_DESCRIPTION,
+ EXPECTED_GITHUB_HOMEPAGE,
+ EXPECTED_GITHUB_TOPICS,
+ )
+
+COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}")
+SOURCE_VERSION_PATTERN = re.compile(r'^__version__ = "([^"]+)"$', re.MULTILINE)
+
+
+@dataclass(frozen=True)
+class PublicationState:
+ """Existing public records for one proposed version."""
+
+ git_tag: bool
+ github_release: bool
+ pypi_version: bool
+
+
+@dataclass(frozen=True)
+class RepositoryMetadata:
+ """Public GitHub identity fields that must agree with the release."""
+
+ description: str
+ homepage: str
+ topics: frozenset[str]
+
+
+def repository_metadata_failures(metadata: RepositoryMetadata) -> list[str]:
+ """Return public GitHub identity mismatches."""
+ failures = []
+ if metadata.description != EXPECTED_DESCRIPTION:
+ failures.append("GitHub description differs from the canonical package description")
+ if metadata.homepage != EXPECTED_GITHUB_HOMEPAGE:
+ failures.append("GitHub homepage differs from the canonical documentation route")
+ if not metadata.topics >= EXPECTED_GITHUB_TOPICS:
+ failures.append("GitHub topics omit required ecosystem or package-specific terms")
+ return failures
+
+
+def version_failure(value: str) -> str | None:
+ """Return why a version cannot identify a new stable release."""
+ try:
+ version = Version(value)
+ except InvalidVersion:
+ return "release version is not PEP 440 compliant"
+ if (
+ str(version) != value
+ or len(version.release) != 3
+ or version.is_prerelease
+ or version.is_devrelease
+ or version.local is not None
+ ):
+ return "release version must be a normalized stable X.Y.Z or X.Y.Z.postN version"
+ return None
+
+
+def preflight_failures(
+ *,
+ version: str,
+ source_version: str,
+ candidate_sha: str,
+ checked_out_sha: str,
+ workflow_sha: str,
+ main_sha: str,
+ publication: PublicationState,
+ repository_metadata: RepositoryMetadata,
+) -> list[str]:
+ """Return every condition that makes publication unsafe."""
+ failures = []
+ if failure := version_failure(version):
+ failures.append(failure)
+ if source_version != version:
+ failures.append("source version differs from the requested release version")
+ if not COMMIT_PATTERN.fullmatch(candidate_sha):
+ failures.append("candidate commit is not a full lowercase SHA")
+ if checked_out_sha != candidate_sha:
+ failures.append("checked-out commit differs from the requested candidate")
+ if workflow_sha != candidate_sha:
+ failures.append("workflow revision differs from the requested candidate")
+ if main_sha != candidate_sha:
+ failures.append("candidate commit is not the current origin/main revision")
+ if publication.git_tag:
+ failures.append(f"Git tag v{version} already exists")
+ if publication.github_release:
+ failures.append(f"GitHub release v{version} already exists")
+ if publication.pypi_version:
+ failures.append(f"PyPI version {version} already exists")
+ failures.extend(repository_metadata_failures(repository_metadata))
+ return failures
+
+
+def _git_output(*arguments: str) -> str:
+ result = subprocess.run(
+ ["git", *arguments],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ return result.stdout.strip()
+
+
+def _resource_exists(url: str, *, token: str | None = None) -> bool:
+ headers = {
+ "Accept": "application/vnd.github+json",
+ "User-Agent": "ml4t-specs-release",
+ }
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ headers["X-GitHub-Api-Version"] = "2022-11-28"
+ request = urllib.request.Request(url, headers=headers)
+ try:
+ with urllib.request.urlopen(request, timeout=30): # noqa: S310
+ return True
+ except urllib.error.HTTPError as error:
+ if error.code == 404:
+ return False
+ raise
+
+
+def publication_state(repository: str, version: str) -> PublicationState:
+ """Read tag, release, and PyPI existence without changing public state."""
+ encoded_tag = urllib.parse.quote(f"tags/v{version}", safe="")
+ encoded_version = urllib.parse.quote(version, safe="")
+ return PublicationState(
+ git_tag=_resource_exists(
+ f"https://api.github.com/repos/{repository}/git/ref/{encoded_tag}",
+ token=os.environ.get("GITHUB_TOKEN"),
+ ),
+ github_release=_resource_exists(
+ f"https://api.github.com/repos/{repository}/releases/tags/v{encoded_version}",
+ token=os.environ.get("GITHUB_TOKEN"),
+ ),
+ pypi_version=_resource_exists(f"https://pypi.org/pypi/ml4t-specs/{encoded_version}/json"),
+ )
+
+
+def github_repository_metadata(repository: str) -> RepositoryMetadata:
+ """Read public repository identity from GitHub."""
+ request = urllib.request.Request(
+ f"https://api.github.com/repos/{repository}",
+ headers={
+ "Accept": "application/vnd.github+json",
+ "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}",
+ "User-Agent": "ml4t-specs-release",
+ "X-GitHub-Api-Version": "2022-11-28",
+ },
+ )
+ with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310
+ payload = json.load(response)
+ if not isinstance(payload, dict):
+ raise ValueError("GitHub repository response must be a JSON object")
+ return RepositoryMetadata(
+ description=str(payload.get("description") or ""),
+ homepage=str(payload.get("homepage") or ""),
+ topics=frozenset(str(topic) for topic in payload.get("topics", [])),
+ )
+
+
+def source_version() -> str:
+ """Read the release version without importing the source package."""
+ source = Path("src/ml4t/specs/__init__.py").read_text(encoding="utf-8")
+ match = SOURCE_VERSION_PATTERN.search(source)
+ if match is None:
+ raise ValueError("source package does not declare __version__")
+ return match.group(1)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--version", required=True)
+ parser.add_argument("--candidate-sha", required=True)
+ parser.add_argument("--repository", default="ml4t/specs")
+ parser.add_argument("--workflow-sha", required=True)
+ args = parser.parse_args()
+ failures = preflight_failures(
+ version=args.version,
+ source_version=source_version(),
+ candidate_sha=args.candidate_sha,
+ checked_out_sha=_git_output("rev-parse", "HEAD"),
+ workflow_sha=args.workflow_sha,
+ main_sha=_git_output("rev-parse", "refs/remotes/origin/main"),
+ publication=publication_state(args.repository, args.version),
+ repository_metadata=github_repository_metadata(args.repository),
+ )
+ print(f"release preflight: {'PASS' if not failures else 'FAIL'}")
+ for failure in failures:
+ print(f"- {failure}")
+ return int(bool(failures))
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/release_candidate.py b/scripts/release_candidate.py
new file mode 100644
index 0000000..1473198
--- /dev/null
+++ b/scripts/release_candidate.py
@@ -0,0 +1,291 @@
+"""Create and verify the immutable ml4t-specs release candidate."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import tarfile
+import zipfile
+from email import policy
+from email.message import Message
+from email.parser import BytesParser
+from pathlib import Path
+from typing import Any
+
+from packaging.specifiers import SpecifierSet
+
+SCHEMA_VERSION = 1
+COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}")
+EXPECTED_DESCRIPTION = (
+ "Serializable, runtime-neutral contracts for market data, artifacts, strategy lifecycles, "
+ "and execution across ML4T libraries."
+)
+EXPECTED_AUTHOR = "Stefan Jansen "
+EXPECTED_MAINTAINER = "Stefan Jansen "
+EXPECTED_URLS = {
+ "Homepage": "https://www.ml4trading.io/",
+ "Documentation": "https://www.ml4trading.io/docs/specs/",
+ "Repository": "https://github.com/ml4t/specs",
+ "Issues": "https://github.com/ml4t/specs/issues",
+ "Changelog": "https://github.com/ml4t/specs/releases",
+}
+EXPECTED_GITHUB_HOMEPAGE = "https://www.ml4trading.io/docs/specs/"
+EXPECTED_GITHUB_TOPICS = {
+ "ml4t",
+ "python",
+ "quantitative-finance",
+ "algorithmic-trading",
+ "data-contracts",
+ "schemas",
+}
+EXPECTED_KEYWORDS = {
+ "finance",
+ "quantitative-finance",
+ "algorithmic-trading",
+ "trading",
+ "market-data",
+ "schemas",
+ "data-contracts",
+ "serialization",
+ "backtesting",
+ "live-trading",
+}
+EXPECTED_CLASSIFIERS = {
+ "Development Status :: 5 - Production/Stable",
+ "License :: OSI Approved :: MIT License",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
+ "Programming Language :: Python :: 3.14",
+ "Typing :: Typed",
+}
+
+
+def _sha256(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _distribution_files(candidate_dir: Path) -> tuple[Path, Path]:
+ dist_dir = candidate_dir / "dist"
+ wheels = tuple(dist_dir.glob("*.whl"))
+ sdists = tuple(dist_dir.glob("*.tar.gz"))
+ if len(wheels) != 1 or len(sdists) != 1:
+ raise ValueError(
+ f"candidate must contain one wheel and one sdist, found {len(wheels)} and {len(sdists)}"
+ )
+ return wheels[0], sdists[0]
+
+
+def _wheel_metadata(wheel: Path) -> Message:
+ with zipfile.ZipFile(wheel) as archive:
+ names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")]
+ if len(names) != 1:
+ raise ValueError(f"wheel must contain one METADATA file: {wheel}")
+ return BytesParser(policy=policy.default).parsebytes(archive.read(names[0]))
+
+
+def _sdist_metadata(sdist: Path) -> Message:
+ with tarfile.open(sdist, "r:gz") as archive:
+ members = [
+ member
+ for member in archive.getmembers()
+ if member.name.count("/") == 1 and member.name.endswith("/PKG-INFO")
+ ]
+ if len(members) != 1:
+ raise ValueError(f"sdist must contain one top-level PKG-INFO file: {sdist}")
+ stream = archive.extractfile(members[0])
+ if stream is None:
+ raise ValueError(f"cannot read sdist metadata: {sdist}")
+ return BytesParser(policy=policy.default).parsebytes(stream.read())
+
+
+def metadata_failures(metadata: Message, *, expected_version: str) -> list[str]:
+ """Return canonical public metadata mismatches."""
+ failures: list[str] = []
+ expected_scalars = {
+ "Name": "ml4t-specs",
+ "Version": expected_version,
+ "Summary": EXPECTED_DESCRIPTION,
+ "Author-email": EXPECTED_AUTHOR,
+ "Maintainer-email": EXPECTED_MAINTAINER,
+ "License-Expression": "MIT",
+ }
+ for field, expected in expected_scalars.items():
+ if metadata.get(field) != expected:
+ failures.append(f"{field} is {metadata.get(field)!r}, expected {expected!r}")
+ if SpecifierSet(str(metadata.get("Requires-Python", ""))) != SpecifierSet(">=3.12"):
+ failures.append("Requires-Python differs from >=3.12")
+ project_urls = {}
+ for value in metadata.get_all("Project-URL", []):
+ label, separator, url = value.partition(", ")
+ if separator:
+ project_urls[label] = url
+ if project_urls != EXPECTED_URLS:
+ failures.append("Project-URL metadata differs from the canonical URLs")
+ keywords = {
+ value.strip() for value in str(metadata.get("Keywords", "")).split(",") if value.strip()
+ }
+ if keywords != EXPECTED_KEYWORDS:
+ failures.append("Keywords differ from the canonical package vocabulary")
+ if not set(metadata.get_all("Classifier", [])) >= EXPECTED_CLASSIFIERS:
+ failures.append("Classifiers omit required support metadata")
+ if "LICENSE" not in metadata.get_all("License-File", []):
+ failures.append("License-File metadata does not identify LICENSE")
+ return failures
+
+
+def _identity(metadata: Message, path: Path) -> tuple[str, str]:
+ name = metadata.get("Name")
+ version = metadata.get("Version")
+ if not name or not version:
+ raise ValueError(f"distribution metadata is missing Name or Version: {path}")
+ return name, version
+
+
+def _public_metadata(metadata: Message) -> dict[str, Any]:
+ project_urls = {}
+ for value in metadata.get_all("Project-URL", []):
+ label, url = value.split(",", maxsplit=1)
+ project_urls[label.strip()] = url.strip()
+ return {
+ "author_email": metadata.get("Author-email"),
+ "classifiers": sorted(metadata.get_all("Classifier", [])),
+ "description": metadata.get("Summary"),
+ "keywords": sorted(
+ keyword.strip() for keyword in (metadata.get("Keywords") or "").split(",") if keyword
+ ),
+ "license": metadata.get("License-Expression") or metadata.get("License"),
+ "maintainer_email": metadata.get("Maintainer-email"),
+ "project_urls": project_urls,
+ "requires_python": metadata.get("Requires-Python"),
+ }
+
+
+def _artifact_record(path: Path) -> dict[str, str | int]:
+ return {"filename": path.name, "sha256": _sha256(path), "size": path.stat().st_size}
+
+
+def create(candidate_dir: Path, commit_sha: str, git_tree: str) -> None:
+ """Write a manifest after validating candidate identity and metadata."""
+ if not COMMIT_PATTERN.fullmatch(commit_sha) or not COMMIT_PATTERN.fullmatch(git_tree):
+ raise ValueError("candidate commit and tree must be full lowercase Git object IDs")
+ wheel, sdist = _distribution_files(candidate_dir)
+ wheel_metadata = _wheel_metadata(wheel)
+ sdist_metadata = _sdist_metadata(sdist)
+ wheel_identity = _identity(wheel_metadata, wheel)
+ if wheel_identity != _identity(sdist_metadata, sdist):
+ raise ValueError("wheel and sdist identities differ")
+ name, version = wheel_identity
+ failures = metadata_failures(wheel_metadata, expected_version=version)
+ failures.extend(metadata_failures(sdist_metadata, expected_version=version))
+ if failures:
+ raise ValueError(f"candidate metadata failed: {failures}")
+ public_metadata = _public_metadata(wheel_metadata)
+ if public_metadata != _public_metadata(sdist_metadata):
+ raise ValueError("wheel and sdist public metadata differ")
+ manifest = {
+ "schema_version": SCHEMA_VERSION,
+ "commit_sha": commit_sha,
+ "git_tree": git_tree,
+ "name": name,
+ "version": version,
+ "metadata": public_metadata,
+ "artifacts": [_artifact_record(wheel), _artifact_record(sdist)],
+ }
+ (candidate_dir / "candidate.json").write_text(
+ json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+ encoding="utf-8",
+ )
+
+
+def _load_manifest(candidate_dir: Path) -> dict[str, Any]:
+ value = json.loads((candidate_dir / "candidate.json").read_text(encoding="utf-8"))
+ if not isinstance(value, dict):
+ raise ValueError("candidate manifest must be a JSON object")
+ return value
+
+
+def verify(
+ candidate_dir: Path,
+ *,
+ expected_commit: str | None = None,
+ expected_tree: str | None = None,
+ expected_tag: str | None = None,
+) -> None:
+ """Verify manifest identity, artifact bytes, and canonical metadata."""
+ manifest = _load_manifest(candidate_dir)
+ if manifest.get("schema_version") != SCHEMA_VERSION:
+ raise ValueError(f"unsupported candidate schema: {manifest.get('schema_version')!r}")
+ if expected_commit is not None and manifest.get("commit_sha") != expected_commit:
+ raise ValueError("candidate commit differs from the requested release commit")
+ if expected_tree is not None and manifest.get("git_tree") != expected_tree:
+ raise ValueError("candidate tree differs from the requested release tree")
+ if expected_tag is not None and expected_tag != f"v{manifest.get('version')}":
+ raise ValueError("release tag differs from the candidate version")
+
+ wheel, sdist = _distribution_files(candidate_dir)
+ actual_files = {wheel.name: wheel, sdist.name: sdist}
+ records = manifest.get("artifacts")
+ if not isinstance(records, list) or len(records) != 2:
+ raise ValueError("candidate manifest must describe exactly two artifacts")
+ recorded_files = {record.get("filename"): record for record in records}
+ if set(recorded_files) != set(actual_files):
+ raise ValueError("candidate manifest artifact inventory differs from the dist directory")
+ for filename, path in actual_files.items():
+ record = recorded_files[filename]
+ if record.get("sha256") != _sha256(path) or record.get("size") != path.stat().st_size:
+ raise ValueError(f"candidate artifact integrity check failed: {filename}")
+
+ wheel_metadata = _wheel_metadata(wheel)
+ sdist_metadata = _sdist_metadata(sdist)
+ identity = _identity(wheel_metadata, wheel)
+ if identity != _identity(sdist_metadata, sdist):
+ raise ValueError("candidate wheel and sdist identities differ")
+ if identity != (manifest.get("name"), manifest.get("version")):
+ raise ValueError("candidate metadata differs from the manifest")
+ failures = metadata_failures(wheel_metadata, expected_version=str(manifest.get("version")))
+ failures.extend(
+ metadata_failures(sdist_metadata, expected_version=str(manifest.get("version")))
+ )
+ if failures:
+ raise ValueError(f"candidate metadata failed: {failures}")
+ if _public_metadata(wheel_metadata) != manifest.get("metadata"):
+ raise ValueError("candidate public metadata differs from the manifest")
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ create_parser = subparsers.add_parser("create")
+ create_parser.add_argument("candidate_dir", type=Path)
+ create_parser.add_argument("--commit-sha", required=True)
+ create_parser.add_argument("--git-tree", required=True)
+ verify_parser = subparsers.add_parser("verify")
+ verify_parser.add_argument("candidate_dir", type=Path)
+ verify_parser.add_argument("--expected-commit")
+ verify_parser.add_argument("--expected-tree")
+ verify_parser.add_argument("--expected-tag")
+ return parser
+
+
+def main() -> None:
+ args = _parser().parse_args()
+ if args.command == "create":
+ create(args.candidate_dir, args.commit_sha, args.git_tree)
+ else:
+ verify(
+ args.candidate_dir,
+ expected_commit=args.expected_commit,
+ expected_tree=args.expected_tree,
+ expected_tag=args.expected_tag,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/run_readme_quickstart.py b/scripts/run_readme_quickstart.py
new file mode 100644
index 0000000..8ae3df5
--- /dev/null
+++ b/scripts/run_readme_quickstart.py
@@ -0,0 +1,34 @@
+"""Execute the Python quick start published in README.md."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).parents[1]
+QUICK_START = re.compile(
+ r"^## Quick Start\s*$\n(?P.*?)(?=^## |\Z)",
+ flags=re.MULTILINE | re.DOTALL,
+)
+PYTHON_BLOCK = re.compile(r"^```python\s*$\n(?P.*?)^```\s*$", re.MULTILINE | re.DOTALL)
+
+
+def extract_quick_start(readme: str) -> str:
+ """Return the single Python block in the README quick-start section."""
+ section = QUICK_START.search(readme)
+ if section is None:
+ raise ValueError("README.md has no '## Quick Start' section")
+ blocks = [match["source"] for match in PYTHON_BLOCK.finditer(section["body"])]
+ if len(blocks) != 1:
+ raise ValueError(f"README quick start must contain one Python block, found {len(blocks)}")
+ return blocks[0]
+
+
+def main() -> None:
+ """Compile and execute the documented quick start."""
+ source = extract_quick_start((ROOT / "README.md").read_text(encoding="utf-8"))
+ exec(compile(source, "README.md quick start", "exec"), {"__name__": "__main__"}) # noqa: S102
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/verify_documentation_identity.py b/scripts/verify_documentation_identity.py
new file mode 100644
index 0000000..89e5a7c
--- /dev/null
+++ b/scripts/verify_documentation_identity.py
@@ -0,0 +1,153 @@
+"""Verify rendered and deployed documentation release identity."""
+
+from __future__ import annotations
+
+import argparse
+import time
+import urllib.error
+import urllib.request
+from html.parser import HTMLParser
+from pathlib import Path
+
+REQUIRED_META = ("ml4t-library", "ml4t-version", "ml4t-commit")
+
+
+class _MetadataParser(HTMLParser):
+ def __init__(self) -> None:
+ super().__init__()
+ self.values: dict[str, str] = {}
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ if tag != "meta":
+ return
+ values = dict(attrs)
+ name = values.get("name")
+ content = values.get("content")
+ if name in REQUIRED_META and content is not None:
+ self.values[name] = content
+
+
+def identity_failures(
+ html: str,
+ *,
+ expected_library: str,
+ expected_version: str,
+ expected_commit: str,
+ source: str,
+) -> list[str]:
+ """Return mismatched or missing documentation identity fields."""
+ parser = _MetadataParser()
+ parser.feed(html)
+ expected = {
+ "ml4t-library": expected_library,
+ "ml4t-version": expected_version,
+ "ml4t-commit": expected_commit,
+ }
+ return [
+ f"{source}: {name} is {parser.values.get(name)!r}, expected {value!r}"
+ for name, value in expected.items()
+ if parser.values.get(name) != value
+ ]
+
+
+def _site_pages(site: Path) -> list[Path]:
+ return sorted(path for path in site.rglob("*.html") if path.is_file())
+
+
+def _read_url(url: str) -> str:
+ request = urllib.request.Request(url, headers={"User-Agent": "ml4t-release-verifier"})
+ with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310
+ if response.status != 200:
+ raise ValueError(f"{url}: HTTP {response.status}")
+ return response.read().decode("utf-8")
+
+
+def deployed_identity_failures(
+ urls: list[str],
+ *,
+ expected_library: str,
+ expected_version: str,
+ expected_commit: str,
+ attempts: int,
+ delay: float,
+) -> list[str]:
+ """Retry deployed pages until every URL reports the expected identity."""
+ failures: list[str] = []
+ for attempt in range(1, attempts + 1):
+ failures = []
+ try:
+ for url in urls:
+ failures.extend(
+ identity_failures(
+ _read_url(url),
+ expected_library=expected_library,
+ expected_version=expected_version,
+ expected_commit=expected_commit,
+ source=url,
+ )
+ )
+ except (OSError, UnicodeDecodeError, urllib.error.URLError, ValueError) as error:
+ failures = [str(error)]
+ if not failures:
+ break
+ if attempt < attempts:
+ time.sleep(delay)
+ return failures
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ source = parser.add_mutually_exclusive_group(required=True)
+ source.add_argument("--site", type=Path)
+ source.add_argument("--url", action="append")
+ parser.add_argument("--expected-library", required=True)
+ parser.add_argument("--expected-version", required=True)
+ parser.add_argument("--expected-commit", required=True)
+ parser.add_argument("--attempts", type=int, default=1)
+ parser.add_argument("--delay", type=float, default=0.0)
+ args = parser.parse_args()
+
+ if args.attempts < 1 or args.delay < 0:
+ parser.error("--attempts must be positive and --delay must be non-negative")
+ if len(args.expected_commit) != 40 or any(
+ character not in "0123456789abcdef" for character in args.expected_commit
+ ):
+ parser.error("--expected-commit must be a full lowercase Git SHA")
+
+ failures: list[str] = []
+ try:
+ if args.site is not None:
+ pages = _site_pages(args.site)
+ if not pages:
+ failures.append(f"{args.site}: no rendered HTML pages found")
+ for page in pages:
+ failures.extend(
+ identity_failures(
+ page.read_text(encoding="utf-8"),
+ expected_library=args.expected_library,
+ expected_version=args.expected_version,
+ expected_commit=args.expected_commit,
+ source=str(page),
+ )
+ )
+ except (OSError, UnicodeDecodeError, urllib.error.URLError, ValueError) as error:
+ failures.append(str(error))
+ if args.url is not None:
+ failures.extend(
+ deployed_identity_failures(
+ args.url,
+ expected_library=args.expected_library,
+ expected_version=args.expected_version,
+ expected_commit=args.expected_commit,
+ attempts=args.attempts,
+ delay=args.delay,
+ )
+ )
+
+ for failure in failures:
+ print(failure)
+ return int(bool(failures))
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/verify_published_release.py b/scripts/verify_published_release.py
new file mode 100644
index 0000000..e858e5d
--- /dev/null
+++ b/scripts/verify_published_release.py
@@ -0,0 +1,188 @@
+"""Verify PyPI and GitHub records against the immutable release candidate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import time
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+try:
+ from scripts.release_candidate import (
+ EXPECTED_DESCRIPTION,
+ EXPECTED_GITHUB_HOMEPAGE,
+ EXPECTED_GITHUB_TOPICS,
+ )
+except ModuleNotFoundError: # Direct execution adds scripts/, not the repository root, to sys.path.
+ from release_candidate import (
+ EXPECTED_DESCRIPTION,
+ EXPECTED_GITHUB_HOMEPAGE,
+ EXPECTED_GITHUB_TOPICS,
+ )
+
+HASH_PATTERN = re.compile(r"[0-9a-f]{64}")
+
+
+def _artifact_records(manifest: dict[str, Any]) -> dict[str, tuple[str, int]]:
+ records = manifest.get("artifacts", [])
+ if not isinstance(records, list):
+ return {}
+ return {
+ str(record.get("filename", "")): (
+ str(record.get("sha256", "")),
+ int(record.get("size", 0)),
+ )
+ for record in records
+ if isinstance(record, dict)
+ }
+
+
+def _pypi_metadata(info: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "author_email": info.get("author_email"),
+ "classifiers": sorted(info.get("classifiers", [])),
+ "description": info.get("summary"),
+ "keywords": sorted(
+ keyword.strip() for keyword in str(info.get("keywords", "")).split(",") if keyword
+ ),
+ "license": info.get("license_expression") or info.get("license"),
+ "maintainer_email": info.get("maintainer_email"),
+ "project_urls": info.get("project_urls"),
+ "requires_python": info.get("requires_python"),
+ }
+
+
+def published_release_failures(
+ manifest: dict[str, Any],
+ pypi: dict[str, Any],
+ release: dict[str, Any],
+ repository_metadata: dict[str, Any],
+) -> list[str]:
+ """Return mismatches across the candidate, PyPI, and GitHub release."""
+ failures: list[str] = []
+ version = str(manifest.get("version", ""))
+ commit = str(manifest.get("commit_sha", ""))
+ artifacts = _artifact_records(manifest)
+ metadata = manifest.get("metadata")
+ if (
+ manifest.get("schema_version") != 1
+ or manifest.get("name") != "ml4t-specs"
+ or not version
+ or len(commit) != 40
+ or not isinstance(metadata, dict)
+ or len(artifacts) != 2
+ or any(
+ not HASH_PATTERN.fullmatch(digest) or size < 1 for digest, size in artifacts.values()
+ )
+ ):
+ return ["release candidate manifest is incomplete"]
+
+ info = pypi.get("info", {})
+ if not isinstance(info, dict):
+ failures.append("PyPI response omits project metadata")
+ elif (
+ info.get("name") != "ml4t-specs"
+ or info.get("version") != version
+ or _pypi_metadata(info) != metadata
+ ):
+ failures.append("PyPI metadata differs from the qualified candidate")
+
+ pypi_files = {
+ str(record.get("filename", "")): (
+ str(record.get("digests", {}).get("sha256", "")),
+ int(record.get("size", 0)),
+ )
+ for record in pypi.get("urls", [])
+ if isinstance(record, dict)
+ }
+ if pypi_files != artifacts:
+ failures.append("PyPI artifact names, sizes, or digests differ from the candidate")
+
+ assets = [asset for asset in release.get("assets", []) if isinstance(asset, dict)]
+ release_assets = {
+ str(asset.get("name", "")): str(asset.get("digest", "")).removeprefix("sha256:")
+ for asset in assets
+ if str(asset.get("name", "")) in artifacts
+ }
+ expected_digests = {filename: digest for filename, (digest, _) in artifacts.items()}
+ if release.get("tag_name") != f"v{version}" or release.get("target_commitish") != commit:
+ failures.append("GitHub release tag or target differs from the candidate")
+ if release_assets != expected_digests:
+ failures.append("GitHub release artifact digests differ from the candidate")
+ if "candidate.json" not in {str(asset.get("name", "")) for asset in assets}:
+ failures.append("GitHub release does not attach the candidate manifest")
+ if (
+ repository_metadata.get("description") != EXPECTED_DESCRIPTION
+ or repository_metadata.get("homepage") != EXPECTED_GITHUB_HOMEPAGE
+ or not set(repository_metadata.get("topics", [])) >= EXPECTED_GITHUB_TOPICS
+ ):
+ failures.append("GitHub repository metadata differs from the qualified candidate")
+ return failures
+
+
+def _read_json(url: str, *, token: str | None = None) -> dict[str, Any]:
+ headers = {
+ "Accept": "application/vnd.github+json",
+ "User-Agent": "ml4t-specs-release",
+ }
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ headers["X-GitHub-Api-Version"] = "2022-11-28"
+ request = urllib.request.Request(url, headers=headers)
+ with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310
+ payload = json.load(response)
+ if not isinstance(payload, dict):
+ raise ValueError(f"{url} did not return a JSON object")
+ return payload
+
+
+def remote_release_failures(
+ manifest: dict[str, Any], repository: str, *, attempts: int = 12, wait_seconds: float = 10.0
+) -> list[str]:
+ """Poll public records until publication propagation completes."""
+ version = urllib.parse.quote(str(manifest.get("version", "")), safe="")
+ tag = urllib.parse.quote(f"v{manifest.get('version', '')}", safe="")
+ failures: list[str] = []
+ for attempt in range(attempts):
+ try:
+ failures = published_release_failures(
+ manifest,
+ _read_json(f"https://pypi.org/pypi/ml4t-specs/{version}/json"),
+ _read_json(
+ f"https://api.github.com/repos/{repository}/releases/tags/{tag}",
+ token=os.environ.get("GITHUB_TOKEN"),
+ ),
+ _read_json(
+ f"https://api.github.com/repos/{repository}",
+ token=os.environ.get("GITHUB_TOKEN"),
+ ),
+ )
+ except Exception as error: # noqa: BLE001
+ failures = [f"public release records are unavailable: {error}"]
+ if not failures:
+ return []
+ if attempt + 1 < attempts:
+ time.sleep(wait_seconds)
+ return failures
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--manifest", type=Path, required=True)
+ parser.add_argument("--repository", default="ml4t/specs")
+ args = parser.parse_args()
+ manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
+ failures = remote_release_failures(manifest, args.repository)
+ print(f"published release identity: {'PASS' if not failures else 'FAIL'}")
+ for failure in failures:
+ print(f"- {failure}")
+ return int(bool(failures))
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/src/ml4t/specs/__init__.py b/src/ml4t/specs/__init__.py
index b81c22f..722e55e 100644
--- a/src/ml4t/specs/__init__.py
+++ b/src/ml4t/specs/__init__.py
@@ -83,7 +83,7 @@
TimestampSemantics,
)
-__version__ = "0.1.4"
+__version__ = "0.1.5"
__all__ = [
"ArtifactKind",
diff --git a/tests/test_release_pipeline.py b/tests/test_release_pipeline.py
new file mode 100644
index 0000000..dfc971c
--- /dev/null
+++ b/tests/test_release_pipeline.py
@@ -0,0 +1,312 @@
+"""Behavioral contracts for the commit-bound release helpers."""
+
+from __future__ import annotations
+
+import io
+import json
+import re
+import tarfile
+import zipfile
+from email.message import Message
+from pathlib import Path
+from urllib.request import Request
+
+import pytest
+import yaml
+
+from scripts.check_readme_links import USER_AGENT, check, targets
+from scripts.check_release_preflight import (
+ PublicationState,
+ RepositoryMetadata,
+ preflight_failures,
+ version_failure,
+)
+from scripts.release_candidate import (
+ EXPECTED_AUTHOR,
+ EXPECTED_CLASSIFIERS,
+ EXPECTED_DESCRIPTION,
+ EXPECTED_GITHUB_HOMEPAGE,
+ EXPECTED_GITHUB_TOPICS,
+ EXPECTED_KEYWORDS,
+ EXPECTED_MAINTAINER,
+ EXPECTED_URLS,
+ create,
+ metadata_failures,
+ verify,
+)
+from scripts.run_readme_quickstart import extract_quick_start
+from scripts.verify_documentation_identity import identity_failures
+from scripts.verify_published_release import published_release_failures
+
+COMMIT = "a" * 40
+TREE = "b" * 40
+VERSION = "1.2.3"
+REPOSITORY_ROOT = Path(__file__).parents[1]
+
+
+@pytest.mark.parametrize("version", ["1.2", "v1.2.3", "1.2.3rc1", "1.2.3+local"])
+def test_release_preflight_rejects_nonstable_versions(version: str) -> None:
+ assert version_failure(version) is not None
+
+
+def test_release_preflight_requires_exact_unpublished_main_candidate() -> None:
+ assert (
+ preflight_failures(
+ version=VERSION,
+ source_version=VERSION,
+ candidate_sha=COMMIT,
+ checked_out_sha=COMMIT,
+ workflow_sha=COMMIT,
+ main_sha=COMMIT,
+ publication=PublicationState(False, False, False),
+ repository_metadata=RepositoryMetadata(
+ EXPECTED_DESCRIPTION,
+ EXPECTED_GITHUB_HOMEPAGE,
+ frozenset(EXPECTED_GITHUB_TOPICS),
+ ),
+ )
+ == []
+ )
+ failures = preflight_failures(
+ version=VERSION,
+ source_version="1.2.2",
+ candidate_sha=COMMIT,
+ checked_out_sha="c" * 40,
+ workflow_sha="d" * 40,
+ main_sha="e" * 40,
+ publication=PublicationState(True, True, True),
+ repository_metadata=RepositoryMetadata("", "", frozenset()),
+ )
+ assert len(failures) == 10
+
+
+def canonical_metadata() -> Message:
+ metadata = Message()
+ for name, value in {
+ "Name": "ml4t-specs",
+ "Version": VERSION,
+ "Summary": EXPECTED_DESCRIPTION,
+ "Author-email": EXPECTED_AUTHOR,
+ "Maintainer-email": EXPECTED_MAINTAINER,
+ "License-Expression": "MIT",
+ "Requires-Python": ">=3.12",
+ "Keywords": ",".join(sorted(EXPECTED_KEYWORDS)),
+ "License-File": "LICENSE",
+ }.items():
+ metadata[name] = value
+ for classifier in sorted(EXPECTED_CLASSIFIERS):
+ metadata["Classifier"] = classifier
+ for label, url in EXPECTED_URLS.items():
+ metadata["Project-URL"] = f"{label}, {url}"
+ return metadata
+
+
+def _candidate(tmp_path: Path) -> Path:
+ candidate = tmp_path / "candidate"
+ dist = candidate / "dist"
+ dist.mkdir(parents=True)
+ metadata = canonical_metadata().as_bytes()
+ wheel = dist / "ml4t_specs-1.2.3-py3-none-any.whl"
+ with zipfile.ZipFile(wheel, "w") as archive:
+ archive.writestr("ml4t_specs-1.2.3.dist-info/METADATA", metadata)
+ sdist = dist / "ml4t_specs-1.2.3.tar.gz"
+ with tarfile.open(sdist, "w:gz") as archive:
+ info = tarfile.TarInfo("ml4t_specs-1.2.3/PKG-INFO")
+ info.size = len(metadata)
+ archive.addfile(info, io.BytesIO(metadata))
+ create(candidate, COMMIT, TREE)
+ return candidate
+
+
+def test_candidate_manifest_binds_identity_metadata_and_exact_bytes(tmp_path: Path) -> None:
+ candidate = _candidate(tmp_path)
+
+ verify(
+ candidate,
+ expected_commit=COMMIT,
+ expected_tree=TREE,
+ expected_tag=f"v{VERSION}",
+ )
+
+ wheel = next((candidate / "dist").glob("*.whl"))
+ wheel.write_bytes(wheel.read_bytes() + b"modified")
+ with pytest.raises(ValueError, match="integrity check failed"):
+ verify(candidate)
+
+
+def test_metadata_validation_detects_identity_drift() -> None:
+ metadata = canonical_metadata()
+ assert metadata_failures(metadata, expected_version=VERSION) == []
+ metadata.replace_header("Maintainer-email", "Template Team ")
+ assert any(
+ "Maintainer-email" in failure
+ for failure in metadata_failures(metadata, expected_version=VERSION)
+ )
+
+
+def published_fixture(candidate: Path) -> tuple[dict, dict, dict, dict]:
+ manifest = json.loads((candidate / "candidate.json").read_text(encoding="utf-8"))
+ artifacts = manifest["artifacts"]
+ metadata = manifest["metadata"]
+ pypi = {
+ "info": {
+ "name": "ml4t-specs",
+ "version": VERSION,
+ "summary": metadata["description"],
+ "author_email": metadata["author_email"],
+ "maintainer_email": metadata["maintainer_email"],
+ "license_expression": metadata["license"],
+ "requires_python": metadata["requires_python"],
+ "project_urls": metadata["project_urls"],
+ "keywords": ",".join(metadata["keywords"]),
+ "classifiers": metadata["classifiers"],
+ },
+ "urls": [
+ {
+ "filename": record["filename"],
+ "digests": {"sha256": record["sha256"]},
+ "size": record["size"],
+ }
+ for record in artifacts
+ ],
+ }
+ release = {
+ "tag_name": f"v{VERSION}",
+ "target_commitish": COMMIT,
+ "assets": [
+ {"name": record["filename"], "digest": f"sha256:{record['sha256']}"}
+ for record in artifacts
+ ]
+ + [{"name": "candidate.json", "digest": "sha256:" + "d" * 64}],
+ }
+ repository = {
+ "description": EXPECTED_DESCRIPTION,
+ "homepage": EXPECTED_GITHUB_HOMEPAGE,
+ "topics": sorted(EXPECTED_GITHUB_TOPICS),
+ }
+ return manifest, pypi, release, repository
+
+
+def test_published_release_must_match_candidate(tmp_path: Path) -> None:
+ manifest, pypi, release, repository = published_fixture(_candidate(tmp_path))
+ assert published_release_failures(manifest, pypi, release, repository) == []
+ release["assets"][0]["digest"] = "sha256:" + "d" * 64
+ assert "GitHub release artifact digests differ from the candidate" in (
+ published_release_failures(manifest, pypi, release, repository)
+ )
+
+
+def test_documentation_identity_and_quickstart_are_executable_contracts() -> None:
+ html = (
+ ''
+ f''
+ f''
+ )
+ assert (
+ identity_failures(
+ html,
+ expected_library="specs",
+ expected_version=VERSION,
+ expected_commit=COMMIT,
+ source="index.html",
+ )
+ == []
+ )
+ assert extract_quick_start("## Quick Start\n\n```python\nvalue = 1\n```\n") == "value = 1\n"
+
+
+def test_readme_link_checker_validates_local_and_http_targets(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ (tmp_path / "LICENSE").write_text("license\n", encoding="utf-8")
+ readme = tmp_path / "README.md"
+ readme.write_text("[local](LICENSE) [remote](https://example.test/docs)\n", encoding="utf-8")
+ requests: list[Request] = []
+
+ class Response(io.BytesIO):
+ status = 200
+
+ def open_url(request: Request, *, timeout: int) -> Response:
+ requests.append(request)
+ assert timeout == 20
+ return Response(b"ok")
+
+ monkeypatch.setattr("scripts.check_readme_links.urlopen", open_url)
+ check(readme)
+ assert requests[0].get_header("User-agent") == USER_AGENT
+
+
+def test_readme_link_checker_includes_badge_image_and_destination(tmp_path: Path) -> None:
+ readme = tmp_path / "README.md"
+ readme.write_text(
+ "[](https://pypi.example.test/project)\n",
+ encoding="utf-8",
+ )
+ assert targets(readme) == (
+ "https://img.example.test/badge.svg",
+ "https://pypi.example.test/project",
+ )
+
+
+def test_workflows_enforce_documented_release_and_docs_contracts() -> None:
+ release = yaml.load(
+ (REPOSITORY_ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8"),
+ Loader=yaml.BaseLoader,
+ )
+ docs = yaml.load(
+ (REPOSITORY_ROOT / ".github/workflows/docs.yml").read_text(encoding="utf-8"),
+ Loader=yaml.BaseLoader,
+ )
+
+ assert set(release["on"]) == {"workflow_dispatch"}
+ assert set(release["on"]["workflow_dispatch"]["inputs"]) == {"version", "candidate-sha"}
+ assert release["concurrency"]["cancel-in-progress"] == "false"
+ assert {
+ "preflight",
+ "ecosystem-qualification",
+ "qualification",
+ "build",
+ "documentation",
+ "publish",
+ "github-release",
+ "post-publication",
+ "recovery",
+ } == set(release["jobs"])
+ assert "pull_request" in docs["on"]
+ assert docs["concurrency"]["cancel-in-progress"] == "true"
+
+ candidate_name = "release-candidate-${{ needs.preflight.outputs.commit }}"
+ docs_name = "docs-${{ needs.preflight.outputs.commit }}"
+
+ def artifact_names(job: str, action: str) -> list[str]:
+ return [
+ step["with"]["name"]
+ for step in release["jobs"][job]["steps"]
+ if action in step.get("uses", "") and "name" in step.get("with", {})
+ ]
+
+ assert candidate_name in artifact_names("build", "actions/upload-artifact")
+ assert docs_name in artifact_names("build", "actions/upload-artifact")
+ assert artifact_names("documentation", "actions/download-artifact") == [docs_name]
+ deployed_docs_verification = next(
+ step["run"]
+ for step in release["jobs"]["documentation"]["steps"]
+ if step.get("name") == "Verify deployed documentation identity"
+ )
+ assert "--attempts 24" in deployed_docs_verification
+ assert "--delay 10" in deployed_docs_verification
+ for job in ("publish", "github-release", "post-publication"):
+ assert artifact_names(job, "actions/download-artifact") == [candidate_name]
+ assert release["jobs"]["publish"]["needs"] == ["preflight", "documentation"]
+ assert release["jobs"]["github-release"]["needs"] == ["preflight", "publish"]
+ assert release["jobs"]["post-publication"]["needs"] == ["preflight", "github-release"]
+
+
+def test_external_workflow_actions_use_full_commit_pins() -> None:
+ action = re.compile(r"^\s*uses:\s*(?!\./)([^\s#]+)@([^\s#]+)", re.MULTILINE)
+ failures = []
+ for workflow in sorted((REPOSITORY_ROOT / ".github/workflows").glob("*.yml")):
+ for name, revision in action.findall(workflow.read_text(encoding="utf-8")):
+ if not re.fullmatch(r"[0-9a-f]{40}", revision):
+ failures.append(f"{workflow.name}: {name}@{revision}")
+ assert failures == []