From 0e26972faba2777b5641e693311c133a37f89cec Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 19 Sep 2026 17:08:52 -0400 Subject: [PATCH 1/4] feat(qualification): classify reusable provider evidence Expand provider identities to cover shared execution behavior and supported runtimes, while preserving verified older evidence through schema upgrades. Return explicit fail-closed classifications for release policy decisions. Closes #95. --- scripts/qualification/provider_contract.py | 155 +++++++++++-- tests/unit/test_provider_contract.py | 241 +++++++++++++++++++++ 2 files changed, 377 insertions(+), 19 deletions(-) create mode 100644 tests/unit/test_provider_contract.py diff --git a/scripts/qualification/provider_contract.py b/scripts/qualification/provider_contract.py index 7e85c87..9877dd5 100644 --- a/scripts/qualification/provider_contract.py +++ b/scripts/qualification/provider_contract.py @@ -6,6 +6,7 @@ import json import subprocess import tomllib +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -33,13 +34,42 @@ "scripts/qualification/qualify_feeds.py", ), } +_PAPER_SHARED_PATHS = ( + "src/ml4t/live/__init__.py", + "src/ml4t/live/orders.py", + "src/ml4t/live/persistence.py", + "src/ml4t/live/protocols.py", + "src/ml4t/live/safety.py", + "src/ml4t/live/state_migration.py", +) +_V3_PATHS = { + "alpaca": (*_V2_PATHS["alpaca"], *_PAPER_SHARED_PATHS), + "ib": (*_V2_PATHS["ib"], *_PAPER_SHARED_PATHS), + "okx": ( + *_V2_PATHS["okx"], + "src/ml4t/live/__init__.py", + "src/ml4t/live/persistence.py", + "src/ml4t/live/protocols.py", + ), +} _DEPENDENCIES = {"alpaca": "alpaca-py", "ib": "ib-async", "okx": "ccxt"} +CURRENT_SCHEMA_VERSION = 3 class ProviderContractError(RuntimeError): """A provider contract cannot be computed or verified.""" +@dataclass(frozen=True) +class ProviderContractClassification: + """Explain whether retained provider evidence covers a candidate.""" + + provider: str + reusable: bool + reason: str + changed_inputs: tuple[str, ...] = () + + def _git_text(checkout_root: Path, ref: str, path: str) -> str: result = subprocess.run( ["git", "show", f"{ref}:{path}"], @@ -67,19 +97,48 @@ def _dependency_version(lock_text: str, dependency: str) -> str: return str(matches.pop()) +def _supported_runtime(pyproject_text: str) -> dict[str, Any]: + project = tomllib.loads(pyproject_text).get("project") + if not isinstance(project, dict): + raise ProviderContractError("pyproject has no project table") + requires_python = project.get("requires-python") + classifiers = project.get("classifiers") + if not isinstance(requires_python, str) or not isinstance(classifiers, list): + raise ProviderContractError("pyproject has no supported Python runtime contract") + python_classifiers = sorted( + classifier + for classifier in classifiers + if isinstance(classifier, str) + and classifier.startswith("Programming Language :: Python ::") + ) + if not python_classifiers: + raise ProviderContractError("pyproject has no Python runtime classifiers") + return { + **RUNTIME_CONTRACT, + "requires_python": requires_python, + "classifiers": python_classifiers, + } + + def provider_contract( provider: str, *, checkout_root: Path, ref: str = "HEAD", - schema_version: int = 2, + schema_version: int = CURRENT_SCHEMA_VERSION, ) -> dict[str, Any]: """Return a deterministic provider contract at a Git revision.""" if provider not in PROVIDERS: raise ProviderContractError(f"unsupported provider: {provider}") - if schema_version not in {1, 2}: + if schema_version not in {1, 2, CURRENT_SCHEMA_VERSION}: raise ProviderContractError(f"unsupported provider contract schema: {schema_version}") - paths = _V1_PATHS[provider] if schema_version == 1 else _V2_PATHS[provider] + paths = ( + _V1_PATHS[provider] + if schema_version == 1 + else _V2_PATHS[provider] + if schema_version == 2 + else _V3_PATHS[provider] + ) inputs = { path: hashlib.sha256(_git_text(checkout_root, ref, path).encode()).hexdigest() for path in paths @@ -93,7 +152,11 @@ def provider_contract( "name": dependency, "version": _dependency_version(_git_text(checkout_root, ref, "uv.lock"), dependency), }, - "runtime": RUNTIME_CONTRACT, + "runtime": ( + RUNTIME_CONTRACT + if schema_version < CURRENT_SCHEMA_VERSION + else _supported_runtime(_git_text(checkout_root, ref, "pyproject.toml")) + ), } digest = hashlib.sha256( json.dumps(payload, separators=(",", ":"), sort_keys=True).encode() @@ -101,38 +164,92 @@ def provider_contract( return {**payload, "sha256": digest} -def provider_contract_matches( +def _contract_changes(evidence: dict[str, Any], candidate: dict[str, Any]) -> tuple[str, ...]: + changed = [ + path + for path in sorted(set(evidence["inputs"]) | set(candidate["inputs"])) + if evidence["inputs"].get(path) != candidate["inputs"].get(path) + ] + if evidence["dependency"] != candidate["dependency"]: + changed.append(f"dependency:{candidate['dependency']['name']}") + if evidence["runtime"] != candidate["runtime"]: + changed.append("supported-runtime") + return tuple(changed) + + +def classify_provider_contract( provider: str, *, evidence_commit: str, candidate_commit: str, checkout_root: Path, reported_contract: dict[str, Any] | None, -) -> bool: - """Return whether extended evidence still covers the candidate contract.""" +) -> ProviderContractClassification: + """Classify retained evidence against the current provider contract.""" + if provider not in PROVIDERS: + return ProviderContractClassification(provider, False, "unsupported provider") if reported_contract is None: if evidence_commit not in LEGACY_EVIDENCE_COMMITS: - return False - schema_version = 1 + return ProviderContractClassification( + provider, False, "retained evidence has no provider contract" + ) + reported_schema = 1 else: - schema_version = reported_contract.get("schema_version") - if schema_version != 2: - return False + reported_schema = reported_contract.get("schema_version") + if not isinstance(reported_schema, int) or reported_schema not in { + 2, + CURRENT_SCHEMA_VERSION, + }: + return ProviderContractClassification( + provider, False, "retained evidence uses an unsupported contract schema" + ) try: + reported_evidence = provider_contract( + provider, + checkout_root=checkout_root, + ref=evidence_commit, + schema_version=reported_schema, + ) + if reported_contract is not None and reported_contract != reported_evidence: + return ProviderContractClassification( + provider, False, "reported contract does not match its evidence commit" + ) evidence = provider_contract( provider, checkout_root=checkout_root, ref=evidence_commit, - schema_version=schema_version, ) candidate = provider_contract( provider, checkout_root=checkout_root, ref=candidate_commit, - schema_version=schema_version, ) - except ProviderContractError: - return False - if reported_contract is not None and reported_contract != evidence: - return False - return evidence["sha256"] == candidate["sha256"] + except (ProviderContractError, tomllib.TOMLDecodeError) as error: + return ProviderContractClassification(provider, False, str(error)) + changed_inputs = _contract_changes(evidence, candidate) + if changed_inputs: + return ProviderContractClassification( + provider, + False, + "provider contract changed", + changed_inputs, + ) + return ProviderContractClassification(provider, True, "provider contract unchanged") + + +def provider_contract_matches( + provider: str, + *, + evidence_commit: str, + candidate_commit: str, + checkout_root: Path, + reported_contract: dict[str, Any] | None, +) -> bool: + """Return whether extended evidence still covers the candidate contract.""" + return classify_provider_contract( + provider, + evidence_commit=evidence_commit, + candidate_commit=candidate_commit, + checkout_root=checkout_root, + reported_contract=reported_contract, + ).reusable diff --git a/tests/unit/test_provider_contract.py b/tests/unit/test_provider_contract.py new file mode 100644 index 0000000..fabe19f --- /dev/null +++ b/tests/unit/test_provider_contract.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from scripts.qualification.provider_contract import ( + CURRENT_SCHEMA_VERSION, + classify_provider_contract, + provider_contract, +) + +PROVIDER_PATHS = { + "src/ml4t/live/__init__.py", + "src/ml4t/live/brokers/alpaca.py", + "src/ml4t/live/brokers/ib.py", + "src/ml4t/live/feeds/events.py", + "src/ml4t/live/feeds/okx_feed.py", + "src/ml4t/live/feeds/queue.py", + "src/ml4t/live/orders.py", + "src/ml4t/live/persistence.py", + "src/ml4t/live/protocols.py", + "src/ml4t/live/safety.py", + "src/ml4t/live/state_migration.py", + "scripts/qualification/qualify_feeds.py", + "scripts/qualification/qualify_paper.py", +} + + +def _run(repository: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=repository, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _commit(repository: Path, message: str) -> str: + _run(repository, "add", ".") + _run(repository, "commit", "-m", message) + return _run(repository, "rev-parse", "HEAD") + + +@pytest.fixture +def contract_repository(tmp_path: Path) -> tuple[Path, str]: + _run(tmp_path, "init", "--initial-branch=main") + _run(tmp_path, "config", "user.name", "Contract Test") + _run(tmp_path, "config", "user.email", "contract@example.test") + for path in PROVIDER_PATHS: + target = tmp_path / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"contract input: {path}\n") + (tmp_path / "README.md").write_text("unrelated metadata\n") + (tmp_path / "pyproject.toml").write_text( + """[project] +name = "ml4t-live" +requires-python = ">=3.12" +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +""" + ) + (tmp_path / "uv.lock").write_text( + """[[package]] +name = "alpaca-py" +version = "0.44.0" + +[[package]] +name = "ib-async" +version = "2.1.0" + +[[package]] +name = "ccxt" +version = "4.5.78" +""" + ) + return tmp_path, _commit(tmp_path, "initial contract") + + +def _classification( + repository: Path, + provider: str, + evidence_commit: str, + candidate_commit: str, + *, + reported_schema: int = CURRENT_SCHEMA_VERSION, +): + return classify_provider_contract( + provider, + evidence_commit=evidence_commit, + candidate_commit=candidate_commit, + checkout_root=repository, + reported_contract=provider_contract( + provider, + checkout_root=repository, + ref=evidence_commit, + schema_version=reported_schema, + ), + ) + + +def test_unrelated_change_preserves_every_provider_contract( + contract_repository: tuple[Path, str], +) -> None: + repository, evidence_commit = contract_repository + (repository / "README.md").write_text("changed metadata\n") + candidate_commit = _commit(repository, "change unrelated metadata") + + for provider in ("alpaca", "ib", "okx"): + classification = _classification(repository, provider, evidence_commit, candidate_commit) + assert classification.reusable is True + assert classification.reason == "provider contract unchanged" + assert classification.changed_inputs == () + + +def test_shared_persistence_change_invalidates_every_reaching_provider( + contract_repository: tuple[Path, str], +) -> None: + repository, evidence_commit = contract_repository + path = repository / "src/ml4t/live/persistence.py" + path.write_text(path.read_text() + "changed = True\n") + candidate_commit = _commit(repository, "change persistence") + + for provider in ("alpaca", "ib", "okx"): + classification = _classification(repository, provider, evidence_commit, candidate_commit) + assert classification.reusable is False + assert classification.reason == "provider contract changed" + assert classification.changed_inputs == ("src/ml4t/live/persistence.py",) + + +def test_provider_change_does_not_invalidate_unrelated_provider( + contract_repository: tuple[Path, str], +) -> None: + repository, evidence_commit = contract_repository + path = repository / "src/ml4t/live/feeds/okx_feed.py" + path.write_text(path.read_text() + "changed = True\n") + candidate_commit = _commit(repository, "change OKX") + + assert not _classification(repository, "okx", evidence_commit, candidate_commit).reusable + assert _classification(repository, "ib", evidence_commit, candidate_commit).reusable + assert _classification(repository, "alpaca", evidence_commit, candidate_commit).reusable + + +def test_supported_runtime_change_invalidates_every_provider( + contract_repository: tuple[Path, str], +) -> None: + repository, evidence_commit = contract_repository + pyproject = repository / "pyproject.toml" + pyproject.write_text(pyproject.read_text().replace(">=3.12", ">=3.13")) + candidate_commit = _commit(repository, "change supported runtime") + + for provider in ("alpaca", "ib", "okx"): + classification = _classification(repository, provider, evidence_commit, candidate_commit) + assert classification.reusable is False + assert classification.changed_inputs == ("supported-runtime",) + + +def test_older_report_schema_is_upgraded_before_comparison( + contract_repository: tuple[Path, str], +) -> None: + repository, evidence_commit = contract_repository + path = repository / "src/ml4t/live/persistence.py" + path.write_text(path.read_text() + "changed = True\n") + candidate_commit = _commit(repository, "change newly covered input") + + classification = _classification( + repository, + "ib", + evidence_commit, + candidate_commit, + reported_schema=2, + ) + + assert classification.reusable is False + assert classification.changed_inputs == ("src/ml4t/live/persistence.py",) + + +@pytest.mark.parametrize( + ("reported_contract", "reason"), + [ + (None, "retained evidence has no provider contract"), + ({"schema_version": 99}, "retained evidence uses an unsupported contract schema"), + ({"schema_version": []}, "retained evidence uses an unsupported contract schema"), + ], +) +def test_unverifiable_contract_fails_closed( + contract_repository: tuple[Path, str], + reported_contract: dict | None, + reason: str, +) -> None: + repository, commit = contract_repository + + classification = classify_provider_contract( + "ib", + evidence_commit=commit, + candidate_commit=commit, + checkout_root=repository, + reported_contract=reported_contract, + ) + + assert classification.reusable is False + assert classification.reason == reason + + +def test_tampered_reported_contract_fails_closed( + contract_repository: tuple[Path, str], +) -> None: + repository, commit = contract_repository + contract = provider_contract("ib", checkout_root=repository, ref=commit) + + classification = classify_provider_contract( + "ib", + evidence_commit=commit, + candidate_commit=commit, + checkout_root=repository, + reported_contract={**contract, "sha256": "0" * 64}, + ) + + assert classification.reusable is False + assert classification.reason == "reported contract does not match its evidence commit" + + +def test_pending_candidate_preserves_v012_ib_contract() -> None: + repository = Path(__file__).resolve().parents[2] + tag = subprocess.run( + ["git", "cat-file", "-e", "v0.1.2^{commit}"], + cwd=repository, + check=False, + capture_output=True, + ) + if tag.returncode != 0: + pytest.skip("v0.1.2 history is required") + classification = _classification(repository, "ib", "v0.1.2", "HEAD") + + assert classification.reusable is True From 6ae41ac55fbd5702963fdc01fdc08bce85b8daf9 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 19 Sep 2026 17:13:01 -0400 Subject: [PATCH 2/4] refactor(qualification): decouple artifacts from provider evidence Derive release hashes from the current stable artifacts while selecting complete matching lifecycle and soak evidence independently for each provider. Fail without writing release outputs when any provider remains unqualified. Closes #96. --- scripts/qualification/check_paper_evidence.py | 109 +++++++++------- tests/unit/test_paper_evidence.py | 122 ++++++++++++++++-- 2 files changed, 172 insertions(+), 59 deletions(-) diff --git a/scripts/qualification/check_paper_evidence.py b/scripts/qualification/check_paper_evidence.py index d947a2b..31cf1af 100644 --- a/scripts/qualification/check_paper_evidence.py +++ b/scripts/qualification/check_paper_evidence.py @@ -35,6 +35,8 @@ _candidate_identity = _paper._candidate_identity _contracts = importlib.import_module("scripts.qualification.provider_contract") provider_contract_matches = _contracts.provider_contract_matches +_release = importlib.import_module("scripts.qualification.verify_release_identity") +artifact_identity = _release.artifact_identity GITHUB_API = "https://api.github.com" @@ -118,37 +120,44 @@ def _optional_json(archive: zipfile.ZipFile, suffix: str) -> dict[str, Any] | No def validate_evidence_archive(payload: bytes, expected_commit: str) -> dict[str, Any]: - """Validate exact-candidate exercises and any retained extended reports.""" + """Validate every complete provider result present in an evidence archive.""" with zipfile.ZipFile(io.BytesIO(payload)) as archive: manifest = _unique_json(archive, "candidate.json") verify_candidate_manifest(manifest) identity = _candidate_identity(manifest) if identity["commit"] != expected_commit: raise PaperQualificationError("paper artifact targets a different commit") + providers: dict[str, dict[str, Any]] = {} for provider in ("alpaca", "ib"): - for phase in ("exercise", "restart"): - validate_provider_report( - _unique_json(archive, f"{provider}-{phase}.json"), - identity, - provider, - phase, - ) - validate_okx_report(_unique_json(archive, "okx.json"), manifest) - soaks: dict[str, dict[str, Any]] = {} - for provider, suffix in { - "alpaca": "alpaca-soak.json", - "ib": "ib-soak.json", - "okx": "okx-soak.json", - }.items(): - report = _optional_json(archive, suffix) - if report is None: + reports = { + phase: _optional_json(archive, f"{provider}-{phase}.json") + for phase in ("exercise", "restart") + } + soak = _optional_json(archive, f"{provider}-soak.json") + present = [*reports.values(), soak] + if not any(present): continue - if provider == "okx": - validate_okx_soak_report(report, manifest) - else: - validate_provider_soak_report(report, identity, provider) - soaks[provider] = report - return {"candidate": identity, "soaks": soaks} + if not all(present): + raise PaperQualificationError(f"paper artifact has incomplete {provider} evidence") + for phase, report in reports.items(): + if report is None: + raise PaperQualificationError( + f"paper artifact has incomplete {provider} evidence" + ) + validate_provider_report(report, identity, provider, phase) + if soak is None: + raise PaperQualificationError(f"paper artifact has incomplete {provider} evidence") + validate_provider_soak_report(soak, identity, provider) + providers[provider] = {**reports, "soak": soak} + okx = _optional_json(archive, "okx.json") + okx_soak = _optional_json(archive, "okx-soak.json") + if okx is not None or okx_soak is not None: + if okx is None or okx_soak is None: + raise FeedQualificationError("paper artifact has incomplete okx evidence") + validate_okx_report(okx, manifest) + validate_okx_soak_report(okx_soak, manifest) + providers["okx"] = {"exercise": okx, "soak": okx_soak} + return {"candidate": identity, "providers": providers} def _archive_source( @@ -192,6 +201,7 @@ def find_paper_evidence( commit: str, token: str, checkout_root: Path, + candidate_artifacts: dict[str, dict[str, str]], fetcher: Callable[[str, str], dict[str, Any]] = fetch_json, downloader: Callable[[str, str], bytes] = fetch_bytes, contract_matcher: Callable[..., bool] = provider_contract_matches, @@ -205,7 +215,6 @@ def find_paper_evidence( ) runs_url = f"{GITHUB_API}/repos/{repository}/actions/workflows/paper.yml/runs?{query}" runs = fetcher(runs_url, token).get("workflow_runs", []) - candidate_evidence: dict[str, Any] | None = None provider_evidence: dict[str, dict[str, Any]] = {} for run in sorted(runs, key=lambda item: item.get("created_at", ""), reverse=True): evidence_commit = run.get("head_sha") @@ -238,20 +247,10 @@ def find_paper_evidence( zipfile.BadZipFile, ): continue - identity = archive["candidate"] - if evidence_commit == commit and candidate_evidence is None: - candidate_evidence = { - "run_id": run["id"], - "run_url": run.get("html_url"), - "created_at": run["created_at"], - "artifact": evidence_name, - "qualification_run_id": identity["qualification_run_id"], - "wheel_sha256": identity["wheel_sha256"], - "sdist_sha256": identity["sdist_sha256"], - } - for provider, report in archive["soaks"].items(): + for provider, reports in archive["providers"].items(): if provider in provider_evidence: continue + report = reports["soak"] if contract_matcher( provider, evidence_commit=evidence_commit, @@ -266,26 +265,43 @@ def find_paper_evidence( "completed_at": report["completed_at"], "artifact": evidence_name, } - if candidate_evidence is None or set(provider_evidence) != {"alpaca", "ib", "okx"}: + if set(provider_evidence) != {"alpaca", "ib", "okx"}: return None - return {**candidate_evidence, "providers": provider_evidence} + return { + "commit": commit, + "wheel_sha256": candidate_artifacts["wheel"]["sha256"], + "sdist_sha256": candidate_artifacts["sdist"]["sha256"], + "providers": provider_evidence, + } + + +def write_github_output(path: Path, evidence: dict[str, Any] | None) -> None: + """Write release artifact hashes only after every provider has valid evidence.""" + if evidence is None: + return + with path.open("a") as output: + output.write(f"wheel_sha256={evidence['wheel_sha256']}\n") + output.write(f"sdist_sha256={evidence['sdist_sha256']}\n") def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--repository", required=True) parser.add_argument("--commit", required=True) + parser.add_argument("--artifacts-dir", type=Path, required=True) parser.add_argument("--output") parser.add_argument("--github-output", type=Path) args = parser.parse_args() token = os.environ.get("GITHUB_TOKEN", "") if not token: raise RuntimeError("GITHUB_TOKEN is required to read workflow evidence") + _, candidate_artifacts = artifact_identity(args.artifacts_dir) evidence = find_paper_evidence( repository=args.repository, commit=args.commit, token=token, checkout_root=REPOSITORY_ROOT, + candidate_artifacts=candidate_artifacts, ) report = { "schema_version": 1, @@ -298,17 +314,16 @@ def main() -> int: output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") - if args.github_output and evidence: - with args.github_output.open("a") as output: - output.write(f"wheel_sha256={evidence['wheel_sha256']}\n") - output.write(f"sdist_sha256={evidence['sdist_sha256']}\n") + if args.github_output: + write_github_output(args.github_output, evidence) print(f"paper evidence: {'PASS' if evidence else 'FAIL'} for commit {args.commit}") if evidence: - print( - f"run_id={evidence['run_id']} created_at={evidence['created_at']} " - f"wheel_sha256={evidence['wheel_sha256']} " - f"sdist_sha256={evidence['sdist_sha256']}" - ) + print(f"wheel_sha256={evidence['wheel_sha256']} sdist_sha256={evidence['sdist_sha256']}") + for provider, source in evidence["providers"].items(): + print( + f"provider={provider} run_id={source['run_id']} " + f"commit={source['commit']} completed_at={source['completed_at']}" + ) return int(evidence is None) diff --git a/tests/unit/test_paper_evidence.py b/tests/unit/test_paper_evidence.py index fa6be1a..e8b7d59 100644 --- a/tests/unit/test_paper_evidence.py +++ b/tests/unit/test_paper_evidence.py @@ -13,6 +13,7 @@ _ArtifactRedirectHandler, fetch_bytes, find_paper_evidence, + write_github_output, ) from scripts.qualification.provider_contract import ( provider_contract, @@ -28,6 +29,10 @@ COMMIT = "a" * 40 LEGACY_COMMIT = "98c414e9d858427c31e1680faccdc8dca498bf6b" +CANDIDATE_ARTIFACTS = { + "wheel": {"filename": "candidate.whl", "sha256": "d" * 64}, + "sdist": {"filename": "candidate.tar.gz", "sha256": "e" * 64}, +} def _fetcher( @@ -244,7 +249,10 @@ def _okx_soak_report(commit: str) -> dict: return report -def _archive(commit: str = COMMIT) -> bytes: +def _archive( + commit: str = COMMIT, + providers: frozenset[str] = frozenset({"alpaca", "ib", "okx"}), +) -> bytes: bundle = { "schema_version": 1, "generated_at": "2026-08-08T02:01:00+00:00", @@ -267,6 +275,8 @@ def _archive(commit: str = COMMIT) -> bytes: with zipfile.ZipFile(payload, "w") as archive: archive.writestr("paper-evidence/candidate.json", json.dumps(_manifest(commit))) for provider in ("alpaca", "ib"): + if provider not in providers: + continue for phase in ("exercise", "restart"): archive.writestr( f"paper-evidence/{provider}-{phase}.json", @@ -276,8 +286,9 @@ def _archive(commit: str = COMMIT) -> bytes: f"paper-evidence/{provider}-soak.json", json.dumps(_soak_report(provider, commit)), ) - archive.writestr("feed-evidence/okx.json", json.dumps(_okx_report(commit))) - archive.writestr("feed-evidence/okx-soak.json", json.dumps(_okx_soak_report(commit))) + if "okx" in providers: + archive.writestr("feed-evidence/okx.json", json.dumps(_okx_report(commit))) + archive.writestr("feed-evidence/okx-soak.json", json.dumps(_okx_soak_report(commit))) archive.writestr("paper-qualification.json", json.dumps(bundle)) archive.writestr( "feed-qualification.json", @@ -329,6 +340,49 @@ def download(url: str, token: str) -> bytes: return download +def _multi_run_services( + evidence: list[tuple[int, str, str, frozenset[str]]], +): + runs = [ + { + "id": run_id, + "head_sha": commit, + "conclusion": "success", + "status": "completed", + "created_at": created_at, + "artifacts_url": f"https://api.github.test/artifacts/{run_id}", + "html_url": f"https://github.test/run/{run_id}", + } + for run_id, commit, created_at, _providers in evidence + ] + payloads = { + f"https://api.github.test/archive/{run_id}": _archive(commit, providers) + for run_id, commit, _created_at, providers in evidence + } + + def fetch(url: str, token: str) -> dict: + assert token == "token" + if "/actions/workflows/" in url: + return {"workflow_runs": runs} + run_id = int(url.rsplit("/", 1)[-1]) + run = next(run for run in runs if run["id"] == run_id) + return { + "artifacts": [ + { + "name": f"paper-{run['head_sha']}-{run_id}", + "expired": False, + "archive_download_url": f"https://api.github.test/archive/{run_id}", + } + ] + } + + def download(url: str, token: str) -> bytes: + assert token == "token" + return payloads[url] + + return fetch, download + + @pytest.mark.parametrize( ("url", "expected_accept"), [ @@ -436,12 +490,13 @@ def test_reported_provider_contract_must_match_its_source_revision() -> None: ) -def test_exact_candidate_and_matching_provider_evidence_passes() -> None: +def test_candidate_hashes_come_from_current_artifacts_not_provider_archive() -> None: evidence = find_paper_evidence( repository="ml4t/live", commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-08-07T20:00:00Z"), downloader=_downloader(), contract_matcher=lambda *_args, **_kwargs: True, @@ -449,13 +504,9 @@ def test_exact_candidate_and_matching_provider_evidence_passes() -> None: assert evidence is not None assert {key: value for key, value in evidence.items() if key != "providers"} == { - "run_id": 42, - "run_url": "https://github.test/run/42", - "created_at": "2026-08-07T20:00:00Z", - "artifact": f"paper-{COMMIT}-42", - "qualification_run_id": 41, - "wheel_sha256": "b" * 64, - "sdist_sha256": "c" * 64, + "commit": COMMIT, + "wheel_sha256": "d" * 64, + "sdist_sha256": "e" * 64, } assert set(evidence["providers"]) == {"alpaca", "ib", "okx"} @@ -466,6 +517,7 @@ def test_age_does_not_invalidate_matching_provider_evidence() -> None: commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-07-01T20:00:00Z"), downloader=_downloader(), contract_matcher=lambda *_args, **_kwargs: True, @@ -480,6 +532,7 @@ def test_changed_provider_contract_fails_closed() -> None: commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-08-07T20:00:00Z"), downloader=_downloader(), contract_matcher=lambda provider, **_kwargs: provider != "ib", @@ -494,6 +547,7 @@ def test_expired_paper_artifact_fails() -> None: commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-08-07T20:00:00Z", expired=True), downloader=_downloader(), contract_matcher=lambda *_args, **_kwargs: True, @@ -508,13 +562,46 @@ def test_durable_release_asset_replaces_an_expired_actions_artifact() -> None: commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-08-07T20:00:00Z", expired=True, release=True), downloader=_downloader(expected_url="https://api.github.test/release-asset/42"), contract_matcher=lambda *_args, **_kwargs: True, ) assert evidence is not None - assert evidence["artifact"] == "provider-evidence-42/provider-evidence.zip" + assert {source["artifact"] for source in evidence["providers"].values()} == { + "provider-evidence-42/provider-evidence.zip" + } + + +def test_providers_can_be_selected_from_independent_matching_runs() -> None: + alpaca_commit = "1" * 40 + ib_commit = "2" * 40 + okx_commit = "3" * 40 + fetcher, downloader = _multi_run_services( + [ + (41, alpaca_commit, "2026-08-09T20:00:00Z", frozenset({"alpaca"})), + (42, ib_commit, "2026-08-08T20:00:00Z", frozenset({"ib"})), + (43, okx_commit, "2026-08-07T20:00:00Z", frozenset({"okx"})), + ] + ) + expected = {"alpaca": alpaca_commit, "ib": ib_commit, "okx": okx_commit} + + evidence = find_paper_evidence( + repository="ml4t/live", + commit=COMMIT, + token="token", + checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, + fetcher=fetcher, + downloader=downloader, + contract_matcher=lambda provider, **kwargs: kwargs["evidence_commit"] == expected[provider], + ) + + assert evidence is not None + assert { + provider: source["commit"] for provider, source in evidence["providers"].items() + } == expected def test_wrong_commit_inside_retained_artifact_fails() -> None: @@ -523,6 +610,7 @@ def test_wrong_commit_inside_retained_artifact_fails() -> None: commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-08-07T20:00:00Z"), downloader=_downloader(_archive("c" * 40)), contract_matcher=lambda *_args, **_kwargs: True, @@ -537,6 +625,7 @@ def test_malformed_retained_artifact_fails() -> None: commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-08-07T20:00:00Z"), downloader=_downloader(b"not a zip"), contract_matcher=lambda *_args, **_kwargs: True, @@ -555,9 +644,18 @@ def test_missing_feed_qualification_fails() -> None: commit=COMMIT, token="token", checkout_root=Path("."), + candidate_artifacts=CANDIDATE_ARTIFACTS, fetcher=_fetcher("2026-08-07T20:00:00Z"), downloader=_downloader(payload.getvalue()), contract_matcher=lambda *_args, **_kwargs: True, ) assert evidence is None + + +def test_failed_lookup_writes_no_release_outputs(tmp_path: Path) -> None: + output = tmp_path / "github-output" + + write_github_output(output, None) + + assert not output.exists() From 1f8aff34f7ff91f0a6d0fe6969e748ba4dbbdcbd Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 19 Sep 2026 17:18:42 -0400 Subject: [PATCH 3/4] feat(workflows): qualify only affected providers Scope manual lifecycle and soak runs to the selected provider, bind release evidence to current artifacts, and add a read-only monthly provider health workflow. Keep all publication paths fail closed through executable workflow-policy tests. Closes #97. --- .github/workflows/paper.yml | 57 ++++++++-- .github/workflows/provider-health.yml | 44 ++++++++ .github/workflows/release-recovery.yml | 7 ++ .github/workflows/release.yml | 5 + scripts/qualification/check_paper_outcomes.py | 22 ++-- scripts/qualification/check_workflows.py | 106 ++++++++++++++++-- tests/unit/test_workflow_policy.py | 62 +++++++--- 7 files changed, 256 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/provider-health.yml diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml index 7bc96d7..c97b3e9 100644 --- a/.github/workflows/paper.yml +++ b/.github/workflows/paper.yml @@ -12,11 +12,11 @@ on: required: true type: string extended-provider: - description: Provider needing a fresh six-hour soak, or all for initial qualification + description: Provider needing fresh evidence, or all for initial qualification required: true - default: none + default: all type: choice - options: [none, alpaca, ib, okx, all] + options: [alpaca, ib, okx, all] permissions: {} @@ -75,6 +75,7 @@ jobs: mkdir -p "${{ runner.temp }}/paper-run" "${{ runner.temp }}/paper-state" - name: Exercise Alpaca paper order lifecycle id: alpaca-exercise + if: ${{ inputs.extended-provider == 'alpaca' || inputs.extended-provider == 'all' }} continue-on-error: true env: ALPACA_API_KEY: ${{ secrets.ALPACA_API_KEY }} @@ -92,7 +93,11 @@ jobs: --output "${{ github.workspace }}/paper-evidence/alpaca-exercise.json" - name: Reconcile Alpaca in a fresh process id: alpaca-restart - if: ${{ always() }} + if: >- + ${{ + always() && + (inputs.extended-provider == 'alpaca' || inputs.extended-provider == 'all') + }} continue-on-error: true env: ALPACA_API_KEY: ${{ secrets.ALPACA_API_KEY }} @@ -110,7 +115,11 @@ jobs: --output "${{ github.workspace }}/paper-evidence/alpaca-restart.json" - name: Exercise IB paper order lifecycle id: ib-exercise - if: ${{ always() }} + if: >- + ${{ + always() && + (inputs.extended-provider == 'ib' || inputs.extended-provider == 'all') + }} continue-on-error: true env: IB_ACCOUNT: ${{ secrets.IB_ACCOUNT }} @@ -130,7 +139,11 @@ jobs: --output "${{ github.workspace }}/paper-evidence/ib-exercise.json" - name: Reconcile IB in a fresh process id: ib-restart - if: ${{ always() }} + if: >- + ${{ + always() && + (inputs.extended-provider == 'ib' || inputs.extended-provider == 'all') + }} continue-on-error: true env: IB_ACCOUNT: ${{ secrets.IB_ACCOUNT }} @@ -150,6 +163,7 @@ jobs: --output "${{ github.workspace }}/paper-evidence/ib-restart.json" - name: Qualify the supported OKX feed against the public service id: okx-external + if: ${{ inputs.extended-provider == 'okx' || inputs.extended-provider == 'all' }} continue-on-error: true run: >- cd "${{ runner.temp }}/paper-run" && @@ -162,12 +176,31 @@ jobs: id: provider-soaks if: >- ${{ - inputs.extended-provider != 'none' && - steps.alpaca-exercise.outcome == 'success' && - steps.alpaca-restart.outcome == 'success' && - steps.ib-exercise.outcome == 'success' && - steps.ib-restart.outcome == 'success' && - steps.okx-external.outcome == 'success' + always() && + ( + ( + inputs.extended-provider == 'alpaca' && + steps.alpaca-exercise.outcome == 'success' && + steps.alpaca-restart.outcome == 'success' + ) || + ( + inputs.extended-provider == 'ib' && + steps.ib-exercise.outcome == 'success' && + steps.ib-restart.outcome == 'success' + ) || + ( + inputs.extended-provider == 'okx' && + steps.okx-external.outcome == 'success' + ) || + ( + inputs.extended-provider == 'all' && + steps.alpaca-exercise.outcome == 'success' && + steps.alpaca-restart.outcome == 'success' && + steps.ib-exercise.outcome == 'success' && + steps.ib-restart.outcome == 'success' && + steps.okx-external.outcome == 'success' + ) + ) }} continue-on-error: true env: diff --git a/.github/workflows/provider-health.yml b/.github/workflows/provider-health.yml new file mode 100644 index 0000000..d170317 --- /dev/null +++ b/.github/workflows/provider-health.yml @@ -0,0 +1,44 @@ +name: Monthly provider health + +on: + schedule: + - cron: "0 14 15 * *" + workflow_dispatch: + +permissions: {} + +env: + UV_NO_SOURCES: "1" + UV_VERSION: "0.10.9" + +jobs: + provider-health: + name: ${{ matrix.provider }} health + runs-on: [self-hosted, linux, x64, ib-paper] + timeout-minutes: 20 + environment: paper + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - provider: alpaca + test_path: tests/integration/test_alpaca_integration.py + - provider: ib + test_path: tests/integration/test_ib_integration.py + - provider: okx + test_path: tests/integration/test_okx_integration.py + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + with: + version: ${{ env.UV_VERSION }} + - run: uv python install 3.12 + - name: Install the default-branch health environment + run: uv sync --python 3.12 --all-extras --dev --locked + - name: Check current provider availability and behavior + env: + ALPACA_API_KEY: ${{ secrets.ALPACA_API_KEY }} + ALPACA_SECRET_KEY: ${{ secrets.ALPACA_SECRET_KEY }} + run: uv run pytest --run-external "${{ matrix.test_path }}" -q diff --git a/.github/workflows/release-recovery.yml b/.github/workflows/release-recovery.yml index c313fea..c1683a6 100644 --- a/.github/workflows/release-recovery.yml +++ b/.github/workflows/release-recovery.yml @@ -37,6 +37,12 @@ jobs: with: version: "0.10.9" - run: uv python install 3.12 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.qualification-run-id }} + name: dist-${{ inputs.candidate-sha }} + path: dist/ - name: Create a clean evidence runtime id: paper-runtime run: | @@ -51,6 +57,7 @@ jobs: scripts/qualification/check_paper_evidence.py --repository "${{ github.repository }}" --commit "${{ inputs.candidate-sha }}" + --artifacts-dir dist --github-output "$GITHUB_OUTPUT" verify: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7b4bc92..bf66744 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,6 +83,10 @@ jobs: with: version: "0.10.9" - run: uv python install 3.12 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist-${{ inputs.candidate-sha }} + path: dist/ - name: Create a clean paper evidence runtime id: paper-runtime run: | @@ -97,6 +101,7 @@ jobs: scripts/qualification/check_paper_evidence.py --repository "${{ github.repository }}" --commit "${{ inputs.candidate-sha }}" + --artifacts-dir dist --github-output "$GITHUB_OUTPUT" documentation: diff --git a/scripts/qualification/check_paper_outcomes.py b/scripts/qualification/check_paper_outcomes.py index b6e74c9..fee1590 100644 --- a/scripts/qualification/check_paper_outcomes.py +++ b/scripts/qualification/check_paper_outcomes.py @@ -5,7 +5,7 @@ import argparse from collections.abc import Sequence -SHORT_OUTCOMES = ( +ALL_OUTCOMES = ( "alpaca-exercise", "alpaca-restart", "ib-exercise", @@ -14,13 +14,21 @@ "paper-evidence-scan", "feed-evidence-scan", ) +PROVIDER_OUTCOMES = { + "alpaca": ("alpaca-exercise", "alpaca-restart"), + "ib": ("ib-exercise", "ib-restart"), + "okx": ("okx-external",), +} def outcome_failures(outcomes: dict[str, str], extended_provider: str) -> list[str]: """Return required qualification stages whose outcomes are not successful.""" - required = list(SHORT_OUTCOMES) - if extended_provider != "none": - required.append("provider-soaks") + selected = ( + tuple(name for names in PROVIDER_OUTCOMES.values() for name in names) + if extended_provider == "all" + else PROVIDER_OUTCOMES[extended_provider] + ) + required = [*selected, "paper-evidence-scan", "feed-evidence-scan", "provider-soaks"] if extended_provider == "all": required.extend(("paper-evidence", "feed-evidence")) elif extended_provider == "okx": @@ -32,15 +40,15 @@ def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( "--extended-provider", - choices=("none", "alpaca", "ib", "okx", "all"), + choices=("alpaca", "ib", "okx", "all"), required=True, ) - for name in (*SHORT_OUTCOMES, "provider-soaks", "paper-evidence", "feed-evidence"): + for name in (*ALL_OUTCOMES, "provider-soaks", "paper-evidence", "feed-evidence"): parser.add_argument(f"--{name}", required=True) args = parser.parse_args(argv) outcomes = { name: getattr(args, name.replace("-", "_")) - for name in (*SHORT_OUTCOMES, "provider-soaks", "paper-evidence", "feed-evidence") + for name in (*ALL_OUTCOMES, "provider-soaks", "paper-evidence", "feed-evidence") } failures = outcome_failures(outcomes, args.extended_provider) if failures: diff --git a/scripts/qualification/check_workflows.py b/scripts/qualification/check_workflows.py index accda04..3a1c036 100644 --- a/scripts/qualification/check_workflows.py +++ b/scripts/qualification/check_workflows.py @@ -23,13 +23,11 @@ "documentation", } POST_BUILD_JOBS = {"artifact-qualification", "security"} -PAPER_QUICK_STEP_IDS = ( - "alpaca-exercise", - "alpaca-restart", - "ib-exercise", - "ib-restart", - "okx-external", -) +PAPER_PROVIDER_STEPS = { + "alpaca": ("alpaca-exercise", "alpaca-restart"), + "ib": ("ib-exercise", "ib-restart"), + "okx": ("okx-external",), +} ACTION_PATTERN = re.compile( r"^\s*-?\s*uses:\s*([^\s#]+)(?:\s+#\s+(v[0-9][A-Za-z0-9_.-]*))?\s*$", re.MULTILINE, @@ -250,6 +248,14 @@ def release_recovery_failures(recovery: dict[str, Any]) -> list[str]: failures.append("release recovery does not require candidate, source run, and tag") jobs = recovery.get("jobs", {}) + paper = jobs.get("paper-evidence", {}) + paper_downloads = _action_steps(paper, "actions/download-artifact") + if ( + len(paper_downloads) != 1 + or paper_downloads[0].get("with", {}).get("name") != "dist-${{ inputs.candidate-sha }}" + or paper_downloads[0].get("with", {}).get("run-id") != "${{ inputs.qualification-run-id }}" + ): + failures.append("release recovery evidence does not bind the exact candidate artifact") verify = jobs.get("verify", {}) if _needs(verify) != {"paper-evidence"}: failures.append("release recovery verification does not require fresh paper evidence") @@ -281,16 +287,44 @@ def release_recovery_failures(recovery: dict[str, Any]) -> list[str]: def paper_soak_failures(paper_job: dict[str, Any]) -> list[str]: - """Reject a long soak that can start after a short provider check fails.""" + """Reject a provider soak that can start without its selected short checks.""" soak_steps = [step for step in _steps(paper_job) if step.get("id") == "provider-soaks"] if len(soak_steps) != 1: return ["paper qualification must define one provider-soaks step"] condition = str(soak_steps[0].get("if", "")) failures = [] - for step_id in PAPER_QUICK_STEP_IDS: - required = f"steps.{step_id}.outcome == 'success'" - if required not in condition: - failures.append(f"paper soak does not require successful {step_id}") + for provider, step_ids in PAPER_PROVIDER_STEPS.items(): + if f"inputs.extended-provider == '{provider}'" not in condition: + failures.append(f"paper soak has no {provider} selection branch") + for step_id in step_ids: + required = f"steps.{step_id}.outcome == 'success'" + if required not in condition: + failures.append(f"paper soak does not require successful {step_id}") + if "inputs.extended-provider == 'all'" not in condition: + failures.append("paper soak has no all-provider selection branch") + return failures + + +def paper_selection_failures(paper_job: dict[str, Any]) -> list[str]: + """Reject short checks that run for an unrelated provider selection.""" + indexed = {step.get("id"): step for step in _steps(paper_job) if step.get("id")} + failures = [] + for provider, step_ids in PAPER_PROVIDER_STEPS.items(): + expected = { + f"inputs.extended-provider == '{provider}'", + "inputs.extended-provider == 'all'", + } + unrelated = { + f"inputs.extended-provider == '{other}'" + for other in PAPER_PROVIDER_STEPS + if other != provider + } + for step_id in step_ids: + condition = str(indexed.get(step_id, {}).get("if", "")) + if any(item not in condition for item in expected): + failures.append(f"{step_id} is not limited to {provider} or all") + if any(item in condition for item in unrelated): + failures.append(f"{step_id} can run for an unrelated provider") return failures @@ -351,6 +385,50 @@ def release_paper_runtime_failures(paper_job: dict[str, Any]) -> list[str]: f"{paper_python} scripts/qualification/check_paper_evidence.py" ): failures.append("release paper evidence does not use the clean validation environment") + if "--artifacts-dir dist" not in str(paper.get("run", "")): + failures.append("release paper evidence does not bind the exact candidate artifacts") + downloads = _action_steps(paper_job, "actions/download-artifact") + if ( + len(downloads) != 1 + or downloads[0].get("with", {}).get("name") != "dist-${{ inputs.candidate-sha }}" + or downloads[0].get("with", {}).get("path") != "dist/" + ): + failures.append("release paper evidence does not download the exact candidate artifacts") + return failures + + +def provider_health_failures(health: dict[str, Any]) -> list[str]: + """Reject a monthly health workflow that is incomplete or can publish.""" + failures = [] + if _triggers(health) != {"schedule", "workflow_dispatch"}: + failures.append("provider health is not monthly and manually dispatchable") + schedules = health.get("on", {}).get("schedule", []) + if schedules != [{"cron": "0 14 15 * *"}]: + failures.append("provider health does not have one monthly schedule") + job = health.get("jobs", {}).get("provider-health", {}) + if job.get("environment") != "paper": + failures.append("provider health does not use the protected paper environment") + matrix = job.get("strategy", {}).get("matrix", {}).get("include", []) + expected = { + ("alpaca", "tests/integration/test_alpaca_integration.py"), + ("ib", "tests/integration/test_ib_integration.py"), + ("okx", "tests/integration/test_okx_integration.py"), + } + actual = { + (entry.get("provider"), entry.get("test_path")) + for entry in matrix + if isinstance(entry, dict) + } + if actual != expected or job.get("strategy", {}).get("fail-fast") != "false": + failures.append("provider health does not independently cover every stable provider") + text = json.dumps(health, sort_keys=True) + if "--run-external" not in text: + failures.append("provider health does not execute external checks") + if any( + forbidden in text + for forbidden in ("gh-action-pypi-publish", "gh release create", "id-token") + ): + failures.append("provider health can mutate a public release") return failures @@ -363,6 +441,7 @@ def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]: "release.yml", "release-recovery.yml", "paper.yml", + "provider-health.yml", "docs.yml", } failures = [] @@ -376,6 +455,7 @@ def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]: release = workflows["release.yml"] recovery = workflows["release-recovery.yml"] paper = workflows["paper.yml"] + health = workflows["provider-health.yml"] for name, workflow in workflows.items(): failures.extend(_permission_failures(name, workflow.get("permissions"), set())) if "pull_request_target" in _triggers(workflow): @@ -523,7 +603,9 @@ def validate_workflows(root: Path = WORKFLOW_ROOT) -> list[str]: failures.append("paper qualification is not manual-only") paper_job = paper.get("jobs", {}).get("paper", {}) failures.extend(paper_soak_failures(paper_job)) + failures.extend(paper_selection_failures(paper_job)) failures.extend(paper_runtime_failures(paper_job)) + failures.extend(provider_health_failures(health)) if paper_job.get("environment") != "paper": failures.append("paper qualification does not use the protected paper environment") if "secrets." not in json.dumps(paper_job, sort_keys=True): diff --git a/tests/unit/test_workflow_policy.py b/tests/unit/test_workflow_policy.py index 6522032..b899455 100644 --- a/tests/unit/test_workflow_policy.py +++ b/tests/unit/test_workflow_policy.py @@ -10,8 +10,10 @@ action_pin_failures, load_workflow, paper_runtime_failures, + paper_selection_failures, paper_soak_failures, promotion_failures, + provider_health_failures, release_paper_runtime_failures, release_recovery_failures, validate_workflows, @@ -26,7 +28,7 @@ def test_every_external_action_is_immutable_and_updateable() -> None: assert action_pin_failures(WORKFLOW_ROOT.glob("*.yml")) == [] -def test_paper_soak_requires_every_short_provider_check() -> None: +def test_paper_soak_requires_each_selected_provider_check() -> None: paper = load_workflow(WORKFLOW_ROOT / "paper.yml") paper_job = paper["jobs"]["paper"] @@ -39,6 +41,19 @@ def test_paper_soak_requires_every_short_provider_check() -> None: assert any("ib-exercise" in failure for failure in paper_soak_failures(seeded_job)) +def test_short_checks_run_only_for_the_selected_provider_or_all() -> None: + paper = load_workflow(WORKFLOW_ROOT / "paper.yml") + paper_job = paper["jobs"]["paper"] + + assert paper_selection_failures(paper_job) == [] + + seeded_job = deepcopy(paper_job) + ib_exercise = next(step for step in seeded_job["steps"] if step.get("id") == "ib-exercise") + del ib_exercise["if"] + + assert paper_selection_failures(seeded_job) == ["ib-exercise is not limited to ib or all"] + + def test_paper_gate_fails_for_each_required_provider_outcome() -> None: outcomes = { "alpaca-exercise": "success", @@ -50,23 +65,15 @@ def test_paper_gate_fails_for_each_required_provider_outcome() -> None: "okx-external": "success", "paper-evidence": "skipped", "paper-evidence-scan": "success", - "provider-soaks": "skipped", + "provider-soaks": "success", } - assert outcome_failures(outcomes, "none") == [] - for variable in ( - "alpaca-exercise", - "alpaca-restart", - "ib-exercise", - "ib-restart", - "okx-external", - "paper-evidence-scan", - "feed-evidence-scan", - ): + assert outcome_failures(outcomes, "alpaca") == [] + for variable in ("alpaca-exercise", "alpaca-restart", "provider-soaks"): seeded = {**outcomes, variable: "failure"} - assert outcome_failures(seeded, "none") == [variable] + assert outcome_failures(seeded, "alpaca") == [variable] + assert outcome_failures({**outcomes, "ib-exercise": "failure"}, "alpaca") == [] - assert outcome_failures(outcomes, "ib") == ["provider-soaks"] all_outcomes = { **outcomes, "feed-evidence": "success", @@ -78,6 +85,10 @@ def test_paper_gate_fails_for_each_required_provider_outcome() -> None: seeded = {**all_outcomes, variable: "failure"} assert outcome_failures(seeded, "all") == [variable] + okx_outcomes = {**outcomes, "feed-evidence": "success"} + assert outcome_failures(okx_outcomes, "okx") == [] + assert outcome_failures({**okx_outcomes, "okx-external": "failure"}, "okx") == ["okx-external"] + def test_paper_qualification_uses_a_clean_explicit_runtime() -> None: paper = load_workflow(WORKFLOW_ROOT / "paper.yml") @@ -96,7 +107,7 @@ def test_paper_qualification_uses_a_clean_explicit_runtime() -> None: ) -@pytest.mark.parametrize("mutation", ["dependency", "interpreter"]) +@pytest.mark.parametrize("mutation", ["dependency", "interpreter", "artifact"]) def test_release_paper_evidence_uses_a_clean_explicit_runtime(mutation: str) -> None: release = load_workflow(WORKFLOW_ROOT / "release.yml") paper_job = release["jobs"]["paper-evidence"] @@ -108,16 +119,35 @@ def test_release_paper_evidence_uses_a_clean_explicit_runtime(mutation: str) -> runtime = next(step for step in seeded_job["steps"] if step.get("id") == "paper-runtime") runtime["run"] = str(runtime["run"]).replace('"psutil==7.2.2"', "") expected = "pinned runtime dependency" - else: + elif mutation == "interpreter": paper = next(step for step in seeded_job["steps"] if step.get("id") == "paper") paper["run"] = str(paper["run"]).replace( '"${RUNNER_TEMP}/paper-evidence-venv/bin/python"', "python" ) expected = "clean validation environment" + else: + download = next( + step + for step in seeded_job["steps"] + if str(step.get("uses", "")).startswith("actions/download-artifact@") + ) + download["with"]["name"] = "dist-wrong" + expected = "exact candidate artifacts" assert any(expected in failure for failure in release_paper_runtime_failures(seeded_job)) +def test_monthly_provider_health_is_visible_but_cannot_publish() -> None: + health = load_workflow(WORKFLOW_ROOT / "provider-health.yml") + + assert provider_health_failures(health) == [] + + seeded = deepcopy(health) + seeded["jobs"]["provider-health"]["permissions"]["id-token"] = "write" + + assert provider_health_failures(seeded) == ["provider health can mutate a public release"] + + @pytest.mark.parametrize( ("mutation", "expected"), [ From 3ee27138121dd174ed392784a6d24560294abf6c Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 19 Sep 2026 17:22:37 -0400 Subject: [PATCH 4/4] docs(qualification): define risk-triggered provider evidence Document exact artifact binding, contract-based provider evidence reuse, monthly health checks, and the no-bypass rule for changed providers during maintenance. Keep public claim and workflow checks aligned with the release policy. Closes #98. --- .github/workflows/release.yml | 2 +- docs/claim-evidence.md | 4 +- docs/qualification.md | 72 +++++++++++-------- scripts/qualification/check_paper_evidence.py | 2 +- scripts/qualification/check_public_claims.py | 4 ++ scripts/qualification/check_workflows.py | 4 +- tests/unit/test_workflow_policy.py | 2 +- 7 files changed, 53 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bf66744..47fc8f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: candidate-version: ${{ inputs.version }} paper-evidence: - name: Require exact exercises and reusable provider evidence + name: Bind exact artifacts to reusable provider evidence runs-on: ubuntu-latest needs: [ecosystem-qualification, qualification] outputs: diff --git a/docs/claim-evidence.md b/docs/claim-evidence.md index b090eff..b4a238e 100644 --- a/docs/claim-evidence.md +++ b/docs/claim-evidence.md @@ -9,10 +9,10 @@ external paper account. | Portability | Lifecycle version 1 callbacks and canonical intents are portable across the qualified backtest and live engines | `tests/contracts/test_causal_strategy_parity.py`; minimum, locked, and maximum dependency profiles | | Safety | Canonical orders fail atomically; live safety may reject or reduce an intent without redefining it | `tests/unit/test_order_contract.py`, `tests/unit/test_safe_broker.py`, `tests/unit/test_secure_persistence.py` | | Feeds | OKX and bar aggregation emit validated UTC events and halt on overload or continuity loss | installed-wheel external OKX qualification; feed contract, queue, continuity, recovery, and stress tests | -| Brokers | IB and Alpaca implement the documented snapshots, state transitions, canonical order input, capabilities, reconciliation, and six-hour paper-session contract | broker contract and adapter suites; `scripts/qualification/qualify_paper.py`; fresh installed-wheel lifecycle and soak evidence | +| Brokers | IB and Alpaca implement the documented snapshots, state transitions, canonical order input, capabilities, reconciliation, and six-hour paper-session contract | broker contract and adapter suites; exact-candidate artifact qualification; complete lifecycle and soak evidence with a matching provider contract | | Performance | The credential-free Linux reference workload sustains 100 events per second for one virtual hour with less than 25 MiB post-warmup RSS growth, no-op dispatch p99 below 10 ms, shutdown below 5 seconds, bounded queues, and exact event and intent checksums | `scripts/qualification/qualify_performance.py`; `tests/stress`; candidate performance report | | Platform | The wheel and source distribution install with Python 3.12, 3.13, and 3.14 on Linux, macOS, and Windows; Python 3.15 prerelease installation and the non-hardware-dependent suite pass on all three operating systems before final support | stable installed-artifact profiles; ecosystem qualification matrix | -| Maturity | IB and Alpaca brokers plus the OKX feed are stable-supported; Alpaca, IB, DataBento, and generic CCXT feeds require experimental opt-in | public claim scan, experimental-feed tests, retained six-hour OKX, IB, and Alpaca evidence tied to the candidate wheel hash | +| Maturity | IB and Alpaca brokers plus the OKX feed are stable-supported; Alpaca, IB, DataBento, and generic CCXT feeds require experimental opt-in | public claim scan, experimental-feed tests, exact candidate artifacts, and retained six-hour OKX, IB, and Alpaca evidence with matching provider contracts | The complete credential-free command is documented in [Candidate Qualification](qualification.md). Paper evidence is separate because untrusted pull requests never receive account credentials. diff --git a/docs/qualification.md b/docs/qualification.md index 0b405d9..35091c5 100644 --- a/docs/qualification.md +++ b/docs/qualification.md @@ -37,10 +37,11 @@ on Linux; they do not state venue latency or capacity. ## Qualify Paper Accounts And External Feeds Separately Credentialed IB and Alpaca checks run only through the manually dispatched `Paper Qualification` -workflow in the protected `paper` environment. Supply the exact candidate commit and the successful -qualification run that retained `dist-CANDIDATE_SHA`. The workflow downloads that artifact instead -of rebuilding it, installs its wheel outside the checkout, and rejects a run whose commit or status -does not match. +workflow in the protected `paper` environment. Run it when a provider contract changes. Supply the +exact candidate commit, the successful qualification run that retained `dist-CANDIDATE_SHA`, and +the changed provider. Use `all` only for initial qualification or when shared changes invalidate all +three providers. The workflow downloads the candidate artifact instead of rebuilding it, installs +its wheel outside the checkout, and rejects a run whose commit or status does not match. Each provider must verify its official paper endpoint and paper account before submission. The workflow then checks positions, all pending orders and an asset-filtered view, cash, account value, @@ -49,9 +50,9 @@ acknowledgement, reconnect reconciliation, replacement, cancellation, cleanup, a from a fresh process. IB accepts only the standard paper ports and an account identified by IB as a paper account. Alpaca accepts only the SDK's official paper endpoint in sandbox mode. -The order-lifecycle checks run for every candidate. Extended Alpaca, IB, and OKX soaks run -independently. Set `extended-provider` to the provider whose contract changed, to `all` for initial -qualification, or to `none` when all three current extended results remain applicable. +The selected provider runs its short lifecycle or external-feed check and its six-hour soak. Other +providers do not need to be online. A release with unchanged provider contracts does not dispatch +this workflow; its release gate selects retained matching provider evidence automatically. Each selected broker remains connected to its paper account for at least six continuous hours. Five-minute snapshots must preserve exact adapter-to-provider positions and pending orders, valid @@ -61,34 +62,45 @@ disconnect within five seconds. Any unexpected disconnect, reconciliation change identity, or retained error fails the run. Every extended report records a provider-contract identity. The identity hashes the provider -adapter, qualification logic reached by that soak, the resolved provider dependency, and the Linux -CPython 3.12 runtime contract. OKX also hashes the shared event and queue contracts reached by its -feed. A changed identity invalidates only that provider's result. Documentation, CI metadata, and -source code outside the extended provider path do not invalidate it. +adapter, the qualification logic reached by the check, the resolved provider dependency, the +supported Python runtime, and the shared behavior that the provider reaches. IB and Alpaca include +canonical orders, safety policy, reconciliation, persistence, protocols, and state migration. OKX +includes event, queue, persistence, and protocol behavior. A changed identity invalidates only that +provider's result. Documentation, metadata, workflows, generic build tools, and source code outside +the provider's reachable contract do not invalidate it. Missing inputs or an unsupported identity +schema fail closed. The workflow stores redacted evidence as a 90-day Actions artifact and as a durable prerelease -asset named `provider-evidence-RUN_ID/provider-evidence.zip`. The release gate validates the exact -candidate's short exercises, then selects a matching extended result for each provider from current -or earlier runs. Evidence has no time-based expiry. Missing, malformed, or mismatched evidence fails -closed. The Aug 10 run at `98c414e9d858427c31e1680faccdc8dca498bf6b` is the only legacy result -accepted without an embedded identity; the verifier recomputes its narrower reviewed identity. - -The same workflow qualifies `OKXFundingFeed` against the public OKX service for every candidate. It compares adapter -events with provider-native observations, observes consecutive complete candles across restart, -rejects stale input, forces fail-closed overload, and requires shutdown within five seconds. The -selected OKX extended run then continues for at least six hours with five-minute event, queue, and -RSS snapshots and one retained-state restart. Every complete minute must remain -contiguous, final state must match a native OKX observation, RSS growth must remain below 25 MiB, -and errors, rejections, or overflows fail the run. The retained feed bundle also proves that Alpaca, -IB, DataBento, and generic CCXT feed construction requires `experimental=True` and records each -adapter's missing guarantees. The release gate requires the paper and feed bundles to identify the -same commit and wheel. +asset named `provider-evidence-RUN_ID/provider-evidence.zip`. The release gate takes wheel and +source-distribution hashes from the exact candidate artifacts. It then selects complete lifecycle, +restart, and soak evidence independently for each matching provider contract. Provider evidence may +come from different current or earlier runs. Evidence has no time-based expiry. Missing, malformed, +mismatched, or unclassifiable evidence fails closed before publication. The Aug 10 run at +`98c414e9d858427c31e1680faccdc8dca498bf6b` is the only legacy result accepted without an embedded +identity; the verifier checks its original identity and compares the current contract inputs across +both revisions. + +When the OKX contract changes, the same workflow compares `OKXFundingFeed` events with +provider-native observations, observes consecutive complete candles across restart, rejects stale +input, forces fail-closed overload, and requires shutdown within five seconds. The selected OKX run +then continues for at least six hours with five-minute event, queue, and RSS snapshots and one +retained-state restart. Every complete minute must remain contiguous, final state must match a +native OKX observation, RSS growth must remain below 25 MiB, and errors, rejections, or overflows +fail the run. The retained feed bundle also proves that Alpaca, IB, DataBento, and generic CCXT feed +construction requires `experimental=True` and records each adapter's missing guarantees. + +The `Monthly provider health` workflow runs the external IB, Alpaca, and OKX integration suites on +the fifteenth day of each month. It detects provider or SDK drift independently of a release. A +failed monthly run is visible for investigation, but it does not revoke matching retained evidence +or block a candidate whose relevant provider contract is unchanged. The health workflow cannot +publish packages or create GitHub releases. IB Gateway or TWS authentication is an operator prerequisite. Once the paper session is authenticated, the self-hosted runner can execute the short check or selected soak without further -operator input. IB does not support fully unattended login, so a changed IB contract may still wait -for authentication. Do not expose broker credentials to pull-request code or point IB qualification -at a live port. +operator input. IB does not support fully unattended login. A changed IB contract waits during IB +maintenance or until authentication is available; there is no manual release bypass. An unchanged +IB contract reuses matching retained evidence. Do not expose broker credentials to pull-request +code or point IB qualification at a live port. ## Retain Candidate Identity diff --git a/scripts/qualification/check_paper_evidence.py b/scripts/qualification/check_paper_evidence.py index 31cf1af..367c77f 100644 --- a/scripts/qualification/check_paper_evidence.py +++ b/scripts/qualification/check_paper_evidence.py @@ -1,4 +1,4 @@ -"""Require exact-candidate exercises and matching retained provider evidence.""" +"""Bind exact candidate artifacts to matching retained provider evidence.""" from __future__ import annotations diff --git a/scripts/qualification/check_public_claims.py b/scripts/qualification/check_public_claims.py index 46e13a2..2165dd2 100644 --- a/scripts/qualification/check_public_claims.py +++ b/scripts/qualification/check_public_claims.py @@ -41,6 +41,7 @@ "identical outputs in backtest and live modes", "same code for backtest and live", "this strategy is **identical** to what we use in backtesting", + "order-lifecycle checks run for every candidate", ) OBSOLETE_ADOPTION_ARGUMENTS = ("shadow_mode=True", "shadow_mode=False") @@ -94,6 +95,9 @@ ), "docs/qualification.md": ( "exact candidate commit", + "matching provider contract", + "Monthly provider health", + "there is no manual release bypass", "does not create a tag", "does not publish", "does not place a live-money order", diff --git a/scripts/qualification/check_workflows.py b/scripts/qualification/check_workflows.py index 3a1c036..4ca28cf 100644 --- a/scripts/qualification/check_workflows.py +++ b/scripts/qualification/check_workflows.py @@ -180,7 +180,7 @@ def promotion_failures(qualification: dict[str, Any], release: dict[str, Any]) - failures.append("a post-qualification release job rebuilds the candidate") paper_evidence = release_jobs.get("paper-evidence", {}) if "check_paper_evidence.py" not in _run_text(paper_evidence): - failures.append("release does not verify fresh paper evidence for the exact commit") + failures.append("release does not verify matching provider evidence") if paper_evidence.get("outputs", {}).get("wheel_sha256") != ( "${{ steps.paper.outputs.wheel_sha256 }}" ): @@ -258,7 +258,7 @@ def release_recovery_failures(recovery: dict[str, Any]) -> list[str]: failures.append("release recovery evidence does not bind the exact candidate artifact") verify = jobs.get("verify", {}) if _needs(verify) != {"paper-evidence"}: - failures.append("release recovery verification does not require fresh paper evidence") + failures.append("release recovery verification does not require provider evidence") downloads = _action_steps(verify, "actions/download-artifact") expected_downloads = { "dist-${{ inputs.candidate-sha }}", diff --git a/tests/unit/test_workflow_policy.py b/tests/unit/test_workflow_policy.py index b899455..48d649a 100644 --- a/tests/unit/test_workflow_policy.py +++ b/tests/unit/test_workflow_policy.py @@ -200,7 +200,7 @@ def test_seeded_mandatory_failure_cannot_reach_publish(mutation: str, expected: @pytest.mark.parametrize( ("mutation", "expected"), [ - ("missing-paper", "fresh paper evidence"), + ("missing-paper", "provider evidence"), ("wrong-source-run", "exact source-run artifact"), ("public-write", "mutate public"), ("missing-policy", "check_release_recovery.py"),