diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 3399d5d..30536e8 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -25,7 +25,8 @@ jobs: - name: Monitor issues and pull requests env: GITHUB_TOKEN: ${{ github.token }} - run: uv run ml4t-ecosystem monitor --output status/monitor.json + # Exit 1 means advisory findings; exit 2 means incomplete evidence. + run: uv run ml4t-ecosystem monitor --output status/monitor.json || test "$?" -eq 1 - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: diff --git a/AGENTS.md b/AGENTS.md index 4123a9e..e8e605a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,7 +103,11 @@ User-visible changes and defect fixes do. ## Issue and Pull-Request Monitoring -The ecosystem monitor covers all open issues and pull requests in the seven repositories. +The shared queue monitor covers all eleven repositories in `config/maintained-repositories.toml`. +It uses full owner/name identities, records actual observation times and PR revisions, and marks +incomplete API evidence explicitly. Stable qualification remains limited to the seven current +libraries in `config/libraries.toml`. Individual maintenance agents own review and executed +verification under their repository's native checks. Recommendations require maintainer approval. - Classify each new item within one hour using the shared type, priority, status, affected-version, and compatibility-impact vocabulary. diff --git a/config/maintained-repositories.toml b/config/maintained-repositories.toml new file mode 100644 index 0000000..971d358 --- /dev/null +++ b/config/maintained-repositories.toml @@ -0,0 +1,35 @@ +schema_version = 1 + +[[repositories]] +key = "data" +repository = "ml4t/data" +[[repositories]] +key = "engineer" +repository = "ml4t/engineer" +[[repositories]] +key = "backtest" +repository = "ml4t/backtest" +[[repositories]] +key = "specs" +repository = "ml4t/specs" +[[repositories]] +key = "live" +repository = "ml4t/live" +[[repositories]] +key = "diagnostic" +repository = "ml4t/diagnostic" +[[repositories]] +key = "models" +repository = "ml4t/models" +[[repositories]] +key = "zipline-reloaded" +repository = "stefan-jansen/zipline-reloaded" +[[repositories]] +key = "pyfolio-reloaded" +repository = "stefan-jansen/pyfolio-reloaded" +[[repositories]] +key = "alphalens-reloaded" +repository = "stefan-jansen/alphalens-reloaded" +[[repositories]] +key = "empyrical-reloaded" +repository = "stefan-jansen/empyrical-reloaded" diff --git a/docs/standards/issues-and-pull-requests.md b/docs/standards/issues-and-pull-requests.md index ececf3e..f1bfe03 100644 --- a/docs/standards/issues-and-pull-requests.md +++ b/docs/standards/issues-and-pull-requests.md @@ -30,6 +30,20 @@ a substantive response or applies `status: pending-review` and states the next r Library-specific labels may extend the vocabulary but may not redefine shared labels. +`uv run ml4t-ecosystem monitor --output status/monitor.json` collects the full open queues in +`config/maintained-repositories.toml`, using each repository's explicit GitHub owner and name. +This maintenance inventory includes the seven current libraries and four Reloaded libraries; +`config/libraries.toml` continues to define only the seven-library qualification scope. +The existing audit workflow runs nominally hourly. Each report records when evidence was actually +collected, complete paginated queues, and current PR head/base revisions. Collection is metadata +inspection, not a code review or executed verification. + +The command exits zero when collection is complete with no response-target findings, one when +findings need attention, and two when API evidence is incomplete. An incomplete repository must not +be interpreted as an empty queue or used to infer that an item closed. The report retains evidence +errors alongside successfully collected repositories. Downstream advisory tools must key items by +full repository identity and number, and re-evaluate a changed PR revision before reusing a review. + ## Implementation Every non-automated user-visible change and defect fix has an associated issue. Its pull request uses diff --git a/src/ml4t_ecosystem/cli.py b/src/ml4t_ecosystem/cli.py index 05eee10..105068b 100644 --- a/src/ml4t_ecosystem/cli.py +++ b/src/ml4t_ecosystem/cli.py @@ -14,7 +14,7 @@ from ml4t_ecosystem.clients import DocumentationClient, GitHubClient, PyPIClient from ml4t_ecosystem.config import load_config from ml4t_ecosystem.labels import load_labels, sync_labels -from ml4t_ecosystem.monitor import monitor_all +from ml4t_ecosystem.maintenance import collect_queues, load_maintained_repositories from ml4t_ecosystem.status import write_current_status from ml4t_ecosystem.templates import sync_templates from ml4t_ecosystem.workspaces import audit_workspaces @@ -43,6 +43,12 @@ def _parser() -> argparse.ArgumentParser: monitor = subparsers.add_parser("monitor", help="check issue and pull-request response targets") monitor.add_argument("--output", type=Path) + monitor.add_argument( + "--inventory", + type=Path, + default=ROOT / "config/maintained-repositories.toml", + help="maintained owner/name inventory, separate from stable qualification", + ) snapshot = subparsers.add_parser("snapshot", help="retain validated current status") snapshot.add_argument("--status-dir", type=Path, default=ROOT / "status") @@ -81,12 +87,12 @@ def _monitor(args: argparse.Namespace) -> int: raise SystemExit("GITHUB_TOKEN is required for accurate maintainer-response monitoring") github = GitHubClient(token=token) try: - findings = monitor_all(config, github) + report = collect_queues(config, load_maintained_repositories(args.inventory), github) finally: github.close() payload = ( json.dumps( - {"schema_version": 1, "findings": [finding.to_dict() for finding in findings]}, + report, indent=2, sort_keys=True, ) @@ -97,7 +103,9 @@ def _monitor(args: argparse.Namespace) -> int: args.output.write_text(payload, encoding="utf-8") else: print(payload, end="") - return 1 if findings else 0 + if not report["complete"]: + return 2 + return 1 if report["findings"] else 0 def _audit_workspaces(args: argparse.Namespace) -> int: diff --git a/src/ml4t_ecosystem/clients.py b/src/ml4t_ecosystem/clients.py index 2fd6495..f858c23 100644 --- a/src/ml4t_ecosystem/clients.py +++ b/src/ml4t_ecosystem/clients.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import re from typing import Any, Protocol import httpx @@ -50,6 +51,12 @@ def issue_comments(self, owner: str, repository: str, number: int) -> list[dict[ def pull_reviews(self, owner: str, repository: str, number: int) -> list[dict[str, Any]]: ... +class QueueGitHub(MonitorGitHub, Protocol): + """Response monitoring plus the revisions required for owner review tasks.""" + + def pull_request(self, owner: str, repository: str, number: int) -> dict[str, Any]: ... + + class GitHubClient: """Minimal read client for GitHub repository evidence.""" @@ -150,33 +157,72 @@ def private_vulnerability_reporting(self, owner: str, repository: str) -> bool | def open_issues(self, owner: str, repository: str) -> list[dict[str, Any]]: """Return open issues and pull requests from the issues endpoint.""" - result = self.get_json( + return self._pages( f"/repos/{owner}/{repository}/issues", params={"state": "open", "per_page": "100", "sort": "created"}, ) - if not isinstance(result, list): - raise EvidenceError("GitHub issues response was not a list") - return [item for item in result if isinstance(item, dict)] def issue_comments(self, owner: str, repository: str, number: int) -> list[dict[str, Any]]: """Return comments for an issue or pull request.""" - result = self.get_json( + return self._pages( f"/repos/{owner}/{repository}/issues/{number}/comments", params={"per_page": "100"}, ) - if not isinstance(result, list): - raise EvidenceError("GitHub comments response was not a list") - return [item for item in result if isinstance(item, dict)] def pull_reviews(self, owner: str, repository: str, number: int) -> list[dict[str, Any]]: """Return reviews for a pull request.""" - result = self.get_json( + return self._pages( f"/repos/{owner}/{repository}/pulls/{number}/reviews", params={"per_page": "100"}, ) - if not isinstance(result, list): - raise EvidenceError("GitHub pull reviews response was not a list") - return [item for item in result if isinstance(item, dict)] + + def _pages(self, path: str, *, params: dict[str, str]) -> list[dict[str, Any]]: + url = str(self._client.base_url.join(path).copy_merge_params(params)) + visited: set[str] = set() + items: list[dict[str, Any]] = [] + resource = "/" + "/".join(path.split("/")[4:]) + canonical_path: str | None = None + while url: + target = httpx.URL(url) + valid_path = target.path == path or target.path == canonical_path + if ( + not valid_path + and canonical_path is None + and re.fullmatch(r"/repositories/[0-9]+" + re.escape(resource), target.path) + ): + canonical_path = target.path + valid_path = True + if ( + (target.scheme, target.host, target.port) != ("https", "api.github.com", None) + or not valid_path + or url in visited + ): + raise EvidenceError(f"GitHub {path} returned an invalid pagination link") + visited.add(url) + try: + response = self._client.get(url) + response.raise_for_status() + page = response.json() + except (httpx.HTTPError, ValueError) as error: + raise EvidenceError(f"GitHub {path} page unavailable: {error}") from error + if not isinstance(page, list) or any(not isinstance(item, dict) for item in page): + raise EvidenceError( + f"GitHub {path.rsplit('/', 1)[-1]} response was not a list of objects" + ) + items.extend(page) + url = response.links.get("next", {}).get("url", "") + return items + + def pull_request(self, owner: str, repository: str, number: int) -> dict[str, Any]: + """Return current pull-request revisions and material status.""" + path = f"/repos/{owner}/{repository}/pulls/{number}" + try: + result = self.get_json(path) + except (httpx.HTTPError, ValueError) as error: + raise EvidenceError(f"GitHub {path} unavailable: {error}") from error + if not isinstance(result, dict): + raise EvidenceError(f"GitHub {path} response was not an object") + return result class PyPIClient: diff --git a/src/ml4t_ecosystem/maintenance.py b/src/ml4t_ecosystem/maintenance.py new file mode 100644 index 0000000..ba44bee --- /dev/null +++ b/src/ml4t_ecosystem/maintenance.py @@ -0,0 +1,170 @@ +"""Collect complete, revision-aware queues independently of package qualification.""" + +from __future__ import annotations + +import hashlib +import re +import tomllib +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from ml4t_ecosystem.clients import EvidenceError, QueueGitHub +from ml4t_ecosystem.models import EcosystemConfig +from ml4t_ecosystem.monitor import monitor_items + + +@dataclass(frozen=True) +class MaintainedRepository: + key: str + repository: str + + +def load_maintained_repositories(path: Path) -> tuple[MaintainedRepository, ...]: + """Validate full GitHub identities without changing stable qualification scope.""" + with path.open("rb") as stream: + raw = tomllib.load(stream) + rows = raw.get("repositories") + if raw.get("schema_version") != 1 or not isinstance(rows, list) or not rows: + raise ValueError("maintained inventory needs schema_version 1 and repositories") + result = [] + for row in rows: + if ( + not isinstance(row, dict) + or not isinstance(row.get("key"), str) + or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", row["key"]) + or not isinstance(row.get("repository"), str) + or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", row["repository"]) + ): + raise ValueError("maintained repository requires a key and owner/name identity") + result.append(MaintainedRepository(row["key"], row["repository"])) + if len({r.key for r in result}) != len(result) or len( + {r.repository.casefold() for r in result} + ) != len(result): + raise ValueError("maintained repository keys and identities must be unique") + return tuple(result) + + +def _queue_item(item: dict[str, Any], repository: str) -> dict[str, Any]: + number = item.get("number") + kind = "pull_request" if "pull_request" in item else "issue" + suffix = "pull" if kind == "pull_request" else "issues" + url = f"https://github.com/{repository}/{suffix}/{number}" + if type(number) is not int or number <= 0 or item.get("html_url") != url: + raise EvidenceError(f"Invalid issue identity in {repository}") + if item.get("state") != "open": + raise EvidenceError(f"Non-open item returned by open queue: {url}") + labels = item.get("labels") + if not isinstance(labels, list) or any( + not isinstance(label, dict) or not isinstance(label.get("name"), str) for label in labels + ): + raise EvidenceError(f"Invalid labels: {url}") + for field in ("created_at", "updated_at"): + value = item.get(field) + try: + if not isinstance(value, str): + raise ValueError("timestamp is not text") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("timestamp lacks timezone") + except (AttributeError, TypeError, ValueError) as error: + raise EvidenceError(f"Invalid {field}: {url}") from error + body = item.get("body") or "" + if not isinstance(body, str) or not isinstance(item.get("title"), str): + raise EvidenceError(f"Invalid issue text: {url}") + return { + "number": number, + "kind": kind, + "title": item.get("title", ""), + "url": url, + "state": "open", + "created_at": item["created_at"], + "updated_at": item["updated_at"], + "labels": sorted(label["name"] for label in labels), + "body_sha256": hashlib.sha256(body.encode()).hexdigest(), + "evidence_level": "metadata_only", + "inspected_revision": None, + } + + +def collect_queues( + config: EcosystemConfig, + repositories: tuple[MaintainedRepository, ...], + github: QueueGitHub, + *, + now: datetime | None = None, +) -> dict[str, Any]: + """Collect every queue, retaining partial failures as missing evidence.""" + observed = now or datetime.now(UTC) + if not repositories: + raise ValueError("No maintained repositories configured") + queues = [] + findings = [] + for entry in repositories: + owner, repository = entry.repository.split("/") + queue: dict[str, Any] = { + "library": entry.key, + "repository": entry.repository, + "observed_at": observed.isoformat(), + "complete": False, + "evidence_level": "metadata_only", + "items": [], + "errors": [], + } + queues.append(queue) + try: + raw_items = github.open_issues(owner, repository) + seen = set() + for raw in raw_items: + item = _queue_item(raw, entry.repository) + if item["number"] in seen: + raise EvidenceError(f"Duplicate queue item: {item['url']}") + seen.add(item["number"]) + queue["items"].append(item) + if item["kind"] == "pull_request": + try: + pull = github.pull_request(owner, repository, item["number"]) + if pull.get("number") != item["number"] or pull.get("state") != "open": + raise EvidenceError(f"PR changed during collection: {item['url']}") + for side in ("head", "base"): + revision = pull.get(side) + sha = revision.get("sha") if isinstance(revision, dict) else None + if not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}", sha): + raise EvidenceError(f"Missing PR {side} revision: {item['url']}") + item[side + "_revision"] = sha + item["draft"] = pull.get("draft") + item["mergeable_state"] = pull.get("mergeable_state") + except EvidenceError as error: + item["evidence_error"] = str(error) + queue["errors"].append(str(error)) + results = monitor_items( + config, entry.key, owner, repository, github, raw_items, observed + ) + findings.extend(result.to_dict() for result in results) + queue["errors"].extend( + result.message for result in results if result.code.startswith("monitor.evidence") + ) + queue["complete"] = not queue["errors"] + except EvidenceError as error: + queue["errors"].append(str(error)) + for message in queue["errors"]: + if any(f["library"] == entry.key and f["message"] == message for f in findings): + continue + findings.append( + { + "library": entry.key, + "number": 0, + "kind": "repository", + "code": "monitor.evidence", + "message": message, + "url": f"https://github.com/{entry.repository}", + } + ) + return { + "schema_version": 2, + "observed_at": observed.isoformat(), + "complete": all(queue["complete"] for queue in queues), + "repositories": queues, + "findings": findings, + } diff --git a/src/ml4t_ecosystem/monitor.py b/src/ml4t_ecosystem/monitor.py index 3984942..5fc5562 100644 --- a/src/ml4t_ecosystem/monitor.py +++ b/src/ml4t_ecosystem/monitor.py @@ -98,7 +98,6 @@ def monitor_library( ) -> list[MonitorFinding]: """Return current response-target findings for a library.""" observed = now or datetime.now(UTC) - findings: list[MonitorFinding] = [] try: items = github.open_issues(config.owner, library.repository) except EvidenceError as error: @@ -113,6 +112,22 @@ def monitor_library( ) ] + return monitor_items( + config, library.key, config.owner, library.repository, github, items, observed + ) + + +def monitor_items( + config: EcosystemConfig, + key: str, + owner: str, + repository: str, + github: MonitorGitHub, + items: list[dict[str, Any]], + observed: datetime, +) -> list[MonitorFinding]: + """Evaluate an already collected queue with its actual repository owner.""" + findings: list[MonitorFinding] = [] for item in items: number = item.get("number") if not isinstance(number, int): @@ -124,9 +139,7 @@ def monitor_library( created = _parse_timestamp(item.get("created_at")) except ValueError: findings.append( - MonitorFinding( - library.key, number, kind, "monitor.timestamp", "Missing creation time", url - ) + MonitorFinding(key, number, kind, "monitor.timestamp", "Missing creation time", url) ) continue age_minutes = (observed - created).total_seconds() / 60 @@ -134,7 +147,7 @@ def monitor_library( if age_minutes > config.policy.classification_target_minutes and not _classified(labels): findings.append( MonitorFinding( - library.key, + key, number, kind, "monitor.classification-overdue", @@ -147,21 +160,19 @@ def monitor_library( if elapsed_business_days <= config.policy.response_target_business_days: continue try: - comments = github.issue_comments(config.owner, library.repository, number) + comments = github.issue_comments(owner, repository, number) responses = list(comments) if is_pull: - responses.extend(github.pull_reviews(config.owner, library.repository, number)) + responses.extend(github.pull_reviews(owner, repository, number)) except EvidenceError as error: - findings.append( - MonitorFinding(library.key, number, kind, "monitor.evidence", str(error), url) - ) + findings.append(MonitorFinding(key, number, kind, "monitor.evidence", str(error), url)) continue responded = _maintainer_response(responses, config.policy.maintainer_logins) pending = "status: pending-review" in labels and _pending_review_has_date(item, comments) if not responded and not pending: findings.append( MonitorFinding( - library.key, + key, number, kind, "monitor.response-overdue", diff --git a/tests/fixtures/monitor-github.json b/tests/fixtures/monitor-github.json new file mode 100644 index 0000000..4978f47 --- /dev/null +++ b/tests/fixtures/monitor-github.json @@ -0,0 +1,132 @@ +{ + "source": "GitHub REST public responses, captured 2026-10-13; selected fields retained.", + "data": { + "repository": "ml4t/data", + "pages": [ + [ + { + "number": 80, + "title": "ci: enforce minimum package coverage", + "html_url": "https://github.com/ml4t/data/pull/80", + "state": "open", + "created_at": "2026-10-10T09:21:16Z", + "updated_at": "2026-10-13T00:29:12Z", + "labels": [ + { + "name": "type: maintenance" + }, + { + "name": "priority: normal" + }, + { + "name": "status: accepted" + }, + { + "name": "compatibility: none" + }, + { + "name": "ecosystem" + } + ], + "user": { + "login": "stefan-jansen" + }, + "body": "## Summary\n\n- set the package coverage floor to 85%, below the fresh 85.98% baseline\n- add a dedicated pull-request coverage job\n- enforce the same coverage lane during release qualification\n- make the README verification command match CI\n\n## Verification\n\n- focused release tests fail at 14.54%, proving the threshold can fail\n- full offline suite passes at 85.98% with 3,759 tests\n- Ruff, formatting, ty, actionlint, and diff checks pass\n\nCloses #79", + "comments": 0, + "pull_request": { + "url": "https://api.github.com/repos/ml4t/data/pulls/80", + "html_url": "https://github.com/ml4t/data/pull/80", + "diff_url": "https://github.com/ml4t/data/pull/80.diff", + "patch_url": "https://github.com/ml4t/data/pull/80.patch", + "merged_at": null + } + } + ], + [ + { + "number": 79, + "title": "Enforce the documented coverage threshold in CI", + "html_url": "https://github.com/ml4t/data/issues/79", + "state": "open", + "created_at": "2026-10-10T09:18:48Z", + "updated_at": "2026-10-13T00:29:08Z", + "labels": [ + { + "name": "type: maintenance" + }, + { + "name": "priority: normal" + }, + { + "name": "status: accepted" + }, + { + "name": "compatibility: none" + }, + { + "name": "ecosystem" + } + ], + "user": { + "login": "stefan-jansen" + }, + "body": "## Problem\n\nThe repository configures coverage collection and documents a local report, but CI and release qualification run pytest without coverage and `pyproject.toml` defines no enforced minimum. The ecosystem standard requires the test lane to enforce its coverage threshold.\n\n## Acceptance criteria\n\n- `pyproject.toml` defines a conservative threshold supported by a fresh full-suite run.\n- Pull-request CI and release quality collect package coverage and fail below that threshold.\n- Contributor and agent verification commands match the enforced lane.\n- A focused test proves the configured threshold is active.\n- Ruff, ty, tests, resource warnings, strict docs, build, actionlint, and pre-commit pass.", + "comments": 0 + } + ] + ], + "pulls": { + "80": { + "number": 80, + "state": "open", + "html_url": "https://github.com/ml4t/data/pull/80", + "draft": false, + "head": { + "sha": "d826295a022746638216172d1d9a31244eac9d5e" + }, + "base": { + "sha": "18551543395368ad9c48a47bdeb1ca6d604c382f" + }, + "mergeable_state": "clean" + } + } + }, + "empyrical": { + "repository": "stefan-jansen/empyrical-reloaded", + "pages": [ + [ + { + "number": 68, + "title": "Select supported tox environments in CI workflows", + "html_url": "https://github.com/stefan-jansen/empyrical-reloaded/issues/68", + "state": "open", + "created_at": "2026-10-10T09:01:20Z", + "updated_at": "2026-10-10T09:01:20Z", + "labels": [], + "user": { + "login": "stefan-jansen" + }, + "body": "The CI workflows select tox environments that cannot satisfy the package's supported dependencies.\n\nThe Python 3.12 matrix includes pandas 1.5, 2.0 and 2.1 even though pyproject.toml requires pandas >=2.2.2 on Python >=3.12. A current public run of #66 fails while trying to build old pandas with missing pkg_resources; its pandas21 environment also ends up testing a different installed version after package dependency installation. Evidence: https://github.com/stefan-jansen/empyrical-reloaded/actions/runs/38039601555/job/114177029798 .\n\nThe scheduled unit_tests.yml additionally calls `tox -e ci_full_tests`, which is not defined in the tox configuration.\n\nLimit Python 3.12's NumPy 1 environments to pandas 2.2, retain the older pandas coverage on Python 3.10/3.11, and have the scheduled workflow select the configured environments using the existing tox-gh-actions mapping. Verify actual environment creation and test execution, not just configuration text. Keep numerical/test compatibility fixes in their separate PRs.\n\nAI assistance: Codex inspected the actual CI job log, reproduced the undefined environment selection and drafted this issue under maintainer authorization.\n", + "comments": 0 + } + ], + [ + { + "number": 67, + "title": "Make performance-attribution concat sorting explicit for pandas 3", + "html_url": "https://github.com/stefan-jansen/empyrical-reloaded/issues/67", + "state": "open", + "created_at": "2026-10-10T08:57:19Z", + "updated_at": "2026-10-10T08:57:19Z", + "labels": [], + "user": { + "login": "stefan-jansen" + }, + "body": "On pandas 3, the public perf_attrib workflow emits a warning that implicit sorting of DatetimeIndex objects during concat is deprecated. #53 and #55 both propose making sort=False explicit at the final concatenation.\n\nMain efe60664f0abd957074b0751ef82228594a659a5 reproduces the warning in the existing performance-attribution fixture regression with Python 3.12.11 / pandas 3.0.6. Treat the warning as an error while retaining the expected output and alignment assertions, then verify the focused fix passes current and older pandas.\n\nThe change must preserve the dates, exposures and attribution values; it is separate from rolling results and SciPy test warning compatibility.\n\nAI assistance: Codex reproduced the baseline warning, inspected #53/#55 and drafted this issue under maintainer authorization.\n", + "comments": 0 + } + ] + ], + "pulls": {} + } +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 69497b1..47b1551 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -82,7 +82,10 @@ def close(self) -> None: finding = MonitorFinding("data", 1, "issue", "overdue", "message", "url") monkeypatch.setenv("GITHUB_TOKEN", "test-token") monkeypatch.setattr("ml4t_ecosystem.cli.GitHubClient", Client) - monkeypatch.setattr("ml4t_ecosystem.cli.monitor_all", lambda config, github: [finding]) + monkeypatch.setattr( + "ml4t_ecosystem.cli.collect_queues", + lambda config, repositories, github: {"complete": True, "findings": [finding.to_dict()]}, + ) output = tmp_path / "monitor.json" assert main(["monitor", "--output", str(output)]) == 1 diff --git a/tests/test_maintained_monitor.py b/tests/test_maintained_monitor.py new file mode 100644 index 0000000..18d610d --- /dev/null +++ b/tests/test_maintained_monitor.py @@ -0,0 +1,291 @@ +import json +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path + +import httpx +import pytest + +from ml4t_ecosystem.cli import main +from ml4t_ecosystem.clients import EvidenceError, GitHubClient +from ml4t_ecosystem.config import load_config +from ml4t_ecosystem.maintenance import ( + MaintainedRepository, + collect_queues, + load_maintained_repositories, +) + +CAPTURE = json.loads(Path("tests/fixtures/monitor-github.json").read_text()) + + +def test_open_queue_does_not_drop_items_after_first_github_page() -> None: + pages = CAPTURE["data"]["pages"] + + def handler(request: httpx.Request) -> httpx.Response: + page = int(request.url.params.get("page", "1")) + headers = {} + if page == 1: + headers["Link"] = ( + "; rel="next"' + ) + return httpx.Response(200, json=pages[page - 1], headers=headers) + + client = GitHubClient(transport=httpx.MockTransport(handler)) + try: + items = client.open_issues("ml4t", "data") + finally: + client.close() + assert [item["number"] for item in items] == [item["number"] for page in pages for item in page] + + +def test_monitor_cli_includes_reloaded_owner_without_expanding_qualification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + requested = set() + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/issues"): + requested.add(request.url.path.removeprefix("/repos/").removesuffix("/issues")) + return httpx.Response(200, json=[]) + + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + monkeypatch.setattr( + "ml4t_ecosystem.cli.GitHubClient", + lambda **kwargs: GitHubClient(transport=httpx.MockTransport(handler)), + ) + assert main(["monitor", "--output", str(tmp_path / "monitor.json")]) == 0 + assert requested == { + "ml4t/data", + "ml4t/engineer", + "ml4t/backtest", + "ml4t/specs", + "ml4t/live", + "ml4t/diagnostic", + "ml4t/models", + "stefan-jansen/zipline-reloaded", + "stefan-jansen/pyfolio-reloaded", + "stefan-jansen/alphalens-reloaded", + "stefan-jansen/empyrical-reloaded", + } + assert main(["validate-config"]) == 0 + assert "Validated 7 libraries" in capsys.readouterr().out + + +def replay(request: httpx.Request) -> httpx.Response: + for name in ("data", "empyrical"): + entry = CAPTURE[name] + root = "/repos/" + entry["repository"] + if request.url.path == root + "/issues": + return httpx.Response(200, json=[item for page in entry["pages"] for item in page]) + for number, pull in entry["pulls"].items(): + if request.url.path == root + "/pulls/" + number: + return httpx.Response(200, json=pull) + if request.url.path.startswith(root) and request.url.path.endswith( + ("/comments", "/reviews") + ): + return httpx.Response(200, json=[]) + return httpx.Response(404) + + +def collect(handler=replay): + client = GitHubClient(transport=httpx.MockTransport(handler)) + try: + return collect_queues( + load_config(Path("config/libraries.toml")), + ( + MaintainedRepository("data", "ml4t/data"), + MaintainedRepository("empyrical-reloaded", "stefan-jansen/empyrical-reloaded"), + ), + client, + now=datetime(2027, 1, 1, tzinfo=UTC), + ) + finally: + client.close() + + +def test_collector_preserves_real_repository_and_pr_revision_without_claiming_review() -> None: + report = collect() + assert report["complete"] is True + assert len(report["repositories"]) == 2 + for queue, name in zip(report["repositories"], ("data", "empyrical"), strict=True): + assert queue["repository"] == CAPTURE[name]["repository"] + assert queue["observed_at"] == "2027-01-01T00:00:00+00:00" + assert len(queue["items"]) == 2 + for item in queue["items"]: + assert item["evidence_level"] == "metadata_only" + assert item["inspected_revision"] is None + if item["kind"] == "pull_request": + assert ( + item["head_revision"] + == CAPTURE[name]["pulls"][str(item["number"])]["head"]["sha"] + ) + + +@pytest.mark.parametrize("failure", ["repository", "pull", "comments", "transport", "json"]) +def test_partial_api_failure_never_becomes_a_clean_complete_queue(failure: str) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if "/repos/ml4t/data" in request.url.path: + if failure == "transport": + raise httpx.ReadTimeout("unavailable", request=request) + if failure == "json": + return httpx.Response(200, content="invalid JSON") + if ( + (failure == "repository" and request.url.path.endswith("/issues")) + or (failure == "pull" and "/pulls/" in request.url.path) + or (failure == "comments" and request.url.path.endswith("/comments")) + ): + return httpx.Response(503) + return replay(request) + + report = collect(handler) + assert report["complete"] is False + assert report["repositories"][0]["complete"] is False + assert report["repositories"][0]["errors"] + assert report["repositories"][1]["complete"] is True + assert any(f["code"] == "monitor.evidence" for f in report["findings"]) + + +def test_second_page_failure_does_not_publish_first_page_as_complete() -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path in ("/repos/ml4t/data/issues", "/repositories/1089877520/issues"): + if request.url.params.get("page") == "2": + return httpx.Response(502) + return httpx.Response( + 200, + json=CAPTURE["data"]["pages"][0], + headers={ + "Link": ( + '; rel="next"' + ) + }, + ) + return replay(request) + + report = collect(handler) + assert report["repositories"][0]["complete"] is False + assert report["repositories"][0]["errors"] + + +@pytest.mark.parametrize("endpoint", ["comments", "reviews"]) +def test_maintainer_response_on_later_page_is_not_reported_overdue(endpoint: str) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/" + endpoint): + if request.url.params.get("page") == "2": + return httpx.Response(200, json=[{"author_association": "OWNER"}]) + resource = "/".join(request.url.path.split("/")[4:]) + repository_id = 1089877520 if "/ml4t/data/" in request.url.path else 341662136 + return httpx.Response( + 200, + json=[{"author_association": "NONE"}], + headers={ + "Link": ( + f"; rel="next"' + ) + }, + ) + return replay(request) + + report = collect(handler) + assert report["complete"] is True + pull_number = int(next(iter(CAPTURE["data"]["pulls"]))) + assert not any( + f["library"] == "data" + and f["number"] == pull_number + and f["code"] == "monitor.response-overdue" + for f in report["findings"] + ) + + +@pytest.mark.parametrize( + "defect", + ["identity", "closed", "labels", "timestamp", "text", "duplicate", "revision", "pr_closed"], +) +def test_invalid_or_racing_github_record_remains_missing_evidence(defect: str) -> None: + def handler(request: httpx.Request) -> httpx.Response: + response = replay(request) + if request.url.path == "/repos/ml4t/data/issues": + records = deepcopy(response.json()) + if defect == "identity": + records[0]["html_url"] = "https://github.com/wrong/repo/pull/80" + if defect == "closed": + records[0]["state"] = "closed" + if defect == "labels": + records[0]["labels"] = None + if defect == "timestamp": + records[0]["updated_at"] = None + if defect == "text": + records[0]["body"] = 123 + if defect == "duplicate": + records.append(records[0]) + return httpx.Response(200, json=records) + if request.url.path.startswith("/repos/ml4t/data/pulls/") and not request.url.path.endswith( + "/reviews" + ): + pull = deepcopy(response.json()) + if defect == "revision": + pull["head"] = None + if defect == "pr_closed": + pull["state"] = "closed" + return httpx.Response(200, json=pull) + return response + + assert collect(handler)["complete"] is False + + +@pytest.mark.parametrize( + "next_url", + [ + "https://example.com/stolen", + "https://api.github.com/repositories/1089877520/pulls?page=2", + "https://api.github.com/repos/ml4t/engineer/issues?page=2", + "https://api.github.com/repos/ml4t/data/issues?per_page=100&state=open&sort=created", + ], +) +def test_pagination_rejects_foreign_and_repeated_links(next_url: str) -> None: + calls = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(str(request.url)) + assert request.url.host == "api.github.com" + return httpx.Response(200, json=[], headers={"Link": f'<{next_url}>; rel="next"'}) + + client = GitHubClient(token="test-token", transport=httpx.MockTransport(handler)) + try: + with pytest.raises(EvidenceError, match="pagination link"): + client.open_issues("ml4t", "data") + finally: + client.close() + assert len(calls) <= 2 + + +@pytest.mark.parametrize( + "contents", + [ + "schema_version=2\nrepositories=[]", + 'schema_version=1\n[[repositories]]\nkey="data"\nrepository="data"', + 'schema_version=1\n[[repositories]]\nkey="data"\nrepository="ml4t/data"\n[[repositories]]\nkey="data"\nrepository="ML4T/Data"', + ], +) +def test_invalid_inventory_is_rejected_before_collecting(contents: str, tmp_path: Path) -> None: + path = tmp_path / "inventory.toml" + path.write_text(contents) + with pytest.raises(ValueError): + load_maintained_repositories(path) + + +def test_cli_evidence_failure_writes_failure_report_and_returns_two(monkeypatch, tmp_path) -> None: + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + monkeypatch.setattr( + "ml4t_ecosystem.cli.GitHubClient", + lambda **kwargs: GitHubClient( + transport=httpx.MockTransport(lambda request: httpx.Response(503)) + ), + ) + output = tmp_path / "monitor.json" + assert main(["monitor", "--output", str(output)]) == 2 + report = json.loads(output.read_text()) + assert report["complete"] is False + assert len(report["repositories"]) == 11 + assert all(queue["errors"] for queue in report["repositories"])