From f96e4d8eb52a39878ca25fbc33512b85ad8403c1 Mon Sep 17 00:00:00 2001 From: Scr4tch587 Date: Fri, 10 Jul 2026 00:04:21 -0400 Subject: [PATCH] feat: dataset proposal service and endpoint Co-Authored-By: Claude Fable 5 --- builders/server/core/api/routes.py | 95 +- builders/server/core/service/proposals.py | 397 +++++ builders/server/pyproject.toml | 1 + .../tests/core/api/test_proposal_route.py | 91 ++ .../tests/core/service/test_proposals.py | 325 ++++ infra/.env.template | 10 + uv.lock | 1431 +++++++++-------- 7 files changed, 1645 insertions(+), 705 deletions(-) create mode 100644 builders/server/core/service/proposals.py create mode 100644 builders/server/tests/core/api/test_proposal_route.py create mode 100644 builders/server/tests/core/service/test_proposals.py diff --git a/builders/server/core/api/routes.py b/builders/server/core/api/routes.py index 9a2f883..b55ed16 100644 --- a/builders/server/core/api/routes.py +++ b/builders/server/core/api/routes.py @@ -1,9 +1,12 @@ from datetime import datetime import structlog -from fastapi import APIRouter, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict, Field +from core.auth import verify_api_key +from core.github import GitHubError from core.service.builder import ( DatasetNotFoundError, NoDataInRangeError, @@ -13,6 +16,13 @@ get_data, ) from core.service.catalog import list_datasets +from core.service.proposals import ( + DatasetProposal, + InvalidProposalError, + ProposalConflictError, + ProposedDependency, + propose_dataset, +) from core.utils.semver import SemVer logger = structlog.get_logger() @@ -45,6 +55,89 @@ def datasets_list() -> dict: } +class DependencyIn(BaseModel): + name: str + version: str + lookback: str | None = None + + +class DatasetProposalIn(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + name: str + version: str + calendar: str + granularity: str + start_date: str + builder_script: str + author_name: str + team: str + discord_user: str + description: str + # 'schema' shadows a deprecated BaseModel attr, so alias it + data_schema: dict[str, str] = Field(alias="schema") + dependencies: list[DependencyIn] = Field(default_factory=list) + env_vars: bool = False + requirements_txt: str | None = None + env_template: str | None = None + + +@router.post("/datasets") +def datasets_propose( + payload: DatasetProposalIn, team: str = Depends(verify_api_key) +) -> dict: + """Propose a new dataset: validate the submission and open a GitHub PR. + + Nothing is written to the server; the dataset goes live only after the + PR is reviewed, merged, and the server restarts. + """ + structlog.contextvars.bind_contextvars( + dataset_name=payload.name, version=payload.version + ) + proposal = DatasetProposal( + name=payload.name, + version=payload.version, + calendar=payload.calendar, + granularity=payload.granularity, + start_date=payload.start_date, + schema=payload.data_schema, + builder_script=payload.builder_script, + author_name=payload.author_name, + team=payload.team, + discord_user=payload.discord_user, + description=payload.description, + dependencies=[ + ProposedDependency(name=d.name, version=d.version, lookback=d.lookback) + for d in payload.dependencies + ], + env_vars=payload.env_vars, + requirements_txt=payload.requirements_txt, + env_template=payload.env_template, + ) + + try: + result = propose_dataset(proposal, requested_by=team) + except InvalidProposalError as e: + logger.warning("proposal rejected", error=str(e)) + raise HTTPException(status_code=400, detail=str(e)) from e + except ProposalConflictError as e: + logger.warning("proposal conflict", error=str(e)) + raise HTTPException(status_code=409, detail=str(e)) from e + except GitHubError as e: + logger.exception("proposal github call failed") + raise HTTPException(status_code=502, detail=f"github error: {e.message}") from e + except Exception as e: + logger.exception("proposal failed") + raise HTTPException(status_code=500, detail=str(e)) from e + + return { + "dataset_name": payload.name, + "dataset_version": payload.version, + "pr_url": result.pr_url, + "branch": result.branch, + } + + @router.post("/build/{dataset_name}/{dataset_version}") def build( dataset_name: str, diff --git a/builders/server/core/service/proposals.py b/builders/server/core/service/proposals.py new file mode 100644 index 0000000..3689fb7 --- /dev/null +++ b/builders/server/core/service/proposals.py @@ -0,0 +1,397 @@ +"""Dataset proposal service: validate a submission and open a GitHub PR. + +Proposed datasets are never written to the server's scripts directory — code +review stays the gate for anything that will execute on the server. This +module turns a submission into the exact files that will land in +builders/scripts///, re-validates the generated config bytes +with the same checks the server runs at startup, and opens the PR via the +github client. +""" + +import ast +import json +import os +import re +import subprocess +import sys +import tempfile +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol + +import structlog + +from core.github import BranchAlreadyExistsError, GitHubClient +from core.runtime import registry +from core.runtime.config import DatasetConfig, normalize_config, validate_config +from core.utils.semver import SemVer + +logger = structlog.get_logger() + +# dataset names become directory names, branch names, and toml keys +_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") +_BARE_TOML_KEY_RE = re.compile(r"^[A-Za-z0-9_-]+$") + +SCRIPTS_PREFIX = "builders/scripts" + +# keep in sync with [tool.ruff.lint] select in the repo-root pyproject.toml: +# proposal prs must pass the same ci lint that runs over builders/scripts/** +RUFF_SELECT = "B,E,F,I,PIE,SIM,T20,UP" +RUFF_TIMEOUT_SECONDS = 30 + +# github logins asked to review every proposal pr (comma-separated env override) +DEFAULT_REVIEWERS = "Blackgaurd,Scr4tch587" + + +class InvalidProposalError(ValueError): + """Submission failed validation; safe to show the message to the caller.""" + + +class ProposalConflictError(Exception): + """The dataset or its proposal branch already exists.""" + + +@dataclass(frozen=True) +class ProposedDependency: + name: str + version: str + lookback: str | None = None + + +@dataclass(frozen=True) +class DatasetProposal: + name: str + version: str + calendar: str + granularity: str + start_date: str + schema: dict[str, str] + builder_script: str + # who is proposing and why -- surfaced in the pr body for reviewers + author_name: str + team: str + discord_user: str + description: str + dependencies: list[ProposedDependency] = field(default_factory=list) + env_vars: bool = False + requirements_txt: str | None = None + env_template: str | None = None + + +@dataclass(frozen=True) +class ProposalResult: + pr_url: str + branch: str + + +class PullRequestOpener(Protocol): + """The slice of GitHubClient the proposal service needs (test-fakeable).""" + + def open_pr_with_files( + self, + branch: str, + base: str, + title: str, + body: str, + commit_message: str, + files: dict[str, str], + reviewers: list[str] | None = None, + ) -> str: ... + + +def _toml_str(value: str) -> str: + """quote a string as a toml basic string + + (json escaping is a valid subset). + """ + return json.dumps(value) + + +def _toml_key(key: str) -> str: + return key if _BARE_TOML_KEY_RE.fullmatch(key) else json.dumps(key) + + +def generate_config_toml(proposal: DatasetProposal) -> str: + """Render the canonical config.toml for a proposal. + + The returned text is exactly what gets committed; callers re-parse and + re-validate these bytes so the PR can never contain a config that fails + server startup. + """ + lines = [ + f"name = {_toml_str(proposal.name)}", + f"version = {_toml_str(proposal.version)}", + 'builder = "builder.py"', + f"calendar = {_toml_str(proposal.calendar)}", + f"granularity = {_toml_str(proposal.granularity)}", + f"start-date = {_toml_str(proposal.start_date)}", + ] + if proposal.env_vars: + lines.append("env-vars = true") + + lines += ["", "[schema]"] + lines += [ + f"{_toml_key(key)} = {_toml_str(type_)}" + for key, type_ in proposal.schema.items() + ] + + if proposal.dependencies: + lines += ["", "[dependencies]"] + for dep in proposal.dependencies: + if dep.lookback: + lines.append( + f"{_toml_key(dep.name)} = {{ version = {_toml_str(dep.version)}," + f" lookback = {_toml_str(dep.lookback)} }}" + ) + else: + lines.append(f"{_toml_key(dep.name)} = {_toml_str(dep.version)}") + + return "\n".join(lines) + "\n" + + +def _validate_builder_script(script: str) -> None: + """Same convention the ci gate enforces: + + parseable, top-level build(dependencies, timestamp). + """ + try: + tree = ast.parse(script) + except SyntaxError as e: + raise InvalidProposalError(f"builder script has a syntax error: {e}") from e + + build_fns = [ + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "build" + ] + if not build_fns: + raise InvalidProposalError( + "builder script must define a top-level build() function" + ) + args = build_fns[0].args + positional = args.posonlyargs + args.args + if len(positional) != 2: + raise InvalidProposalError( + "build() must take exactly two arguments (dependencies, timestamp), " + f"got {[a.arg for a in positional]}" + ) + + +def _lint_builder_script(script: str) -> str: + """Autofix + format the script with ruff so the proposal pr passes repo ci. + + Returns the cleaned script. Raises InvalidProposalError when violations + remain after autofix (the message includes ruff's output). + """ + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "builder.py" + path.write_text(script) + common = [sys.executable, "-m", "ruff"] + check = subprocess.run( + [*common, "check", "--isolated", "--select", RUFF_SELECT, "--fix", str(path)], + capture_output=True, + text=True, + timeout=RUFF_TIMEOUT_SECONDS, + ) + if check.returncode != 0: + raise InvalidProposalError( + f"builder script fails lint:\n{check.stdout.strip()}" + ) + fmt = subprocess.run( + [*common, "format", "--isolated", str(path)], + capture_output=True, + text=True, + timeout=RUFF_TIMEOUT_SECONDS, + ) + if fmt.returncode != 0: + raise InvalidProposalError( + f"builder script fails formatting:\n{fmt.stderr.strip()}" + ) + return path.read_text() + + +def _validate_against_registry(cfg: DatasetConfig) -> None: + """Cross-checks against currently registered datasets. + + A new dataset is a leaf no existing config references, so it cannot + introduce a cycle — only dep existence, granularity, and start-date + ordering need checking (mirrors registry startup validation). + """ + for dep_name, dep_info in cfg.dependencies.items(): + try: + dep_cfg = registry.get_config(dep_name, dep_info.version) + except ValueError as e: + raise InvalidProposalError( + f"dependency {dep_name}/{dep_info.version} is not a known dataset" + ) from e + if cfg.granularity < dep_cfg.granularity: + raise InvalidProposalError( + f"granularity is finer than dependency {dep_name}/{dep_info.version}" + ) + if cfg.start_date < dep_cfg.start_date: + raise InvalidProposalError( + f"start-date is before dependency {dep_name}/{dep_info.version}'s " + f"start-date ({dep_cfg.start_date.date()})" + ) + + +def _build_pr_body( + proposal: DatasetProposal, requested_by: str, dataset_dir: str +) -> str: + schema_lines = "\n".join( + f"- `{key}`: `{type_}`" for key, type_ in proposal.schema.items() + ) + if proposal.dependencies: + dep_lines = "\n".join( + f"- `{dep.name}/{dep.version}`" + + (f" (lookback `{dep.lookback}`)" if dep.lookback else "") + for dep in proposal.dependencies + ) + else: + dep_lines = "- none (root dataset)" + + body = f"""## Dataset proposal: `{proposal.name}/{proposal.version}` + +{proposal.description.strip()} + +**Proposed by:** {proposal.author_name.strip()} · team **{proposal.team.strip()}** \ +· discord `{proposal.discord_user.strip()}` · api key label `{requested_by}` + +| field | value | +|-------|-------| +| calendar | `{proposal.calendar}` | +| granularity | `{proposal.granularity}` | +| start-date | `{proposal.start_date}` | +| env-vars | `{str(proposal.env_vars).lower()}` | + +**Schema** +{schema_lines} + +**Dependencies** +{dep_lines} + +### Review checklist + +- [ ] builder logic reviewed — this code will run on the datastream server +- [ ] schema and dependencies make sense for the data +""" + if proposal.requirements_txt: + body += "- [ ] `requirements.txt` packages reviewed\n" + if proposal.env_vars: + body += ( + f"- [ ] **before first build**: place the real `.env` at " + f"`{dataset_dir}/.env` on the server " + f"(only `.env.template` is committed; see it for required vars)\n" + ) + return body + + +def propose_dataset( + proposal: DatasetProposal, + requested_by: str, + client: PullRequestOpener | None = None, +) -> ProposalResult: + """Validate a proposal and open a PR adding the dataset directory. + + Raises InvalidProposalError (bad submission), ProposalConflictError + (dataset or proposal branch already exists), or GitHubError (github + unreachable / misconfigured). + """ + if not _NAME_RE.fullmatch(proposal.name): + raise InvalidProposalError( + "dataset name must be lowercase alphanumeric with '-' or '_' " + "(it becomes a directory and branch name)" + ) + for field_name, value in ( + ("author_name", proposal.author_name), + ("team", proposal.team), + ("discord_user", proposal.discord_user), + ("description", proposal.description), + ): + if not value.strip(): + raise InvalidProposalError(f"{field_name} must not be empty") + try: + version = SemVer.parse(proposal.version) + except ValueError as e: + raise InvalidProposalError(f"invalid version: {e}") from e + + try: + registry.get_config(proposal.name, version) + except ValueError: + pass + else: + raise ProposalConflictError( + f"dataset {proposal.name}/{proposal.version} already exists" + ) + + # validate the exact bytes that will be committed, with the same code + # paths the server runs at startup + config_toml = generate_config_toml(proposal) + try: + raw = tomllib.loads(config_toml) + validate_config(raw, proposal.name, version) + normalize_config(raw) + cfg = DatasetConfig.from_raw(raw) + except ValueError as e: + raise InvalidProposalError(str(e)) from e + + _validate_against_registry(cfg) + _validate_builder_script(proposal.builder_script) + # commit the linted/formatted script so the pr passes repo ci + builder_script = _lint_builder_script(proposal.builder_script) + + dataset_dir = f"{SCRIPTS_PREFIX}/{proposal.name}/{proposal.version}" + files = { + f"{dataset_dir}/config.toml": config_toml, + f"{dataset_dir}/builder.py": _ensure_trailing_newline(builder_script), + } + if proposal.requirements_txt and proposal.requirements_txt.strip(): + files[f"{dataset_dir}/requirements.txt"] = _ensure_trailing_newline( + proposal.requirements_txt + ) + if proposal.env_template and proposal.env_template.strip(): + files[f"{dataset_dir}/.env.template"] = _ensure_trailing_newline( + proposal.env_template + ) + + branch = f"add-dataset/{proposal.name}-{proposal.version}" + title = f"feat: add dataset {proposal.name}/{proposal.version}" + body = _build_pr_body(proposal, requested_by, dataset_dir) + + github: PullRequestOpener = ( + client if client is not None else GitHubClient.from_env() + ) + reviewers = [ + login.strip() + for login in os.environ.get("GITHUB_REVIEWERS", DEFAULT_REVIEWERS).split(",") + if login.strip() + ] + try: + pr_url = github.open_pr_with_files( + branch=branch, + base="main", + title=title, + body=body, + commit_message=title, + files=files, + reviewers=reviewers, + ) + except BranchAlreadyExistsError as e: + raise ProposalConflictError( + f"a proposal for {proposal.name}/{proposal.version} is already open " + f"(branch '{branch}' exists)" + ) from e + + logger.info( + "dataset proposal submitted", + dataset=proposal.name, + version=proposal.version, + requested_by=requested_by, + pr_url=pr_url, + ) + return ProposalResult(pr_url=pr_url, branch=branch) + + +def _ensure_trailing_newline(text: str) -> str: + return text if text.endswith("\n") else text + "\n" diff --git a/builders/server/pyproject.toml b/builders/server/pyproject.toml index b264677..f46cb5d 100644 --- a/builders/server/pyproject.toml +++ b/builders/server/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "exchange-calendars>=4.13", "python-dotenv>=1.0.0", "structlog>=24.0.0", + "ruff>=0.15.21", ] [dependency-groups] diff --git a/builders/server/tests/core/api/test_proposal_route.py b/builders/server/tests/core/api/test_proposal_route.py new file mode 100644 index 0000000..e0e30e3 --- /dev/null +++ b/builders/server/tests/core/api/test_proposal_route.py @@ -0,0 +1,91 @@ +import core.api.routes as routes +import pytest +from core.github.client import GitHubError +from core.service.proposals import ( + InvalidProposalError, + ProposalConflictError, + ProposalResult, +) +from fastapi.testclient import TestClient +from main import app + +client: TestClient = TestClient(app) + +PAYLOAD = { + "name": "my-dataset", + "version": "0.1.0", + "calendar": "everyday", + "granularity": "1d", + "start_date": "2022-01-01", + "schema": {"price": "float"}, + "builder_script": "def build(deps, ts):\n return []\n", + "author_name": "Kai Zhang", + "team": "quant", + "discord_user": "kai#1234", + "description": "test data", +} + + +def _propose_stub(result=None, error=None): + def stub(proposal, requested_by, client=None): + if error is not None: + raise error + return result + + return stub + + +def test_propose_returns_pr_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub( + result=ProposalResult( + pr_url="https://github.com/acme/data/pull/42", + branch="add-dataset/my-dataset-0.1.0", + ) + ), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 200 + body = res.json() + assert body["pr_url"] == "https://github.com/acme/data/pull/42" + assert body["branch"] == "add-dataset/my-dataset-0.1.0" + assert body["dataset_name"] == "my-dataset" + + +def test_invalid_proposal_maps_to_400(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=InvalidProposalError("bad schema")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 400 + assert "bad schema" in res.json()["detail"] + + +def test_conflict_maps_to_409(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=ProposalConflictError("already exists")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 409 + + +def test_github_failure_maps_to_502(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=GitHubError(500, "boom")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 502 + assert "github error" in res.json()["detail"] + + +def test_missing_fields_rejected_by_validation() -> None: + res = client.post("/api/v1/datasets", json={"name": "x"}) + assert res.status_code == 422 diff --git a/builders/server/tests/core/service/test_proposals.py b/builders/server/tests/core/service/test_proposals.py new file mode 100644 index 0000000..f5350b4 --- /dev/null +++ b/builders/server/tests/core/service/test_proposals.py @@ -0,0 +1,325 @@ +import tomllib +from pathlib import Path +from typing import Any + +import core.runtime.registry as registry +import pytest +from core.github.client import BranchAlreadyExistsError +from core.service.proposals import ( + DatasetProposal, + InvalidProposalError, + ProposalConflictError, + ProposedDependency, + generate_config_toml, + propose_dataset, +) + +_DEP_CONFIG = """\ +name = "mock-dep" +version = "0.1.0" +granularity = "1d" +start-date = "2021-06-01" +calendar = "everyday" + +[schema] +price = "int" +""" + +VALID_BUILDER = """\ +from datetime import datetime + + +def build(dependencies, timestamp: datetime) -> list[dict]: + return [{"ticker": "AAPL", "price": 1}] +""" + + +class FakeGitHub: + """stands in for GitHubClient; records the pr call or raises.""" + + def __init__(self, error: Exception | None = None): + self.error = error + self.calls: list[dict[str, Any]] = [] + + def open_pr_with_files( + self, + branch: str, + base: str, + title: str, + body: str, + commit_message: str, + files: dict[str, str], + reviewers: list[str] | None = None, + ) -> str: + if self.error is not None: + raise self.error + self.calls.append( + { + "branch": branch, + "base": base, + "title": title, + "body": body, + "commit_message": commit_message, + "files": files, + "reviewers": reviewers, + } + ) + return "https://github.com/acme/data/pull/42" + + +@pytest.fixture(autouse=True) +def _registry_with_dep(tmp_path: Path): + """populate the registry with mock-dep/0.1.0; reset afterwards.""" + dep_dir = tmp_path / "mock-dep" / "0.1.0" + dep_dir.mkdir(parents=True) + (dep_dir / "config.toml").write_text(_DEP_CONFIG) + registry.load_all_configs(tmp_path) + yield + registry._CONFIG_REGISTRY = {} + + +def _proposal(**overrides: Any) -> DatasetProposal: + defaults: dict[str, Any] = dict( + name="my-dataset", + version="0.1.0", + calendar="everyday", + granularity="1d", + start_date="2022-01-01", + schema={"ticker": "str", "price": "float"}, + builder_script=VALID_BUILDER, + author_name="Kai Zhang", + team="quant", + discord_user="kai#1234", + description="daily test data for the proposal flow", + ) + defaults.update(overrides) + return DatasetProposal(**defaults) + + +def test_happy_path_opens_pr_with_files() -> None: + github = FakeGitHub() + result = propose_dataset(_proposal(), requested_by="team-a", client=github) + + assert result.pr_url == "https://github.com/acme/data/pull/42" + assert result.branch == "add-dataset/my-dataset-0.1.0" + + call = github.calls[0] + assert call["base"] == "main" + assert call["title"] == "feat: add dataset my-dataset/0.1.0" + files = call["files"] + assert set(files) == { + "builders/scripts/my-dataset/0.1.0/config.toml", + "builders/scripts/my-dataset/0.1.0/builder.py", + } + assert "team-a" in call["body"] + # proposer identity and purpose are surfaced for reviewers + assert "Kai Zhang" in call["body"] + assert "quant" in call["body"] + assert "kai#1234" in call["body"] + assert "daily test data for the proposal flow" in call["body"] + + # the committed config parses and carries the submitted fields + raw = tomllib.loads(files["builders/scripts/my-dataset/0.1.0/config.toml"]) + assert raw["name"] == "my-dataset" + assert raw["schema"] == {"ticker": "str", "price": "float"} + + +def test_dependency_with_lookback_round_trips() -> None: + github = FakeGitHub() + proposal = _proposal( + dependencies=[ + ProposedDependency(name="mock-dep", version="0.1.0", lookback="5d") + ] + ) + propose_dataset(proposal, requested_by="team-a", client=github) + + config = github.calls[0]["files"][ + "builders/scripts/my-dataset/0.1.0/config.toml" + ] + raw = tomllib.loads(config) + assert raw["dependencies"]["mock-dep"] == {"version": "0.1.0", "lookback": "5d"} + + +def test_optional_files_included_when_present() -> None: + github = FakeGitHub() + proposal = _proposal( + env_vars=True, + requirements_txt="pandas>=2.0\n", + env_template="API_KEY=\n", + ) + propose_dataset(proposal, requested_by="team-a", client=github) + + call = github.calls[0] + files = call["files"] + assert "builders/scripts/my-dataset/0.1.0/requirements.txt" in files + assert "builders/scripts/my-dataset/0.1.0/.env.template" in files + # the env checklist reminds reviewers secrets are placed manually + assert ".env" in call["body"] + assert "before first build" in call["body"] + + +def test_env_file_itself_is_never_committed() -> None: + github = FakeGitHub() + propose_dataset( + _proposal(env_vars=True, env_template="API_KEY=\n"), + requested_by="team-a", + client=github, + ) + files = github.calls[0]["files"] + assert "builders/scripts/my-dataset/0.1.0/.env" not in files + + +def test_unknown_dependency_rejected() -> None: + proposal = _proposal( + dependencies=[ProposedDependency(name="nope", version="9.9.9")] + ) + with pytest.raises(InvalidProposalError, match="not a known dataset"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_granularity_finer_than_dependency_rejected() -> None: + proposal = _proposal( + granularity="1h", + dependencies=[ProposedDependency(name="mock-dep", version="0.1.0")], + ) + with pytest.raises(InvalidProposalError, match="finer than dependency"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_start_date_before_dependency_rejected() -> None: + proposal = _proposal( + start_date="2020-01-01", + dependencies=[ProposedDependency(name="mock-dep", version="0.1.0")], + ) + with pytest.raises(InvalidProposalError, match="start-date"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_existing_dataset_conflicts() -> None: + proposal = _proposal(name="mock-dep", version="0.1.0", start_date="2021-06-01") + with pytest.raises(ProposalConflictError, match="already exists"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_existing_branch_conflicts() -> None: + github = FakeGitHub(error=BranchAlreadyExistsError(422, "Reference already exists")) + with pytest.raises(ProposalConflictError, match="already open"): + propose_dataset(_proposal(), requested_by="t", client=github) + + +@pytest.mark.parametrize( + "bad_name", ["MyDataset", "my dataset", "../escape", "-leading", ""] +) +def test_invalid_names_rejected(bad_name: str) -> None: + with pytest.raises(InvalidProposalError, match="dataset name"): + propose_dataset( + _proposal(name=bad_name), requested_by="t", client=FakeGitHub() + ) + + +@pytest.mark.parametrize( + "field_name", ["author_name", "team", "discord_user", "description"] +) +def test_blank_proposer_fields_rejected(field_name: str) -> None: + with pytest.raises(InvalidProposalError, match=field_name): + propose_dataset( + _proposal(**{field_name: " "}), requested_by="t", client=FakeGitHub() + ) + + +def test_invalid_version_rejected() -> None: + with pytest.raises(InvalidProposalError, match="version"): + propose_dataset( + _proposal(version="not-semver"), requested_by="t", client=FakeGitHub() + ) + + +def test_invalid_schema_type_rejected() -> None: + with pytest.raises(InvalidProposalError): + propose_dataset( + _proposal(schema={"price": "decimal"}), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_unknown_calendar_rejected() -> None: + with pytest.raises(InvalidProposalError): + propose_dataset( + _proposal(calendar="lunar"), requested_by="t", client=FakeGitHub() + ) + + +def test_builder_syntax_error_rejected() -> None: + with pytest.raises(InvalidProposalError, match="syntax error"): + propose_dataset( + _proposal(builder_script="def build(:\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_builder_without_build_rejected() -> None: + with pytest.raises(InvalidProposalError, match="build\\(\\)"): + propose_dataset( + _proposal(builder_script="def make(a, b):\n return []\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_builder_wrong_arity_rejected() -> None: + with pytest.raises(InvalidProposalError, match="two arguments"): + propose_dataset( + _proposal(builder_script="def build(only_one):\n return []\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_default_reviewers_requested(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_REVIEWERS", raising=False) + github = FakeGitHub() + propose_dataset(_proposal(), requested_by="t", client=github) + assert github.calls[0]["reviewers"] == ["Blackgaurd", "Scr4tch587"] + + +def test_reviewers_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_REVIEWERS", "alice, bob") + github = FakeGitHub() + propose_dataset(_proposal(), requested_by="t", client=github) + assert github.calls[0]["reviewers"] == ["alice", "bob"] + + +def test_builder_script_is_autofixed_before_commit() -> None: + """unused imports and formatting are cleaned server-side so the pr passes ci.""" + github = FakeGitHub() + script = ( + "from datetime import datetime\n" + "from typing import Any\n\n\n" + "def build(dependencies, timestamp):\n" + " return [{'ticker': 'AAPL', 'price': 1.0}]\n" + ) + propose_dataset(_proposal(builder_script=script), requested_by="t", client=github) + committed = github.calls[0]["files"][ + "builders/scripts/my-dataset/0.1.0/builder.py" + ] + assert "from typing import Any" not in committed # unused import removed + assert "from datetime import datetime" not in committed + assert '"AAPL"' in committed # ruff format normalizes quotes + + +def test_unfixable_lint_error_rejected() -> None: + """violations ruff cannot autofix (e.g. undefined name) reject the proposal.""" + script = "def build(dependencies, timestamp):\n return [undefined_var]\n" + with pytest.raises(InvalidProposalError, match="fails lint"): + propose_dataset( + _proposal(builder_script=script), requested_by="t", client=FakeGitHub() + ) + + +def test_generated_toml_quotes_awkward_schema_keys() -> None: + """schema keys that aren't bare toml keys are quoted, not mangled.""" + proposal = _proposal(schema={"has space": "str", "ok_key": "int"}) + raw = tomllib.loads(generate_config_toml(proposal)) + assert raw["schema"] == {"has space": "str", "ok_key": "int"} diff --git a/infra/.env.template b/infra/.env.template index 89ced49..cfeee7a 100644 --- a/infra/.env.template +++ b/infra/.env.template @@ -14,3 +14,13 @@ DOMAIN=localhost # cd builders/server && uv run python -m core.auth generate