diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d1a03a..01ccba5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,20 @@ jobs: # what this step enforces. - name: Pyright type check run: uv run pyright + - name: FrontierChallenge release gates + run: | + uv run pytest -q \ + -c benchmarks/frontierchallenge/pyproject.toml \ + benchmarks/frontierchallenge/tests + uv run python benchmarks/frontierchallenge/scripts/check_public_leaks.py \ + benchmarks/frontierchallenge --allow-empty + uv run python benchmarks/frontierchallenge/scripts/check_restricted_software.py \ + benchmarks/frontierchallenge + for script in \ + benchmarks/frontierchallenge/scripts/*.sh \ + benchmarks/frontierchallenge/shared_images/*.sh; do + bash -n "$script" + done - name: Unit and exec-bundle tests env: SANDBOX_BACKEND: bwrap diff --git a/benchmarks/frontierchallenge/.github/workflows/checks.yml b/benchmarks/frontierchallenge/.github/workflows/checks.yml deleted file mode 100644 index 3949f12..0000000 --- a/benchmarks/frontierchallenge/.github/workflows/checks.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: checks - -# Guard the boundary between this runtime repository and the two HF datasets. - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -jobs: - leak-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: No task payload committed - run: | - found=$(git ls-files 'tasks/**' | head -20) - if [ -n "$found" ]; then - echo "::error::task payload belongs on Hugging Face, not in Git:" - echo "$found" - exit 1 - fi - echo "no tracked task payload" - - - name: No answer material readable in the clear - run: python3 scripts/check_public_leaks.py . --allow-empty - - - name: Task set matches the registry - run: | - python3 - <<'PY' - import json - registry = json.load(open("registry.json")) - assert registry["n_tasks"] == len(registry["tasks"]) == 97 - assert len({row["id"] for row in registry["tasks"]}) == 97 - print("public registry: 97 unique task commitments") - PY - - - name: No restricted software redistribution path - run: python3 scripts/check_restricted_software.py - - tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install - run: python -m pip install -q -e ".[dev]" - - name: Unit tests - run: python -m pytest - - shell: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Shell scripts parse - run: | - for script in scripts/*.sh shared_images/*.sh; do - bash -n "$script" || exit 1 - done - echo "all shell scripts parse" - - - name: Python tooling parses - run: | - for script in scripts/*.py tests/*.py; do - python3 -m py_compile "$script" || exit 1 - done - echo "all python tooling parses" diff --git a/benchmarks/frontierchallenge/README.md b/benchmarks/frontierchallenge/README.md index 67332ca..ca1554d 100644 --- a/benchmarks/frontierchallenge/README.md +++ b/benchmarks/frontierchallenge/README.md @@ -1,7 +1,12 @@ -

- FrontierChallenge   - Apodex AI -

+
+ + Apodex AI + +
+ +--- + +
@@ -9,10 +14,13 @@ [Tasks](TASKS.md) · [Quickstart](docs/quickstart.md) · [Hugging Face](https://huggingface.co/datasets/apodex/FrontierChallenge) · -[Website](https://urban-chainsaw-mnormyp.pages.github.io/) · [Citation](#citation) +[Website](https://apodexai.github.io/FrontierAgent/benchmarks/FrontierChallenge/) · +[Citation](#citation)
+# FrontierChallenge + FrontierChallenge asks an agent to turn raw scientific data into specified, machine-checkable deliverables. It covers diffraction, spectroscopy, molecular simulation, electrochemistry, quantitative imaging, and molecular biology. diff --git a/benchmarks/frontierchallenge/site/README.md b/benchmarks/frontierchallenge/site/README.md index b2e9a55..98b4517 100644 --- a/benchmarks/frontierchallenge/site/README.md +++ b/benchmarks/frontierchallenge/site/README.md @@ -4,7 +4,7 @@ This directory contains the dependency-free static website for the benchmark. It lives in the public FrontierChallenge repository so website content, task documentation, and leaderboard links can evolve together. -Published preview: +Published site: ## Preview locally